From 12ffd463b41c0819573ea2692d035914c47841ab Mon Sep 17 00:00:00 2001 From: Dave Grote Date: Tue, 7 May 2019 15:19:09 -0700 Subject: Added code for RZ multimode for e and b advance, deposition, gather --- Source/Particles/PhysicalParticleContainer.cpp | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 17e6d98d9..e50aa957f 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1065,6 +1065,7 @@ PhysicalParticleContainer::FieldGather (int lev, BL_TO_FORTRAN_ANYD(bxfab), BL_TO_FORTRAN_ANYD(byfab), BL_TO_FORTRAN_ANYD(bzfab), + &WarpX::nmodes, &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, &lvect_fieldgathe, &WarpX::field_gathering_algo); @@ -1365,6 +1366,7 @@ PhysicalParticleContainer::Evolve (int lev, BL_TO_FORTRAN_ANYD(*bxfab), BL_TO_FORTRAN_ANYD(*byfab), BL_TO_FORTRAN_ANYD(*bzfab), + &WarpX::nmodes, &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, &lvect_fieldgathe, &WarpX::field_gathering_algo); @@ -1462,6 +1464,7 @@ PhysicalParticleContainer::Evolve (int lev, BL_TO_FORTRAN_ANYD(*cbxfab), BL_TO_FORTRAN_ANYD(*cbyfab), BL_TO_FORTRAN_ANYD(*cbzfab), + &WarpX::nmodes, &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, &lvect_fieldgathe, &WarpX::field_gathering_algo); } @@ -1794,6 +1797,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, BL_TO_FORTRAN_ANYD(bxfab), BL_TO_FORTRAN_ANYD(byfab), BL_TO_FORTRAN_ANYD(bzfab), + &WarpX::nmodes, &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, &lvect_fieldgathe, &WarpX::field_gathering_algo); -- cgit v1.2.3 From 21ff79766454d6e386d1b80bbf3b177a252e8a87 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 22 May 2019 11:56:41 -0700 Subject: Convert particle pusher to C++ --- Source/Particles/PhysicalParticleContainer.cpp | 50 ++++++++++++++------------ Source/Particles/Pusher/Make.package | 1 + Source/Particles/Pusher/UpdateMomentumBoris.H | 46 ++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 23 deletions(-) create mode 100644 Source/Particles/Pusher/UpdateMomentumBoris.H (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 37c136a3d..f497ef4e4 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1703,19 +1703,21 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti, Cuda::ManagedDeviceVector& giv, Real dt) { - // This wraps the call to warpx_particle_pusher so that inheritors can modify the call. auto& attribs = pti.GetAttribs(); - auto& uxp = attribs[PIdx::ux]; - auto& uyp = attribs[PIdx::uy]; - auto& uzp = attribs[PIdx::uz]; - auto& Exp = attribs[PIdx::Ex]; - auto& Eyp = attribs[PIdx::Ey]; - auto& Ezp = attribs[PIdx::Ez]; - auto& Bxp = attribs[PIdx::Bx]; - auto& Byp = attribs[PIdx::By]; - auto& Bzp = attribs[PIdx::Bz]; - const long np = pti.numParticles(); + // Extract pointers to the different particle quantities + Real* AMREX_RESTRICT x = xp.dataPtr(); + Real* AMREX_RESTRICT y = yp.dataPtr(); + Real* AMREX_RESTRICT z = zp.dataPtr(); + Real* AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); + Real* AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); + Real* AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); + Real* AMREX_RESTRICT Ex = attribs[PIdx::Ex].dataPtr(); + Real* AMREX_RESTRICT Ey = attribs[PIdx::Ey].dataPtr(); + Real* AMREX_RESTRICT Ez = attribs[PIdx::Ez].dataPtr(); + Real* AMREX_RESTRICT Bx = attribs[PIdx::Bx].dataPtr(); + Real* AMREX_RESTRICT By = attribs[PIdx::By].dataPtr(); + Real* AMREX_RESTRICT Bz = attribs[PIdx::Bz].dataPtr(); if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) { @@ -1726,23 +1728,25 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti, auto& uypold = pti.GetAttribs(particle_comps["uyold"]); auto& uzpold = pti.GetAttribs(particle_comps["uzold"]); + const long np = pti.numParticles(); warpx_copy_attribs(&np, xp.dataPtr(), yp.dataPtr(), zp.dataPtr(), - uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(), + ux, uy, uz, xpold.dataPtr(), ypold.dataPtr(), zpold.dataPtr(), uxpold.dataPtr(), uypold.dataPtr(), uzpold.dataPtr()); } - warpx_particle_pusher(&np, - xp.dataPtr(), - yp.dataPtr(), - zp.dataPtr(), - uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(), - giv.dataPtr(), - Exp.dataPtr(), Eyp.dataPtr(), Ezp.dataPtr(), - Bxp.dataPtr(), Byp.dataPtr(), Bzp.dataPtr(), - &this->charge, &this->mass, &dt, - &WarpX::particle_pusher_algo); - + // Loop over the particles and update their momentum + // TODO: Choose particle pusher algorithm + const Real q = this->charge; + const Real m = this-> mass; + amrex::ParallelFor( pti.numParticles(), + [=] AMREX_GPU_DEVICE (long i) { + UpdateMomentumBoris( ux[i], uy[i], uz[i], + Ex[i], Ey[i], Ez[i], Bx[i], By[i], Bz[i], q, m, dt); + UpdatePosition( x[i], y[i], z[i], + ux[i], uy[i], uz[i], dt ); + } + ); } void diff --git a/Source/Particles/Pusher/Make.package b/Source/Particles/Pusher/Make.package index 8c8e77905..9e720f9eb 100644 --- a/Source/Particles/Pusher/Make.package +++ b/Source/Particles/Pusher/Make.package @@ -1,4 +1,5 @@ CEXE_headers += GetAndSetPosition.H CEXE_headers += UpdatePosition.H +CEXE_headers += UpdateMomentumBoris.H INCLUDE_LOCATIONS += $(WARPX_HOME)/Source/Particles/Pusher VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/Pusher diff --git a/Source/Particles/Pusher/UpdateMomentumBoris.H b/Source/Particles/Pusher/UpdateMomentumBoris.H new file mode 100644 index 000000000..d1d85b5df --- /dev/null +++ b/Source/Particles/Pusher/UpdateMomentumBoris.H @@ -0,0 +1,46 @@ +#ifndef WARPX_PARTICLES_PUSHER_UPDATEMOMENTUM_BORIS_H_ +#define WARPX_PARTICLES_PUSHER_UPDATEMOMENTUM_BORIS_H_ + +#include + +/* \brief Push the particle's positions over one timestep, + * given the value of its momenta `ux`, `uy`, `uz` */ +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void UpdatePosition( + amrex::Real& ux, amrex::Real& uy, amrex::Real& uz, + const amrex::Real Ex, const amrex::Real Ey, const amrex::Real Ez, + const amrex::Real Bx, const amrex::Real By, const amrex::Real Bz, + const amrex::Real q, const amrex::Real m, const amrex::Real dt ) +{ + const amrex::Real econst = 0.5*q*dt/m; + + // First half-push for E + ux += econst*Ex; + uy += econst*Ey; + uz += econst*Ez; + // Compute temporary gamma factor + constexpr amrex::Real inv_c2 = 1./(PhysConst::c*PhysConst::c); + const amrex::Real inv_gamma = 1./std::sqrt(1. + (ux*ux + uy*uy + uz*uz)*inv_c2); + // Magnetic rotation + // - Compute temporary variables + const amrex::Real tx = econst*inv_gamma*Bx; + const amrex::Real ty = econst*inv_gamma*By; + const amrex::Real tz = econst*inv_gamma*Bz; + const amrex::Real tsqi = 2./(1. + tx*tx + ty*ty + tz*tz); + const amrex::Real sx = tx*tsqi; + const amrex::Real sy = ty*tsqi; + const amrex::Real sz = tz*tsqi; + const amrex::Real ux_p = ux + uy*tz - uz*ty; + const amrex::Real uy_p = uy + uz*tx - ux*tz; + const amrex::Real uz_p = uz + ux*ty - uy*tx; + // - Update momentum + ux += uy_p*sz - uz_p*sy; + uy += uz_p*sx - ux_p*sz; + uz += ux_p*sy - uy_p*sx; + // Second half-push for E + ux += econst*Ex; + uy += econst*Ey; + uz += econst*Ez; +} + +#endif // WARPX_PARTICLES_PUSHER_UPDATEMOMENTUM_BORIS_H_ -- cgit v1.2.3 From 6532be2a87c16e69f8025ad678b483e9575122d7 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 22 May 2019 15:17:18 -0700 Subject: Fix compilation errors --- Source/Particles/PhysicalParticleContainer.cpp | 3 +++ Source/Particles/Pusher/UpdateMomentumBoris.H | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index f497ef4e4..c6364c027 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -7,6 +7,9 @@ #include #include +// Import low-level single-particle kernels +#include +#include using namespace amrex; diff --git a/Source/Particles/Pusher/UpdateMomentumBoris.H b/Source/Particles/Pusher/UpdateMomentumBoris.H index d1d85b5df..a33058347 100644 --- a/Source/Particles/Pusher/UpdateMomentumBoris.H +++ b/Source/Particles/Pusher/UpdateMomentumBoris.H @@ -6,7 +6,7 @@ /* \brief Push the particle's positions over one timestep, * given the value of its momenta `ux`, `uy`, `uz` */ AMREX_GPU_HOST_DEVICE AMREX_INLINE -void UpdatePosition( +void UpdateMomentumBoris( amrex::Real& ux, amrex::Real& uy, amrex::Real& uz, const amrex::Real Ex, const amrex::Real Ey, const amrex::Real Ez, const amrex::Real Bx, const amrex::Real By, const amrex::Real Bz, -- cgit v1.2.3 From 89854c7c5cfab93b71d56fbcc674c82da29b5a83 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 23 May 2019 14:49:26 -0700 Subject: Add Vay pusher --- Source/Particles/PhysicalParticleContainer.cpp | 93 +++++++++++++++----------- Source/Particles/Pusher/UpdateMomentumVay.H | 50 ++++++++------ Source/Utils/Make.package | 1 + Source/Utils/WarpXAlgorithmSelection.H | 11 +++ 4 files changed, 97 insertions(+), 58 deletions(-) create mode 100644 Source/Utils/WarpXAlgorithmSelection.H (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index c6364c027..04c53ce34 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -7,9 +7,12 @@ #include #include +#include + // Import low-level single-particle kernels #include #include +#include using namespace amrex; @@ -41,7 +44,7 @@ NumParticlesToAdd(const Box& overlap_box, const RealBox& overlap_realbox, } else { fac = 1.0; } - + int ref_num_ppc = num_ppc * AMREX_D_TERM(fac, *fac, *fac); for (int i_part=0; i_part r; @@ -65,7 +68,7 @@ NumParticlesToAdd(const Box& overlap_box, const RealBox& overlap_realbox, ++np; } } - + return np; } @@ -85,7 +88,7 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp pp.query("do_splitting", do_splitting); pp.query("split_type", split_type); pp.query("do_continuous_injection", do_continuous_injection); - // Whether to plot back-transformed (lab-frame) diagnostics + // Whether to plot back-transformed (lab-frame) diagnostics // for this species. pp.query("do_boosted_frame_diags", do_boosted_frame_diags); @@ -94,7 +97,7 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp do_user_plot_vars = pp.queryarr("plot_vars", plot_vars); if (not do_user_plot_vars){ // By default, all particle variables are dumped to plotfiles, - // including {x,y,z,ux,uy,uz}old variables when running in a + // including {x,y,z,ux,uy,uz}old variables when running in a // boosted frame if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags){ plot_flags.resize(PIdx::nattribs + 6, 1); @@ -111,9 +114,9 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp // If not none, set plot_flags values to 1 for elements in plot_vars. if (plot_vars[0] != "none"){ for (const auto& var : plot_vars){ - // Return error if var not in PIdx. - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( - ParticleStringNames::to_index.count(var), + // Return error if var not in PIdx. + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + ParticleStringNames::to_index.count(var), "plot_vars argument not in ParticleStringNames"); plot_flags[ParticleStringNames::to_index.at(var)] = 1; } @@ -228,12 +231,12 @@ PhysicalParticleContainer::AddGaussianBeam(Real x_m, Real y_m, Real z_m, particle_tile.push_back_real(particle_comps["xold"], x); particle_tile.push_back_real(particle_comps["yold"], y); particle_tile.push_back_real(particle_comps["zold"], z); - + particle_tile.push_back_real(particle_comps["uxold"], u[0]); particle_tile.push_back_real(particle_comps["uyold"], u[1]); particle_tile.push_back_real(particle_comps["uzold"], u[2]); } - + AddOneParticle(0, 0, 0, x, y, z, attribs); } } @@ -505,7 +508,7 @@ PhysicalParticleContainer::AddPlasmaCPU (int lev, RealBox part_realbox) attribs[PIdx::ux] = u[0]; attribs[PIdx::uy] = u[1]; attribs[PIdx::uz] = u[2]; - + if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) { auto& particle_tile = DefineAndReturnParticleTile(lev, grid_id, tile_id); @@ -633,7 +636,7 @@ PhysicalParticleContainer::AddPlasmaGPU (int lev, RealBox part_realbox) Cuda::HostVector host_particles; std::array, PIdx::nattribs> host_attribs; - + // Loop through the cells of overlap_box and inject // the corresponding particles const auto& overlap_corner = overlap_realbox.lo(); @@ -795,7 +798,7 @@ PhysicalParticleContainer::AddPlasmaGPU (int lev, RealBox part_realbox) host_attribs[kk].end(), particle_tile.GetStructOfArrays().GetRealData(kk).begin() + old_size); } - + if (cost) { wt = (amrex::second() - wt) / tile_box.d_numPts(); FArrayBox* costfab = cost->fabPtr(mfi); @@ -804,7 +807,7 @@ PhysicalParticleContainer::AddPlasmaGPU (int lev, RealBox part_realbox) costfab->plus(wt, work_box); }); } - } + } } } #endif @@ -1134,7 +1137,7 @@ PhysicalParticleContainer::Evolve (int lev, BL_PROFILE_VAR_NS("PICSAR::FieldGather", blp_pxr_fg); BL_PROFILE_VAR_NS("PICSAR::ParticlePush", blp_pxr_pp); BL_PROFILE_VAR_NS("PPC::Evolve::partition", blp_partition); - + const std::array& dx = WarpX::CellSize(lev); const std::array& cdx = WarpX::CellSize(std::max(lev-1,0)); @@ -1151,7 +1154,7 @@ PhysicalParticleContainer::Evolve (int lev, bool has_buffer = cEx || cjx; #ifdef _OPENMP -#pragma omp parallel +#pragma omp parallel #endif { #ifdef _OPENMP @@ -1360,7 +1363,7 @@ PhysicalParticleContainer::Evolve (int lev, } const long np_current = (cjx) ? nfine_current : np; - + // // copy data from particle container to temp arrays // @@ -1369,7 +1372,7 @@ PhysicalParticleContainer::Evolve (int lev, BL_PROFILE_VAR_STOP(blp_copy); if (rho) DepositCharge(pti, wp, rho, crho, 0, np_current, np, thread_num, lev); - + if (! do_not_push) { // @@ -1380,7 +1383,7 @@ PhysicalParticleContainer::Evolve (int lev, const std::array& xyzmin_grid = WarpX::LowerCorner(box, lev); const int* ixyzmin_grid = box.loVect(); - + const long np_gather = (cEx) ? nfine_gather : np; BL_PROFILE_VAR_START(blp_pxr_fg); @@ -1462,7 +1465,7 @@ PhysicalParticleContainer::Evolve (int lev, mypc.fdtd_nci_stencilz_ex[lev-1].data(), &nstencilz_fdtd_nci_corr); ceyfab = &filtered_Ey; - + filtered_Bx.resize(amrex::convert(tbox,WarpX::Bx_nodal_flag)); WRPX_PXR_GODFREY_FILTER(BL_TO_FORTRAN_BOX(filtered_Bx), BL_TO_FORTRAN_ANYD(filtered_Bx), @@ -1470,7 +1473,7 @@ PhysicalParticleContainer::Evolve (int lev, mypc.fdtd_nci_stencilz_by[lev-1].data(), &nstencilz_fdtd_nci_corr); cbxfab = &filtered_Bx; - + filtered_Bz.resize(amrex::convert(tbox,WarpX::Bz_nodal_flag)); WRPX_PXR_GODFREY_FILTER(BL_TO_FORTRAN_BOX(filtered_Bz), BL_TO_FORTRAN_ANYD(filtered_Bz), @@ -1480,7 +1483,7 @@ PhysicalParticleContainer::Evolve (int lev, cbzfab = &filtered_Bz; #endif } - + long ncrse = np - nfine_gather; warpx_geteb_energy_conserving( &ncrse, @@ -1509,7 +1512,7 @@ PhysicalParticleContainer::Evolve (int lev, // Particle Push // BL_PROFILE_VAR_START(blp_pxr_pp); - PushPX(pti, m_xp[thread_num], m_yp[thread_num], m_zp[thread_num], + PushPX(pti, m_xp[thread_num], m_yp[thread_num], m_zp[thread_num], m_giv[thread_num], dt); BL_PROFILE_VAR_STOP(blp_pxr_pp); @@ -1518,7 +1521,7 @@ PhysicalParticleContainer::Evolve (int lev, // DepositCurrent(pti, wp, uxp, uyp, uzp, jx, jy, jz, cjx, cjy, cjz, np_current, np, thread_num, lev, dt); - + // // copy particle data back // @@ -1526,7 +1529,7 @@ PhysicalParticleContainer::Evolve (int lev, pti.SetPosition(m_xp[thread_num], m_yp[thread_num], m_zp[thread_num]); BL_PROFILE_VAR_STOP(blp_copy); } - + if (rho) DepositCharge(pti, wp, rho, crho, 1, np_current, np, thread_num, lev); if (cost) { @@ -1580,7 +1583,7 @@ PhysicalParticleContainer::SplitParticles(int lev) for(int i=0; icharge; const Real m = this-> mass; - amrex::ParallelFor( pti.numParticles(), - [=] AMREX_GPU_DEVICE (long i) { - UpdateMomentumBoris( ux[i], uy[i], uz[i], - Ex[i], Ey[i], Ez[i], Bx[i], By[i], Bz[i], q, m, dt); - UpdatePosition( x[i], y[i], z[i], - ux[i], uy[i], uz[i], dt ); - } - ); + if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Boris){ + amrex::ParallelFor( pti.numParticles(), + [=] AMREX_GPU_DEVICE (long i) { + UpdateMomentumBoris( ux[i], uy[i], uz[i], + Ex[i], Ey[i], Ez[i], Bx[i], By[i], Bz[i], q, m, dt); + UpdatePosition( x[i], y[i], z[i], + ux[i], uy[i], uz[i], dt ); + } + ); + } else if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Vay) { + amrex::ParallelFor( pti.numParticles(), + [=] AMREX_GPU_DEVICE (long i) { + UpdateMomentumVay( ux[i], uy[i], uz[i], + Ex[i], Ey[i], Ez[i], Bx[i], By[i], Bz[i], q, m, dt); + UpdatePosition( x[i], y[i], z[i], + ux[i], uy[i], uz[i], dt ); + } + ); + } else { + amrex::Abort("Unknown particle pusher"); + }; } void @@ -1771,7 +1786,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, int thread_num = omp_get_thread_num(); #else int thread_num = 0; -#endif +#endif for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { const Box& box = pti.validbox(); @@ -1884,7 +1899,7 @@ void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real slice_box.setHi(direction, z_max); diagnostic_particles.resize(finestLevel()+1); - + for (int lev = 0; lev < nlevs; ++lev) { const Real* dx = Geom(lev).CellSize(); @@ -1966,7 +1981,7 @@ void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real Real uzp = uz_old_p *weight_old + uz_new_p *weight_new; diagnostic_particles[lev][index].GetRealData(DiagIdx::w).push_back(wp[i]); - + diagnostic_particles[lev][index].GetRealData(DiagIdx::x).push_back(xp); diagnostic_particles[lev][index].GetRealData(DiagIdx::y).push_back(yp); diagnostic_particles[lev][index].GetRealData(DiagIdx::z).push_back(zp); diff --git a/Source/Particles/Pusher/UpdateMomentumVay.H b/Source/Particles/Pusher/UpdateMomentumVay.H index d28c1a9a2..1f0f19e63 100644 --- a/Source/Particles/Pusher/UpdateMomentumVay.H +++ b/Source/Particles/Pusher/UpdateMomentumVay.H @@ -17,25 +17,37 @@ void UpdateMomentumVay( // Constants const amrex::Real econst = q*dt/m; const amrex::Real bconst = 0.5*q*dt/m; - constexpr amrex::Real inv_c2 = 1./(PhysConst::c*PhysConst::c); - // Compute temporary gamma factor - const amrex::Real inv_gamma = 1./std::sqrt(1. + (ux*ux + uy*uy + uz*uz)*inv_c2); - // Compute temporary variables - const amrex::Real taux = econst*Bx; - const amrex::Real tauy = econst*By; - const amrex::Real tauz = econst*Bz; - const amrex::Real tausq = taux*taux + tauy*tauy + tauz*tauz; - const amrex::Real ux_p = ux + econst*Ex + (uy*tauz - uz*tauy)*inv_gamma; - const amrex::Real uy_p = uy + econst*Ey + (uz*taux - ux*tauz)*inv_gamma; - const amrex::Real uz_p = uz + econst*Ez + (ux*tauy - uy*taux)*inv_gamma; - // - Update momentum - ux += uy_p*sz - uz_p*sy; - uy += uz_p*sx - ux_p*sz; - uz += ux_p*sy - uy_p*sx; - // Second half-push for E - ux += econst*Ex; - uy += econst*Ey; - uz += econst*Ez; + constexpr amrex::Real invclight = 1./PhysConst::c; + constexpr amrex::Real invclightsq = 1./(PhysConst::c*PhysConst::c); + // Compute initial gamma + const amrex::Real inv_gamma = 1./std::sqrt(1. + (ux*ux + uy*uy + uz*uz)*invclightsq); + // Get tau + const amrex::Real taux = bconst*Bx; + const amrex::Real tauy = bconst*By; + const amrex::Real tauz = bconst*Bz; + const amrex::Real tausq = taux*taux+tauy*tauy+tauz*tauz; + // Get U', gamma'^2 + const amrex::Real uxpr = ux + econst*Ex + (uy*tauz-uz*tauy)*inv_gamma; + const amrex::Real uypr = uy + econst*Ey + (uz*taux-ux*tauz)*inv_gamma; + const amrex::Real uzpr = uz + econst*Ez + (ux*tauy-uy*taux)*inv_gamma; + const amrex::Real gprsq = (1. + (uxpr*uxpr + uypr*uypr + uzpr*uzpr)*invclightsq); + // Get u* + const amrex::Real ust = (uxpr*taux + uypr*tauy + uzpr*tauz)*invclight; + // Get new gamma + const amrex::Real sigma = gprsq-tausq; + const amrex::Real gisq = 2./(sigma + std::sqrt(sigma*sigma + 4.*(tausq + ust*ust)) ); + // Get t, s + const amrex::Real bg = bconst*std::sqrt(gisq); + const amrex::Real tx = bg*Bx; + const amrex::Real ty = bg*By; + const amrex::Real tz = bg*Bz; + const amrex::Real s = 1./(1.+tausq*gisq); + // Get t.u' + const amrex::Real tu = tx*uxpr + ty*uypr + tz*uzpr; + // Get new U + ux = s*(uxpr+tx*tu+uypr*tz-uzpr*ty); + uy = s*(uypr+ty*tu+uzpr*tx-uxpr*tz); + uz = s*(uzpr+tz*tu+uxpr*ty-uypr*tx); } #endif // WARPX_PARTICLES_PUSHER_UPDATEMOMENTUM_VAY_H_ diff --git a/Source/Utils/Make.package b/Source/Utils/Make.package index d8e2d2dab..4a7d75a3e 100644 --- a/Source/Utils/Make.package +++ b/Source/Utils/Make.package @@ -4,6 +4,7 @@ CEXE_sources += WarpXTagging.cpp CEXE_sources += WarpXUtil.cpp CEXE_headers += WarpXConst.H CEXE_headers += WarpXUtil.H +CEXE_headers += WarpXAlgorithmSelection.H INCLUDE_LOCATIONS += $(WARPX_HOME)/Source/Utils VPATH_LOCATIONS += $(WARPX_HOME)/Source/Utils diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H new file mode 100644 index 000000000..824a622f8 --- /dev/null +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -0,0 +1,11 @@ +#ifndef UTILS_WARPXALGORITHMSELECTION_H_ +#define UTILS_WARPXALGORITHMSELECTION_H_ + +struct ParticlePusherAlgo { + enum { + Boris = 0, + Vay = 1 + }; + }; + +#endif // UTILS_WARPXALGORITHMSELECTION_H_ -- cgit v1.2.3 From 7d608324b222aca93866e8eb8946bcb7e6a36aaf Mon Sep 17 00:00:00 2001 From: Dave Grote Date: Fri, 24 May 2019 15:54:47 -0700 Subject: Updated particle injection for RZ multimode --- Source/Initialization/PlasmaInjector.cpp | 7 +-- Source/Particles/PhysicalParticleContainer.cpp | 77 +++++++++++++++++--------- Source/Particles/WarpXParticleContainer.H | 3 + 3 files changed, 55 insertions(+), 32 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Initialization/PlasmaInjector.cpp b/Source/Initialization/PlasmaInjector.cpp index 52b5d8b61..18840fdba 100644 --- a/Source/Initialization/PlasmaInjector.cpp +++ b/Source/Initialization/PlasmaInjector.cpp @@ -218,7 +218,7 @@ void RegularPosition::getPositionUnitBox(vec3& r, int i_part, int ref_fac) { int nx = ref_fac*_num_particles_per_cell_each_dim[0]; int ny = ref_fac*_num_particles_per_cell_each_dim[1]; -#if AMREX_SPACEDIM == 3 +#if (AMREX_SPACEDIM == 3) || (defined WARPX_RZ) int nz = ref_fac*_num_particles_per_cell_each_dim[2]; #else int nz = 1; @@ -295,11 +295,8 @@ PlasmaInjector::PlasmaInjector(int ispecies, const std::string& name) parseDensity(pp); parseMomentum(pp); } else if (part_pos_s == "nuniformpercell") { - num_particles_per_cell_each_dim.resize(3); + num_particles_per_cell_each_dim.assign(3, 1); pp.getarr("num_particles_per_cell_each_dim", num_particles_per_cell_each_dim); -#if ( AMREX_SPACEDIM == 2 ) - num_particles_per_cell_each_dim[2] = 1; -#endif part_pos.reset(new RegularPosition(num_particles_per_cell_each_dim)); num_particles_per_cell = num_particles_per_cell_each_dim[0] * num_particles_per_cell_each_dim[1] * diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index e1b012464..b4c308c84 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -41,16 +41,19 @@ NumParticlesToAdd(const Box& overlap_box, const RealBox& overlap_realbox, int ref_num_ppc = num_ppc * AMREX_D_TERM(fac, *fac, *fac); for (int i_part=0; i_part r; - plasma_injector->getPositionUnitBox(r, i_part, fac); + std::array point; + plasma_injector->getPositionUnitBox(point, i_part, fac); + Real x = overlap_corner[0] + (iv[0] + point[0])*dx[0]; #if ( AMREX_SPACEDIM == 3 ) - Real x = overlap_corner[0] + (iv[0] + r[0])*dx[0]; - Real y = overlap_corner[1] + (iv[1] + r[1])*dx[1]; - Real z = overlap_corner[2] + (iv[2] + r[2])*dx[2]; + Real y = overlap_corner[1] + (iv[1] + point[1])*dx[1]; + Real z = overlap_corner[2] + (iv[2] + point[2])*dx[2]; #elif ( AMREX_SPACEDIM == 2 ) - Real x = overlap_corner[0] + (iv[0] + r[0])*dx[0]; Real y = 0; - Real z = overlap_corner[1] + (iv[1] + r[1])*dx[1]; +#ifdef WARPX_RZ + Real z = overlap_corner[1] + (iv[1] + point[2])*dx[1]; +#else + Real z = overlap_corner[1] + (iv[1] + point[1])*dx[1]; +#endif #endif // If the new particle is not inside the tile box, // go to the next generated particle. @@ -413,16 +416,19 @@ PhysicalParticleContainer::AddPlasmaCPU (int lev, RealBox part_realbox) int ref_num_ppc = num_ppc * AMREX_D_TERM(fac, *fac, *fac); for (int i_part=0; i_part r; - plasma_injector->getPositionUnitBox(r, i_part, fac); + std::array point; + plasma_injector->getPositionUnitBox(point, i_part, fac); + Real x = overlap_corner[0] + (iv[0] + point[0])*dx[0]; #if ( AMREX_SPACEDIM == 3 ) - Real x = overlap_corner[0] + (iv[0] + r[0])*dx[0]; - Real y = overlap_corner[1] + (iv[1] + r[1])*dx[1]; - Real z = overlap_corner[2] + (iv[2] + r[2])*dx[2]; + Real y = overlap_corner[1] + (iv[1] + point[1])*dx[1]; + Real z = overlap_corner[2] + (iv[2] + point[2])*dx[2]; #elif ( AMREX_SPACEDIM == 2 ) - Real x = overlap_corner[0] + (iv[0] + r[0])*dx[0]; Real y = 0; - Real z = overlap_corner[1] + (iv[1] + r[1])*dx[1]; +#ifdef WARPX_RZ + Real z = overlap_corner[1] + (iv[1] + point[2])*dx[1]; +#else + Real z = overlap_corner[1] + (iv[1] + point[1])*dx[1]; +#endif #endif // If the new particle is not inside the tile box, // go to the next generated particle. @@ -438,11 +444,18 @@ PhysicalParticleContainer::AddPlasmaCPU (int lev, RealBox part_realbox) Real yb = y; #ifdef WARPX_RZ - // Replace the x and y, choosing the angle randomly. + // Replace the x and y, setting an angle theta. // These x and y are used to get the momentum and density - Real theta = 2.*MathConst::pi*amrex::Random(); - y = x*std::sin(theta); - x = x*std::cos(theta); + Real theta; + if (WarpX::nmodes == 1) { + // With only 1 mode, the angle doesn't matter so + // choose it randomly. + theta = 2.*MathConst::pi*amrex::Random(); + } else { + theta = 2.*MathConst::pi*point[1]; + } + x = xb*std::cos(theta); + y = xb*std::sin(theta); #endif Real dens; @@ -654,16 +667,19 @@ PhysicalParticleContainer::AddPlasmaGPU (int lev, RealBox part_realbox) int ref_num_ppc = num_ppc * AMREX_D_TERM(fac, *fac, *fac); for (int i_part=0; i_part r; - plasma_injector->getPositionUnitBox(r, i_part, fac); + std::array point; + plasma_injector->getPositionUnitBox(point, i_part, fac); + Real x = overlap_corner[0] + (iv[0] + point[0])*dx[0]; #if ( AMREX_SPACEDIM == 3 ) - Real x = overlap_corner[0] + (iv[0] + r[0])*dx[0]; - Real y = overlap_corner[1] + (iv[1] + r[1])*dx[1]; - Real z = overlap_corner[2] + (iv[2] + r[2])*dx[2]; + Real y = overlap_corner[1] + (iv[1] + point[1])*dx[1]; + Real z = overlap_corner[2] + (iv[2] + point[2])*dx[2]; #elif ( AMREX_SPACEDIM == 2 ) - Real x = overlap_corner[0] + (iv[0] + r[0])*dx[0]; Real y = 0; - Real z = overlap_corner[1] + (iv[1] + r[1])*dx[1]; +#ifdef WARPX_RZ + Real z = overlap_corner[1] + (iv[1] + point[2])*dx[1]; +#else + Real z = overlap_corner[1] + (iv[1] + point[1])*dx[1]; +#endif #endif // If the new particle is not inside the tile box, // go to the next generated particle. @@ -679,9 +695,16 @@ PhysicalParticleContainer::AddPlasmaGPU (int lev, RealBox part_realbox) Real yb = y; #ifdef WARPX_RZ - // Replace the x and y, choosing the angle randomly. + // Replace the x and y, setting an angle theta. // These x and y are used to get the momentum and density - Real theta = 2.*MathConst::pi*amrex::Random(); + Real theta; + if (WarpX::nmodes == 1) { + // With only 1 mode, the angle doesn't matter so + // choose it randomly. + theta = 2.*MathConst::pi*amrex::Random(); + } else { + theta = 2.*MathConst::pi*point[1]; + } x = xb*std::cos(theta); y = xb*std::sin(theta); #endif diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index c24a6a3dc..0f4f9dc6f 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -42,6 +42,9 @@ namespace ParticleStringNames {"Bx", PIdx::Bx }, {"By", PIdx::By }, {"Bz", PIdx::Bz } +#ifdef WARPX_RZ + ,{"theta", PIdx::theta} +#endif }; } -- cgit v1.2.3 From a9dde5192caae19bc4198b4ae404c7015c104cf5 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Sun, 7 Jul 2019 11:20:28 -0700 Subject: avoid duplication in current deposition when using buffers --- Source/Laser/LaserParticleContainer.cpp | 10 +- Source/Particles/PhysicalParticleContainer.cpp | 10 +- Source/Particles/WarpXParticleContainer.H | 51 +++-- Source/Particles/WarpXParticleContainer.cpp | 281 ++++++++++--------------- 4 files changed, 156 insertions(+), 196 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Laser/LaserParticleContainer.cpp b/Source/Laser/LaserParticleContainer.cpp index de410b31f..eb1eaa2f0 100644 --- a/Source/Laser/LaserParticleContainer.cpp +++ b/Source/Laser/LaserParticleContainer.cpp @@ -503,8 +503,14 @@ LaserParticleContainer::Evolve (int lev, // // Current Deposition // - DepositCurrent(pti, wp, uxp, uyp, uzp, jx, jy, jz, - cjx, cjy, cjz, np_current, np, thread_num, lev, dt); + // Deposit inside domains + DepositCurrent(pti, wp, uxp, uyp, uzp, &jx, &jy, &jz, + 0, np_current, thread_num, + lev, lev, dt, jx.nGrow()); + // Deposit in buffers + DepositCurrent(pti, wp, uxp, uyp, uzp, cjx, cjy, cjz, + np_current, np-np_current, thread_num, + lev, lev-1, dt, jx.nGrow()); // // copy particle data back diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 7e7c9534e..1ed318c13 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1529,8 +1529,14 @@ PhysicalParticleContainer::Evolve (int lev, // // Current Deposition // - DepositCurrent(pti, wp, uxp, uyp, uzp, jx, jy, jz, - cjx, cjy, cjz, np_current, np, thread_num, lev, dt); + // Deposit inside domains + DepositCurrent(pti, wp, uxp, uyp, uzp, &jx, &jy, &jz, + 0, np_current, thread_num, + lev, lev, dt, jx.nGrow()); + // Deposit in buffers + DepositCurrent(pti, wp, uxp, uyp, uzp, cjx, cjy, cjz, + np_current, np-np_current, thread_num, + lev, lev-1, dt, jx.nGrow()); // // copy particle data back diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 1e3dfb2ae..3098e9296 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -164,32 +164,31 @@ public: bool local = false); std::unique_ptr GetChargeDensity(int lev, bool local = false); - virtual void DepositCharge(WarpXParIter& pti, - RealVector& wp, - amrex::MultiFab* rhomf, - amrex::MultiFab* crhomf, - int icomp, - const long np_current, - const long np, - int thread_num, - int lev ); - - virtual void DepositCurrent(WarpXParIter& pti, - RealVector& wp, - RealVector& uxp, - RealVector& uyp, - RealVector& uzp, - amrex::MultiFab& jx, - amrex::MultiFab& jy, - amrex::MultiFab& jz, - amrex::MultiFab* cjx, - amrex::MultiFab* cjy, - amrex::MultiFab* cjz, - const long np_current, - const long np, - int thread_num, - int lev, - amrex::Real dt ); + virtual void DepositCharge(WarpXParIter& pti, + RealVector& wp, + amrex::MultiFab* rhomf, + amrex::MultiFab* crhomf, + int icomp, + const long np_current, + const long np, + int thread_num, + int lev ); + + virtual void DepositCurrent(WarpXParIter& pti, + RealVector& wp, + RealVector& uxp, + RealVector& uyp, + RealVector& uzp, + amrex::MultiFab* jx, + amrex::MultiFab* jy, + amrex::MultiFab* jz, + const long offset, + const long np_to_depose, + int thread_num, + int lev, + int depos_lev, + amrex::Real dt, + const long ngJ); // If particles start outside of the domain, ContinuousInjection // makes sure that they are initialized when they enter the domain, and diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index ac532912d..a02c9cfcd 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -279,187 +279,136 @@ WarpXParticleContainer::AddNParticles (int lev, Redistribute(); } - +/* \brief Current Deposition for thread thread_num + * \param pti : Particle iterator + * \param wp : Array of particle weights + * \param uxp uyp uzp : Array of particle + * \param jx jy jz : Full array of current density + * \param offset : Index of first particle for which current is deposited + * \param np_to_depose: Number of particles for which current is deposited. + Particles [offset,offset+np_tp_depose] deposit their + current + * \param thread_num : Thread number (if tiling) + * \param lev : Level of box that contains particles + * \param depos_lev : Level on which particles deposit (if buffers are used) + * \param dt : Time step for particle level + * \param ngJ : Number of ghosts cells (of lev) + */ void WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, RealVector& wp, RealVector& uxp, RealVector& uyp, RealVector& uzp, - MultiFab& jx, MultiFab& jy, MultiFab& jz, - MultiFab* cjx, MultiFab* cjy, MultiFab* cjz, - const long np_current, const long np, - int thread_num, int lev, Real dt ) + MultiFab* jx, MultiFab* jy, MultiFab* jz, + const long offset, const long np_to_depose, + int thread_num, int lev, int depos_lev, + Real dt, const long ngJ) { - Real *jx_ptr, *jy_ptr, *jz_ptr; - const std::array& xyzmin_tile = WarpX::LowerCorner(pti.tilebox(), lev); - const std::array& dx = WarpX::CellSize(lev); - const std::array& cdx = WarpX::CellSize(std::max(lev-1,0)); - const std::array& xyzmin = xyzmin_tile; - const long lvect = 8; - - BL_PROFILE_VAR_NS("PICSAR::CurrentDeposition", blp_pxr_cd); - BL_PROFILE_VAR_NS("PPC::Evolve::Accumulate", blp_accumulate); - - Box tbx = convert(pti.tilebox(), WarpX::jx_nodal_flag); - Box tby = convert(pti.tilebox(), WarpX::jy_nodal_flag); - Box tbz = convert(pti.tilebox(), WarpX::jz_nodal_flag); - - // WarpX assumes the same number of guard cells for Jx, Jy, Jz - long ngJ = jx.nGrow(); - - int j_is_nodal = jx.is_nodal() and jy.is_nodal() and jz.is_nodal(); - - // Deposit charge for particles that are not in the current buffers - if (np_current > 0) - { -#ifdef AMREX_USE_GPU - jx_ptr = jx[pti].dataPtr(); - jy_ptr = jy[pti].dataPtr(); - jz_ptr = jz[pti].dataPtr(); - - auto jxntot = jx[pti].length(); - auto jyntot = jy[pti].length(); - auto jzntot = jz[pti].length(); -#else - tbx.grow(ngJ); - tby.grow(ngJ); - tbz.grow(ngJ); - - local_jx[thread_num].resize(tbx); - local_jy[thread_num].resize(tby); - local_jz[thread_num].resize(tbz); - - jx_ptr = local_jx[thread_num].dataPtr(); - jy_ptr = local_jy[thread_num].dataPtr(); - jz_ptr = local_jz[thread_num].dataPtr(); - - local_jx[thread_num].setVal(0.0); - local_jy[thread_num].setVal(0.0); - local_jz[thread_num].setVal(0.0); - - auto jxntot = local_jx[thread_num].length(); - auto jyntot = local_jy[thread_num].length(); - auto jzntot = local_jz[thread_num].length(); -#endif - - BL_PROFILE_VAR_START(blp_pxr_cd); - warpx_current_deposition( - jx_ptr, &ngJ, jxntot.getVect(), - jy_ptr, &ngJ, jyntot.getVect(), - jz_ptr, &ngJ, jzntot.getVect(), - &np_current, - m_xp[thread_num].dataPtr(), - m_yp[thread_num].dataPtr(), - m_zp[thread_num].dataPtr(), - uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(), - m_giv[thread_num].dataPtr(), - wp.dataPtr(), &this->charge, - &xyzmin[0], &xyzmin[1], &xyzmin[2], - &dt, &dx[0], &dx[1], &dx[2], - &WarpX::nox,&WarpX::noy,&WarpX::noz, &j_is_nodal, - &lvect,&WarpX::current_deposition_algo); - -#ifdef WARPX_RZ - warpx_current_deposition_rz_volume_scaling( - jx_ptr, &ngJ, jxntot.getVect(), - jy_ptr, &ngJ, jyntot.getVect(), - jz_ptr, &ngJ, jzntot.getVect(), - &xyzmin[0], &dx[0]); -#endif - BL_PROFILE_VAR_STOP(blp_pxr_cd); - -#ifndef AMREX_USE_GPU - BL_PROFILE_VAR_START(blp_accumulate); - - jx[pti].atomicAdd(local_jx[thread_num], tbx, tbx, 0, 0, 1); - jy[pti].atomicAdd(local_jy[thread_num], tby, tby, 0, 0, 1); - jz[pti].atomicAdd(local_jz[thread_num], tbz, tbz, 0, 0, 1); - - BL_PROFILE_VAR_STOP(blp_accumulate); -#endif - } + AMREX_ALWAYS_ASSERT_WITH_MESSAGE((depos_lev==(lev-1)) || + (depos_lev==(lev )), + "Deposition buffers only work for lev-1"); + // If no particles, do not do anything + if (np_to_depose == 0) return; + + const std::array& dx = WarpX::CellSize(std::max(depos_lev,0)); + const long lvect = 8; + int j_is_nodal = jx->is_nodal() and jy->is_nodal() and jz->is_nodal(); + + BL_PROFILE_VAR_NS("PICSAR::CurrentDeposition", blp_pxr_cd); + BL_PROFILE_VAR_NS("PPC::Evolve::Accumulate", blp_accumulate); + + // Get tile box where current is deposited + Box tilebox; + if (lev == depos_lev) { + tilebox = pti.tilebox(); + } else { + const IntVect& ref_ratio = WarpX::RefRatio(depos_lev); + tilebox = amrex::coarsen(pti.tilebox(),ref_ratio); + } + + // Staggered tile boxes (different in each direction) + Box tbx = convert(tilebox, WarpX::jx_nodal_flag); + Box tby = convert(tilebox, WarpX::jy_nodal_flag); + Box tbz = convert(tilebox, WarpX::jz_nodal_flag); - // Deposit charge for particles that are in the current buffers - if (np_current < np) - { - const IntVect& ref_ratio = WarpX::RefRatio(lev-1); - const Box& ctilebox = amrex::coarsen(pti.tilebox(),ref_ratio); - const std::array& cxyzmin_tile = WarpX::LowerCorner(ctilebox, lev-1); + // Lower corner of tile box physical domain + const std::array& xyzmin_tile = WarpX::LowerCorner(tilebox, depos_lev); + const std::array& xyzmin = xyzmin_tile; #ifdef AMREX_USE_GPU - jx_ptr = (*cjx)[pti].dataPtr(); - jy_ptr = (*cjy)[pti].dataPtr(); - jz_ptr = (*cjz)[pti].dataPtr(); - - auto jxntot = jx[pti].length(); - auto jyntot = jy[pti].length(); - auto jzntot = jz[pti].length(); + // No tiling on GPU: jx_ptr points to the full + // jx array (same for jy_ptr and jz_ptr). + Real* jx_ptr = (*jx)[pti].dataPtr(); + Real* jy_ptr = (*jy)[pti].dataPtr(); + Real* jz_ptr = (*jz)[pti].dataPtr(); + + auto jxntot = jx[pti].length(); + auto jyntot = jy[pti].length(); + auto jzntot = jz[pti].length(); #else + // Tiling is on: jx_ptr points to local_jx[thread_num] + // (same for jy_ptr and jz_ptr) + tbx.grow(ngJ); + tby.grow(ngJ); + tbz.grow(ngJ); + + local_jx[thread_num].resize(tbx); + local_jy[thread_num].resize(tby); + local_jz[thread_num].resize(tbz); + + Real* jx_ptr = local_jx[thread_num].dataPtr(); + Real* jy_ptr = local_jy[thread_num].dataPtr(); + Real* jz_ptr = local_jz[thread_num].dataPtr(); + + // local_jx[thread_num] is set to zero + local_jx[thread_num].setVal(0.0); + local_jy[thread_num].setVal(0.0); + local_jz[thread_num].setVal(0.0); + + auto jxntot = local_jx[thread_num].length(); + auto jyntot = local_jy[thread_num].length(); + auto jzntot = local_jz[thread_num].length(); +#endif + // GPU, no tiling: deposit directly in jx + // CPU, tiling: deposit into local_jx + // (same for jx and jz) + BL_PROFILE_VAR_START(blp_pxr_cd); + warpx_current_deposition( + jx_ptr, &ngJ, jxntot.getVect(), + jy_ptr, &ngJ, jyntot.getVect(), + jz_ptr, &ngJ, jzntot.getVect(), + &np_to_depose, + m_xp[thread_num].dataPtr() + offset, + m_yp[thread_num].dataPtr() + offset, + m_zp[thread_num].dataPtr() + offset, + uxp.dataPtr() + offset, + uyp.dataPtr() + offset, + uzp.dataPtr() + offset, + m_giv[thread_num].dataPtr() + offset, + wp.dataPtr() + offset, &this->charge, + &xyzmin[0], &xyzmin[1], &xyzmin[2], + &dt, &dx[0], &dx[1], &dx[2], + &WarpX::nox,&WarpX::noy,&WarpX::noz, &j_is_nodal, + &lvect,&WarpX::current_deposition_algo); - tbx = amrex::convert(ctilebox, WarpX::jx_nodal_flag); - tby = amrex::convert(ctilebox, WarpX::jy_nodal_flag); - tbz = amrex::convert(ctilebox, WarpX::jz_nodal_flag); - tbx.grow(ngJ); - tby.grow(ngJ); - tbz.grow(ngJ); - - local_jx[thread_num].resize(tbx); - local_jy[thread_num].resize(tby); - local_jz[thread_num].resize(tbz); - - jx_ptr = local_jx[thread_num].dataPtr(); - jy_ptr = local_jy[thread_num].dataPtr(); - jz_ptr = local_jz[thread_num].dataPtr(); - - local_jx[thread_num].setVal(0.0); - local_jy[thread_num].setVal(0.0); - local_jz[thread_num].setVal(0.0); - - auto jxntot = local_jx[thread_num].length(); - auto jyntot = local_jy[thread_num].length(); - auto jzntot = local_jz[thread_num].length(); -#endif - - long ncrse = np - np_current; - BL_PROFILE_VAR_START(blp_pxr_cd); - warpx_current_deposition( - jx_ptr, &ngJ, jxntot.getVect(), - jy_ptr, &ngJ, jyntot.getVect(), - jz_ptr, &ngJ, jzntot.getVect(), - &ncrse, - m_xp[thread_num].dataPtr() +np_current, - m_yp[thread_num].dataPtr() +np_current, - m_zp[thread_num].dataPtr() +np_current, - uxp.dataPtr()+np_current, - uyp.dataPtr()+np_current, - uzp.dataPtr()+np_current, - m_giv[thread_num].dataPtr()+np_current, - wp.dataPtr()+np_current, &this->charge, - &cxyzmin_tile[0], &cxyzmin_tile[1], &cxyzmin_tile[2], - &dt, &cdx[0], &cdx[1], &cdx[2], - &WarpX::nox,&WarpX::noy,&WarpX::noz, &j_is_nodal, - &lvect,&WarpX::current_deposition_algo); #ifdef WARPX_RZ - warpx_current_deposition_rz_volume_scaling( - jx_ptr, &ngJ, jxntot.getVect(), - jy_ptr, &ngJ, jyntot.getVect(), - jz_ptr, &ngJ, jzntot.getVect(), - &xyzmin[0], &dx[0]); + warpx_current_deposition_rz_volume_scaling( + jx_ptr, &ngJ, jxntot.getVect(), + jy_ptr, &ngJ, jyntot.getVect(), + jz_ptr, &ngJ, jzntot.getVect(), + &xyzmin[0], &dx[0]); #endif - - BL_PROFILE_VAR_STOP(blp_pxr_cd); + BL_PROFILE_VAR_STOP(blp_pxr_cd); #ifndef AMREX_USE_GPU - BL_PROFILE_VAR_START(blp_accumulate); - - (*cjx)[pti].atomicAdd(local_jx[thread_num], tbx, tbx, 0, 0, 1); - (*cjy)[pti].atomicAdd(local_jy[thread_num], tby, tby, 0, 0, 1); - (*cjz)[pti].atomicAdd(local_jz[thread_num], tbz, tbz, 0, 0, 1); - - BL_PROFILE_VAR_STOP(blp_accumulate); + BL_PROFILE_VAR_START(blp_accumulate); + // CPU, tiling: atomicAdd local_jx into jx + // (same for jx and jz) + (*jx)[pti].atomicAdd(local_jx[thread_num], tbx, tbx, 0, 0, 1); + (*jy)[pti].atomicAdd(local_jy[thread_num], tby, tby, 0, 0, 1); + (*jz)[pti].atomicAdd(local_jz[thread_num], tbz, tbz, 0, 0, 1); + BL_PROFILE_VAR_STOP(blp_accumulate); #endif - } -}; - +} void WarpXParticleContainer::DepositCharge ( WarpXParIter& pti, RealVector& wp, -- cgit v1.2.3 From 3ad66d17dcbca4b94ecab60e3ab53282e7552547 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Mon, 8 Jul 2019 09:51:47 -0700 Subject: number of guard cells in current j multifab --- Source/Laser/LaserParticleContainer.cpp | 13 ++++++++----- Source/Particles/PhysicalParticleContainer.cpp | 12 +++++++----- Source/Particles/WarpXParticleContainer.H | 3 +-- Source/Particles/WarpXParticleContainer.cpp | 3 ++- 4 files changed, 18 insertions(+), 13 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Laser/LaserParticleContainer.cpp b/Source/Laser/LaserParticleContainer.cpp index eb1eaa2f0..515aa1f5d 100644 --- a/Source/Laser/LaserParticleContainer.cpp +++ b/Source/Laser/LaserParticleContainer.cpp @@ -506,11 +506,14 @@ LaserParticleContainer::Evolve (int lev, // Deposit inside domains DepositCurrent(pti, wp, uxp, uyp, uzp, &jx, &jy, &jz, 0, np_current, thread_num, - lev, lev, dt, jx.nGrow()); - // Deposit in buffers - DepositCurrent(pti, wp, uxp, uyp, uzp, cjx, cjy, cjz, - np_current, np-np_current, thread_num, - lev, lev-1, dt, jx.nGrow()); + lev, lev, dt); + bool has_buffer = cjx; + if (has_buffer){ + // Deposit in buffers + DepositCurrent(pti, wp, uxp, uyp, uzp, cjx, cjy, cjz, + np_current, np-np_current, thread_num, + lev, lev-1, dt); + } // // copy particle data back diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 1ed318c13..43b46ec49 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1532,11 +1532,13 @@ PhysicalParticleContainer::Evolve (int lev, // Deposit inside domains DepositCurrent(pti, wp, uxp, uyp, uzp, &jx, &jy, &jz, 0, np_current, thread_num, - lev, lev, dt, jx.nGrow()); - // Deposit in buffers - DepositCurrent(pti, wp, uxp, uyp, uzp, cjx, cjy, cjz, - np_current, np-np_current, thread_num, - lev, lev-1, dt, jx.nGrow()); + lev, lev, dt); + if (has_buffer){ + // Deposit in buffers + DepositCurrent(pti, wp, uxp, uyp, uzp, cjx, cjy, cjz, + np_current, np-np_current, thread_num, + lev, lev-1, dt); + } // // copy particle data back diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 3098e9296..0e800bf1d 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -187,8 +187,7 @@ public: int thread_num, int lev, int depos_lev, - amrex::Real dt, - const long ngJ); + amrex::Real dt); // If particles start outside of the domain, ContinuousInjection // makes sure that they are initialized when they enter the domain, and diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 20d4049a2..5cdcdd1e3 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -301,7 +301,7 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, MultiFab* jx, MultiFab* jy, MultiFab* jz, const long offset, const long np_to_depose, int thread_num, int lev, int depos_lev, - Real dt, const long ngJ) + Real dt) { AMREX_ALWAYS_ASSERT_WITH_MESSAGE((depos_lev==(lev-1)) || (depos_lev==(lev )), @@ -309,6 +309,7 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, // If no particles, do not do anything if (np_to_depose == 0) return; + const long ngJ = jx->nGrow(); const std::array& dx = WarpX::CellSize(std::max(depos_lev,0)); const long lvect = 8; int j_is_nodal = jx->is_nodal() and jy->is_nodal() and jz->is_nodal(); -- cgit v1.2.3 From 6d3848aa366d40a3b8f48131da172a6aef5ebae0 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Mon, 8 Jul 2019 16:01:37 -0700 Subject: bug when tiling on CPU --- Examples/Tests/SingleParticle/Untitled.ipynb | 114 +++++++++++++++++++++++++ Examples/Tests/SingleParticle/inputs | 10 +-- Source/Particles/PhysicalParticleContainer.cpp | 2 +- Source/Particles/WarpXParticleContainer.cpp | 66 +++++++++++--- 4 files changed, 172 insertions(+), 20 deletions(-) create mode 100644 Examples/Tests/SingleParticle/Untitled.ipynb (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Examples/Tests/SingleParticle/Untitled.ipynb b/Examples/Tests/SingleParticle/Untitled.ipynb new file mode 100644 index 000000000..7a98be71a --- /dev/null +++ b/Examples/Tests/SingleParticle/Untitled.ipynb @@ -0,0 +1,114 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# Import statements\n", + "import yt ; yt.funcs.mylog.setLevel(50)\n", + "import numpy as np\n", + "import scipy.constants as scc\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib notebook" + ] + }, + { + "cell_type": "code", + "execution_count": 82, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/mthevenet/anaconda3/lib/python3.7/site-packages/yt/units/yt_array.py:1394: RuntimeWarning: invalid value encountered in true_divide\n", + " out=out, **kwargs)\n" + ] + }, + { + "data": { + "text/html": [ + "
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "ds = yt.load( './old_diags/plt00001' ) # Create a dataset object\n", + "sl = yt.SlicePlot(ds, 2, 'jz', aspect=.4) # Create a sliceplot object\n", + "sl.annotate_grids() # Show grids\n", + "sl.show() # Show the plot" + ] + }, + { + "cell_type": "code", + "execution_count": 83, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/mthevenet/anaconda3/lib/python3.7/site-packages/yt/units/yt_array.py:1394: RuntimeWarning: invalid value encountered in true_divide\n", + " out=out, **kwargs)\n" + ] + }, + { + "data": { + "text/html": [ + "
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "ds = yt.load( './new_diags/plt00001' ) # Create a dataset object\n", + "sl = yt.SlicePlot(ds, 2, 'jx', aspect=.4) # Create a sliceplot object\n", + "sl.annotate_grids() # Show grids\n", + "sl.show() # Show the plot" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + " " + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.1" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Examples/Tests/SingleParticle/inputs b/Examples/Tests/SingleParticle/inputs index 89e3eb1c6..feecb996c 100644 --- a/Examples/Tests/SingleParticle/inputs +++ b/Examples/Tests/SingleParticle/inputs @@ -1,11 +1,11 @@ -max_step = 1 -amr.n_cell = 8 8 +max_step = 10 +amr.n_cell = 8 16 amr.max_level = 0 amr.blocking_factor = 8 amr.max_grid_size = 16 amr.plot_int = 1 geometry.coord_sys = 0 -geometry.is_periodic = 0 0 +geometry.is_periodic = 1 1 geometry.prob_lo = -8 -12 geometry.prob_hi = 8 12 warpx.do_pml = 0 @@ -23,6 +23,6 @@ particles.species_names = electron electron.charge = -q_e electron.mass = 1.e10 electron.injection_style = "SingleParticle" -electron.single_particle_pos = 0.0 0.0 0.0 -electron.single_particle_vel = 1.e20 0.0 1.e20 # gamma*beta +electron.single_particle_pos = 4 0 6 +electron.single_particle_vel = -2 0 -5 # -2. 1. 4. electron.single_particle_weight = 1.0 diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 43b46ec49..b78d36c87 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1530,6 +1530,7 @@ PhysicalParticleContainer::Evolve (int lev, // Current Deposition // // Deposit inside domains + Print()<<"np_current "<& xyzmin_tile = WarpX::LowerCorner(tilebox, depos_lev); - const std::array& xyzmin = xyzmin_tile; - Print()<<" xyzmin_tile "<& xyzmin_tile = WarpX::LowerCorner(tilebox, depos_lev); + // const std::array& xyzmin = xyzmin_tile; + const std::array& xyzmin = WarpX::LowerCorner(tilebox, depos_lev);; + + // Print()<<" xyzmin_tile "<array(pti); auto jy_arr = jy->array(pti); auto jz_arr = jz->array(pti); - - // auto const& jx_arr = jx->array(); - // auto const& jy_arr = jy->array(); - // auto const& jz_arr = jz->array(); + */ + + // const auto& jx_arr = local_jx[thread_num].array(); + // const auto& jy_arr = local_jy[thread_num].array(); + // const auto& jz_arr = local_jz[thread_num].array(); + + //Array4 const& jx_arr = local_jx[thread_num].array(tbx); + //Array4 const& jy_arr = local_jy[thread_num].array(tby); + //Array4 const& jz_arr = local_jz[thread_num].array(tbz); + Array4 const& jx_arr = local_jx[thread_num].array(); + Array4 const& jy_arr = local_jy[thread_num].array(); + Array4 const& jz_arr = local_jz[thread_num].array(); + + Dim3 lox = lbound(jx_arr); + Dim3 loy = lbound(jy_arr); + Dim3 loz = lbound(jz_arr); + /* + Dim3 loy = lbound(local_jy[thread_num]); + Dim3 loz = lbound(local_jz[thread_num]); + Dim3 hix = hbound(local_jx[thread_num]); + Dim3 hiy = hbound(local_jy[thread_num]); + Dim3 hiz = hbound(local_jz[thread_num]); + */ + Print()<<" lox "< Vx, Vy, Vz; //pti.GetPosition(Vx, Vy, Vz); @@ -446,7 +474,7 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, std::cout<<" vx "< Date: Tue, 9 Jul 2019 16:22:09 -0700 Subject: Changed warpx_copy_attribs function from F90 to CPP inside PhysicalParticleContainer class. --- .DS_Store | Bin 0 -> 6148 bytes Source/.DS_Store | Bin 0 -> 6148 bytes Source/FortranInterface/WarpX_f.F90 | 33 ------------ Source/FortranInterface/WarpX_f.H | 6 --- Source/Particles/PhysicalParticleContainer.H | 3 ++ Source/Particles/PhysicalParticleContainer.cpp | 56 ++++++++++++++------- .../Particles/RigidInjectedParticleContainer.cpp | 20 ++------ 7 files changed, 47 insertions(+), 71 deletions(-) create mode 100644 .DS_Store create mode 100644 Source/.DS_Store (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 000000000..b96111fac Binary files /dev/null and b/.DS_Store differ diff --git a/Source/.DS_Store b/Source/.DS_Store new file mode 100644 index 000000000..01640e062 Binary files /dev/null and b/Source/.DS_Store differ diff --git a/Source/FortranInterface/WarpX_f.F90 b/Source/FortranInterface/WarpX_f.F90 index 0560a9981..542cc95a1 100644 --- a/Source/FortranInterface/WarpX_f.F90 +++ b/Source/FortranInterface/WarpX_f.F90 @@ -8,39 +8,6 @@ module warpx_module contains - subroutine warpx_copy_attribs(np, xp, yp, zp, uxp, uyp, uzp, & - xpold, ypold, zpold, uxpold, uypold, uzpold) & - bind(c,name='warpx_copy_attribs') - integer(c_long), intent(in) :: np - real(amrex_real), intent(in) :: xp(np) - real(amrex_real), intent(in) :: yp(np) - real(amrex_real), intent(in) :: zp(np) - real(amrex_real), intent(in) :: uxp(np) - real(amrex_real), intent(in) :: uyp(np) - real(amrex_real), intent(in) :: uzp(np) - real(amrex_real), intent(inout) :: xpold(np) - real(amrex_real), intent(inout) :: ypold(np) - real(amrex_real), intent(inout) :: zpold(np) - real(amrex_real), intent(inout) :: uxpold(np) - real(amrex_real), intent(inout) :: uypold(np) - real(amrex_real), intent(inout) :: uzpold(np) - - integer n - - do n = 1, np - - xpold(n) = xp(n) - ypold(n) = yp(n) - zpold(n) = zp(n) - - uxpold(n) = uxp(n) - uypold(n) = uyp(n) - uzpold(n) = uzp(n) - - end do - - end subroutine warpx_copy_attribs - subroutine warpx_compute_E (lo, hi, & phi, phlo, phhi, & Ex, Exlo, Exhi, & diff --git a/Source/FortranInterface/WarpX_f.H b/Source/FortranInterface/WarpX_f.H index 98dd8685d..0440148eb 100644 --- a/Source/FortranInterface/WarpX_f.H +++ b/Source/FortranInterface/WarpX_f.H @@ -75,12 +75,6 @@ extern "C" { #endif - void warpx_copy_attribs(const long* np, - const amrex_real* xp, const amrex_real* yp, const amrex_real* zp, - const amrex_real* uxp, const amrex_real* uyp, const amrex_real* uzp, - amrex_real* xpold, amrex_real* ypold, amrex_real* zpold, - amrex_real* uxpold, amrex_real* uypold, amrex_real* uzpold); - // Charge deposition void warpx_charge_deposition(amrex::Real* rho, const long* np, const amrex::Real* xp, const amrex::Real* yp, const amrex::Real* zp, const amrex::Real* w, diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index bd8cfb47e..99fc0f8da 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -77,6 +77,9 @@ public: const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz) override; + + void warpx_copy_attribs(WarpXParIter& pti,const amrex::Real* xp, + const amrex::Real* yp, const amrex::Real* zp); virtual void PostRestart () final {} diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 43b46ec49..69c27cd21 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1729,7 +1729,13 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti, Real dt) { - // This wraps the call to warpx_particle_pusher so that inheritors can modify the call. + if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) + { + warpx_copy_attribs(pti, xp.dataPtr(), yp.dataPtr(), zp.dataPtr()); + } + + // The following attributes should be included in CPP version of warpx_particle_pusher + // This wraps the call to warpx_particle_pusher so that inheritors can modify the call. auto& attribs = pti.GetAttribs(); auto& uxp = attribs[PIdx::ux]; auto& uyp = attribs[PIdx::uy]; @@ -1741,22 +1747,7 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti, auto& Byp = attribs[PIdx::By]; auto& Bzp = attribs[PIdx::Bz]; const long np = pti.numParticles(); - - if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) - { - auto& xpold = pti.GetAttribs(particle_comps["xold"]); - auto& ypold = pti.GetAttribs(particle_comps["yold"]); - auto& zpold = pti.GetAttribs(particle_comps["zold"]); - auto& uxpold = pti.GetAttribs(particle_comps["uxold"]); - auto& uypold = pti.GetAttribs(particle_comps["uyold"]); - auto& uzpold = pti.GetAttribs(particle_comps["uzold"]); - - warpx_copy_attribs(&np, xp.dataPtr(), yp.dataPtr(), zp.dataPtr(), - uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(), - xpold.dataPtr(), ypold.dataPtr(), zpold.dataPtr(), - uxpold.dataPtr(), uypold.dataPtr(), uzpold.dataPtr()); - } - + warpx_particle_pusher(&np, xp.dataPtr(), yp.dataPtr(), @@ -1870,6 +1861,37 @@ PhysicalParticleContainer::PushP (int lev, Real dt, } } +void PhysicalParticleContainer::warpx_copy_attribs(WarpXParIter& pti,const amrex::Real* xp, + const amrex::Real* yp, const amrex::Real* zp) +{ + auto& attribs = pti.GetAttribs(); + + Real* AMREX_RESTRICT uxp = attribs[PIdx::ux].dataPtr(); + Real* AMREX_RESTRICT uyp = attribs[PIdx::uy].dataPtr(); + Real* AMREX_RESTRICT uzp = attribs[PIdx::uz].dataPtr(); + + Real* AMREX_RESTRICT xpold = attribs[PIdx::nattribs].dataPtr(); + Real* AMREX_RESTRICT ypold = attribs[PIdx::nattribs+1].dataPtr(); + Real* AMREX_RESTRICT zpold = attribs[PIdx::nattribs+2].dataPtr(); + Real* AMREX_RESTRICT uxpold = attribs[PIdx::nattribs+3].dataPtr(); + Real* AMREX_RESTRICT uypold = attribs[PIdx::nattribs+4].dataPtr(); + Real* AMREX_RESTRICT uzpold = attribs[PIdx::nattribs+5].dataPtr(); + + const long np = pti.numParticles(); + + ParallelFor( np, + [=] AMREX_GPU_DEVICE (long i) { + xpold[i]=xp[i]; + ypold[i]=yp[i]; + zpold[i]=zp[i]; + + uxpold[i]=uxp[i]; + uypold[i]=uyp[i]; + uzpold[i]=uzp[i]; + } + ); +} + void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real z_old, const Real z_new, const Real t_boost, const Real t_lab, const Real dt, diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index 2a3e8dd0d..ca02d1458 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -211,6 +211,11 @@ RigidInjectedParticleContainer::PushPX(WarpXParIter& pti, Real dt) { + if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) + { + warpx_copy_attribs(pti, xp.dataPtr(), yp.dataPtr(), zp.dataPtr()); + } + // This wraps the call to warpx_particle_pusher so that inheritors can modify the call. auto& attribs = pti.GetAttribs(); auto& uxp = attribs[PIdx::ux]; @@ -224,21 +229,6 @@ RigidInjectedParticleContainer::PushPX(WarpXParIter& pti, auto& Bzp = attribs[PIdx::Bz]; const long np = pti.numParticles(); - if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) - { - auto& xpold = pti.GetAttribs(particle_comps["xold"]); - auto& ypold = pti.GetAttribs(particle_comps["yold"]); - auto& zpold = pti.GetAttribs(particle_comps["zold"]); - auto& uxpold = pti.GetAttribs(particle_comps["uxold"]); - auto& uypold = pti.GetAttribs(particle_comps["uyold"]); - auto& uzpold = pti.GetAttribs(particle_comps["uzold"]); - - warpx_copy_attribs(&np, xp.dataPtr(), yp.dataPtr(), zp.dataPtr(), - uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(), - xpold.dataPtr(), ypold.dataPtr(), zpold.dataPtr(), - uxpold.dataPtr(), uypold.dataPtr(), uzpold.dataPtr()); - } - // Save the position and momenta, making copies Cuda::ManagedDeviceVector xp_save, yp_save, zp_save; RealVector uxp_save, uyp_save, uzp_save; -- cgit v1.2.3 From 9957d80bce7bf685f96ccffe15b420b2216d7f11 Mon Sep 17 00:00:00 2001 From: Diana Amorim Date: Tue, 9 Jul 2019 18:06:24 -0700 Subject: Attempt to fix segfault issue - Unsuccessful. Tested alternatives to . --- .DS_Store | Bin 6148 -> 0 bytes .gitignore | 1 + Source/Particles/PhysicalParticleContainer.cpp | 22 ++++++++++++---------- 3 files changed, 13 insertions(+), 10 deletions(-) delete mode 100644 .DS_Store (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index b96111fac..000000000 Binary files a/.DS_Store and /dev/null differ diff --git a/.gitignore b/.gitignore index 02bd9eafa..13147b8da 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.DS_Store plt* chk* main?d.*.ex diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 69c27cd21..e8654e573 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1866,20 +1866,22 @@ void PhysicalParticleContainer::warpx_copy_attribs(WarpXParIter& pti,const amrex { auto& attribs = pti.GetAttribs(); - Real* AMREX_RESTRICT uxp = attribs[PIdx::ux].dataPtr(); - Real* AMREX_RESTRICT uyp = attribs[PIdx::uy].dataPtr(); - Real* AMREX_RESTRICT uzp = attribs[PIdx::uz].dataPtr(); + //Real* AMREX_RESTRICT - Real* AMREX_RESTRICT xpold = attribs[PIdx::nattribs].dataPtr(); - Real* AMREX_RESTRICT ypold = attribs[PIdx::nattribs+1].dataPtr(); - Real* AMREX_RESTRICT zpold = attribs[PIdx::nattribs+2].dataPtr(); - Real* AMREX_RESTRICT uxpold = attribs[PIdx::nattribs+3].dataPtr(); - Real* AMREX_RESTRICT uypold = attribs[PIdx::nattribs+4].dataPtr(); - Real* AMREX_RESTRICT uzpold = attribs[PIdx::nattribs+5].dataPtr(); + const amrex::Real* uxp = attribs[PIdx::ux].dataPtr(); + const amrex::Real* uyp = attribs[PIdx::uy].dataPtr(); + const amrex::Real* uzp = attribs[PIdx::uz].dataPtr(); + + amrex::Real* xpold = attribs[PIdx::nattribs].dataPtr(); + amrex::Real* ypold = attribs[PIdx::nattribs+1].dataPtr(); + amrex::Real* zpold = attribs[PIdx::nattribs+2].dataPtr(); + amrex::Real* uxpold = attribs[PIdx::nattribs+3].dataPtr(); + amrex::Real* uypold = attribs[PIdx::nattribs+4].dataPtr(); + amrex::Real* uzpold = attribs[PIdx::nattribs+5].dataPtr(); const long np = pti.numParticles(); - ParallelFor( np, + amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (long i) { xpold[i]=xp[i]; ypold[i]=yp[i]; -- cgit v1.2.3 From b69ea2ccf999d80c4f2bb5e295a5ae170bb58a34 Mon Sep 17 00:00:00 2001 From: Diana Amorim Date: Wed, 10 Jul 2019 13:33:48 -0700 Subject: Fixed segfault error. --- Source/Particles/PhysicalParticleContainer.cpp | 27 +++++++++++++------------- 1 file changed, 13 insertions(+), 14 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index e8654e573..c45f73f0f 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1861,27 +1861,26 @@ PhysicalParticleContainer::PushP (int lev, Real dt, } } -void PhysicalParticleContainer::warpx_copy_attribs(WarpXParIter& pti,const amrex::Real* xp, - const amrex::Real* yp, const amrex::Real* zp) +void PhysicalParticleContainer::warpx_copy_attribs(WarpXParIter& pti,const Real* xp, + const Real* yp, const Real* zp) { + auto& attribs = pti.GetAttribs(); - //Real* AMREX_RESTRICT - - const amrex::Real* uxp = attribs[PIdx::ux].dataPtr(); - const amrex::Real* uyp = attribs[PIdx::uy].dataPtr(); - const amrex::Real* uzp = attribs[PIdx::uz].dataPtr(); + Real* AMREX_RESTRICT uxp = attribs[PIdx::ux].dataPtr(); + Real* AMREX_RESTRICT uyp = attribs[PIdx::uy].dataPtr(); + Real* AMREX_RESTRICT uzp = attribs[PIdx::uz].dataPtr(); - amrex::Real* xpold = attribs[PIdx::nattribs].dataPtr(); - amrex::Real* ypold = attribs[PIdx::nattribs+1].dataPtr(); - amrex::Real* zpold = attribs[PIdx::nattribs+2].dataPtr(); - amrex::Real* uxpold = attribs[PIdx::nattribs+3].dataPtr(); - amrex::Real* uypold = attribs[PIdx::nattribs+4].dataPtr(); - amrex::Real* uzpold = attribs[PIdx::nattribs+5].dataPtr(); + Real* AMREX_RESTRICT xpold = pti.GetAttribs(particle_comps["xold"]).dataPtr(); + Real* AMREX_RESTRICT ypold = pti.GetAttribs(particle_comps["xold"]).dataPtr(); + Real* AMREX_RESTRICT zpold = pti.GetAttribs(particle_comps["xold"]).dataPtr(); + Real* AMREX_RESTRICT uxpold = pti.GetAttribs(particle_comps["uxold"]).dataPtr(); + Real* AMREX_RESTRICT uypold = pti.GetAttribs(particle_comps["uyold"]).dataPtr(); + Real* AMREX_RESTRICT uzpold = pti.GetAttribs(particle_comps["uzold"]).dataPtr(); const long np = pti.numParticles(); - amrex::ParallelFor( np, + ParallelFor( np, [=] AMREX_GPU_DEVICE (long i) { xpold[i]=xp[i]; ypold[i]=yp[i]; -- cgit v1.2.3 From fb25dde6d6770f01bdfee71e10137c5fe8a8e035 Mon Sep 17 00:00:00 2001 From: Diana Amorim Date: Wed, 10 Jul 2019 16:56:58 -0700 Subject: Changed name of function to copy_attribs() --- Source/Particles/PhysicalParticleContainer.H | 2 +- Source/Particles/PhysicalParticleContainer.cpp | 4 ++-- Source/Particles/RigidInjectedParticleContainer.cpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index 99fc0f8da..d55764682 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -78,7 +78,7 @@ public: const amrex::MultiFab& By, const amrex::MultiFab& Bz) override; - void warpx_copy_attribs(WarpXParIter& pti,const amrex::Real* xp, + void copy_attribs(WarpXParIter& pti,const amrex::Real* xp, const amrex::Real* yp, const amrex::Real* zp); virtual void PostRestart () final {} diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index c45f73f0f..addde5e37 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1731,7 +1731,7 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti, if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) { - warpx_copy_attribs(pti, xp.dataPtr(), yp.dataPtr(), zp.dataPtr()); + copy_attribs(pti, xp.dataPtr(), yp.dataPtr(), zp.dataPtr()); } // The following attributes should be included in CPP version of warpx_particle_pusher @@ -1861,7 +1861,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, } } -void PhysicalParticleContainer::warpx_copy_attribs(WarpXParIter& pti,const Real* xp, +void PhysicalParticleContainer::copy_attribs(WarpXParIter& pti,const Real* xp, const Real* yp, const Real* zp) { diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index ca02d1458..9bd4cb4fc 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -213,7 +213,7 @@ RigidInjectedParticleContainer::PushPX(WarpXParIter& pti, if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) { - warpx_copy_attribs(pti, xp.dataPtr(), yp.dataPtr(), zp.dataPtr()); + copy_attribs(pti, xp.dataPtr(), yp.dataPtr(), zp.dataPtr()); } // This wraps the call to warpx_particle_pusher so that inheritors can modify the call. -- cgit v1.2.3 From 5baf28e5115f450d4f2fd821c33df5a37054792d Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Thu, 11 Jul 2019 09:07:26 -0700 Subject: remove print statements --- Source/Particles/PhysicalParticleContainer.cpp | 1 - Source/Particles/WarpXParticleContainer.cpp | 4 ---- 2 files changed, 5 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index b78d36c87..df08d77a7 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1530,7 +1530,6 @@ PhysicalParticleContainer::Evolve (int lev, // Current Deposition // // Deposit inside domains - Print()<<"np_current "< Date: Thu, 11 Jul 2019 14:26:38 -0700 Subject: option use_picsar_deposition --- Source/Particles/Deposition/CurrentDeposition.H | 3 ++- Source/Particles/PhysicalParticleContainer.cpp | 32 ++++++++++++++++++------- Source/Particles/WarpXParticleContainer.H | 15 ++++++++++++ Source/Particles/WarpXParticleContainer.cpp | 2 +- Source/WarpX.cpp | 6 ++--- 5 files changed, 44 insertions(+), 14 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 006ff05fe..66db003a3 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -109,7 +109,7 @@ void doDepositionShapeN(Real* xp, Real* yp, Real* zp, const amrex::Real q) { BL_PROFILE("PICSAR::CurrentDeposition"); - + Real dxi = 1.0/dx[0]; Real dzi = 1.0/dx[2]; Real dts2dx = 0.5*dt*dxi; @@ -130,6 +130,7 @@ void doDepositionShapeN(Real* xp, Real* yp, Real* zp, compute_shape_factor_lo(shape_lo); Real clightsq = 1.0/PhysConst::c/PhysConst::c; + // Loop over particles and deposit into jx_arr, jy_arr and jz_arr ParallelFor( np_to_depose, [=] AMREX_GPU_DEVICE (long ip) { diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index df08d77a7..86cd0827f 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1529,16 +1529,30 @@ PhysicalParticleContainer::Evolve (int lev, // // Current Deposition // - // Deposit inside domains - DepositCurrent(pti, wp, uxp, uyp, uzp, &jx, &jy, &jz, - 0, np_current, thread_num, - lev, lev, dt); - if (has_buffer){ - // Deposit in buffers - DepositCurrent(pti, wp, uxp, uyp, uzp, cjx, cjy, cjz, - np_current, np-np_current, thread_num, - lev, lev-1, dt); + if (WarpX::use_picsar_deposition) { + // Deposit inside domains + DepositCurrentFortran(pti, wp, uxp, uyp, uzp, &jx, &jy, &jz, + 0, np_current, thread_num, + lev, lev, dt); + if (has_buffer){ + // Deposit in buffers + DepositCurrentFortran(pti, wp, uxp, uyp, uzp, cjx, cjy, cjz, + np_current, np-np_current, thread_num, + lev, lev-1, dt); + } + } else { + // Deposit inside domains + DepositCurrent(pti, wp, uxp, uyp, uzp, &jx, &jy, &jz, + 0, np_current, thread_num, + lev, lev, dt); + if (has_buffer){ + // Deposit in buffers + DepositCurrent(pti, wp, uxp, uyp, uzp, cjx, cjy, cjz, + np_current, np-np_current, thread_num, + lev, lev-1, dt); + } } + // // copy particle data back // diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 6182ec5bd..662b2e1b8 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -189,6 +189,21 @@ public: int depos_lev, amrex::Real dt); + virtual void DepositCurrentFortran(WarpXParIter& pti, + RealVector& wp, + RealVector& uxp, + RealVector& uyp, + RealVector& uzp, + amrex::MultiFab* jx, + amrex::MultiFab* jy, + amrex::MultiFab* jz, + const long offset, + const long np_to_depose, + int thread_num, + int lev, + int depos_lev, + amrex::Real dt); + // If particles start outside of the domain, ContinuousInjection // makes sure that they are initialized when they enter the domain, and // NOT before. Virtual function, overriden by derived classes. diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 42b99efcb..ac123f9b6 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -435,7 +435,7 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, const long offset, const long np_to_depose, int thread_num, int lev, int depos_lev, Real dt) -{ +{ AMREX_ALWAYS_ASSERT_WITH_MESSAGE((depos_lev==(lev-1)) || (depos_lev==(lev )), "Deposition buffers only work for lev-1"); diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 24011fb2f..fd4840b8d 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -485,11 +485,11 @@ WarpX::ReadParameters () { ParmParse pp("algo"); - pp.query("use_picsar_deposition", use_picsar_deposition) + pp.query("use_picsar_deposition", use_picsar_deposition); current_deposition_algo = GetAlgorithmInteger(pp, "current_deposition"); - if (!use_picsar_algo){ + if (!use_picsar_deposition){ AMREX_ALWAYS_ASSERT_WITH_MESSAGE( current_deposition_algo >= 2, - "if not use_picsar_deposition, cannot use Esirkepov deposition.") + "if not use_picsar_deposition, cannot use Esirkepov deposition."); } charge_deposition_algo = GetAlgorithmInteger(pp, "charge_deposition"); field_gathering_algo = GetAlgorithmInteger(pp, "field_gathering"); -- cgit v1.2.3 From 31edd3bad5d2747c7e045240b488d7fd101f1a30 Mon Sep 17 00:00:00 2001 From: Dave Grote Date: Fri, 12 Jul 2019 17:44:25 -0700 Subject: Changed nmodes to n_rz_azimuthal_modes --- Python/pywarpx/picmi.py | 2 +- Source/Diagnostics/FieldIO.cpp | 24 ++-- Source/Diagnostics/WarpXIO.cpp | 4 +- Source/Evolve/WarpXEvolveEM.cpp | 14 +- Source/FieldSolver/WarpXPushFieldsEM.cpp | 4 +- Source/FortranInterface/WarpX_f.H | 10 +- Source/FortranInterface/WarpX_picsar.F90 | 150 ++++++++++----------- Source/Particles/PhysicalParticleContainer.cpp | 12 +- .../Particles/RigidInjectedParticleContainer.cpp | 2 +- Source/Particles/WarpXParticleContainer.cpp | 4 +- Source/WarpX.H | 2 +- Source/WarpX.cpp | 8 +- 12 files changed, 118 insertions(+), 118 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index b69a3ab21..67318c802 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -280,7 +280,7 @@ class CylindricalGrid(picmistandard.PICMI_CylindricalGrid): pywarpx.geometry.is_periodic = '0 %d'%(self.bc_zmin=='periodic') # Is periodic? pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound - pywarpx.warpx.nmodes = self.n_azimuthal_modes + pywarpx.warpx.n_rz_azimuthal_modes = self.n_azimuthal_modes if self.moving_window_velocity is not None and np.any(np.not_equal(self.moving_window_velocity, 0.)): pywarpx.warpx.do_moving_window = 1 diff --git a/Source/Diagnostics/FieldIO.cpp b/Source/Diagnostics/FieldIO.cpp index c8eaa8ee0..135b7ff19 100644 --- a/Source/Diagnostics/FieldIO.cpp +++ b/Source/Diagnostics/FieldIO.cpp @@ -388,14 +388,14 @@ AverageAndPackScalarField( MultiFab& mf_avg, void AddToVarNames( Vector& varnames, std::string name, std::string suffix, - int nmodes ) { + int n_rz_azimuthal_modes ) { auto coords = {"x", "y", "z"}; auto coordsRZ = {"r", "theta", "z"}; for(auto coord:coords) varnames.push_back(name+coord+suffix); - if (nmodes > 1) { + if (n_rz_azimuthal_modes > 1) { // Note that the names are added in the same order as the fields // are packed in AverageAndPackVectorField - for (int i = 0 ; i < nmodes ; i++) { + for (int i = 0 ; i < n_rz_azimuthal_modes ; i++) { for(auto coord:coordsRZ) { varnames.push_back(name + coord + suffix + std::to_string(i) + "_real"); } @@ -415,12 +415,12 @@ WarpX::AverageAndPackFields ( Vector& varnames, amrex::Vector& mf_avg, const int ngrow) const { // Factor to account for quantities that have multiple components. - // If nmodes > 1, allow space for total field and the real and + // If n_rz_azimuthal_modes > 1, allow space for total field and the real and // imaginary part of each node. For now, also include the // imaginary part of mode 0 for code symmetry, even though // it is always zero. int modes_factor = 1; - if (nmodes > 1) modes_factor = 2*nmodes + 1; + if (n_rz_azimuthal_modes > 1) modes_factor = 2*n_rz_azimuthal_modes + 1; // Count how many different fields should be written (ncomp) const int ncomp = 0 @@ -450,17 +450,17 @@ WarpX::AverageAndPackFields ( Vector& varnames, int dcomp = 0; if (plot_J_field) { AverageAndPackVectorField(mf_avg[lev], current_fp[lev], dmap[lev], dcomp, ngrow); - if (lev == 0) AddToVarNames(varnames, "j", "", nmodes); + if (lev == 0) AddToVarNames(varnames, "j", "", n_rz_azimuthal_modes); dcomp += 3*modes_factor; } if (plot_E_field) { AverageAndPackVectorField(mf_avg[lev], Efield_aux[lev], dmap[lev], dcomp, ngrow); - if (lev == 0) AddToVarNames(varnames, "E", "", nmodes); + if (lev == 0) AddToVarNames(varnames, "E", "", n_rz_azimuthal_modes); dcomp += 3*modes_factor; } if (plot_B_field) { AverageAndPackVectorField(mf_avg[lev], Bfield_aux[lev], dmap[lev], dcomp, ngrow); - if (lev == 0) AddToVarNames(varnames, "B", "", nmodes); + if (lev == 0) AddToVarNames(varnames, "B", "", n_rz_azimuthal_modes); dcomp += 3*modes_factor; } @@ -569,10 +569,10 @@ WarpX::AverageAndPackFields ( Vector& varnames, if (plot_finepatch) { AverageAndPackVectorField( mf_avg[lev], Efield_fp[lev], dmap[lev], dcomp, ngrow ); - if (lev == 0) AddToVarNames(varnames, "E", "_fp", nmodes); + if (lev == 0) AddToVarNames(varnames, "E", "_fp", n_rz_azimuthal_modes); dcomp += 3*modes_factor; AverageAndPackVectorField( mf_avg[lev], Bfield_fp[lev], dmap[lev], dcomp, ngrow ); - if (lev == 0) AddToVarNames(varnames, "B", "_fp", nmodes); + if (lev == 0) AddToVarNames(varnames, "B", "_fp", n_rz_azimuthal_modes); dcomp += 3*modes_factor; } @@ -589,7 +589,7 @@ WarpX::AverageAndPackFields ( Vector& varnames, AverageAndPackVectorField( mf_avg[lev], E, dmap[lev], dcomp, ngrow ); } - if (lev == 0) AddToVarNames(varnames, "E", "_cp", nmodes); + if (lev == 0) AddToVarNames(varnames, "E", "_cp", n_rz_azimuthal_modes); dcomp += 3*modes_factor; // now the magnetic field @@ -603,7 +603,7 @@ WarpX::AverageAndPackFields ( Vector& varnames, std::array, 3> B = getInterpolatedB(lev); AverageAndPackVectorField( mf_avg[lev], B, dmap[lev], dcomp, ngrow ); } - if (lev == 0) AddToVarNames(varnames, "B", "_cp", nmodes); + if (lev == 0) AddToVarNames(varnames, "B", "_cp", n_rz_azimuthal_modes); dcomp += 3*modes_factor; } diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index 214948b2b..bb6775680 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -416,12 +416,12 @@ WarpX::GetCellCenteredData() { Vector > cc(finest_level+1); // Factor to account for quantities that have multiple components. - // If nmodes > 1, allow space for total field and the real and + // If n_rz_azimuthal_modes > 1, allow space for total field and the real and // imaginary part of each node. For now, also include the // imaginary part of mode 0 for code symmetry, even though // it is always zero. int modes_factor = 1; - if (nmodes > 1) modes_factor = 2*nmodes + 1; + if (n_rz_azimuthal_modes > 1) modes_factor = 2*n_rz_azimuthal_modes + 1; for (int lev = 0; lev <= finest_level; ++lev) { diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index c5dbd73a7..110d07a6d 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -348,7 +348,7 @@ WarpX::OneStep_sub1 (Real curtime) RestrictRhoFromFineToCoarsePatch(fine_lev); ApplyFilterandSumBoundaryJ(fine_lev, PatchType::fine); NodalSyncJ(fine_lev, PatchType::fine); - ApplyFilterandSumBoundaryRho(fine_lev, PatchType::fine, 0, 2*nmodes); + ApplyFilterandSumBoundaryRho(fine_lev, PatchType::fine, 0, 2*n_rz_azimuthal_modes); NodalSyncRho(fine_lev, PatchType::fine, 0, 2); EvolveB(fine_lev, PatchType::fine, 0.5*dt[fine_lev]); @@ -375,7 +375,7 @@ WarpX::OneStep_sub1 (Real curtime) PushParticlesandDepose(coarse_lev, curtime); StoreCurrent(coarse_lev); AddCurrentFromFineLevelandSumBoundary(coarse_lev); - AddRhoFromFineLevelandSumBoundary(coarse_lev, 0, nmodes); + AddRhoFromFineLevelandSumBoundary(coarse_lev, 0, n_rz_azimuthal_modes); EvolveB(fine_lev, PatchType::coarse, dt[fine_lev]); EvolveF(fine_lev, PatchType::coarse, dt[fine_lev], DtType::FirstHalf); @@ -402,7 +402,7 @@ WarpX::OneStep_sub1 (Real curtime) RestrictRhoFromFineToCoarsePatch(fine_lev); ApplyFilterandSumBoundaryJ(fine_lev, PatchType::fine); NodalSyncJ(fine_lev, PatchType::fine); - ApplyFilterandSumBoundaryRho(fine_lev, PatchType::fine, 0, 2*nmodes); + ApplyFilterandSumBoundaryRho(fine_lev, PatchType::fine, 0, 2*n_rz_azimuthal_modes); NodalSyncRho(fine_lev, PatchType::fine, 0, 2); EvolveB(fine_lev, PatchType::fine, 0.5*dt[fine_lev]); @@ -428,7 +428,7 @@ WarpX::OneStep_sub1 (Real curtime) // by only half a coarse step (second half) RestoreCurrent(coarse_lev); AddCurrentFromFineLevelandSumBoundary(coarse_lev); - AddRhoFromFineLevelandSumBoundary(coarse_lev, nmodes, nmodes); + AddRhoFromFineLevelandSumBoundary(coarse_lev, n_rz_azimuthal_modes, n_rz_azimuthal_modes); EvolveE(fine_lev, PatchType::coarse, dt[fine_lev]); FillBoundaryE(fine_lev, PatchType::coarse); @@ -501,12 +501,12 @@ WarpX::ComputeDt () // high enough (The simulation became unstable). Real multimode_coeffs[6] = { 0.2105, 1.0, 3.5234, 8.5104, 15.5059, 24.5037 }; Real multimode_alpha; - if (nmodes < 7) { + if (n_rz_azimuthal_modes < 7) { // Use the table of the coefficients - multimode_alpha = multimode_coeffs[nmodes-1]; + multimode_alpha = multimode_coeffs[n_rz_azimuthal_modes-1]; } else { // Use a realistic extrapolation - multimode_alpha = (nmodes - 1)*(nmodes - 1) - 0.4; + multimode_alpha = (n_rz_azimuthal_modes - 1)*(n_rz_azimuthal_modes - 1) - 0.4; } deltat = cfl * 1./( std::sqrt((1+multimode_alpha)/(dx[0]*dx[0]) + 1./(dx[1]*dx[1])) * PhysConst::c ); #else diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index fc4fb902b..f08b44adb 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -109,7 +109,7 @@ WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real a_dt) tbx.loVect(), tbx.hiVect(), tby.loVect(), tby.hiVect(), tbz.loVect(), tbz.hiVect(), - &nmodes, + &n_rz_azimuthal_modes, BL_TO_FORTRAN_3D((*Ex)[mfi]), BL_TO_FORTRAN_3D((*Ey)[mfi]), BL_TO_FORTRAN_3D((*Ez)[mfi]), @@ -272,7 +272,7 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) tex.loVect(), tex.hiVect(), tey.loVect(), tey.hiVect(), tez.loVect(), tez.hiVect(), - &nmodes, + &n_rz_azimuthal_modes, BL_TO_FORTRAN_3D((*Ex)[mfi]), BL_TO_FORTRAN_3D((*Ey)[mfi]), BL_TO_FORTRAN_3D((*Ez)[mfi]), diff --git a/Source/FortranInterface/WarpX_f.H b/Source/FortranInterface/WarpX_f.H index 8dcd4222b..f193f30c4 100644 --- a/Source/FortranInterface/WarpX_f.H +++ b/Source/FortranInterface/WarpX_f.H @@ -102,7 +102,7 @@ extern "C" amrex::Real* jx, const long* jx_ng, const int* jx_ntot, amrex::Real* jy, const long* jy_ng, const int* jy_ntot, amrex::Real* jz, const long* jz_ng, const int* jz_ntot, - const long* nmodes, + const long* n_rz_azimuthal_modes, const long* np, const amrex::Real* xp, const amrex::Real* yp, const amrex::Real* zp, const amrex::Real* uxp, const amrex::Real* uyp,const amrex::Real* uzp, @@ -118,7 +118,7 @@ extern "C" amrex::Real* jx, const long* jx_ng, const int* jx_ntot, amrex::Real* jy, const long* jy_ng, const int* jy_ntot, amrex::Real* jz, const long* jz_ng, const int* jz_ntot, - const long* nmodes, + const long* n_rz_azimuthal_modes, const amrex::Real* rmin, const amrex::Real* dr); @@ -138,7 +138,7 @@ extern "C" const amrex::Real* bxg, const int* bxg_lo, const int* bxg_hi, const amrex::Real* byg, const int* byg_lo, const int* byg_hi, const amrex::Real* bzg, const int* bzg_lo, const int* bzg_hi, - const long* nmodes, + const long* n_rz_azimuthal_modes, const int* ll4symtry, const int* l_lower_order_in_v, const int* l_nodal, const long* lvect, const long* field_gathe_algo); @@ -197,7 +197,7 @@ extern "C" const int* xlo, const int* xhi, const int* ylo, const int* yhi, const int* zlo, const int* zhi, - const long* nmodes, + const long* n_rz_azimuthal_modes, BL_FORT_FAB_ARG_3D(ex), BL_FORT_FAB_ARG_3D(ey), BL_FORT_FAB_ARG_3D(ez), @@ -218,7 +218,7 @@ extern "C" const int* xlo, const int* xhi, const int* ylo, const int* yhi, const int* zlo, const int* zhi, - const long* nmodes, + const long* n_rz_azimuthal_modes, const BL_FORT_FAB_ARG_3D(ex), const BL_FORT_FAB_ARG_3D(ey), const BL_FORT_FAB_ARG_3D(ez), diff --git a/Source/FortranInterface/WarpX_picsar.F90 b/Source/FortranInterface/WarpX_picsar.F90 index e6e61b734..c94d8459f 100644 --- a/Source/FortranInterface/WarpX_picsar.F90 +++ b/Source/FortranInterface/WarpX_picsar.F90 @@ -87,7 +87,7 @@ contains ex,ey,ez,bx,by,bz,ixyzmin,xmin,ymin,zmin,dx,dy,dz,nox,noy,noz, & exg,exg_lo,exg_hi,eyg,eyg_lo,eyg_hi,ezg,ezg_lo,ezg_hi, & bxg,bxg_lo,bxg_hi,byg,byg_lo,byg_hi,bzg,bzg_lo,bzg_hi, & - nmodes, & + n_rz_azimuthal_modes, & ll4symtry,l_lower_order_in_v, l_nodal,& lvect,field_gathe_algo) & bind(C, name="warpx_geteb_energy_conserving") @@ -99,19 +99,19 @@ contains integer, intent(in) :: ixyzmin(AMREX_SPACEDIM) real(amrex_real), intent(in) :: xmin,ymin,zmin,dx,dy,dz integer(c_long), intent(in) :: field_gathe_algo - integer(c_long), intent(in) :: np,nmodes,nox,noy,noz + integer(c_long), intent(in) :: np,n_rz_azimuthal_modes,nox,noy,noz integer(c_int), intent(in) :: ll4symtry,l_lower_order_in_v, l_nodal integer(c_long),intent(in) :: lvect real(amrex_real), intent(in), dimension(np) :: xp,yp,zp real(amrex_real), intent(out), dimension(np) :: ex,ey,ez,bx,by,bz #ifdef WARPX_RZ ! The dimensions must be specified to allow the transpose - real(amrex_real),intent(in):: exg(exg_lo(1):exg_hi(1),exg_lo(2):exg_hi(2),2,nmodes) - real(amrex_real),intent(in):: eyg(eyg_lo(1):eyg_hi(1),eyg_lo(2):eyg_hi(2),2,nmodes) - real(amrex_real),intent(in):: ezg(ezg_lo(1):ezg_hi(1),ezg_lo(2):ezg_hi(2),2,nmodes) - real(amrex_real),intent(in):: bxg(bxg_lo(1):bxg_hi(1),bxg_lo(2):bxg_hi(2),2,nmodes) - real(amrex_real),intent(in):: byg(byg_lo(1):byg_hi(1),byg_lo(2):byg_hi(2),2,nmodes) - real(amrex_real),intent(in):: bzg(bzg_lo(1):bzg_hi(1),bzg_lo(2):bzg_hi(2),2,nmodes) + real(amrex_real),intent(in):: exg(exg_lo(1):exg_hi(1),exg_lo(2):exg_hi(2),2,n_rz_azimuthal_modes) + real(amrex_real),intent(in):: eyg(eyg_lo(1):eyg_hi(1),eyg_lo(2):eyg_hi(2),2,n_rz_azimuthal_modes) + real(amrex_real),intent(in):: ezg(ezg_lo(1):ezg_hi(1),ezg_lo(2):ezg_hi(2),2,n_rz_azimuthal_modes) + real(amrex_real),intent(in):: bxg(bxg_lo(1):bxg_hi(1),bxg_lo(2):bxg_hi(2),2,n_rz_azimuthal_modes) + real(amrex_real),intent(in):: byg(byg_lo(1):byg_hi(1),byg_lo(2):byg_hi(2),2,n_rz_azimuthal_modes) + real(amrex_real),intent(in):: bzg(bzg_lo(1):bzg_hi(1),bzg_lo(2):bzg_hi(2),2,n_rz_azimuthal_modes) #else ! These could be either 2d or 3d arrays real(amrex_real),intent(in):: exg(*), eyg(*), ezg(*), bxg(*), byg(*), bzg(*) @@ -148,14 +148,14 @@ contains bzg_nvalid = bzg_lo + bzg_hi - 2_c_long*ixyzmin + 1_c_long #ifdef WARPX_RZ - if (nmodes > 1) then - - allocate(erg_c(exg_lo(1):exg_hi(1),exg_lo(2):exg_hi(2),nmodes), & - etg_c(eyg_lo(1):eyg_hi(1),eyg_lo(2):eyg_hi(2),nmodes), & - ezg_c(ezg_lo(1):ezg_hi(1),ezg_lo(2):ezg_hi(2),nmodes), & - brg_c(bxg_lo(1):bxg_hi(1),bxg_lo(2):bxg_hi(2),nmodes), & - btg_c(byg_lo(1):byg_hi(1),byg_lo(2):byg_hi(2),nmodes), & - bzg_c(bzg_lo(1):bzg_hi(1),bzg_lo(2):bzg_hi(2),nmodes), stat=alloc_status) + if (n_rz_azimuthal_modes > 1) then + + allocate(erg_c(exg_lo(1):exg_hi(1),exg_lo(2):exg_hi(2),n_rz_azimuthal_modes), & + etg_c(eyg_lo(1):eyg_hi(1),eyg_lo(2):eyg_hi(2),n_rz_azimuthal_modes), & + ezg_c(ezg_lo(1):ezg_hi(1),ezg_lo(2):ezg_hi(2),n_rz_azimuthal_modes), & + brg_c(bxg_lo(1):bxg_hi(1),bxg_lo(2):bxg_hi(2),n_rz_azimuthal_modes), & + btg_c(byg_lo(1):byg_hi(1),byg_lo(2):byg_hi(2),n_rz_azimuthal_modes), & + bzg_c(bzg_lo(1):bzg_hi(1),bzg_lo(2):bzg_hi(2),n_rz_azimuthal_modes), stat=alloc_status) if (alloc_status /= 0) then print*,"Error: warpx_geteb_energy_conserving: complex arrays could not be allocated" stop @@ -173,7 +173,7 @@ contains bzg_c(:,:,:) = cmplx(bzg(:,:,1,:), bzg(:,:,2,:), amrex_real) call geteb2dcirc_energy_conserving_generic(np, xp, yp, zp, ex, ey, ez, bx, by, bz, & - xmin, zmin, dx, dz, nmodes, nox, noz, & + xmin, zmin, dx, dz, n_rz_azimuthal_modes, nox, noz, & pxr_l_lower_order_in_v, pxr_l_nodal, & erg_c, exg_nguards, exg_nvalid, & etg_c, eyg_nguards, eyg_nvalid, & @@ -372,22 +372,22 @@ subroutine warpx_charge_deposition(rho,np,xp,yp,zp,w,q,xmin,ymin,zmin,dx,dy,dz,n !> @param[in] charge_depo_algo algorithm choice for the charge deposition !> subroutine warpx_current_deposition( & - jx,jx_ng,jx_ntot,jy,jy_ng,jy_ntot,jz,jz_ng,jz_ntot,nmodes, & + jx,jx_ng,jx_ntot,jy,jy_ng,jy_ntot,jz,jz_ng,jz_ntot,n_rz_azimuthal_modes, & np,xp,yp,zp,uxp,uyp,uzp,gaminv,w,q,xmin,ymin,zmin,dt,dx,dy,dz,nox,noy,noz,& l_nodal,lvect,current_depo_algo) & bind(C, name="warpx_current_deposition") integer, intent(in) :: jx_ntot(AMREX_SPACEDIM), jy_ntot(AMREX_SPACEDIM), jz_ntot(AMREX_SPACEDIM) integer(c_long), intent(in) :: jx_ng, jy_ng, jz_ng - integer(c_long), intent(IN) :: nmodes + integer(c_long), intent(IN) :: n_rz_azimuthal_modes integer(c_long), intent(IN) :: np integer(c_long), intent(IN) :: nox,noy,noz integer(c_int), intent(in) :: l_nodal #ifdef WARPX_RZ - real(amrex_real), intent(IN OUT):: jx(jx_ntot(1),jx_ntot(2),2,nmodes) - real(amrex_real), intent(IN OUT):: jy(jy_ntot(1),jy_ntot(2),2,nmodes) - real(amrex_real), intent(IN OUT):: jz(jz_ntot(1),jz_ntot(2),2,nmodes) + real(amrex_real), intent(IN OUT):: jx(jx_ntot(1),jx_ntot(2),2,n_rz_azimuthal_modes) + real(amrex_real), intent(IN OUT):: jy(jy_ntot(1),jy_ntot(2),2,n_rz_azimuthal_modes) + real(amrex_real), intent(IN OUT):: jz(jz_ntot(1),jz_ntot(2),2,n_rz_azimuthal_modes) #else real(amrex_real), intent(IN OUT):: jx(*), jy(*), jz(*) #endif @@ -432,11 +432,11 @@ subroutine warpx_charge_deposition(rho,np,xp,yp,zp,w,q,xmin,ymin,zmin,dx,dy,dz,n ! Dimension 2 #elif (AMREX_SPACEDIM==2) #ifdef WARPX_RZ - if (nmodes > 1) then + if (n_rz_azimuthal_modes > 1) then - allocate(jr_c(jx_ntot(1),jx_ntot(2),nmodes), & - jt_c(jy_ntot(1),jy_ntot(2),nmodes), & - jz_c(jz_ntot(1),jz_ntot(2),nmodes), stat=alloc_status) + allocate(jr_c(jx_ntot(1),jx_ntot(2),n_rz_azimuthal_modes), & + jt_c(jy_ntot(1),jy_ntot(2),n_rz_azimuthal_modes), & + jz_c(jz_ntot(1),jz_ntot(2),n_rz_azimuthal_modes), stat=alloc_status) if (alloc_status /= 0) then print*,"Error: warpx_current_deposition: complex arrays could not be allocated" stop @@ -450,7 +450,7 @@ subroutine warpx_charge_deposition(rho,np,xp,yp,zp,w,q,xmin,ymin,zmin,dx,dy,dz,n jr_c,jx_nguards,jx_nvalid, & jt_c,jy_nguards,jy_nvalid, & jz_c,jz_nguards,jz_nvalid, & - nmodes, & + n_rz_azimuthal_modes, & np,xp,yp,zp,uxp,uyp,uzp,gaminv,w,q, & xmin,zmin,dt,dx,dz, & nox,noz,l_particles_weight,type_rz_depose) @@ -500,15 +500,15 @@ subroutine warpx_charge_deposition(rho,np,xp,yp,zp,w,q,xmin,ymin,zmin,dx,dy,dz,n !> subroutine warpx_current_deposition_rz_volume_scaling( & jr,jr_ng,jr_ntot,jt,jt_ng,jt_ntot,jz,jz_ng,jz_ntot, & - nmodes,rmin,dr) & + n_rz_azimuthal_modes,rmin,dr) & bind(C, name="warpx_current_deposition_rz_volume_scaling") integer, intent(in) :: jr_ntot(AMREX_SPACEDIM), jt_ntot(AMREX_SPACEDIM), jz_ntot(AMREX_SPACEDIM) integer(c_long), intent(in) :: jr_ng, jt_ng, jz_ng - integer(c_long), intent(in) :: nmodes - real(amrex_real), intent(IN OUT):: jr(jr_ntot(1),jr_ntot(2),2,nmodes) - real(amrex_real), intent(IN OUT):: jt(jt_ntot(1),jt_ntot(2),2,nmodes) - real(amrex_real), intent(IN OUT):: jz(jz_ntot(1),jz_ntot(2),2,nmodes) + integer(c_long), intent(in) :: n_rz_azimuthal_modes + real(amrex_real), intent(IN OUT):: jr(jr_ntot(1),jr_ntot(2),2,n_rz_azimuthal_modes) + real(amrex_real), intent(IN OUT):: jt(jt_ntot(1),jt_ntot(2),2,n_rz_azimuthal_modes) + real(amrex_real), intent(IN OUT):: jz(jz_ntot(1),jz_ntot(2),2,n_rz_azimuthal_modes) real(amrex_real), intent(IN) :: rmin, dr #ifdef WARPX_RZ @@ -528,11 +528,11 @@ subroutine warpx_charge_deposition(rho,np,xp,yp,zp,w,q,xmin,ymin,zmin,dx,dy,dz,n jt_nguards = jt_ng jz_nguards = jz_ng - if (nmodes > 1) then + if (n_rz_azimuthal_modes > 1) then - allocate(jr_c(jr_ntot(1),jr_ntot(2),nmodes), & - jt_c(jt_ntot(1),jt_ntot(2),nmodes), & - jz_c(jz_ntot(1),jz_ntot(2),nmodes), stat=alloc_status) + allocate(jr_c(jr_ntot(1),jr_ntot(2),n_rz_azimuthal_modes), & + jt_c(jt_ntot(1),jt_ntot(2),n_rz_azimuthal_modes), & + jz_c(jz_ntot(1),jz_ntot(2),n_rz_azimuthal_modes), stat=alloc_status) if (alloc_status /= 0) then print*,"Error: warpx_current_deposition_rz_volume_scaling: complex arrays could not be allocated" stop @@ -550,7 +550,7 @@ subroutine warpx_charge_deposition(rho,np,xp,yp,zp,w,q,xmin,ymin,zmin,dx,dy,dz,n jr_c, jr_nguards, jr_nvalid, & jt_c, jt_nguards, jt_nvalid, & jz_c, jz_nguards, jz_nvalid, & - nmodes, & + n_rz_azimuthal_modes, & rmin,dr, & type_rz_depose) @@ -710,7 +710,7 @@ subroutine warpx_charge_deposition(rho,np,xp,yp,zp,w,q,xmin,ymin,zmin,dx,dy,dz,n !> @param[in] dtsdx, dtsdy, dtsdz factors c**2 * dt/(dx, dy, dz) subroutine warpx_push_evec( & xlo, xhi, ylo, yhi, zlo, zhi, & - nmodes, & + n_rz_azimuthal_modes, & ex, exlo, exhi, & ey, eylo, eyhi, & ez, ezlo, ezhi, & @@ -730,19 +730,19 @@ subroutine warpx_charge_deposition(rho,np,xp,yp,zp,w,q,xmin,ymin,zmin,dx,dy,dz,n jxlo(BL_SPACEDIM), jxhi(BL_SPACEDIM), jylo(BL_SPACEDIM), jyhi(BL_SPACEDIM), & jzlo(BL_SPACEDIM), jzhi(BL_SPACEDIM) - integer(c_long), intent(in) :: nmodes + integer(c_long), intent(in) :: n_rz_azimuthal_modes #ifdef WARPX_RZ ! The dimensions must be specified to allow the transpose - real(amrex_real), intent(IN OUT):: ex(exlo(1):exhi(1),exlo(2):exhi(2),2,nmodes) - real(amrex_real), intent(IN OUT):: ey(eylo(1):eyhi(1),eylo(2):eyhi(2),2,nmodes) - real(amrex_real), intent(IN OUT):: ez(ezlo(1):ezhi(1),ezlo(2):ezhi(2),2,nmodes) - real(amrex_real), intent(IN):: bx(bxlo(1):bxhi(1),bxlo(2):bxhi(2),2,nmodes) - real(amrex_real), intent(IN):: by(bylo(1):byhi(1),bylo(2):byhi(2),2,nmodes) - real(amrex_real), intent(IN):: bz(bzlo(1):bzhi(1),bzlo(2):bzhi(2),2,nmodes) - real(amrex_real), intent(IN):: jx(jxlo(1):jxhi(1),jxlo(2):jxhi(2),2,nmodes) - real(amrex_real), intent(IN):: jy(jylo(1):jyhi(1),jylo(2):jyhi(2),2,nmodes) - real(amrex_real), intent(IN):: jz(jzlo(1):jzhi(1),jzlo(2):jzhi(2),2,nmodes) + real(amrex_real), intent(IN OUT):: ex(exlo(1):exhi(1),exlo(2):exhi(2),2,n_rz_azimuthal_modes) + real(amrex_real), intent(IN OUT):: ey(eylo(1):eyhi(1),eylo(2):eyhi(2),2,n_rz_azimuthal_modes) + real(amrex_real), intent(IN OUT):: ez(ezlo(1):ezhi(1),ezlo(2):ezhi(2),2,n_rz_azimuthal_modes) + real(amrex_real), intent(IN):: bx(bxlo(1):bxhi(1),bxlo(2):bxhi(2),2,n_rz_azimuthal_modes) + real(amrex_real), intent(IN):: by(bylo(1):byhi(1),bylo(2):byhi(2),2,n_rz_azimuthal_modes) + real(amrex_real), intent(IN):: bz(bzlo(1):bzhi(1),bzlo(2):bzhi(2),2,n_rz_azimuthal_modes) + real(amrex_real), intent(IN):: jx(jxlo(1):jxhi(1),jxlo(2):jxhi(2),2,n_rz_azimuthal_modes) + real(amrex_real), intent(IN):: jy(jylo(1):jyhi(1),jylo(2):jyhi(2),2,n_rz_azimuthal_modes) + real(amrex_real), intent(IN):: jz(jzlo(1):jzhi(1),jzlo(2):jzhi(2),2,n_rz_azimuthal_modes) #else ! These can be either 2d or 3d real(amrex_real), intent(IN OUT):: ex(*) @@ -766,7 +766,7 @@ subroutine warpx_charge_deposition(rho,np,xp,yp,zp,w,q,xmin,ymin,zmin,dx,dy,dz,n complex(amrex_real), allocatable, dimension(:,:,:) :: jr_c, jt_c, jz_c integer :: alloc_status - if (nmodes == 1) then + if (n_rz_azimuthal_modes == 1) then #endif CALL WRPX_PXR_PUSH_EVEC(& xlo, xhi, ylo, yhi, zlo, zhi, & @@ -788,15 +788,15 @@ subroutine warpx_charge_deposition(rho,np,xp,yp,zp,w,q,xmin,ymin,zmin,dx,dy,dz,n #ifdef WARPX_RZ else - allocate(er_c(exlo(1):exhi(1),exlo(2):exhi(2),nmodes), & - et_c(eylo(1):eyhi(1),eylo(2):eyhi(2),nmodes), & - ez_c(ezlo(1):ezhi(1),ezlo(2):ezhi(2),nmodes), & - br_c(bxlo(1):bxhi(1),bxlo(2):bxhi(2),nmodes), & - bt_c(bylo(1):byhi(1),bylo(2):byhi(2),nmodes), & - bz_c(bzlo(1):bzhi(1),bzlo(2):bzhi(2),nmodes), & - jr_c(jxlo(1):jxhi(1),jxlo(2):jxhi(2),nmodes), & - jt_c(jylo(1):jyhi(1),jylo(2):jyhi(2),nmodes), & - jz_c(jzlo(1):jzhi(1),jzlo(2):jzhi(2),nmodes), stat=alloc_status) + allocate(er_c(exlo(1):exhi(1),exlo(2):exhi(2),n_rz_azimuthal_modes), & + et_c(eylo(1):eyhi(1),eylo(2):eyhi(2),n_rz_azimuthal_modes), & + ez_c(ezlo(1):ezhi(1),ezlo(2):ezhi(2),n_rz_azimuthal_modes), & + br_c(bxlo(1):bxhi(1),bxlo(2):bxhi(2),n_rz_azimuthal_modes), & + bt_c(bylo(1):byhi(1),bylo(2):byhi(2),n_rz_azimuthal_modes), & + bz_c(bzlo(1):bzhi(1),bzlo(2):bzhi(2),n_rz_azimuthal_modes), & + jr_c(jxlo(1):jxhi(1),jxlo(2):jxhi(2),n_rz_azimuthal_modes), & + jt_c(jylo(1):jyhi(1),jylo(2):jyhi(2),n_rz_azimuthal_modes), & + jz_c(jzlo(1):jzhi(1),jzlo(2):jzhi(2),n_rz_azimuthal_modes), stat=alloc_status) if (alloc_status /= 0) then print*,"Error: warpx_push_evec: complex arrays could not be allocated" stop @@ -818,7 +818,7 @@ subroutine warpx_charge_deposition(rho,np,xp,yp,zp,w,q,xmin,ymin,zmin,dx,dy,dz,n CALL pxrpush_emrz_evec_multimode(& xlo, xhi, ylo, yhi, zlo, zhi, & - nmodes, & + n_rz_azimuthal_modes, & er_c, exlo, exhi,& et_c, eylo, eyhi,& ez_c, ezlo, ezhi,& @@ -863,7 +863,7 @@ subroutine warpx_charge_deposition(rho,np,xp,yp,zp,w,q,xmin,ymin,zmin,dx,dy,dz,n !> @param[in] dtsdx, dtsdy, dtsdz factors 0.5 * dt/(dx, dy, dz) subroutine warpx_push_bvec( & xlo, xhi, ylo, yhi, zlo, zhi, & - nmodes, & + n_rz_azimuthal_modes, & ex, exlo, exhi, & ey, eylo, eyhi, & ez, ezlo, ezhi, & @@ -880,16 +880,16 @@ subroutine warpx_charge_deposition(rho,np,xp,yp,zp,w,q,xmin,ymin,zmin,dx,dy,dz,n bylo(BL_SPACEDIM), byhi(BL_SPACEDIM), bzlo(BL_SPACEDIM), bzhi(BL_SPACEDIM), & maxwell_fdtd_solver_id - integer(c_long), intent(in) :: nmodes + integer(c_long), intent(in) :: n_rz_azimuthal_modes #ifdef WARPX_RZ ! The dimensions must be specified to allow the transpose - real(amrex_real), intent(IN):: ex(exlo(1):exhi(1),exlo(2):exhi(2),2,nmodes) - real(amrex_real), intent(IN):: ey(eylo(1):eyhi(1),eylo(2):eyhi(2),2,nmodes) - real(amrex_real), intent(IN):: ez(ezlo(1):ezhi(1),ezlo(2):ezhi(2),2,nmodes) - real(amrex_real), intent(IN OUT):: bx(bxlo(1):bxhi(1),bxlo(2):bxhi(2),2,nmodes) - real(amrex_real), intent(IN OUT):: by(bylo(1):byhi(1),bylo(2):byhi(2),2,nmodes) - real(amrex_real), intent(IN OUT):: bz(bzlo(1):bzhi(1),bzlo(2):bzhi(2),2,nmodes) + real(amrex_real), intent(IN):: ex(exlo(1):exhi(1),exlo(2):exhi(2),2,n_rz_azimuthal_modes) + real(amrex_real), intent(IN):: ey(eylo(1):eyhi(1),eylo(2):eyhi(2),2,n_rz_azimuthal_modes) + real(amrex_real), intent(IN):: ez(ezlo(1):ezhi(1),ezlo(2):ezhi(2),2,n_rz_azimuthal_modes) + real(amrex_real), intent(IN OUT):: bx(bxlo(1):bxhi(1),bxlo(2):bxhi(2),2,n_rz_azimuthal_modes) + real(amrex_real), intent(IN OUT):: by(bylo(1):byhi(1),bylo(2):byhi(2),2,n_rz_azimuthal_modes) + real(amrex_real), intent(IN OUT):: bz(bzlo(1):bzhi(1),bzlo(2):bzhi(2),2,n_rz_azimuthal_modes) #else real(amrex_real), intent(IN):: ex(*) real(amrex_real), intent(IN):: ey(*) @@ -909,7 +909,7 @@ subroutine warpx_charge_deposition(rho,np,xp,yp,zp,w,q,xmin,ymin,zmin,dx,dy,dz,n integer :: alloc_status #endif - if (nmodes == 1) then + if (n_rz_azimuthal_modes == 1) then IF (maxwell_fdtd_solver_id .eq. 0) THEN ! Yee FDTD solver @@ -942,12 +942,12 @@ subroutine warpx_charge_deposition(rho,np,xp,yp,zp,w,q,xmin,ymin,zmin,dx,dy,dz,n #ifdef WARPX_RZ else - allocate(er_c(exlo(1):exhi(1),exlo(2):exhi(2),nmodes), & - et_c(eylo(1):eyhi(1),eylo(2):eyhi(2),nmodes), & - ez_c(ezlo(1):ezhi(1),ezlo(2):ezhi(2),nmodes), & - br_c(bxlo(1):bxhi(1),bxlo(2):bxhi(2),nmodes), & - bt_c(bylo(1):byhi(1),bylo(2):byhi(2),nmodes), & - bz_c(bzlo(1):bzhi(1),bzlo(2):bzhi(2),nmodes), stat=alloc_status) + allocate(er_c(exlo(1):exhi(1),exlo(2):exhi(2),n_rz_azimuthal_modes), & + et_c(eylo(1):eyhi(1),eylo(2):eyhi(2),n_rz_azimuthal_modes), & + ez_c(ezlo(1):ezhi(1),ezlo(2):ezhi(2),n_rz_azimuthal_modes), & + br_c(bxlo(1):bxhi(1),bxlo(2):bxhi(2),n_rz_azimuthal_modes), & + bt_c(bylo(1):byhi(1),bylo(2):byhi(2),n_rz_azimuthal_modes), & + bz_c(bzlo(1):bzhi(1),bzlo(2):bzhi(2),n_rz_azimuthal_modes), stat=alloc_status) if (alloc_status /= 0) then print*,"Error: warpx_push_bvec: complex arrays could not be allocated" stop @@ -966,7 +966,7 @@ subroutine warpx_charge_deposition(rho,np,xp,yp,zp,w,q,xmin,ymin,zmin,dx,dy,dz,n CALL pxrpush_emrz_bvec_multimode(& xlo, xhi, ylo, yhi, zlo, zhi, & - nmodes, & + n_rz_azimuthal_modes, & er_c, exlo, exhi,& et_c, eylo, eyhi,& ez_c, ezlo, ezhi,& diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 93a0ad7ea..aa9fbada7 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -482,7 +482,7 @@ PhysicalParticleContainer::AddPlasmaCPU (int lev, RealBox part_realbox) // Replace the x and y, setting an angle theta. // These x and y are used to get the momentum and density Real theta; - if (WarpX::nmodes == 1) { + if (WarpX::n_rz_azimuthal_modes == 1) { // With only 1 mode, the angle doesn't matter so // choose it randomly. theta = 2.*MathConst::pi*amrex::Random(); @@ -734,7 +734,7 @@ PhysicalParticleContainer::AddPlasmaGPU (int lev, RealBox part_realbox) // Replace the x and y, setting an angle theta. // These x and y are used to get the momentum and density Real theta; - if (WarpX::nmodes == 1) { + if (WarpX::n_rz_azimuthal_modes == 1) { // With only 1 mode, the angle doesn't matter so // choose it randomly. theta = 2.*MathConst::pi*amrex::Random(); @@ -1156,7 +1156,7 @@ PhysicalParticleContainer::FieldGather (int lev, BL_TO_FORTRAN_ANYD(bxfab), BL_TO_FORTRAN_ANYD(byfab), BL_TO_FORTRAN_ANYD(bzfab), - &WarpX::nmodes, + &WarpX::n_rz_azimuthal_modes, &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, &lvect_fieldgathe, &WarpX::field_gathering_algo); @@ -1449,7 +1449,7 @@ PhysicalParticleContainer::Evolve (int lev, BL_TO_FORTRAN_ANYD(*bxfab), BL_TO_FORTRAN_ANYD(*byfab), BL_TO_FORTRAN_ANYD(*bzfab), - &WarpX::nmodes, + &WarpX::n_rz_azimuthal_modes, &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, &lvect_fieldgathe, &WarpX::field_gathering_algo); @@ -1537,7 +1537,7 @@ PhysicalParticleContainer::Evolve (int lev, BL_TO_FORTRAN_ANYD(*cbxfab), BL_TO_FORTRAN_ANYD(*cbyfab), BL_TO_FORTRAN_ANYD(*cbzfab), - &WarpX::nmodes, + &WarpX::n_rz_azimuthal_modes, &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, &lvect_fieldgathe, &WarpX::field_gathering_algo); } @@ -1879,7 +1879,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, BL_TO_FORTRAN_ANYD(bxfab), BL_TO_FORTRAN_ANYD(byfab), BL_TO_FORTRAN_ANYD(bzfab), - &WarpX::nmodes, + &WarpX::n_rz_azimuthal_modes, &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, &lvect_fieldgathe, &WarpX::field_gathering_algo); diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index d659b3854..ff07527aa 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -426,7 +426,7 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, BL_TO_FORTRAN_ANYD(bxfab), BL_TO_FORTRAN_ANYD(byfab), BL_TO_FORTRAN_ANYD(bzfab), - &WarpX::nmodes, + &WarpX::n_rz_azimuthal_modes, &ll4symtry, &l_lower_order_in_v, &WarpX::do_nodal, &lvect_fieldgathe, &WarpX::field_gathering_algo); diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index a1517cb44..6ecf0fe98 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -379,7 +379,7 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, jx_ptr, &ngJ, jxntot.getVect(), jy_ptr, &ngJ, jyntot.getVect(), jz_ptr, &ngJ, jzntot.getVect(), - &WarpX::nmodes, + &WarpX::n_rz_azimuthal_modes, &np_to_depose, m_xp[thread_num].dataPtr() + offset, m_yp[thread_num].dataPtr() + offset, @@ -400,7 +400,7 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, jx_ptr, &ngJ, jxntot.getVect(), jy_ptr, &ngJ, jyntot.getVect(), jz_ptr, &ngJ, jzntot.getVect(), - &WarpX::nmodes, + &WarpX::n_rz_azimuthal_modes, &xyzmin[0], &dx[0]); #endif BL_PROFILE_VAR_STOP(blp_pxr_cd); diff --git a/Source/WarpX.H b/Source/WarpX.H index 5d3008972..581ca39ce 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -93,7 +93,7 @@ public: static long noz; // Number of modes for the RZ multimode version - static long nmodes; + static long n_rz_azimuthal_modes; static long ncomps; static bool use_fdtd_nci_corr; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 4ad2cf75d..a693952dd 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -43,7 +43,7 @@ long WarpX::field_gathering_algo; long WarpX::particle_pusher_algo; int WarpX::maxwell_fdtd_solver_id; -long WarpX::nmodes = 1; +long WarpX::n_rz_azimuthal_modes = 1; long WarpX::ncomps = 1; long WarpX::nox = 1; @@ -475,7 +475,7 @@ WarpX::ReadParameters () } // Only needs to be set with WARPX_RZ, otherwise defaults to 1. - pp.query("nmodes", nmodes); + pp.query("n_rz_azimuthal_modes", n_rz_azimuthal_modes); } @@ -709,9 +709,9 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm { #if defined WARPX_RZ - if (nmodes > 1) { + if (n_rz_azimuthal_modes > 1) { // There is a real and imaginary component for each mode - ncomps = nmodes*2; + ncomps = n_rz_azimuthal_modes*2; } else { // With only mode 0, only reals are used ncomps = 1; -- cgit v1.2.3 From f86512d788477a8eab5ed3c3c92ce9a7453ae4c7 Mon Sep 17 00:00:00 2001 From: Dave Grote Date: Fri, 12 Jul 2019 18:09:34 -0700 Subject: For RZ, updated documentation regarding num_particles_per_cell_each_dim --- Docs/source/running_cpp/parameters.rst | 7 +++++++ Source/Initialization/PlasmaInjector.cpp | 2 ++ Source/Particles/PhysicalParticleContainer.cpp | 2 ++ 3 files changed, 11 insertions(+) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Docs/source/running_cpp/parameters.rst b/Docs/source/running_cpp/parameters.rst index c159509b4..ec3baee86 100644 --- a/Docs/source/running_cpp/parameters.rst +++ b/Docs/source/running_cpp/parameters.rst @@ -91,6 +91,9 @@ Setting up the field mesh in the list will deposit their charge/current directly on the main grid (i.e. the coarsest level), even if they are inside a refinement patch. +* ``warpx.n_rz_azimuthal_modes`` (`integer`; 1 by default) + When using the RZ version, this is the number of azimuthal modes. + Distribution across MPI ranks and parallelization ------------------------------------------------- @@ -201,6 +204,10 @@ Particle initialization and optional argument ``.do_symmetrize`` (whether to symmetrize the beam in the x and y directions). +* ``.num_particles_per_cell_each_dim`` (`3 integers in 3D and RZ, 2 integers in 2D`) + With the NUniformPerCell injection style, this specifies the number of particles along each axis + within a cell. Note that for RZ, the three axis are radius, theta, and z. + * ``.do_continuous_injection`` (`0` or `1`) Whether to inject particles during the simulation, and not only at initialization. This can be required whith a moving window and/or when diff --git a/Source/Initialization/PlasmaInjector.cpp b/Source/Initialization/PlasmaInjector.cpp index a7dbe728a..bb603a4bd 100644 --- a/Source/Initialization/PlasmaInjector.cpp +++ b/Source/Initialization/PlasmaInjector.cpp @@ -296,6 +296,8 @@ PlasmaInjector::PlasmaInjector(int ispecies, const std::string& name) parseDensity(pp); parseMomentum(pp); } else if (part_pos_s == "nuniformpercell") { + // Note that for RZ, three numbers are expected, r, theta, and z. + // For 2D, only two are expected. The third is overwritten with 1. num_particles_per_cell_each_dim.assign(3, 1); pp.getarr("num_particles_per_cell_each_dim", num_particles_per_cell_each_dim); #if ( AMREX_SPACEDIM == 2 ) && !defined(WARPX_RZ) diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index aa9fbada7..751f4e1eb 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -50,6 +50,7 @@ NumParticlesToAdd(const Box& overlap_box, const RealBox& overlap_realbox, #elif ( AMREX_SPACEDIM == 2 ) Real y = 0; #ifdef WARPX_RZ + // Note that for RZ, point[1] will be theta Real z = overlap_corner[1] + (iv[1] + point[2])*dx[1]; #else Real z = overlap_corner[1] + (iv[1] + point[1])*dx[1]; @@ -460,6 +461,7 @@ PhysicalParticleContainer::AddPlasmaCPU (int lev, RealBox part_realbox) #elif ( AMREX_SPACEDIM == 2 ) Real y = 0; #ifdef WARPX_RZ + // Note that for RZ, point[1] will be theta Real z = overlap_corner[1] + (iv[1] + point[2])*dx[1]; #else Real z = overlap_corner[1] + (iv[1] + point[1])*dx[1]; -- cgit v1.2.3 From cd7f25974133d0aef31a90af6d79dd36372fc3fd Mon Sep 17 00:00:00 2001 From: Diana Amorim Date: Fri, 12 Jul 2019 20:24:28 -0700 Subject: Corrected typo of xold incorrect repetition. --- .../laser_acceleration/inputs.2d.boost | 21 +++++++++++---------- Source/Particles/PhysicalParticleContainer.cpp | 4 ++-- 2 files changed, 13 insertions(+), 12 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Examples/Physics_applications/laser_acceleration/inputs.2d.boost b/Examples/Physics_applications/laser_acceleration/inputs.2d.boost index 5fe4bfcf6..75a246ea6 100644 --- a/Examples/Physics_applications/laser_acceleration/inputs.2d.boost +++ b/Examples/Physics_applications/laser_acceleration/inputs.2d.boost @@ -1,8 +1,8 @@ ################################# ######### BOX PARAMETERS ######## ################################# -max_step = 1000 -# stop_time = 1.9e-12 +# max_step = 2700 +stop_time = 1.9e-12 amr.n_cell = 128 1024 amr.max_grid_size = 64 amr.blocking_factor = 32 @@ -19,8 +19,10 @@ geometry.prob_hi = 128.e-6 0.96e-6 ################################# warpx.verbose = 1 amrex.v = 1 -algo.current_deposition = direct -algo.particle_pusher = vay +algo.current_deposition = direct #2 +algo.charge_deposition = standard # 0 +algo.field_gathering = standard # 0 +algo.particle_pusher = vay # 1 algo.maxwell_fdtd_solver = ckc interpolation.nox = 3 interpolation.noy = 3 @@ -37,12 +39,11 @@ warpx.serialize_ics = 1 ################################# ####### BOOST PARAMETERS ######## ################################# -warpx.gamma_boost = 30. +warpx.gamma_boost = 10. warpx.boost_direction = z warpx.do_boosted_frame_diagnostic = 1 warpx.num_snapshots_lab = 7 warpx.dt_snapshots_lab = 1.6678204759907604e-12 -warpx.boosted_frame_diag_fields = Ex Ez By jz ################################# ############ PLASMA ############# @@ -60,11 +61,11 @@ electrons.momentum_distribution_type = "gaussian" electrons.xmin = -120.e-6 electrons.xmax = 120.e-6 electrons.zmin = 0.5e-3 -electrons.zmax = 1. +electrons.zmax = .0035 electrons.profile = "predefined" electrons.predefined_profile_name = "parabolic_channel" # predefined_profile_params = z_start ramp_up plateau ramp_down rc n0 -electrons.predefined_profile_params = .5e-3 .5e-3 2.e-3 .5e-3 50.e-6 3.5e25 +electrons.predefined_profile_params = .5e-3 .5e-3 2.e-3 .5e-3 50.e-6 3.5e24 electrons.do_continuous_injection = 1 ions.charge = q_e @@ -75,11 +76,11 @@ ions.momentum_distribution_type = "gaussian" ions.xmin = -120.e-6 ions.xmax = 120.e-6 ions.zmin = 0.5e-3 -ions.zmax = 1. +ions.zmax = .0035 ions.profile = "predefined" ions.predefined_profile_name = "parabolic_channel" # predefined_profile_params = z_start ramp_up plateau ramp_down rc n0 -ions.predefined_profile_params = .5e-3 .5e-3 2.e-3 .5e-3 50.e-6 3.5e25 +ions.predefined_profile_params = .5e-3 .5e-3 2.e-3 .5e-3 50.e-6 3.5e24 ions.do_continuous_injection = 1 beam.charge = -q_e diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index addde5e37..d6c4973c3 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1872,8 +1872,8 @@ void PhysicalParticleContainer::copy_attribs(WarpXParIter& pti,const Real* xp, Real* AMREX_RESTRICT uzp = attribs[PIdx::uz].dataPtr(); Real* AMREX_RESTRICT xpold = pti.GetAttribs(particle_comps["xold"]).dataPtr(); - Real* AMREX_RESTRICT ypold = pti.GetAttribs(particle_comps["xold"]).dataPtr(); - Real* AMREX_RESTRICT zpold = pti.GetAttribs(particle_comps["xold"]).dataPtr(); + Real* AMREX_RESTRICT ypold = pti.GetAttribs(particle_comps["yold"]).dataPtr(); + Real* AMREX_RESTRICT zpold = pti.GetAttribs(particle_comps["zold"]).dataPtr(); Real* AMREX_RESTRICT uxpold = pti.GetAttribs(particle_comps["uxold"]).dataPtr(); Real* AMREX_RESTRICT uypold = pti.GetAttribs(particle_comps["uyold"]).dataPtr(); Real* AMREX_RESTRICT uzpold = pti.GetAttribs(particle_comps["uzold"]).dataPtr(); -- cgit v1.2.3 From 7a450310547e94df56a7dd089b40bec47f2ba772 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Mon, 15 Jul 2019 17:59:13 -0700 Subject: fix indentation in physicalparticlecontainer.cpp, couldn't read --- Source/Particles/PhysicalParticleContainer.cpp | 588 ++++++++++++------------- 1 file changed, 294 insertions(+), 294 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 28d611020..d47a7b220 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -12,7 +12,7 @@ using namespace amrex; long PhysicalParticleContainer:: NumParticlesToAdd(const Box& overlap_box, const RealBox& overlap_realbox, - const RealBox& tile_realbox, const RealBox& particle_real_box) + const RealBox& tile_realbox, const RealBox& particle_real_box) { const int lev = 0; const Geometry& geom = Geom(lev); @@ -24,43 +24,43 @@ NumParticlesToAdd(const Box& overlap_box, const RealBox& overlap_realbox, for (IntVect iv = overlap_box.smallEnd(); iv <= overlap_box.bigEnd(); overlap_box.next(iv)) { int fac; - if (do_continuous_injection) { + if (do_continuous_injection) { #if ( AMREX_SPACEDIM == 3 ) - Real x = overlap_corner[0] + (iv[0] + 0.5)*dx[0]; - Real y = overlap_corner[1] + (iv[1] + 0.5)*dx[1]; - Real z = overlap_corner[2] + (iv[2] + 0.5)*dx[2]; + Real x = overlap_corner[0] + (iv[0] + 0.5)*dx[0]; + Real y = overlap_corner[1] + (iv[1] + 0.5)*dx[1]; + Real z = overlap_corner[2] + (iv[2] + 0.5)*dx[2]; #elif ( AMREX_SPACEDIM == 2 ) - Real x = overlap_corner[0] + (iv[0] + 0.5)*dx[0]; - Real y = 0; - Real z = overlap_corner[1] + (iv[1] + 0.5)*dx[1]; + Real x = overlap_corner[0] + (iv[0] + 0.5)*dx[0]; + Real y = 0; + Real z = overlap_corner[1] + (iv[1] + 0.5)*dx[1]; #endif - fac = GetRefineFac(x, y, z); - } else { - fac = 1.0; - } + fac = GetRefineFac(x, y, z); + } else { + fac = 1.0; + } - int ref_num_ppc = num_ppc * AMREX_D_TERM(fac, *fac, *fac); - for (int i_part=0; i_part r; - plasma_injector->getPositionUnitBox(r, i_part, fac); + int ref_num_ppc = num_ppc * AMREX_D_TERM(fac, *fac, *fac); + for (int i_part=0; i_part r; + plasma_injector->getPositionUnitBox(r, i_part, fac); #if ( AMREX_SPACEDIM == 3 ) - Real x = overlap_corner[0] + (iv[0] + r[0])*dx[0]; - Real y = overlap_corner[1] + (iv[1] + r[1])*dx[1]; - Real z = overlap_corner[2] + (iv[2] + r[2])*dx[2]; + Real x = overlap_corner[0] + (iv[0] + r[0])*dx[0]; + Real y = overlap_corner[1] + (iv[1] + r[1])*dx[1]; + Real z = overlap_corner[2] + (iv[2] + r[2])*dx[2]; #elif ( AMREX_SPACEDIM == 2 ) - Real x = overlap_corner[0] + (iv[0] + r[0])*dx[0]; - Real y = 0; - Real z = overlap_corner[1] + (iv[1] + r[1])*dx[1]; + Real x = overlap_corner[0] + (iv[0] + r[0])*dx[0]; + Real y = 0; + Real z = overlap_corner[1] + (iv[1] + r[1])*dx[1]; #endif - // If the new particle is not inside the tile box, - // go to the next generated particle. + // If the new particle is not inside the tile box, + // go to the next generated particle. #if ( AMREX_SPACEDIM == 3 ) - if(!tile_realbox.contains( RealVect{x, y, z} )) continue; + if(!tile_realbox.contains( RealVect{x, y, z} )) continue; #elif ( AMREX_SPACEDIM == 2 ) - if(!tile_realbox.contains( RealVect{x, z} )) continue; + if(!tile_realbox.contains( RealVect{x, z} )) continue; #endif - ++np; - } + ++np; + } } return np; @@ -170,8 +170,8 @@ void PhysicalParticleContainer::MapParticletoBoostedFrame(Real& x, Real& y, Real // Move the particles to where they will be at t = 0 in the boosted frame if (boost_adjust_transverse_positions) { - x = xpr - tpr*vxpr; - y = ypr - tpr*vypr; + x = xpr - tpr*vxpr; + y = ypr - tpr*vypr; } z = zpr - tpr*vzpr; @@ -323,9 +323,9 @@ void PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) { #ifdef AMREX_USE_GPU - AddPlasmaGPU(lev, part_realbox); + AddPlasmaGPU(lev, part_realbox); #else - AddPlasmaCPU(lev, part_realbox); + AddPlasmaCPU(lev, part_realbox); #endif } @@ -416,7 +416,7 @@ PhysicalParticleContainer::AddPlasmaCPU (int lev, RealBox part_realbox) // Count the number of cells in this direction in overlap_realbox overlap_box.setSmall( dir, 0 ); overlap_box.setBig( dir, - int( round((overlap_realbox.hi(dir)-overlap_realbox.lo(dir))/dx[dir] )) - 1); + int( round((overlap_realbox.hi(dir)-overlap_realbox.lo(dir))/dx[dir] )) - 1); } if (no_overlap == 1) { continue; // Go to the next tile @@ -483,54 +483,54 @@ PhysicalParticleContainer::AddPlasmaCPU (int lev, RealBox part_realbox) Real dens; std::array u; if (WarpX::gamma_boost == 1.){ - // Lab-frame simulation - // If the particle is not within the species's - // xmin, xmax, ymin, ymax, zmin, zmax, go to - // the next generated particle. - if (!plasma_injector->insideBounds(xb, yb, z)) continue; - plasma_injector->getMomentum(u, x, y, z); - dens = plasma_injector->getDensity(x, y, z); + // Lab-frame simulation + // If the particle is not within the species's + // xmin, xmax, ymin, ymax, zmin, zmax, go to + // the next generated particle. + if (!plasma_injector->insideBounds(xb, yb, z)) continue; + plasma_injector->getMomentum(u, x, y, z); + dens = plasma_injector->getDensity(x, y, z); } else { - // Boosted-frame simulation - Real c = PhysConst::c; - Real gamma_boost = WarpX::gamma_boost; - Real beta_boost = WarpX::beta_boost; - // Since the user provides the density distribution - // at t_lab=0 and in the lab-frame coordinates, - // we need to find the lab-frame position of this - // particle at t_lab=0, from its boosted-frame coordinates - // Assuming ballistic motion, this is given by: - // z0_lab = gamma*( z_boost*(1-beta*betaz_lab) - ct_boost*(betaz_lab-beta) ) - // where betaz_lab is the speed of the particle in the lab frame - // - // In order for this equation to be solvable, betaz_lab - // is explicitly assumed to have no dependency on z0_lab - plasma_injector->getMomentum(u, x, y, 0.); // No z0_lab dependency - // At this point u is the lab-frame momentum - // => Apply the above formula for z0_lab - Real gamma_lab = std::sqrt( 1 + (u[0]*u[0] + u[1]*u[1] + u[2]*u[2])/(c*c) ); - Real betaz_lab = u[2]/gamma_lab/c; - Real t = WarpX::GetInstance().gett_new(lev); - Real z0_lab = gamma_boost * ( z*(1-beta_boost*betaz_lab) - c*t*(betaz_lab-beta_boost) ); - // If the particle is not within the lab-frame zmin, zmax, etc. - // go to the next generated particle. - if (!plasma_injector->insideBounds(xb, yb, z0_lab)) continue; - // call `getDensity` with lab-frame parameters - dens = plasma_injector->getDensity(x, y, z0_lab); - // At this point u and dens are the lab-frame quantities - // => Perform Lorentz transform - dens = gamma_boost * dens * ( 1 - beta_boost*betaz_lab ); - u[2] = gamma_boost * ( u[2] -beta_boost*c*gamma_lab ); + // Boosted-frame simulation + Real c = PhysConst::c; + Real gamma_boost = WarpX::gamma_boost; + Real beta_boost = WarpX::beta_boost; + // Since the user provides the density distribution + // at t_lab=0 and in the lab-frame coordinates, + // we need to find the lab-frame position of this + // particle at t_lab=0, from its boosted-frame coordinates + // Assuming ballistic motion, this is given by: + // z0_lab = gamma*( z_boost*(1-beta*betaz_lab) - ct_boost*(betaz_lab-beta) ) + // where betaz_lab is the speed of the particle in the lab frame + // + // In order for this equation to be solvable, betaz_lab + // is explicitly assumed to have no dependency on z0_lab + plasma_injector->getMomentum(u, x, y, 0.); // No z0_lab dependency + // At this point u is the lab-frame momentum + // => Apply the above formula for z0_lab + Real gamma_lab = std::sqrt( 1 + (u[0]*u[0] + u[1]*u[1] + u[2]*u[2])/(c*c) ); + Real betaz_lab = u[2]/gamma_lab/c; + Real t = WarpX::GetInstance().gett_new(lev); + Real z0_lab = gamma_boost * ( z*(1-beta_boost*betaz_lab) - c*t*(betaz_lab-beta_boost) ); + // If the particle is not within the lab-frame zmin, zmax, etc. + // go to the next generated particle. + if (!plasma_injector->insideBounds(xb, yb, z0_lab)) continue; + // call `getDensity` with lab-frame parameters + dens = plasma_injector->getDensity(x, y, z0_lab); + // At this point u and dens are the lab-frame quantities + // => Perform Lorentz transform + dens = gamma_boost * dens * ( 1 - beta_boost*betaz_lab ); + u[2] = gamma_boost * ( u[2] -beta_boost*c*gamma_lab ); } Real weight = dens * scale_fac / (AMREX_D_TERM(fac, *fac, *fac)); #ifdef WARPX_RZ if (plasma_injector->radially_weighted) { - weight *= 2*MathConst::pi*xb; + weight *= 2*MathConst::pi*xb; } else { - // This is not correct since it might shift the particle - // out of the local grid - x = std::sqrt(xb*rmax); - weight *= dx[0]; + // This is not correct since it might shift the particle + // out of the local grid + x = std::sqrt(xb*rmax); + weight *= dx[0]; } #endif attribs[PIdx::w ] = weight; @@ -550,18 +550,18 @@ PhysicalParticleContainer::AddPlasmaCPU (int lev, RealBox part_realbox) particle_tile.push_back_real(particle_comps["uzold"], u[2]); } - AddOneParticle(lev, grid_id, tile_id, x, y, z, attribs); + AddOneParticle(lev, grid_id, tile_id, x, y, z, attribs); } } if (cost) { - wt = (amrex::second() - wt) / tile_box.d_numPts(); + wt = (amrex::second() - wt) / tile_box.d_numPts(); Array4 const& costarr = cost->array(mfi); amrex::ParallelFor(tile_box, - [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - costarr(i,j,k) += wt; - }); + [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + costarr(i,j,k) += wt; + }); } } } @@ -655,7 +655,7 @@ PhysicalParticleContainer::AddPlasmaGPU (int lev, RealBox part_realbox) // Count the number of cells in this direction in overlap_realbox overlap_box.setSmall( dir, 0 ); overlap_box.setBig( dir, - int( round((overlap_realbox.hi(dir)-overlap_realbox.lo(dir))/dx[dir] )) - 1); + int( round((overlap_realbox.hi(dir)-overlap_realbox.lo(dir))/dx[dir] )) - 1); } if (no_overlap == 1) { continue; // Go to the next tile @@ -664,8 +664,8 @@ PhysicalParticleContainer::AddPlasmaGPU (int lev, RealBox part_realbox) const int grid_id = mfi.index(); const int tile_id = mfi.LocalTileIndex(); - Cuda::HostVector host_particles; - std::array, PIdx::nattribs> host_attribs; + Cuda::HostVector host_particles; + std::array, PIdx::nattribs> host_attribs; // Loop through the cells of overlap_box and inject // the corresponding particles @@ -725,54 +725,54 @@ PhysicalParticleContainer::AddPlasmaGPU (int lev, RealBox part_realbox) Real dens; std::array u; if (WarpX::gamma_boost == 1.){ - // Lab-frame simulation - // If the particle is not within the species's - // xmin, xmax, ymin, ymax, zmin, zmax, go to - // the next generated particle. - if (!plasma_injector->insideBounds(xb, yb, z)) continue; - plasma_injector->getMomentum(u, x, y, z); - dens = plasma_injector->getDensity(x, y, z); + // Lab-frame simulation + // If the particle is not within the species's + // xmin, xmax, ymin, ymax, zmin, zmax, go to + // the next generated particle. + if (!plasma_injector->insideBounds(xb, yb, z)) continue; + plasma_injector->getMomentum(u, x, y, z); + dens = plasma_injector->getDensity(x, y, z); } else { - // Boosted-frame simulation - Real c = PhysConst::c; - Real gamma_boost = WarpX::gamma_boost; - Real beta_boost = WarpX::beta_boost; - // Since the user provides the density distribution - // at t_lab=0 and in the lab-frame coordinates, - // we need to find the lab-frame position of this - // particle at t_lab=0, from its boosted-frame coordinates - // Assuming ballistic motion, this is given by: - // z0_lab = gamma*( z_boost*(1-beta*betaz_lab) - ct_boost*(betaz_lab-beta) ) - // where betaz_lab is the speed of the particle in the lab frame - // - // In order for this equation to be solvable, betaz_lab - // is explicitly assumed to have no dependency on z0_lab - plasma_injector->getMomentum(u, x, y, 0.); // No z0_lab dependency - // At this point u is the lab-frame momentum - // => Apply the above formula for z0_lab - Real gamma_lab = std::sqrt( 1 + (u[0]*u[0] + u[1]*u[1] + u[2]*u[2])/(c*c) ); - Real betaz_lab = u[2]/gamma_lab/c; - Real t = WarpX::GetInstance().gett_new(lev); - Real z0_lab = gamma_boost * ( z*(1-beta_boost*betaz_lab) - c*t*(betaz_lab-beta_boost) ); - // If the particle is not within the lab-frame zmin, zmax, etc. - // go to the next generated particle. - if (!plasma_injector->insideBounds(xb, yb, z0_lab)) continue; - // call `getDensity` with lab-frame parameters - dens = plasma_injector->getDensity(x, y, z0_lab); - // At this point u and dens are the lab-frame quantities - // => Perform Lorentz transform - dens = gamma_boost * dens * ( 1 - beta_boost*betaz_lab ); - u[2] = gamma_boost * ( u[2] -beta_boost*c*gamma_lab ); + // Boosted-frame simulation + Real c = PhysConst::c; + Real gamma_boost = WarpX::gamma_boost; + Real beta_boost = WarpX::beta_boost; + // Since the user provides the density distribution + // at t_lab=0 and in the lab-frame coordinates, + // we need to find the lab-frame position of this + // particle at t_lab=0, from its boosted-frame coordinates + // Assuming ballistic motion, this is given by: + // z0_lab = gamma*( z_boost*(1-beta*betaz_lab) - ct_boost*(betaz_lab-beta) ) + // where betaz_lab is the speed of the particle in the lab frame + // + // In order for this equation to be solvable, betaz_lab + // is explicitly assumed to have no dependency on z0_lab + plasma_injector->getMomentum(u, x, y, 0.); // No z0_lab dependency + // At this point u is the lab-frame momentum + // => Apply the above formula for z0_lab + Real gamma_lab = std::sqrt( 1 + (u[0]*u[0] + u[1]*u[1] + u[2]*u[2])/(c*c) ); + Real betaz_lab = u[2]/gamma_lab/c; + Real t = WarpX::GetInstance().gett_new(lev); + Real z0_lab = gamma_boost * ( z*(1-beta_boost*betaz_lab) - c*t*(betaz_lab-beta_boost) ); + // If the particle is not within the lab-frame zmin, zmax, etc. + // go to the next generated particle. + if (!plasma_injector->insideBounds(xb, yb, z0_lab)) continue; + // call `getDensity` with lab-frame parameters + dens = plasma_injector->getDensity(x, y, z0_lab); + // At this point u and dens are the lab-frame quantities + // => Perform Lorentz transform + dens = gamma_boost * dens * ( 1 - beta_boost*betaz_lab ); + u[2] = gamma_boost * ( u[2] -beta_boost*c*gamma_lab ); } Real weight = dens * scale_fac / (AMREX_D_TERM(fac, *fac, *fac)); #ifdef WARPX_RZ if (plasma_injector->radially_weighted) { - weight *= 2*MathConst::pi*xb; + weight *= 2*MathConst::pi*xb; } else { - // This is not correct since it might shift the particle - // out of the local grid - x = std::sqrt(xb*rmax); - weight *= dx[0]; + // This is not correct since it might shift the particle + // out of the local grid + x = std::sqrt(xb*rmax); + weight *= dx[0]; } #endif attribs[PIdx::w ] = weight; @@ -793,50 +793,50 @@ PhysicalParticleContainer::AddPlasmaGPU (int lev, RealBox part_realbox) particle_tile.push_back_real(particle_comps["uzold"], u[2]); } - ParticleType p; - p.id() = ParticleType::NextID(); - p.cpu() = ParallelDescriptor::MyProc(); + ParticleType p; + p.id() = ParticleType::NextID(); + p.cpu() = ParallelDescriptor::MyProc(); #if (AMREX_SPACEDIM == 3) - p.pos(0) = x; - p.pos(1) = y; - p.pos(2) = z; + p.pos(0) = x; + p.pos(1) = y; + p.pos(2) = z; #elif (AMREX_SPACEDIM == 2) #ifdef WARPX_RZ attribs[PIdx::theta] = theta; #endif - p.pos(0) = xb; - p.pos(1) = z; + p.pos(0) = xb; + p.pos(1) = z; #endif - host_particles.push_back(p); - for (int kk = 0; kk < PIdx::nattribs; ++kk) - host_attribs[kk].push_back(attribs[kk]); + host_particles.push_back(p); + for (int kk = 0; kk < PIdx::nattribs; ++kk) + host_attribs[kk].push_back(attribs[kk]); } } - auto& particle_tile = GetParticles(lev)[std::make_pair(grid_id,tile_id)]; + auto& particle_tile = GetParticles(lev)[std::make_pair(grid_id,tile_id)]; auto old_size = particle_tile.GetArrayOfStructs().size(); auto new_size = old_size + host_particles.size(); - particle_tile.resize(new_size); + particle_tile.resize(new_size); Cuda::thrust_copy(host_particles.begin(), host_particles.end(), particle_tile.GetArrayOfStructs().begin() + old_size); - for (int kk = 0; kk < PIdx::nattribs; ++kk) { + for (int kk = 0; kk < PIdx::nattribs; ++kk) { Cuda::thrust_copy(host_attribs[kk].begin(), host_attribs[kk].end(), particle_tile.GetStructOfArrays().GetRealData(kk).begin() + old_size); - } + } if (cost) { - wt = (amrex::second() - wt) / tile_box.d_numPts(); + wt = (amrex::second() - wt) / tile_box.d_numPts(); Array4 const& costarr = cost->array(mfi); amrex::ParallelFor(tile_box, - [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - costarr(i,j,k) += wt; - }); + [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + costarr(i,j,k) += wt; + }); } } } @@ -963,13 +963,13 @@ FieldGatherES (const amrex::Vector, WRPX_INTERPOLATE_CIC(particles.dataPtr(), nstride, np, Exp.dataPtr(), Eyp.dataPtr(), #if AMREX_SPACEDIM == 3 - Ezp.dataPtr(), + Ezp.dataPtr(), #endif - exfab.dataPtr(), eyfab.dataPtr(), + exfab.dataPtr(), eyfab.dataPtr(), #if AMREX_SPACEDIM == 3 - ezfab.dataPtr(), + ezfab.dataPtr(), #endif - box.loVect(), box.hiVect(), plo, dx, &ng); + box.loVect(), box.hiVect(), plo, dx, &ng); } else { const FArrayBox& exfab_coarse = coarse_Ex[pti]; @@ -1004,7 +1004,7 @@ FieldGatherES (const amrex::Vector, void PhysicalParticleContainer::EvolveES (const Vector, 3> >& E, - Vector >& rho, + Vector >& rho, Real t, Real dt) { BL_PROFILE("PPC::EvolveES()"); @@ -1014,7 +1014,7 @@ PhysicalParticleContainer::EvolveES (const VectorGeom(lev); const RealBox& prob_domain = gm.ProbDomain(); - for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { + for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { // Particle structs auto& particles = pti.GetArrayOfStructs(); int nstride = particles.dataShape().first; @@ -1071,11 +1071,11 @@ PhysicalParticleContainer::FieldGather (int lev, { Cuda::ManagedDeviceVector xp, yp, zp; - for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) - { + for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) + { Real wt = amrex::second(); - const Box& box = pti.validbox(); + const Box& box = pti.validbox(); auto& attribs = pti.GetAttribs(); @@ -1088,63 +1088,63 @@ PhysicalParticleContainer::FieldGather (int lev, const long np = pti.numParticles(); - // Data on the grid - const FArrayBox& exfab = Ex[pti]; - const FArrayBox& eyfab = Ey[pti]; - const FArrayBox& ezfab = Ez[pti]; - const FArrayBox& bxfab = Bx[pti]; - const FArrayBox& byfab = By[pti]; - const FArrayBox& bzfab = Bz[pti]; - - Exp.assign(np,0.0); - Eyp.assign(np,0.0); - Ezp.assign(np,0.0); - Bxp.assign(np,0.0); - Byp.assign(np,0.0); - Bzp.assign(np,0.0); - - // - // copy data from particle container to temp arrays - // + // Data on the grid + const FArrayBox& exfab = Ex[pti]; + const FArrayBox& eyfab = Ey[pti]; + const FArrayBox& ezfab = Ez[pti]; + const FArrayBox& bxfab = Bx[pti]; + const FArrayBox& byfab = By[pti]; + const FArrayBox& bzfab = Bz[pti]; + + Exp.assign(np,0.0); + Eyp.assign(np,0.0); + Ezp.assign(np,0.0); + Bxp.assign(np,0.0); + Byp.assign(np,0.0); + Bzp.assign(np,0.0); + + // + // copy data from particle container to temp arrays + // pti.GetPosition(xp, yp, zp); const std::array& xyzmin = WarpX::LowerCorner(box, lev); const int* ixyzmin = box.loVect(); - // - // Field Gather - // - const int ll4symtry = false; + // + // Field Gather + // + const int ll4symtry = false; long lvect_fieldgathe = 64; - warpx_geteb_energy_conserving( - &np, - xp.dataPtr(), - yp.dataPtr(), - zp.dataPtr(), - Exp.dataPtr(),Eyp.dataPtr(),Ezp.dataPtr(), - Bxp.dataPtr(),Byp.dataPtr(),Bzp.dataPtr(), - ixyzmin, - &xyzmin[0], &xyzmin[1], &xyzmin[2], - &dx[0], &dx[1], &dx[2], - &WarpX::nox, &WarpX::noy, &WarpX::noz, - BL_TO_FORTRAN_ANYD(exfab), - BL_TO_FORTRAN_ANYD(eyfab), - BL_TO_FORTRAN_ANYD(ezfab), - BL_TO_FORTRAN_ANYD(bxfab), - BL_TO_FORTRAN_ANYD(byfab), - BL_TO_FORTRAN_ANYD(bzfab), - &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, - &lvect_fieldgathe, &WarpX::field_gathering_algo); + warpx_geteb_energy_conserving( + &np, + xp.dataPtr(), + yp.dataPtr(), + zp.dataPtr(), + Exp.dataPtr(),Eyp.dataPtr(),Ezp.dataPtr(), + Bxp.dataPtr(),Byp.dataPtr(),Bzp.dataPtr(), + ixyzmin, + &xyzmin[0], &xyzmin[1], &xyzmin[2], + &dx[0], &dx[1], &dx[2], + &WarpX::nox, &WarpX::noy, &WarpX::noz, + BL_TO_FORTRAN_ANYD(exfab), + BL_TO_FORTRAN_ANYD(eyfab), + BL_TO_FORTRAN_ANYD(ezfab), + BL_TO_FORTRAN_ANYD(bxfab), + BL_TO_FORTRAN_ANYD(byfab), + BL_TO_FORTRAN_ANYD(bzfab), + &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, + &lvect_fieldgathe, &WarpX::field_gathering_algo); if (cost) { const Box& tbx = pti.tilebox(); wt = (amrex::second() - wt) / tbx.d_numPts(); Array4 const& costarr = cost->array(pti); amrex::ParallelFor(tbx, - [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - costarr(i,j,k) += wt; - }); + [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + costarr(i,j,k) += wt; + }); } } } @@ -1152,9 +1152,9 @@ PhysicalParticleContainer::FieldGather (int lev, void PhysicalParticleContainer::Evolve (int lev, - const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, - const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz, - MultiFab& jx, MultiFab& jy, MultiFab& jz, + const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, + const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz, + MultiFab& jx, MultiFab& jy, MultiFab& jz, MultiFab* cjx, MultiFab* cjy, MultiFab* cjz, MultiFab* rho, MultiFab* crho, const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, @@ -1200,8 +1200,8 @@ PhysicalParticleContainer::Evolve (int lev, RealVector tmp; ParticleVector particle_tmp; - for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) - { + for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) + { Real wt = amrex::second(); const Box& box = pti.validbox(); @@ -1282,14 +1282,14 @@ PhysicalParticleContainer::Evolve (int lev, #endif } - Exp.assign(np,0.0); - Eyp.assign(np,0.0); - Ezp.assign(np,0.0); - Bxp.assign(np,WarpX::B_external[0]); - Byp.assign(np,WarpX::B_external[1]); - Bzp.assign(np,WarpX::B_external[2]); + Exp.assign(np,0.0); + Eyp.assign(np,0.0); + Ezp.assign(np,0.0); + Bxp.assign(np,WarpX::B_external[0]); + Byp.assign(np,WarpX::B_external[1]); + Bzp.assign(np,WarpX::B_external[2]); - m_giv[thread_num].resize(np); + m_giv[thread_num].resize(np); long nfine_current = np; long nfine_gather = np; @@ -1384,12 +1384,12 @@ PhysicalParticleContainer::Evolve (int lev, const long np_current = (cjx) ? nfine_current : np; - // - // copy data from particle container to temp arrays - // - BL_PROFILE_VAR_START(blp_copy); + // + // copy data from particle container to temp arrays + // + BL_PROFILE_VAR_START(blp_copy); pti.GetPosition(m_xp[thread_num], m_yp[thread_num], m_zp[thread_num]); - BL_PROFILE_VAR_STOP(blp_copy); + BL_PROFILE_VAR_STOP(blp_copy); if (rho) DepositCharge(pti, wp, rho, crho, 0, np_current, np, thread_num, lev); @@ -1568,10 +1568,10 @@ PhysicalParticleContainer::Evolve (int lev, wt = (amrex::second() - wt) / tbx.d_numPts(); Array4 const& costarr = cost->array(pti); amrex::ParallelFor(tbx, - [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - costarr(i,j,k) += wt; - }); + [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + costarr(i,j,k) += wt; + }); } } } @@ -1603,9 +1603,9 @@ PhysicalParticleContainer::SplitParticles(int lev) { pti.GetPosition(xp, yp, zp); const std::array& dx = WarpX::CellSize(lev); - // particle Array Of Structs data + // particle Array Of Structs data auto& particles = pti.GetArrayOfStructs(); - // particle Struct Of Arrays data + // particle Struct Of Arrays data auto& attribs = pti.GetAttribs(); auto& wp = attribs[PIdx::w ]; auto& uxp = attribs[PIdx::ux]; @@ -1615,13 +1615,13 @@ PhysicalParticleContainer::SplitParticles(int lev) for(int i=0; i& xp, + Cuda::ManagedDeviceVector& xp, Cuda::ManagedDeviceVector& yp, Cuda::ManagedDeviceVector& zp, Cuda::ManagedDeviceVector& giv, @@ -1795,8 +1795,8 @@ PhysicalParticleContainer::PushP (int lev, Real dt, int thread_num = 0; #endif for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) - { - const Box& box = pti.validbox(); + { + const Box& box = pti.validbox(); auto& attribs = pti.GetAttribs(); @@ -1812,26 +1812,26 @@ PhysicalParticleContainer::PushP (int lev, Real dt, const long np = pti.numParticles(); - // Data on the grid - const FArrayBox& exfab = Ex[pti]; - const FArrayBox& eyfab = Ey[pti]; - const FArrayBox& ezfab = Ez[pti]; - const FArrayBox& bxfab = Bx[pti]; - const FArrayBox& byfab = By[pti]; - const FArrayBox& bzfab = Bz[pti]; - - Exp.assign(np,0.0); - Eyp.assign(np,0.0); - Ezp.assign(np,0.0); - Bxp.assign(np,WarpX::B_external[0]); - Byp.assign(np,WarpX::B_external[1]); - Bzp.assign(np,WarpX::B_external[2]); - - m_giv[thread_num].resize(np); - - // - // copy data from particle container to temp arrays - // + // Data on the grid + const FArrayBox& exfab = Ex[pti]; + const FArrayBox& eyfab = Ey[pti]; + const FArrayBox& ezfab = Ez[pti]; + const FArrayBox& bxfab = Bx[pti]; + const FArrayBox& byfab = By[pti]; + const FArrayBox& bzfab = Bz[pti]; + + Exp.assign(np,0.0); + Eyp.assign(np,0.0); + Ezp.assign(np,0.0); + Bxp.assign(np,WarpX::B_external[0]); + Byp.assign(np,WarpX::B_external[1]); + Bzp.assign(np,WarpX::B_external[2]); + + m_giv[thread_num].resize(np); + + // + // copy data from particle container to temp arrays + // pti.GetPosition(m_xp[thread_num], m_yp[thread_num], m_zp[thread_num]); const std::array& xyzmin_grid = WarpX::LowerCorner(box, lev); @@ -1875,7 +1875,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, } void PhysicalParticleContainer::copy_attribs(WarpXParIter& pti,const Real* xp, - const Real* yp, const Real* zp) + const Real* yp, const Real* zp) { auto& attribs = pti.GetAttribs(); @@ -1894,16 +1894,16 @@ void PhysicalParticleContainer::copy_attribs(WarpXParIter& pti,const Real* xp, const long np = pti.numParticles(); ParallelFor( np, - [=] AMREX_GPU_DEVICE (long i) { - xpold[i]=xp[i]; - ypold[i]=yp[i]; - zpold[i]=zp[i]; + [=] AMREX_GPU_DEVICE (long i) { + xpold[i]=xp[i]; + ypold[i]=yp[i]; + zpold[i]=zp[i]; - uxpold[i]=uxp[i]; - uypold[i]=uyp[i]; - uzpold[i]=uzp[i]; - } - ); + uxpold[i]=uxp[i]; + uypold[i]=uyp[i]; + uzpold[i]=uzp[i]; + } + ); } void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real z_old, -- cgit v1.2.3 From f2b14c9e3daf0b3896c497572cdb1a1c088b5596 Mon Sep 17 00:00:00 2001 From: Dave Grote Date: Tue, 16 Jul 2019 10:38:33 -0700 Subject: Update gaminv in the momentum push (in c++) --- Source/Particles/PhysicalParticleContainer.cpp | 5 +++-- Source/Particles/Pusher/UpdateMomentumBoris.H | 3 ++- Source/Particles/Pusher/UpdateMomentumVay.H | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index a1d9a9afb..9441a4d91 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1754,6 +1754,7 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti, Real* AMREX_RESTRICT x = xp.dataPtr(); Real* AMREX_RESTRICT y = yp.dataPtr(); Real* AMREX_RESTRICT z = zp.dataPtr(); + Real* AMREX_RESTRICT gi = giv.dataPtr(); Real* AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); Real* AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); Real* AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); @@ -1775,7 +1776,7 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti, if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Boris){ amrex::ParallelFor( pti.numParticles(), [=] AMREX_GPU_DEVICE (long i) { - UpdateMomentumBoris( ux[i], uy[i], uz[i], + UpdateMomentumBoris( ux[i], uy[i], uz[i], gi[i], Ex[i], Ey[i], Ez[i], Bx[i], By[i], Bz[i], q, m, dt); UpdatePosition( x[i], y[i], z[i], ux[i], uy[i], uz[i], dt ); @@ -1784,7 +1785,7 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti, } else if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Vay) { amrex::ParallelFor( pti.numParticles(), [=] AMREX_GPU_DEVICE (long i) { - UpdateMomentumVay( ux[i], uy[i], uz[i], + UpdateMomentumVay( ux[i], uy[i], uz[i], gi[i], Ex[i], Ey[i], Ez[i], Bx[i], By[i], Bz[i], q, m, dt); UpdatePosition( x[i], y[i], z[i], ux[i], uy[i], uz[i], dt ); diff --git a/Source/Particles/Pusher/UpdateMomentumBoris.H b/Source/Particles/Pusher/UpdateMomentumBoris.H index a33058347..71e9a8ed1 100644 --- a/Source/Particles/Pusher/UpdateMomentumBoris.H +++ b/Source/Particles/Pusher/UpdateMomentumBoris.H @@ -7,7 +7,7 @@ * given the value of its momenta `ux`, `uy`, `uz` */ AMREX_GPU_HOST_DEVICE AMREX_INLINE void UpdateMomentumBoris( - amrex::Real& ux, amrex::Real& uy, amrex::Real& uz, + amrex::Real& ux, amrex::Real& uy, amrex::Real& uz, amrex::Real& gaminv, const amrex::Real Ex, const amrex::Real Ey, const amrex::Real Ez, const amrex::Real Bx, const amrex::Real By, const amrex::Real Bz, const amrex::Real q, const amrex::Real m, const amrex::Real dt ) @@ -41,6 +41,7 @@ void UpdateMomentumBoris( ux += econst*Ex; uy += econst*Ey; uz += econst*Ez; + gaminv = 1./std::sqrt(1. + (ux*ux + uy*uy + uz*uz)*inv_c2); } #endif // WARPX_PARTICLES_PUSHER_UPDATEMOMENTUM_BORIS_H_ diff --git a/Source/Particles/Pusher/UpdateMomentumVay.H b/Source/Particles/Pusher/UpdateMomentumVay.H index 1f0f19e63..044297e22 100644 --- a/Source/Particles/Pusher/UpdateMomentumVay.H +++ b/Source/Particles/Pusher/UpdateMomentumVay.H @@ -9,7 +9,7 @@ * given the value of its momenta `ux`, `uy`, `uz` */ AMREX_GPU_HOST_DEVICE AMREX_INLINE void UpdateMomentumVay( - amrex::Real& ux, amrex::Real& uy, amrex::Real& uz, + amrex::Real& ux, amrex::Real& uy, amrex::Real& uz, amrex::Real& gaminv, const amrex::Real Ex, const amrex::Real Ey, const amrex::Real Ez, const amrex::Real Bx, const amrex::Real By, const amrex::Real Bz, const amrex::Real q, const amrex::Real m, const amrex::Real dt ) @@ -48,6 +48,7 @@ void UpdateMomentumVay( ux = s*(uxpr+tx*tu+uypr*tz-uzpr*ty); uy = s*(uypr+ty*tu+uzpr*tx-uxpr*tz); uz = s*(uzpr+tz*tu+uxpr*ty-uypr*tx); + gaminv = 1./std::sqrt(1. + (ux*ux + uy*uy + uz*uz)*invclightsq); } #endif // WARPX_PARTICLES_PUSHER_UPDATEMOMENTUM_VAY_H_ -- cgit v1.2.3 From 6f23f443585034a413059074830d7ffd3052520e Mon Sep 17 00:00:00 2001 From: Dave Grote Date: Wed, 17 Jul 2019 09:28:50 -0700 Subject: Convert PushP and RigidPC to c++ --- Source/Particles/PhysicalParticleContainer.cpp | 45 ++++-- .../Particles/RigidInjectedParticleContainer.cpp | 164 ++++++++++++++------- 2 files changed, 147 insertions(+), 62 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 3a512d9dc..4b4747d0e 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1748,7 +1748,7 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti, Real dt) { - // This wraps the call to warpx_particle_pusher so that inheritors can modify the call. + // This wraps the momentum and position advance so that inheritors can modify the call. auto& attribs = pti.GetAttribs(); // Extract pointers to the different particle quantities Real* AMREX_RESTRICT x = xp.dataPtr(); @@ -1883,16 +1883,39 @@ PhysicalParticleContainer::PushP (int lev, Real dt, &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, &lvect_fieldgathe, &WarpX::field_gathering_algo); - warpx_particle_pusher_momenta(&np, - m_xp[thread_num].dataPtr(), - m_yp[thread_num].dataPtr(), - m_zp[thread_num].dataPtr(), - uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(), - m_giv[thread_num].dataPtr(), - Exp.dataPtr(), Eyp.dataPtr(), Ezp.dataPtr(), - Bxp.dataPtr(), Byp.dataPtr(), Bzp.dataPtr(), - &this->charge, &this->mass, &dt, - &WarpX::particle_pusher_algo); + // This wraps the momentum advance so that inheritors can modify the call. + // Extract pointers to the different particle quantities + Real* AMREX_RESTRICT gi = m_giv[thread_num].dataPtr(); + Real* AMREX_RESTRICT uxpp = uxp.dataPtr(); + Real* AMREX_RESTRICT uypp = uyp.dataPtr(); + Real* AMREX_RESTRICT uzpp = uzp.dataPtr(); + Real* AMREX_RESTRICT Expp = Exp.dataPtr(); + Real* AMREX_RESTRICT Eypp = Eyp.dataPtr(); + Real* AMREX_RESTRICT Ezpp = Ezp.dataPtr(); + Real* AMREX_RESTRICT Bxpp = Bxp.dataPtr(); + Real* AMREX_RESTRICT Bypp = Byp.dataPtr(); + Real* AMREX_RESTRICT Bzpp = Bzp.dataPtr(); + + // Loop over the particles and update their momentum + const Real q = this->charge; + const Real m = this-> mass; + if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Boris){ + amrex::ParallelFor( pti.numParticles(), + [=] AMREX_GPU_DEVICE (long i) { + UpdateMomentumBoris( uxpp[i], uypp[i], uzpp[i], gi[i], + Expp[i], Eypp[i], Ezpp[i], Bxpp[i], Bypp[i], Bzpp[i], q, m, dt); + } + ); + } else if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Vay) { + amrex::ParallelFor( pti.numParticles(), + [=] AMREX_GPU_DEVICE (long i) { + UpdateMomentumVay( uxpp[i], uypp[i], uzpp[i], gi[i], + Expp[i], Eypp[i], Ezpp[i], Bxpp[i], Bypp[i], Bzpp[i], q, m, dt); + } + ); + } else { + amrex::Abort("Unknown particle pusher"); + }; } } } diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index 9bd4cb4fc..a0aadb57d 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -10,6 +10,9 @@ #include #include #include +#include +#include +#include using namespace amrex; @@ -233,6 +236,20 @@ RigidInjectedParticleContainer::PushPX(WarpXParIter& pti, Cuda::ManagedDeviceVector xp_save, yp_save, zp_save; RealVector uxp_save, uyp_save, uzp_save; + Real* AMREX_RESTRICT x = xp.dataPtr(); + Real* AMREX_RESTRICT y = yp.dataPtr(); + Real* AMREX_RESTRICT z = zp.dataPtr(); + Real* AMREX_RESTRICT gi = giv.dataPtr(); + Real* AMREX_RESTRICT uxpp = uxp.dataPtr(); + Real* AMREX_RESTRICT uypp = uyp.dataPtr(); + Real* AMREX_RESTRICT uzpp = uzp.dataPtr(); + Real* AMREX_RESTRICT Expp = Exp.dataPtr(); + Real* AMREX_RESTRICT Eypp = Eyp.dataPtr(); + Real* AMREX_RESTRICT Ezpp = Ezp.dataPtr(); + Real* AMREX_RESTRICT Bxpp = Bxp.dataPtr(); + Real* AMREX_RESTRICT Bypp = Byp.dataPtr(); + Real* AMREX_RESTRICT Bzpp = Bzp.dataPtr(); + if (!done_injecting_lev) { xp_save = xp; yp_save = yp; @@ -240,33 +257,34 @@ RigidInjectedParticleContainer::PushPX(WarpXParIter& pti, uxp_save = uxp; uyp_save = uyp; uzp_save = uzp; + + Real* AMREX_RESTRICT xp_savep = xp_save.dataPtr(); + Real* AMREX_RESTRICT yp_savep = yp_save.dataPtr(); + Real* AMREX_RESTRICT zp_savep = zp_save.dataPtr(); + Real* AMREX_RESTRICT uxp_savep = uxp_save.dataPtr(); + Real* AMREX_RESTRICT uyp_savep = uyp_save.dataPtr(); + Real* AMREX_RESTRICT uzp_savep = uzp_save.dataPtr(); + // Scale the fields of particles about to cross the injection plane. // This only approximates what should be happening. The particles // should by advanced a fraction of a time step instead. // Scaling the fields is much easier and may be good enough. - for (int i=0 ; i < zp.size() ; i++) { - const Real dtscale = dt - (zinject_plane_lev_previous - zp[i])/(vzbeam_ave_boosted + WarpX::beta_boost*PhysConst::c); + amrex::ParallelFor( pti.numParticles(), + [=] AMREX_GPU_DEVICE (long i) { + const Real dtscale = dt - (zinject_plane_lev_previous - z[i])/(vzbeam_ave_boosted + WarpX::beta_boost*PhysConst::c); if (0. < dtscale && dtscale < dt) { - Exp[i] *= dtscale; - Eyp[i] *= dtscale; - Ezp[i] *= dtscale; - Bxp[i] *= dtscale; - Byp[i] *= dtscale; - Bzp[i] *= dtscale; + Expp[i] *= dtscale; + Eypp[i] *= dtscale; + Ezpp[i] *= dtscale; + Bxpp[i] *= dtscale; + Bypp[i] *= dtscale; + Bzpp[i] *= dtscale; } } + ); } - warpx_particle_pusher(&np, - xp.dataPtr(), - yp.dataPtr(), - zp.dataPtr(), - uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(), - giv.dataPtr(), - Exp.dataPtr(), Eyp.dataPtr(), Ezp.dataPtr(), - Bxp.dataPtr(), Byp.dataPtr(), Bzp.dataPtr(), - &this->charge, &this->mass, &dt, - &WarpX::particle_pusher_algo); + PhysicalParticleContainer::PushPX(pti, xp, yp, zp, giv, dt); if (!done_injecting_lev) { #ifdef _OPENMP @@ -274,25 +292,35 @@ RigidInjectedParticleContainer::PushPX(WarpXParIter& pti, #else const int tid = 0; #endif + + Real* AMREX_RESTRICT xp_savep = xp_save.dataPtr(); + Real* AMREX_RESTRICT yp_savep = yp_save.dataPtr(); + Real* AMREX_RESTRICT zp_savep = zp_save.dataPtr(); + Real* AMREX_RESTRICT uxp_savep = uxp_save.dataPtr(); + Real* AMREX_RESTRICT uyp_savep = uyp_save.dataPtr(); + Real* AMREX_RESTRICT uzp_savep = uzp_save.dataPtr(); + // Undo the push for particles not injected yet. // The zp are advanced a fixed amount. - for (int i=0 ; i < zp.size() ; i++) { - if (zp[i] <= zinject_plane_lev) { - uxp[i] = uxp_save[i]; - uyp[i] = uyp_save[i]; - uzp[i] = uzp_save[i]; - giv[i] = 1./std::sqrt(1. + (uxp[i]*uxp[i] + uyp[i]*uyp[i] + uzp[i]*uzp[i])/(PhysConst::c*PhysConst::c)); - xp[i] = xp_save[i]; - yp[i] = yp_save[i]; + amrex::ParallelFor( pti.numParticles(), + [=] AMREX_GPU_DEVICE (long i) { + if (z[i] <= zinject_plane_lev) { + uxpp[i] = uxp_savep[i]; + uypp[i] = uyp_savep[i]; + uzpp[i] = uzp_savep[i]; + gi[i] = 1./std::sqrt(1. + (uxpp[i]*uxpp[i] + uypp[i]*uypp[i] + uzpp[i]*uzpp[i])/(PhysConst::c*PhysConst::c)); + x[i] = xp_savep[i]; + y[i] = yp_savep[i]; if (rigid_advance) { - zp[i] = zp_save[i] + dt*vzbeam_ave_boosted; + z[i] = zp_savep[i] + dt*vzbeam_ave_boosted; } else { - zp[i] = zp_save[i] + dt*uzp[i]*giv[i]; + z[i] = zp_savep[i] + dt*uzpp[i]*gi[i]; } done_injecting_temp[tid] = 0; } } + ); } } @@ -343,6 +371,8 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz) { + BL_PROFILE("RigidInjectedParticleContainer::PushP"); + if (do_not_push) return; const std::array& dx = WarpX::CellSize(lev); @@ -351,8 +381,11 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, #pragma omp parallel #endif { - Cuda::ManagedDeviceVector xp, yp, zp, giv; - +#ifdef _OPENMP + int thread_num = omp_get_thread_num(); +#else + int thread_num = 0; +#endif for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { const Box& box = pti.validbox(); @@ -386,24 +419,24 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, Byp.assign(np,WarpX::B_external[1]); Bzp.assign(np,WarpX::B_external[2]); - giv.resize(np); + m_giv[thread_num].resize(np); // // copy data from particle container to temp arrays // - pti.GetPosition(xp, yp, zp); + pti.GetPosition(m_xp[thread_num], m_yp[thread_num], m_zp[thread_num]); const std::array& xyzmin_grid = WarpX::LowerCorner(box, lev); const int* ixyzmin_grid = box.loVect(); const int ll4symtry = false; - const int l_lower_order_in_v = true; long lvect_fieldgathe = 64; + warpx_geteb_energy_conserving( &np, - xp.dataPtr(), - yp.dataPtr(), - zp.dataPtr(), + m_xp[thread_num].dataPtr(), + m_yp[thread_num].dataPtr(), + m_zp[thread_num].dataPtr(), Exp.dataPtr(),Eyp.dataPtr(),Ezp.dataPtr(), Bxp.dataPtr(),Byp.dataPtr(),Bzp.dataPtr(), ixyzmin_grid, @@ -416,7 +449,7 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, BL_TO_FORTRAN_ANYD(bxfab), BL_TO_FORTRAN_ANYD(byfab), BL_TO_FORTRAN_ANYD(bzfab), - &ll4symtry, &l_lower_order_in_v, &WarpX::do_nodal, + &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, &lvect_fieldgathe, &WarpX::field_gathering_algo); // Save the position and momenta, making copies @@ -424,27 +457,56 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, auto uyp_save = uyp; auto uzp_save = uzp; - warpx_particle_pusher_momenta(&np, - xp.dataPtr(), - yp.dataPtr(), - zp.dataPtr(), - uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(), - giv.dataPtr(), - Exp.dataPtr(), Eyp.dataPtr(), Ezp.dataPtr(), - Bxp.dataPtr(), Byp.dataPtr(), Bzp.dataPtr(), - &this->charge, &this->mass, &dt, - &WarpX::particle_pusher_algo); + // This wraps the momentum advance so that inheritors can modify the call. + // Extract pointers to the different particle quantities + Real* AMREX_RESTRICT zp = m_zp[thread_num].dataPtr(); + Real* AMREX_RESTRICT gi = m_giv[thread_num].dataPtr(); + Real* AMREX_RESTRICT uxpp = uxp.dataPtr(); + Real* AMREX_RESTRICT uypp = uyp.dataPtr(); + Real* AMREX_RESTRICT uzpp = uzp.dataPtr(); + Real* AMREX_RESTRICT uxp_savep = uxp_save.dataPtr(); + Real* AMREX_RESTRICT uyp_savep = uyp_save.dataPtr(); + Real* AMREX_RESTRICT uzp_savep = uzp_save.dataPtr(); + Real* AMREX_RESTRICT Expp = Exp.dataPtr(); + Real* AMREX_RESTRICT Eypp = Eyp.dataPtr(); + Real* AMREX_RESTRICT Ezpp = Ezp.dataPtr(); + Real* AMREX_RESTRICT Bxpp = Bxp.dataPtr(); + Real* AMREX_RESTRICT Bypp = Byp.dataPtr(); + Real* AMREX_RESTRICT Bzpp = Bzp.dataPtr(); + + // Loop over the particles and update their momentum + const Real q = this->charge; + const Real m = this-> mass; + if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Boris){ + amrex::ParallelFor( pti.numParticles(), + [=] AMREX_GPU_DEVICE (long i) { + UpdateMomentumBoris( uxpp[i], uypp[i], uzpp[i], gi[i], + Expp[i], Eypp[i], Ezpp[i], Bxpp[i], Bypp[i], Bzpp[i], q, m, dt); + } + ); + } else if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Vay) { + amrex::ParallelFor( pti.numParticles(), + [=] AMREX_GPU_DEVICE (long i) { + UpdateMomentumVay( uxpp[i], uypp[i], uzpp[i], gi[i], + Expp[i], Eypp[i], Ezpp[i], Bxpp[i], Bypp[i], Bzpp[i], q, m, dt); + } + ); + } else { + amrex::Abort("Unknown particle pusher"); + }; // Undo the push for particles not injected yet. // It is assumed that PushP will only be called on the first and last steps // and that no particles will cross zinject_plane. - for (int i=0 ; i < zp.size() ; i++) { + amrex::ParallelFor( pti.numParticles(), + [=] AMREX_GPU_DEVICE (long i) { if (zp[i] <= zinject_plane_levels[lev]) { - uxp[i] = uxp_save[i]; - uyp[i] = uyp_save[i]; - uzp[i] = uzp_save[i]; + uxpp[i] = uxp_save[i]; + uypp[i] = uyp_save[i]; + uzpp[i] = uzp_save[i]; } } + ); } } -- cgit v1.2.3 From f96dde8721bc1e5d71084d864fa674c1eac9bf47 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Wed, 17 Jul 2019 13:31:18 -0700 Subject: replace FieldGather by FieldGatherFortran to keep old version --- Source/Particles/MultiParticleContainer.H | 6 +++--- Source/Particles/MultiParticleContainer.cpp | 8 ++++---- Source/Particles/PhysicalParticleContainer.H | 14 +++++++------- Source/Particles/PhysicalParticleContainer.cpp | 6 +++--- Source/Particles/WarpXParticleContainer.H | 6 +++--- 5 files changed, 20 insertions(+), 20 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 869126fef..76e3c44bc 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -84,9 +84,9 @@ public: /// Performs the field gather operation using the input fields E and B, for all the species /// in the MultiParticleContainer. This is the electromagnetic version of the field gather. /// - void FieldGather (int lev, - const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz); + void FieldGatherFortran (int lev, + const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, + const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz); /// /// This evolves all the particles by one PIC time step, including current deposition, the diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 9d39ec2f9..ab4400dad 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -239,12 +239,12 @@ MultiParticleContainer::sumParticleCharge (bool local) #endif // WARPX_DO_ELECTROSTATIC void -MultiParticleContainer::FieldGather (int lev, - const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, - const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz) +MultiParticleContainer::FieldGatherFortran (int lev, + const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, + const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz) { for (auto& pc : allcontainers) { - pc->FieldGather(lev, Ex, Ey, Ez, Bx, By, Bz); + pc->FieldGatherFortran(lev, Ex, Ey, Ez, Bx, By, Bz); } } diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index d55764682..25d79c8fd 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -31,13 +31,13 @@ public: amrex::Real t, amrex::Real dt) override; #endif // WARPX_DO_ELECTROSTATIC - virtual void FieldGather(int lev, - const amrex::MultiFab& Ex, - const amrex::MultiFab& Ey, - const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, - const amrex::MultiFab& By, - const amrex::MultiFab& Bz) final; + virtual void FieldGatherFortran(int lev, + const amrex::MultiFab& Ex, + const amrex::MultiFab& Ey, + const amrex::MultiFab& Ez, + const amrex::MultiFab& Bx, + const amrex::MultiFab& By, + const amrex::MultiFab& Bz) final; virtual void Evolve (int lev, const amrex::MultiFab& Ex, diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index d47a7b220..24a8904e9 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1055,9 +1055,9 @@ PhysicalParticleContainer::EvolveES (const Vector& dx = WarpX::CellSize(lev); diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 662b2e1b8..1b2a06171 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -103,9 +103,9 @@ public: virtual void FieldGatherES (const amrex::Vector, 3> >& E, const amrex::Vector > > >& masks) {} - virtual void FieldGather (int lev, - const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz) {} + virtual void FieldGatherFortran (int lev, + const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, + const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz) {} #ifdef WARPX_DO_ELECTROSTATIC virtual void EvolveES (const amrex::Vector, 3> >& E, -- cgit v1.2.3 From 8430b823965088a3f67efdbb7c99f71c2cda7d31 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Wed, 17 Jul 2019 13:40:29 -0700 Subject: field gather function in PPC --- Source/Particles/PhysicalParticleContainer.H | 20 +++ Source/Particles/PhysicalParticleContainer.cpp | 193 +++++++++++++++++-------- 2 files changed, 156 insertions(+), 57 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index 25d79c8fd..ec9d7f98b 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -39,6 +39,26 @@ public: const amrex::MultiFab& By, const amrex::MultiFab& Bz) final; + void FieldGather(WarpXParIter& pti, + RealVector& Exp, + RealVector& Eyp, + RealVector& Ezp, + RealVector& Bxp, + RealVector& Byp, + RealVector& Bzp, + amrex::FArrayBox const * exfab, + amrex::FArrayBox const * eyfab, + amrex::FArrayBox const * ezfab, + amrex::FArrayBox const * bxfab, + amrex::FArrayBox const * byfab, + amrex::FArrayBox const * bzfab, + const int ngE, const int e_is_nodal, + const long offset, + const long np_to_gather, + int thread_num, + int lev, + int depos_lev); + virtual void Evolve (int lev, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 24a8904e9..e46fa0560 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -6,7 +6,7 @@ #include #include #include - +#include using namespace amrex; @@ -1395,38 +1395,16 @@ PhysicalParticleContainer::Evolve (int lev, if (! do_not_push) { + const long np_gather = (cEx) ? nfine_gather : np; + + int e_is_nodal = Ex.is_nodal() and Ey.is_nodal() and Ez.is_nodal(); + // // Field Gather of Aux Data (i.e., the full solution) // - const int ll4symtry = false; - long lvect_fieldgathe = 64; - - const std::array& xyzmin_grid = WarpX::LowerCorner(box, lev); - const int* ixyzmin_grid = box.loVect(); - - const long np_gather = (cEx) ? nfine_gather : np; - - BL_PROFILE_VAR_START(blp_pxr_fg); - - warpx_geteb_energy_conserving( - &np_gather, - m_xp[thread_num].dataPtr(), - m_yp[thread_num].dataPtr(), - m_zp[thread_num].dataPtr(), - Exp.dataPtr(),Eyp.dataPtr(),Ezp.dataPtr(), - Bxp.dataPtr(),Byp.dataPtr(),Bzp.dataPtr(), - ixyzmin_grid, - &xyzmin_grid[0], &xyzmin_grid[1], &xyzmin_grid[2], - &dx[0], &dx[1], &dx[2], - &WarpX::nox, &WarpX::noy, &WarpX::noz, - BL_TO_FORTRAN_ANYD(*exfab), - BL_TO_FORTRAN_ANYD(*eyfab), - BL_TO_FORTRAN_ANYD(*ezfab), - BL_TO_FORTRAN_ANYD(*bxfab), - BL_TO_FORTRAN_ANYD(*byfab), - BL_TO_FORTRAN_ANYD(*bzfab), - &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, - &lvect_fieldgathe, &WarpX::field_gathering_algo); + FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + exfab, eyfab, ezfab, bxfab, byfab, bzfab, + Ex.nGrow(), e_is_nodal, 0, np_gather, thread_num, lev, lev); if (np_gather < np) { @@ -1435,13 +1413,14 @@ PhysicalParticleContainer::Evolve (int lev, const std::array& cxyzmin_grid = WarpX::LowerCorner(cbox, lev-1); const int* cixyzmin_grid = cbox.loVect(); - const FArrayBox* cexfab = &(*cEx)[pti]; - const FArrayBox* ceyfab = &(*cEy)[pti]; - const FArrayBox* cezfab = &(*cEz)[pti]; - const FArrayBox* cbxfab = &(*cBx)[pti]; - const FArrayBox* cbyfab = &(*cBy)[pti]; - const FArrayBox* cbzfab = &(*cBz)[pti]; - + // Data on the grid + FArrayBox const* cexfab = &(*cEx)[pti]; + FArrayBox const* ceyfab = &(*cEy)[pti]; + FArrayBox const* cezfab = &(*cEz)[pti]; + FArrayBox const* cbxfab = &(*cBx)[pti]; + FArrayBox const* cbyfab = &(*cBy)[pti]; + FArrayBox const* cbzfab = &(*cBz)[pti]; + if (WarpX::use_fdtd_nci_corr) { #if (AMREX_SPACEDIM == 2) @@ -1494,26 +1473,13 @@ PhysicalParticleContainer::Evolve (int lev, #endif } - long ncrse = np - nfine_gather; - warpx_geteb_energy_conserving( - &ncrse, - m_xp[thread_num].dataPtr()+nfine_gather, - m_yp[thread_num].dataPtr()+nfine_gather, - m_zp[thread_num].dataPtr()+nfine_gather, - Exp.dataPtr()+nfine_gather, Eyp.dataPtr()+nfine_gather, Ezp.dataPtr()+nfine_gather, - Bxp.dataPtr()+nfine_gather, Byp.dataPtr()+nfine_gather, Bzp.dataPtr()+nfine_gather, - cixyzmin_grid, - &cxyzmin_grid[0], &cxyzmin_grid[1], &cxyzmin_grid[2], - &cdx[0], &cdx[1], &cdx[2], - &WarpX::nox, &WarpX::noy, &WarpX::noz, - BL_TO_FORTRAN_ANYD(*cexfab), - BL_TO_FORTRAN_ANYD(*ceyfab), - BL_TO_FORTRAN_ANYD(*cezfab), - BL_TO_FORTRAN_ANYD(*cbxfab), - BL_TO_FORTRAN_ANYD(*cbyfab), - BL_TO_FORTRAN_ANYD(*cbzfab), - &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, - &lvect_fieldgathe, &WarpX::field_gathering_algo); + e_is_nodal = cEx->is_nodal() and cEy->is_nodal() and cEz->is_nodal(); + FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + cexfab, ceyfab, cezfab, + cbxfab, cbyfab, cbzfab, + cEx->nGrow(), e_is_nodal, + nfine_gather, np-nfine_gather, + thread_num, lev, lev-1); } BL_PROFILE_VAR_STOP(blp_pxr_fg); @@ -2112,3 +2078,116 @@ PhysicalParticleContainer::ContinuousInjection(const RealBox& injection_box) const int lev=0; AddPlasma(lev, injection_box); } + +void +PhysicalParticleContainer::FieldGather(WarpXParIter& pti, + RealVector& Exp, + RealVector& Eyp, + RealVector& Ezp, + RealVector& Bxp, + RealVector& Byp, + RealVector& Bzp, + FArrayBox const * exfab, + FArrayBox const * eyfab, + FArrayBox const * ezfab, + FArrayBox const * bxfab, + FArrayBox const * byfab, + FArrayBox const * bzfab, + const int ngE, const int e_is_nodal, + const long offset, + const long np_to_gather, + int thread_num, + int lev, + int gather_lev) +{ + AMREX_ALWAYS_ASSERT_WITH_MESSAGE((gather_lev==(lev-1)) || + (gather_lev==(lev )), + "Gather buffers only work for lev-1"); + + // If no particles, do not do anything + if (np_to_gather == 0) return; + + const std::array& dx = WarpX::CellSize(std::max(gather_lev,0)); + const Real stagger_shift = e_is_nodal ? 0.0 : 0.5; + + // Get box =from which field is gathered. + Box box; + if (lev == gather_lev) { + box = pti.tilebox(); + } else { + const IntVect& ref_ratio = WarpX::RefRatio(gather_lev); + box = amrex::coarsen(pti.tilebox(),ref_ratio); + } + + box.grow(ngE); + + const Array4& ex_arr = exfab->array(); + const Array4& ey_arr = eyfab->array(); + const Array4& ez_arr = ezfab->array(); + const Array4& bx_arr = bxfab->array(); + const Array4& by_arr = byfab->array(); + const Array4& bz_arr = bzfab->array(); + + const Real * const AMREX_RESTRICT xp = m_xp[thread_num].dataPtr() + offset; + const Real * const AMREX_RESTRICT zp = m_zp[thread_num].dataPtr() + offset; + const Real * const AMREX_RESTRICT yp = m_yp[thread_num].dataPtr() + offset; + + // Lower corner of tile box physical domain + const std::array& xyzmin = WarpX::LowerCorner(box, gather_lev); + + const Dim3 lo = lbound(box); + + if (WarpX::l_lower_order_in_v){ + if (WarpX::nox == 1){ + doGatherShapeN<1,1>(xp, yp, zp, + Exp.dataPtr() + offset, Eyp.dataPtr() + offset, + Ezp.dataPtr() + offset, Bxp.dataPtr() + offset, + Byp.dataPtr() + offset, Bzp.dataPtr() + offset, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + np_to_gather, dx, + xyzmin, lo, stagger_shift); + } else if (WarpX::nox == 2){ + doGatherShapeN<2,1>(xp, yp, zp, + Exp.dataPtr() + offset, Eyp.dataPtr() + offset, + Ezp.dataPtr() + offset, Bxp.dataPtr() + offset, + Byp.dataPtr() + offset, Bzp.dataPtr() + offset, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + np_to_gather, dx, + xyzmin, lo, stagger_shift); + } else if (WarpX::nox == 3){ + doGatherShapeN<3,1>(xp, yp, zp, + Exp.dataPtr() + offset, Eyp.dataPtr() + offset, + Ezp.dataPtr() + offset, Bxp.dataPtr() + offset, + Byp.dataPtr() + offset, Bzp.dataPtr() + offset, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + np_to_gather, dx, + xyzmin, lo, stagger_shift); + } + } else { + if (WarpX::nox == 1){ + doGatherShapeN<1,0>(xp, yp, zp, + Exp.dataPtr() + offset, Eyp.dataPtr() + offset, + Ezp.dataPtr() + offset, Bxp.dataPtr() + offset, + Byp.dataPtr() + offset, Bzp.dataPtr() + offset, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + np_to_gather, dx, + xyzmin, lo, stagger_shift); + } else if (WarpX::nox == 2){ + doGatherShapeN<2,0>(xp, yp, zp, + Exp.dataPtr() + offset, Eyp.dataPtr() + offset, + Ezp.dataPtr() + offset, Bxp.dataPtr() + offset, + Byp.dataPtr() + offset, Bzp.dataPtr() + offset, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + np_to_gather, dx, + xyzmin, lo, stagger_shift); + } else if (WarpX::nox == 3){ + doGatherShapeN<3,0>(xp, yp, zp, + Exp.dataPtr() + offset, Eyp.dataPtr() + offset, + Ezp.dataPtr() + offset, Bxp.dataPtr() + offset, + Byp.dataPtr() + offset, Bzp.dataPtr() + offset, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + np_to_gather, dx, + xyzmin, lo, stagger_shift); + } + } +} -- cgit v1.2.3 From d0f799ada37b37245b884f064989f0f498b8dc7c Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Wed, 17 Jul 2019 14:05:40 -0700 Subject: docstring for FieldGether function --- Source/Particles/PhysicalParticleContainer.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index e46fa0560..0e1204dc4 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1473,6 +1473,7 @@ PhysicalParticleContainer::Evolve (int lev, #endif } + // Field gather for particles in gather buffers e_is_nodal = cEx->is_nodal() and cEy->is_nodal() and cEz->is_nodal(); FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, cexfab, ceyfab, cezfab, @@ -2079,6 +2080,19 @@ PhysicalParticleContainer::ContinuousInjection(const RealBox& injection_box) AddPlasma(lev, injection_box); } +/* \brief Gather fields from FArrayBox exfab, eyfab, ezfab, bxfab, byfab, + * bzfab into arrays of fields on particles Exp, Eyp, Ezp, Bxp, Byp, Bzp. + * \param Exp-Bzp: fields on particles. + * \param exfab-bzfab: FAB of electric and magnetic fields for particles in pti + * \param ngE: number of guard cells for E + * \param e_is_nodal: 0 if E is staggered, 1 if E is nodal + * \param offset: index of first particle for which fields are gathered + * \param np_to_gather: number of particles onto which fields are gathered + * \param thread_num: if using OpenMP, thread number + * \param lev: level on which particles are located + * \param gather_lev: level from which particles gather fields (lev-1) for + particles in buffers. + */ void PhysicalParticleContainer::FieldGather(WarpXParIter& pti, RealVector& Exp, -- cgit v1.2.3 From a9bf7c5d24596d88e4c96e9ae5d2b5efce694b51 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Wed, 17 Jul 2019 14:30:08 -0700 Subject: use old version when running rz --- Source/Particles/PhysicalParticleContainer.cpp | 51 ++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 0e1204dc4..3f9e8da67 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1402,9 +1402,36 @@ PhysicalParticleContainer::Evolve (int lev, // // Field Gather of Aux Data (i.e., the full solution) // + BL_PROFILE_VAR_START(blp_pxr_fg); +#ifdef WARPX_RZ + const int ll4symtry = false; + long lvect_fieldgathe = 64; + const std::array& xyzmin_grid = WarpX::LowerCorner(box, lev); + const int* ixyzmin_grid = box.loVect(); + warpx_geteb_energy_conserving( + &np_gather, + m_xp[thread_num].dataPtr(), + m_yp[thread_num].dataPtr(), + m_zp[thread_num].dataPtr(), + Exp.dataPtr(),Eyp.dataPtr(),Ezp.dataPtr(), + Bxp.dataPtr(),Byp.dataPtr(),Bzp.dataPtr(), + ixyzmin_grid, + &xyzmin_grid[0], &xyzmin_grid[1], &xyzmin_grid[2], + &dx[0], &dx[1], &dx[2], + &WarpX::nox, &WarpX::noy, &WarpX::noz, + BL_TO_FORTRAN_ANYD(*exfab), + BL_TO_FORTRAN_ANYD(*eyfab), + BL_TO_FORTRAN_ANYD(*ezfab), + BL_TO_FORTRAN_ANYD(*bxfab), + BL_TO_FORTRAN_ANYD(*byfab), + BL_TO_FORTRAN_ANYD(*bzfab), + &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, + &lvect_fieldgathe, &WarpX::field_gathering_algo); +#else FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, exfab, eyfab, ezfab, bxfab, byfab, bzfab, Ex.nGrow(), e_is_nodal, 0, np_gather, thread_num, lev, lev); +#endif if (np_gather < np) { @@ -1474,6 +1501,29 @@ PhysicalParticleContainer::Evolve (int lev, } // Field gather for particles in gather buffers +#ifdef WARPX_RZ + + long ncrse = np - nfine_gather; + warpx_geteb_energy_conserving( + &ncrse, + m_xp[thread_num].dataPtr()+nfine_gather, + m_yp[thread_num].dataPtr()+nfine_gather, + m_zp[thread_num].dataPtr()+nfine_gather, + Exp.dataPtr()+nfine_gather, Eyp.dataPtr()+nfine_gather, Ezp.dataPtr()+nfine_gather, + Bxp.dataPtr()+nfine_gather, Byp.dataPtr()+nfine_gather, Bzp.dataPtr()+nfine_gather, + cixyzmin_grid, + &cxyzmin_grid[0], &cxyzmin_grid[1], &cxyzmin_grid[2], + &cdx[0], &cdx[1], &cdx[2], + &WarpX::nox, &WarpX::noy, &WarpX::noz, + BL_TO_FORTRAN_ANYD(*cexfab), + BL_TO_FORTRAN_ANYD(*ceyfab), + BL_TO_FORTRAN_ANYD(*cezfab), + BL_TO_FORTRAN_ANYD(*cbxfab), + BL_TO_FORTRAN_ANYD(*cbyfab), + BL_TO_FORTRAN_ANYD(*cbzfab), + &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, + &lvect_fieldgathe, &WarpX::field_gathering_algo);# +else e_is_nodal = cEx->is_nodal() and cEy->is_nodal() and cEz->is_nodal(); FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, cexfab, ceyfab, cezfab, @@ -1481,6 +1531,7 @@ PhysicalParticleContainer::Evolve (int lev, cEx->nGrow(), e_is_nodal, nfine_gather, np-nfine_gather, thread_num, lev, lev-1); +#endif } BL_PROFILE_VAR_STOP(blp_pxr_fg); -- cgit v1.2.3 From e44075f29c45398a09a72848d8592b0a80fee06d Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Wed, 17 Jul 2019 14:36:45 -0700 Subject: minor fix. Now RZ should test passes --- Source/Particles/PhysicalParticleContainer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 3f9e8da67..357a187fd 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1522,8 +1522,8 @@ PhysicalParticleContainer::Evolve (int lev, BL_TO_FORTRAN_ANYD(*cbyfab), BL_TO_FORTRAN_ANYD(*cbzfab), &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, - &lvect_fieldgathe, &WarpX::field_gathering_algo);# -else + &lvect_fieldgathe, &WarpX::field_gathering_algo); +#else e_is_nodal = cEx->is_nodal() and cEy->is_nodal() and cEz->is_nodal(); FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, cexfab, ceyfab, cezfab, -- cgit v1.2.3 From f9a6c067a8c861089dba00e54228086e9501aaec Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 18 Jul 2019 20:46:27 -0700 Subject: Restore spaces --- Source/Particles/PhysicalParticleContainer.cpp | 82 +++++++++++++------------- 1 file changed, 41 insertions(+), 41 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 4b4747d0e..132224db2 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -68,7 +68,7 @@ NumParticlesToAdd(const Box& overlap_box, const RealBox& overlap_realbox, ++np; } } - + return np; } @@ -88,7 +88,7 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp pp.query("do_splitting", do_splitting); pp.query("split_type", split_type); pp.query("do_continuous_injection", do_continuous_injection); - // Whether to plot back-transformed (lab-frame) diagnostics + // Whether to plot back-transformed (lab-frame) diagnostics // for this species. pp.query("do_boosted_frame_diags", do_boosted_frame_diags); @@ -97,7 +97,7 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp do_user_plot_vars = pp.queryarr("plot_vars", plot_vars); if (not do_user_plot_vars){ // By default, all particle variables are dumped to plotfiles, - // including {x,y,z,ux,uy,uz}old variables when running in a + // including {x,y,z,ux,uy,uz}old variables when running in a // boosted frame if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags){ plot_flags.resize(PIdx::nattribs + 6, 1); @@ -114,9 +114,9 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp // If not none, set plot_flags values to 1 for elements in plot_vars. if (plot_vars[0] != "none"){ for (const auto& var : plot_vars){ - // Return error if var not in PIdx. - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( - ParticleStringNames::to_index.count(var), + // Return error if var not in PIdx. + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + ParticleStringNames::to_index.count(var), "plot_vars argument not in ParticleStringNames"); plot_flags[ParticleStringNames::to_index.at(var)] = 1; } @@ -187,7 +187,7 @@ void PhysicalParticleContainer::MapParticletoBoostedFrame(Real& x, Real& y, Real void PhysicalParticleContainer::AddGaussianBeam(Real x_m, Real y_m, Real z_m, Real x_rms, Real y_rms, Real z_rms, - Real q_tot, long npart, + Real q_tot, long npart, int do_symmetrize) { const Geometry& geom = m_gdb->Geom(0); @@ -201,7 +201,7 @@ PhysicalParticleContainer::AddGaussianBeam(Real x_m, Real y_m, Real z_m, if (ParallelDescriptor::IOProcessor()) { std::array u; Real weight; - // If do_symmetrize, create 4x fewer particles, and + // If do_symmetrize, create 4x fewer particles, and // Replicate each particle 4 times (x,y) (-x,y) (x,-y) (-x,-y) if (do_symmetrize){ npart /= 4; @@ -232,7 +232,7 @@ PhysicalParticleContainer::AddGaussianBeam(Real x_m, Real y_m, Real z_m, u_tmp[0] *= std::pow(-1,ix); y_tmp = y*std::pow(-1,iy); u_tmp[1] *= std::pow(-1,iy); - CheckAndAddParticle(x_tmp, y_tmp, z, + CheckAndAddParticle(x_tmp, y_tmp, z, u_tmp, weight/4); } } @@ -269,7 +269,7 @@ PhysicalParticleContainer::CheckAndAddParticle(Real x, Real y, Real z, particle_tile.push_back_real(particle_comps["xold"], x); particle_tile.push_back_real(particle_comps["yold"], y); particle_tile.push_back_real(particle_comps["zold"], z); - + particle_tile.push_back_real(particle_comps["uxold"], u[0]); particle_tile.push_back_real(particle_comps["uyold"], u[1]); particle_tile.push_back_real(particle_comps["uzold"], u[2]); @@ -543,7 +543,7 @@ PhysicalParticleContainer::AddPlasmaCPU (int lev, RealBox part_realbox) attribs[PIdx::ux] = u[0]; attribs[PIdx::uy] = u[1]; attribs[PIdx::uz] = u[2]; - + if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) { auto& particle_tile = DefineAndReturnParticleTile(lev, grid_id, tile_id); @@ -844,7 +844,7 @@ PhysicalParticleContainer::AddPlasmaGPU (int lev, RealBox part_realbox) costarr(i,j,k) += wt; }); } - } + } } } #endif @@ -1172,11 +1172,11 @@ PhysicalParticleContainer::Evolve (int lev, BL_PROFILE_VAR_NS("PICSAR::FieldGather", blp_pxr_fg); BL_PROFILE_VAR_NS("PICSAR::ParticlePush", blp_pxr_pp); BL_PROFILE_VAR_NS("PPC::Evolve::partition", blp_partition); - + const std::array& dx = WarpX::CellSize(lev); const std::array& cdx = WarpX::CellSize(std::max(lev-1,0)); - // Get instances of NCI Godfrey filters + // Get instances of NCI Godfrey filters const auto& nci_godfrey_filter_exeybz = WarpX::GetInstance().nci_godfrey_filter_exeybz; const auto& nci_godfrey_filter_bxbyez = WarpX::GetInstance().nci_godfrey_filter_bxbyez; @@ -1190,7 +1190,7 @@ PhysicalParticleContainer::Evolve (int lev, bool has_buffer = cEx || cjx; #ifdef _OPENMP -#pragma omp parallel +#pragma omp parallel #endif { #ifdef _OPENMP @@ -1398,7 +1398,7 @@ PhysicalParticleContainer::Evolve (int lev, BL_PROFILE_VAR_STOP(blp_copy); if (rho) DepositCharge(pti, wp, rho, crho, 0, np_current, np, thread_num, lev); - + if (! do_not_push) { // @@ -1409,7 +1409,7 @@ PhysicalParticleContainer::Evolve (int lev, const std::array& xyzmin_grid = WarpX::LowerCorner(box, lev); const int* ixyzmin_grid = box.loVect(); - + const long np_gather = (cEx) ? nfine_gather : np; BL_PROFILE_VAR_START(blp_pxr_fg); @@ -1485,13 +1485,13 @@ PhysicalParticleContainer::Evolve (int lev, eyeli = filtered_Ey.elixir(); nci_godfrey_filter_exeybz[lev-1]->ApplyStencil(filtered_Ey, (*cEy)[pti], filtered_Ey.box()); ceyfab = &filtered_Ey; - + // Filter Bx filtered_Bx.resize(amrex::convert(tbox,WarpX::Bx_nodal_flag)); bxeli = filtered_Bx.elixir(); nci_godfrey_filter_bxbyez[lev-1]->ApplyStencil(filtered_Bx, (*cBx)[pti], filtered_Bx.box()); cbxfab = &filtered_Bx; - + // Filter Bz filtered_Bz.resize(amrex::convert(tbox,WarpX::Bz_nodal_flag)); bzeli = filtered_Bz.elixir(); @@ -1499,7 +1499,7 @@ PhysicalParticleContainer::Evolve (int lev, cbzfab = &filtered_Bz; #endif } - + long ncrse = np - nfine_gather; warpx_geteb_energy_conserving( &ncrse, @@ -1528,7 +1528,7 @@ PhysicalParticleContainer::Evolve (int lev, // Particle Push // BL_PROFILE_VAR_START(blp_pxr_pp); - PushPX(pti, m_xp[thread_num], m_yp[thread_num], m_zp[thread_num], + PushPX(pti, m_xp[thread_num], m_yp[thread_num], m_zp[thread_num], m_giv[thread_num], dt); BL_PROFILE_VAR_STOP(blp_pxr_pp); @@ -1566,7 +1566,7 @@ PhysicalParticleContainer::Evolve (int lev, pti.SetPosition(m_xp[thread_num], m_yp[thread_num], m_zp[thread_num]); BL_PROFILE_VAR_STOP(blp_copy); } - + if (rho) DepositCharge(pti, wp, rho, crho, 1, np_current, np, thread_num, lev); if (cost) { @@ -1718,21 +1718,21 @@ PhysicalParticleContainer::SplitParticles(int lev) } // Add local arrays psplit_x etc. to the temporary // particle container pctmp_split. Split particles - // are tagged with p.id()=NoSplitParticleID so that + // are tagged with p.id()=NoSplitParticleID so that // they are not re-split when entering a higher level // AddNParticles calls Redistribute, so that particles // in pctmp_split are in the proper grids and tiles - pctmp_split.AddNParticles(lev, - np_split_to_add, - psplit_x.dataPtr(), - psplit_y.dataPtr(), - psplit_z.dataPtr(), - psplit_ux.dataPtr(), - psplit_uy.dataPtr(), - psplit_uz.dataPtr(), - 1, - psplit_w.dataPtr(), - 1, NoSplitParticleID); + pctmp_split.AddNParticles(lev, + np_split_to_add, + psplit_x.dataPtr(), + psplit_y.dataPtr(), + psplit_z.dataPtr(), + psplit_ux.dataPtr(), + psplit_uy.dataPtr(), + psplit_uz.dataPtr(), + 1, + psplit_w.dataPtr(), + 1, NoSplitParticleID); // Copy particles from tmp to current particle container addParticles(pctmp_split,1); // Clear tmp container @@ -1816,7 +1816,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, int thread_num = omp_get_thread_num(); #else int thread_num = 0; -#endif +#endif for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { const Box& box = pti.validbox(); @@ -1925,20 +1925,20 @@ void PhysicalParticleContainer::copy_attribs(WarpXParIter& pti,const Real* xp, { auto& attribs = pti.GetAttribs(); - + Real* AMREX_RESTRICT uxp = attribs[PIdx::ux].dataPtr(); Real* AMREX_RESTRICT uyp = attribs[PIdx::uy].dataPtr(); Real* AMREX_RESTRICT uzp = attribs[PIdx::uz].dataPtr(); - + Real* AMREX_RESTRICT xpold = pti.GetAttribs(particle_comps["xold"]).dataPtr(); Real* AMREX_RESTRICT ypold = pti.GetAttribs(particle_comps["yold"]).dataPtr(); Real* AMREX_RESTRICT zpold = pti.GetAttribs(particle_comps["zold"]).dataPtr(); Real* AMREX_RESTRICT uxpold = pti.GetAttribs(particle_comps["uxold"]).dataPtr(); Real* AMREX_RESTRICT uypold = pti.GetAttribs(particle_comps["uyold"]).dataPtr(); Real* AMREX_RESTRICT uzpold = pti.GetAttribs(particle_comps["uzold"]).dataPtr(); - + const long np = pti.numParticles(); - + ParallelFor( np, [=] AMREX_GPU_DEVICE (long i) { xpold[i]=xp[i]; @@ -1984,7 +1984,7 @@ void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real slice_box.setHi(direction, z_max); diagnostic_particles.resize(finestLevel()+1); - + for (int lev = 0; lev < nlevs; ++lev) { const Real* dx = Geom(lev).CellSize(); @@ -2066,7 +2066,7 @@ void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real Real uzp = uz_old_p *weight_old + uz_new_p *weight_new; diagnostic_particles[lev][index].GetRealData(DiagIdx::w).push_back(wp[i]); - + diagnostic_particles[lev][index].GetRealData(DiagIdx::x).push_back(xp); diagnostic_particles[lev][index].GetRealData(DiagIdx::y).push_back(yp); diagnostic_particles[lev][index].GetRealData(DiagIdx::z).push_back(zp); -- cgit v1.2.3 From 2b2d1c40ac995951661acfe03c8f13993ece741e Mon Sep 17 00:00:00 2001 From: Dave Grote Date: Mon, 22 Jul 2019 16:08:51 -0700 Subject: Momentum push conversion clean up for clarity and optimization --- Source/Particles/PhysicalParticleContainer.cpp | 54 ++++---- .../Particles/RigidInjectedParticleContainer.cpp | 146 ++++++++++----------- 2 files changed, 97 insertions(+), 103 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 132224db2..5ee2ef5b6 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1751,19 +1751,19 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti, // This wraps the momentum and position advance so that inheritors can modify the call. auto& attribs = pti.GetAttribs(); // Extract pointers to the different particle quantities - Real* AMREX_RESTRICT x = xp.dataPtr(); - Real* AMREX_RESTRICT y = yp.dataPtr(); - Real* AMREX_RESTRICT z = zp.dataPtr(); - Real* AMREX_RESTRICT gi = giv.dataPtr(); - Real* AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); - Real* AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); - Real* AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); - Real* AMREX_RESTRICT Ex = attribs[PIdx::Ex].dataPtr(); - Real* AMREX_RESTRICT Ey = attribs[PIdx::Ey].dataPtr(); - Real* AMREX_RESTRICT Ez = attribs[PIdx::Ez].dataPtr(); - Real* AMREX_RESTRICT Bx = attribs[PIdx::Bx].dataPtr(); - Real* AMREX_RESTRICT By = attribs[PIdx::By].dataPtr(); - Real* AMREX_RESTRICT Bz = attribs[PIdx::Bz].dataPtr(); + Real* const AMREX_RESTRICT x = xp.dataPtr(); + Real* const AMREX_RESTRICT y = yp.dataPtr(); + Real* const AMREX_RESTRICT z = zp.dataPtr(); + Real* const AMREX_RESTRICT gi = giv.dataPtr(); + Real* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); + Real* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); + Real* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); + const Real* AMREX_RESTRICT Ex = attribs[PIdx::Ex].dataPtr(); + const Real* AMREX_RESTRICT Ey = attribs[PIdx::Ey].dataPtr(); + const Real* AMREX_RESTRICT Ez = attribs[PIdx::Ez].dataPtr(); + const Real* AMREX_RESTRICT Bx = attribs[PIdx::Bx].dataPtr(); + const Real* AMREX_RESTRICT By = attribs[PIdx::By].dataPtr(); + const Real* AMREX_RESTRICT Bz = attribs[PIdx::Bz].dataPtr(); if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) { @@ -1794,7 +1794,6 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti, } else { amrex::Abort("Unknown particle pusher"); }; - } void @@ -1823,9 +1822,6 @@ PhysicalParticleContainer::PushP (int lev, Real dt, auto& attribs = pti.GetAttribs(); - auto& uxp = attribs[PIdx::ux]; - auto& uyp = attribs[PIdx::uy]; - auto& uzp = attribs[PIdx::uz]; auto& Exp = attribs[PIdx::Ex]; auto& Eyp = attribs[PIdx::Ey]; auto& Ezp = attribs[PIdx::Ez]; @@ -1885,16 +1881,16 @@ PhysicalParticleContainer::PushP (int lev, Real dt, // This wraps the momentum advance so that inheritors can modify the call. // Extract pointers to the different particle quantities - Real* AMREX_RESTRICT gi = m_giv[thread_num].dataPtr(); - Real* AMREX_RESTRICT uxpp = uxp.dataPtr(); - Real* AMREX_RESTRICT uypp = uyp.dataPtr(); - Real* AMREX_RESTRICT uzpp = uzp.dataPtr(); - Real* AMREX_RESTRICT Expp = Exp.dataPtr(); - Real* AMREX_RESTRICT Eypp = Eyp.dataPtr(); - Real* AMREX_RESTRICT Ezpp = Ezp.dataPtr(); - Real* AMREX_RESTRICT Bxpp = Bxp.dataPtr(); - Real* AMREX_RESTRICT Bypp = Byp.dataPtr(); - Real* AMREX_RESTRICT Bzpp = Bzp.dataPtr(); + Real* const AMREX_RESTRICT gi = m_giv[thread_num].dataPtr(); + Real* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); + Real* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); + Real* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); + const Real* AMREX_RESTRICT Expp = Exp.dataPtr(); + const Real* AMREX_RESTRICT Eypp = Eyp.dataPtr(); + const Real* AMREX_RESTRICT Ezpp = Ezp.dataPtr(); + const Real* AMREX_RESTRICT Bxpp = Bxp.dataPtr(); + const Real* AMREX_RESTRICT Bypp = Byp.dataPtr(); + const Real* AMREX_RESTRICT Bzpp = Bzp.dataPtr(); // Loop over the particles and update their momentum const Real q = this->charge; @@ -1902,14 +1898,14 @@ PhysicalParticleContainer::PushP (int lev, Real dt, if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Boris){ amrex::ParallelFor( pti.numParticles(), [=] AMREX_GPU_DEVICE (long i) { - UpdateMomentumBoris( uxpp[i], uypp[i], uzpp[i], gi[i], + UpdateMomentumBoris( ux[i], uy[i], uz[i], gi[i], Expp[i], Eypp[i], Ezpp[i], Bxpp[i], Bypp[i], Bzpp[i], q, m, dt); } ); } else if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Vay) { amrex::ParallelFor( pti.numParticles(), [=] AMREX_GPU_DEVICE (long i) { - UpdateMomentumVay( uxpp[i], uypp[i], uzpp[i], gi[i], + UpdateMomentumVay( ux[i], uy[i], uz[i], gi[i], Expp[i], Eypp[i], Ezpp[i], Bxpp[i], Bypp[i], Bzpp[i], q, m, dt); } ); diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index 7e4434278..09df32db4 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -214,56 +214,40 @@ RigidInjectedParticleContainer::PushPX(WarpXParIter& pti, Real dt) { - if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) - { - copy_attribs(pti, xp.dataPtr(), yp.dataPtr(), zp.dataPtr()); - } - // This wraps the momentum and position advance so that inheritors can modify the call. auto& attribs = pti.GetAttribs(); auto& uxp = attribs[PIdx::ux]; auto& uyp = attribs[PIdx::uy]; auto& uzp = attribs[PIdx::uz]; - auto& Exp = attribs[PIdx::Ex]; - auto& Eyp = attribs[PIdx::Ey]; - auto& Ezp = attribs[PIdx::Ez]; - auto& Bxp = attribs[PIdx::Bx]; - auto& Byp = attribs[PIdx::By]; - auto& Bzp = attribs[PIdx::Bz]; - const long np = pti.numParticles(); // Save the position and momenta, making copies Cuda::ManagedDeviceVector xp_save, yp_save, zp_save; RealVector uxp_save, uyp_save, uzp_save; - Real* AMREX_RESTRICT x = xp.dataPtr(); - Real* AMREX_RESTRICT y = yp.dataPtr(); - Real* AMREX_RESTRICT z = zp.dataPtr(); - Real* AMREX_RESTRICT gi = giv.dataPtr(); - Real* AMREX_RESTRICT uxpp = uxp.dataPtr(); - Real* AMREX_RESTRICT uypp = uyp.dataPtr(); - Real* AMREX_RESTRICT uzpp = uzp.dataPtr(); - Real* AMREX_RESTRICT Expp = Exp.dataPtr(); - Real* AMREX_RESTRICT Eypp = Eyp.dataPtr(); - Real* AMREX_RESTRICT Ezpp = Ezp.dataPtr(); - Real* AMREX_RESTRICT Bxpp = Bxp.dataPtr(); - Real* AMREX_RESTRICT Bypp = Byp.dataPtr(); - Real* AMREX_RESTRICT Bzpp = Bzp.dataPtr(); + Real* const AMREX_RESTRICT x = xp.dataPtr(); + Real* const AMREX_RESTRICT y = yp.dataPtr(); + Real* const AMREX_RESTRICT z = zp.dataPtr(); + Real* const AMREX_RESTRICT gi = giv.dataPtr(); + Real* const AMREX_RESTRICT ux = uxp.dataPtr(); + Real* const AMREX_RESTRICT uy = uyp.dataPtr(); + Real* const AMREX_RESTRICT uz = uzp.dataPtr(); + Real* const AMREX_RESTRICT Exp = attribs[PIdx::Ex].dataPtr(); + Real* const AMREX_RESTRICT Eyp = attribs[PIdx::Ey].dataPtr(); + Real* const AMREX_RESTRICT Ezp = attribs[PIdx::Ez].dataPtr(); + Real* const AMREX_RESTRICT Bxp = attribs[PIdx::Bx].dataPtr(); + Real* const AMREX_RESTRICT Byp = attribs[PIdx::By].dataPtr(); + Real* const AMREX_RESTRICT Bzp = attribs[PIdx::Bz].dataPtr(); if (!done_injecting_lev) { - xp_save = xp; - yp_save = yp; - zp_save = zp; - uxp_save = uxp; - uyp_save = uyp; - uzp_save = uzp; - - Real* AMREX_RESTRICT xp_savep = xp_save.dataPtr(); - Real* AMREX_RESTRICT yp_savep = yp_save.dataPtr(); - Real* AMREX_RESTRICT zp_savep = zp_save.dataPtr(); - Real* AMREX_RESTRICT uxp_savep = uxp_save.dataPtr(); - Real* AMREX_RESTRICT uyp_savep = uyp_save.dataPtr(); - Real* AMREX_RESTRICT uzp_savep = uzp_save.dataPtr(); + if (!(WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags)) { + // If the old values are not already saved, create copies here. + xp_save = xp; + yp_save = yp; + zp_save = zp; + uxp_save = uxp; + uyp_save = uyp; + uzp_save = uzp; + } // Scale the fields of particles about to cross the injection plane. // This only approximates what should be happening. The particles @@ -273,12 +257,12 @@ RigidInjectedParticleContainer::PushPX(WarpXParIter& pti, [=] AMREX_GPU_DEVICE (long i) { const Real dtscale = dt - (zinject_plane_lev_previous - z[i])/(vzbeam_ave_boosted + WarpX::beta_boost*PhysConst::c); if (0. < dtscale && dtscale < dt) { - Expp[i] *= dtscale; - Eypp[i] *= dtscale; - Ezpp[i] *= dtscale; - Bxpp[i] *= dtscale; - Bypp[i] *= dtscale; - Bzpp[i] *= dtscale; + Exp[i] *= dtscale; + Eyp[i] *= dtscale; + Ezp[i] *= dtscale; + Bxp[i] *= dtscale; + Byp[i] *= dtscale; + Bzp[i] *= dtscale; } } ); @@ -293,36 +277,50 @@ RigidInjectedParticleContainer::PushPX(WarpXParIter& pti, const int tid = 0; #endif - Real* AMREX_RESTRICT xp_savep = xp_save.dataPtr(); - Real* AMREX_RESTRICT yp_savep = yp_save.dataPtr(); - Real* AMREX_RESTRICT zp_savep = zp_save.dataPtr(); - Real* AMREX_RESTRICT uxp_savep = uxp_save.dataPtr(); - Real* AMREX_RESTRICT uyp_savep = uyp_save.dataPtr(); - Real* AMREX_RESTRICT uzp_savep = uzp_save.dataPtr(); + Real* AMREX_RESTRICT x_save; + Real* AMREX_RESTRICT y_save; + Real* AMREX_RESTRICT z_save; + Real* AMREX_RESTRICT ux_save; + Real* AMREX_RESTRICT uy_save; + Real* AMREX_RESTRICT uz_save; + if (!(WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags)) { + x_save = xp_save.dataPtr(); + y_save = yp_save.dataPtr(); + z_save = zp_save.dataPtr(); + ux_save = uxp_save.dataPtr(); + uy_save = uyp_save.dataPtr(); + uz_save = uzp_save.dataPtr(); + } else { + x_save = pti.GetAttribs(particle_comps["xold"]).dataPtr(); + y_save = pti.GetAttribs(particle_comps["yold"]).dataPtr(); + z_save = pti.GetAttribs(particle_comps["zold"]).dataPtr(); + ux_save = pti.GetAttribs(particle_comps["uxold"]).dataPtr(); + uy_save = pti.GetAttribs(particle_comps["uyold"]).dataPtr(); + uz_save = pti.GetAttribs(particle_comps["uzold"]).dataPtr(); + } // Undo the push for particles not injected yet. // The zp are advanced a fixed amount. amrex::ParallelFor( pti.numParticles(), [=] AMREX_GPU_DEVICE (long i) { if (z[i] <= zinject_plane_lev) { - uxpp[i] = uxp_savep[i]; - uypp[i] = uyp_savep[i]; - uzpp[i] = uzp_savep[i]; - gi[i] = 1./std::sqrt(1. + (uxpp[i]*uxpp[i] + uypp[i]*uypp[i] + uzpp[i]*uzpp[i])/(PhysConst::c*PhysConst::c)); - x[i] = xp_savep[i]; - y[i] = yp_savep[i]; + ux[i] = ux_save[i]; + uy[i] = uy_save[i]; + uz[i] = uz_save[i]; + gi[i] = 1./std::sqrt(1. + (ux[i]*ux[i] + uy[i]*uy[i] + uz[i]*uz[i])/(PhysConst::c*PhysConst::c)); + x[i] = x_save[i]; + y[i] = y_save[i]; if (rigid_advance) { - z[i] = zp_savep[i] + dt*vzbeam_ave_boosted; + z[i] = z_save[i] + dt*vzbeam_ave_boosted; } else { - z[i] = zp_savep[i] + dt*uzpp[i]*gi[i]; + z[i] = z_save[i] + dt*uz[i]*gi[i]; } done_injecting_temp[tid] = 0; } } ); } - } void @@ -459,24 +457,24 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, // This wraps the momentum advance so that inheritors can modify the call. // Extract pointers to the different particle quantities - Real* AMREX_RESTRICT zp = m_zp[thread_num].dataPtr(); - Real* AMREX_RESTRICT gi = m_giv[thread_num].dataPtr(); - Real* AMREX_RESTRICT uxpp = uxp.dataPtr(); - Real* AMREX_RESTRICT uypp = uyp.dataPtr(); - Real* AMREX_RESTRICT uzpp = uzp.dataPtr(); - Real* AMREX_RESTRICT uxp_savep = uxp_save.dataPtr(); - Real* AMREX_RESTRICT uyp_savep = uyp_save.dataPtr(); - Real* AMREX_RESTRICT uzp_savep = uzp_save.dataPtr(); - Real* AMREX_RESTRICT Expp = Exp.dataPtr(); - Real* AMREX_RESTRICT Eypp = Eyp.dataPtr(); - Real* AMREX_RESTRICT Ezpp = Ezp.dataPtr(); - Real* AMREX_RESTRICT Bxpp = Bxp.dataPtr(); - Real* AMREX_RESTRICT Bypp = Byp.dataPtr(); - Real* AMREX_RESTRICT Bzpp = Bzp.dataPtr(); + const Real* AMREX_RESTRICT zp = m_zp[thread_num].dataPtr(); + Real* const AMREX_RESTRICT gi = m_giv[thread_num].dataPtr(); + Real* const AMREX_RESTRICT uxpp = uxp.dataPtr(); + Real* const AMREX_RESTRICT uypp = uyp.dataPtr(); + Real* const AMREX_RESTRICT uzpp = uzp.dataPtr(); + const Real* AMREX_RESTRICT uxp_savep = uxp_save.dataPtr(); + const Real* AMREX_RESTRICT uyp_savep = uyp_save.dataPtr(); + const Real* AMREX_RESTRICT uzp_savep = uzp_save.dataPtr(); + const Real* AMREX_RESTRICT Expp = Exp.dataPtr(); + const Real* AMREX_RESTRICT Eypp = Eyp.dataPtr(); + const Real* AMREX_RESTRICT Ezpp = Ezp.dataPtr(); + const Real* AMREX_RESTRICT Bxpp = Bxp.dataPtr(); + const Real* AMREX_RESTRICT Bypp = Byp.dataPtr(); + const Real* AMREX_RESTRICT Bzpp = Bzp.dataPtr(); // Loop over the particles and update their momentum const Real q = this->charge; - const Real m = this-> mass; + const Real m = this->mass; if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Boris){ amrex::ParallelFor( pti.numParticles(), [=] AMREX_GPU_DEVICE (long i) { -- cgit v1.2.3 From 9105ca79ac7a115cc3348d937462044a7c55bf2e Mon Sep 17 00:00:00 2001 From: Dave Grote Date: Wed, 24 Jul 2019 10:16:30 -0700 Subject: For momentum push conversion, added more consts --- Source/Particles/PhysicalParticleContainer.cpp | 24 +++++++++++----------- .../Particles/RigidInjectedParticleContainer.cpp | 20 +++++++++--------- 2 files changed, 22 insertions(+), 22 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 5ee2ef5b6..b913ed5cd 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1758,12 +1758,12 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti, Real* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); Real* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); Real* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); - const Real* AMREX_RESTRICT Ex = attribs[PIdx::Ex].dataPtr(); - const Real* AMREX_RESTRICT Ey = attribs[PIdx::Ey].dataPtr(); - const Real* AMREX_RESTRICT Ez = attribs[PIdx::Ez].dataPtr(); - const Real* AMREX_RESTRICT Bx = attribs[PIdx::Bx].dataPtr(); - const Real* AMREX_RESTRICT By = attribs[PIdx::By].dataPtr(); - const Real* AMREX_RESTRICT Bz = attribs[PIdx::Bz].dataPtr(); + const Real* const AMREX_RESTRICT Ex = attribs[PIdx::Ex].dataPtr(); + const Real* const AMREX_RESTRICT Ey = attribs[PIdx::Ey].dataPtr(); + const Real* const AMREX_RESTRICT Ez = attribs[PIdx::Ez].dataPtr(); + const Real* const AMREX_RESTRICT Bx = attribs[PIdx::Bx].dataPtr(); + const Real* const AMREX_RESTRICT By = attribs[PIdx::By].dataPtr(); + const Real* const AMREX_RESTRICT Bz = attribs[PIdx::Bz].dataPtr(); if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) { @@ -1885,12 +1885,12 @@ PhysicalParticleContainer::PushP (int lev, Real dt, Real* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); Real* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); Real* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); - const Real* AMREX_RESTRICT Expp = Exp.dataPtr(); - const Real* AMREX_RESTRICT Eypp = Eyp.dataPtr(); - const Real* AMREX_RESTRICT Ezpp = Ezp.dataPtr(); - const Real* AMREX_RESTRICT Bxpp = Bxp.dataPtr(); - const Real* AMREX_RESTRICT Bypp = Byp.dataPtr(); - const Real* AMREX_RESTRICT Bzpp = Bzp.dataPtr(); + const Real* const AMREX_RESTRICT Expp = Exp.dataPtr(); + const Real* const AMREX_RESTRICT Eypp = Eyp.dataPtr(); + const Real* const AMREX_RESTRICT Ezpp = Ezp.dataPtr(); + const Real* const AMREX_RESTRICT Bxpp = Bxp.dataPtr(); + const Real* const AMREX_RESTRICT Bypp = Byp.dataPtr(); + const Real* const AMREX_RESTRICT Bzpp = Bzp.dataPtr(); // Loop over the particles and update their momentum const Real q = this->charge; diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index dc594f5a1..cad82d6d4 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -453,20 +453,20 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, // This wraps the momentum advance so that inheritors can modify the call. // Extract pointers to the different particle quantities - const Real* AMREX_RESTRICT zp = m_zp[thread_num].dataPtr(); + const Real* const AMREX_RESTRICT zp = m_zp[thread_num].dataPtr(); Real* const AMREX_RESTRICT gi = m_giv[thread_num].dataPtr(); Real* const AMREX_RESTRICT uxpp = uxp.dataPtr(); Real* const AMREX_RESTRICT uypp = uyp.dataPtr(); Real* const AMREX_RESTRICT uzpp = uzp.dataPtr(); - const Real* AMREX_RESTRICT uxp_savep = uxp_save.dataPtr(); - const Real* AMREX_RESTRICT uyp_savep = uyp_save.dataPtr(); - const Real* AMREX_RESTRICT uzp_savep = uzp_save.dataPtr(); - const Real* AMREX_RESTRICT Expp = Exp.dataPtr(); - const Real* AMREX_RESTRICT Eypp = Eyp.dataPtr(); - const Real* AMREX_RESTRICT Ezpp = Ezp.dataPtr(); - const Real* AMREX_RESTRICT Bxpp = Bxp.dataPtr(); - const Real* AMREX_RESTRICT Bypp = Byp.dataPtr(); - const Real* AMREX_RESTRICT Bzpp = Bzp.dataPtr(); + const Real* const AMREX_RESTRICT uxp_savep = uxp_save.dataPtr(); + const Real* const AMREX_RESTRICT uyp_savep = uyp_save.dataPtr(); + const Real* const AMREX_RESTRICT uzp_savep = uzp_save.dataPtr(); + const Real* const AMREX_RESTRICT Expp = Exp.dataPtr(); + const Real* const AMREX_RESTRICT Eypp = Eyp.dataPtr(); + const Real* const AMREX_RESTRICT Ezpp = Ezp.dataPtr(); + const Real* const AMREX_RESTRICT Bxpp = Bxp.dataPtr(); + const Real* const AMREX_RESTRICT Bypp = Byp.dataPtr(); + const Real* const AMREX_RESTRICT Bzpp = Bzp.dataPtr(); // Loop over the particles and update their momentum const Real q = this->charge; -- cgit v1.2.3 From 571aa14bb468a7f51fc664ef8f50f29c5397b97a Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Wed, 24 Jul 2019 14:37:29 -0700 Subject: add some more comments --- Source/Particles/PhysicalParticleContainer.cpp | 57 ++++++++++++++------------ 1 file changed, 31 insertions(+), 26 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 357a187fd..ec54305bb 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -2171,11 +2171,13 @@ PhysicalParticleContainer::FieldGather(WarpXParIter& pti, // If no particles, do not do anything if (np_to_gather == 0) return; - + // Get cell size on gather_lev const std::array& dx = WarpX::CellSize(std::max(gather_lev,0)); + // Set staggering shift depending on e_is_nodal const Real stagger_shift = e_is_nodal ? 0.0 : 0.5; - // Get box =from which field is gathered. + // Get box from which field is gathered. + // If not gathering from the finest level, the box is coarsened. Box box; if (lev == gather_lev) { box = pti.tilebox(); @@ -2184,6 +2186,7 @@ PhysicalParticleContainer::FieldGather(WarpXParIter& pti, box = amrex::coarsen(pti.tilebox(),ref_ratio); } + // Add guard cells to the box. box.grow(ngE); const Array4& ex_arr = exfab->array(); @@ -2202,55 +2205,57 @@ PhysicalParticleContainer::FieldGather(WarpXParIter& pti, const Dim3 lo = lbound(box); + // Depending on l_lower_in_v and WarpX::nox, call + // different versions of template function doGatherShapeN if (WarpX::l_lower_order_in_v){ if (WarpX::nox == 1){ - doGatherShapeN<1,1>(xp, yp, zp, - Exp.dataPtr() + offset, Eyp.dataPtr() + offset, + doGatherShapeN<1,1>(xp, yp, zp, + Exp.dataPtr() + offset, Eyp.dataPtr() + offset, Ezp.dataPtr() + offset, Bxp.dataPtr() + offset, - Byp.dataPtr() + offset, Bzp.dataPtr() + offset, - ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + Byp.dataPtr() + offset, Bzp.dataPtr() + offset, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, np_to_gather, dx, xyzmin, lo, stagger_shift); } else if (WarpX::nox == 2){ - doGatherShapeN<2,1>(xp, yp, zp, - Exp.dataPtr() + offset, Eyp.dataPtr() + offset, + doGatherShapeN<2,1>(xp, yp, zp, + Exp.dataPtr() + offset, Eyp.dataPtr() + offset, Ezp.dataPtr() + offset, Bxp.dataPtr() + offset, - Byp.dataPtr() + offset, Bzp.dataPtr() + offset, - ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + Byp.dataPtr() + offset, Bzp.dataPtr() + offset, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, np_to_gather, dx, xyzmin, lo, stagger_shift); } else if (WarpX::nox == 3){ - doGatherShapeN<3,1>(xp, yp, zp, - Exp.dataPtr() + offset, Eyp.dataPtr() + offset, + doGatherShapeN<3,1>(xp, yp, zp, + Exp.dataPtr() + offset, Eyp.dataPtr() + offset, Ezp.dataPtr() + offset, Bxp.dataPtr() + offset, - Byp.dataPtr() + offset, Bzp.dataPtr() + offset, - ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + Byp.dataPtr() + offset, Bzp.dataPtr() + offset, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, np_to_gather, dx, xyzmin, lo, stagger_shift); } } else { if (WarpX::nox == 1){ - doGatherShapeN<1,0>(xp, yp, zp, - Exp.dataPtr() + offset, Eyp.dataPtr() + offset, + doGatherShapeN<1,0>(xp, yp, zp, + Exp.dataPtr() + offset, Eyp.dataPtr() + offset, Ezp.dataPtr() + offset, Bxp.dataPtr() + offset, - Byp.dataPtr() + offset, Bzp.dataPtr() + offset, - ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + Byp.dataPtr() + offset, Bzp.dataPtr() + offset, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, np_to_gather, dx, xyzmin, lo, stagger_shift); } else if (WarpX::nox == 2){ - doGatherShapeN<2,0>(xp, yp, zp, - Exp.dataPtr() + offset, Eyp.dataPtr() + offset, + doGatherShapeN<2,0>(xp, yp, zp, + Exp.dataPtr() + offset, Eyp.dataPtr() + offset, Ezp.dataPtr() + offset, Bxp.dataPtr() + offset, - Byp.dataPtr() + offset, Bzp.dataPtr() + offset, - ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + Byp.dataPtr() + offset, Bzp.dataPtr() + offset, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, np_to_gather, dx, xyzmin, lo, stagger_shift); } else if (WarpX::nox == 3){ - doGatherShapeN<3,0>(xp, yp, zp, - Exp.dataPtr() + offset, Eyp.dataPtr() + offset, + doGatherShapeN<3,0>(xp, yp, zp, + Exp.dataPtr() + offset, Eyp.dataPtr() + offset, Ezp.dataPtr() + offset, Bxp.dataPtr() + offset, - Byp.dataPtr() + offset, Bzp.dataPtr() + offset, - ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + Byp.dataPtr() + offset, Bzp.dataPtr() + offset, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, np_to_gather, dx, xyzmin, lo, stagger_shift); } -- cgit v1.2.3 From c51759f954fc58b15efeeb492e56d1f58ce6b892 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Wed, 24 Jul 2019 14:40:27 -0700 Subject: use Weiqun convention for function definition --- Source/Particles/MultiParticleContainer.cpp | 2 +- Source/Particles/PhysicalParticleContainer.H | 56 +++++++++++++------------- Source/Particles/PhysicalParticleContainer.cpp | 38 ++++++++--------- 3 files changed, 48 insertions(+), 48 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index ab4400dad..f9d29e254 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -331,7 +331,7 @@ MultiParticleContainer::RedistributeLocal (const int num_ghost) } Vector -MultiParticleContainer::NumberOfParticlesInGrid(int lev) const +MultiParticleContainer::NumberOfParticlesInGrid (int lev) const { const bool only_valid=true, only_local=true; Vector r = allcontainers[0]->NumberOfParticlesInGrid(lev,only_valid,only_local); diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index ec9d7f98b..eebc9b55d 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -27,37 +27,37 @@ public: const amrex::Vector > > >& masks) override; virtual void EvolveES (const amrex::Vector, 3> >& E, - amrex::Vector >& rho, + amrex::Vector >& rho, amrex::Real t, amrex::Real dt) override; #endif // WARPX_DO_ELECTROSTATIC - virtual void FieldGatherFortran(int lev, - const amrex::MultiFab& Ex, - const amrex::MultiFab& Ey, - const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, - const amrex::MultiFab& By, - const amrex::MultiFab& Bz) final; - - void FieldGather(WarpXParIter& pti, - RealVector& Exp, - RealVector& Eyp, - RealVector& Ezp, - RealVector& Bxp, - RealVector& Byp, - RealVector& Bzp, - amrex::FArrayBox const * exfab, - amrex::FArrayBox const * eyfab, - amrex::FArrayBox const * ezfab, - amrex::FArrayBox const * bxfab, - amrex::FArrayBox const * byfab, - amrex::FArrayBox const * bzfab, - const int ngE, const int e_is_nodal, - const long offset, - const long np_to_gather, - int thread_num, - int lev, - int depos_lev); + virtual void FieldGatherFortran (int lev, + const amrex::MultiFab& Ex, + const amrex::MultiFab& Ey, + const amrex::MultiFab& Ez, + const amrex::MultiFab& Bx, + const amrex::MultiFab& By, + const amrex::MultiFab& Bz) final; + + void FieldGather (WarpXParIter& pti, + RealVector& Exp, + RealVector& Eyp, + RealVector& Ezp, + RealVector& Bxp, + RealVector& Byp, + RealVector& Bzp, + amrex::FArrayBox const * exfab, + amrex::FArrayBox const * eyfab, + amrex::FArrayBox const * ezfab, + amrex::FArrayBox const * bxfab, + amrex::FArrayBox const * byfab, + amrex::FArrayBox const * bzfab, + const int ngE, const int e_is_nodal, + const long offset, + const long np_to_gather, + int thread_num, + int lev, + int depos_lev); virtual void Evolve (int lev, const amrex::MultiFab& Ex, diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index ec54305bb..b98797b56 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -2145,25 +2145,25 @@ PhysicalParticleContainer::ContinuousInjection(const RealBox& injection_box) particles in buffers. */ void -PhysicalParticleContainer::FieldGather(WarpXParIter& pti, - RealVector& Exp, - RealVector& Eyp, - RealVector& Ezp, - RealVector& Bxp, - RealVector& Byp, - RealVector& Bzp, - FArrayBox const * exfab, - FArrayBox const * eyfab, - FArrayBox const * ezfab, - FArrayBox const * bxfab, - FArrayBox const * byfab, - FArrayBox const * bzfab, - const int ngE, const int e_is_nodal, - const long offset, - const long np_to_gather, - int thread_num, - int lev, - int gather_lev) +PhysicalParticleContainer::FieldGather (WarpXParIter& pti, + RealVector& Exp, + RealVector& Eyp, + RealVector& Ezp, + RealVector& Bxp, + RealVector& Byp, + RealVector& Bzp, + FArrayBox const * exfab, + FArrayBox const * eyfab, + FArrayBox const * ezfab, + FArrayBox const * bxfab, + FArrayBox const * byfab, + FArrayBox const * bzfab, + const int ngE, const int e_is_nodal, + const long offset, + const long np_to_gather, + int thread_num, + int lev, + int gather_lev) { AMREX_ALWAYS_ASSERT_WITH_MESSAGE((gather_lev==(lev-1)) || (gather_lev==(lev )), -- cgit v1.2.3 From d59fa46d24417b67554132bc666e45886160bd09 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Wed, 24 Jul 2019 19:43:18 -0700 Subject: Reimplement AddPlasma. Commits related to AddPlasma in hackathon branch are squashed into one. --- Source/Initialization/CustomDensityProb.H | 43 ++ Source/Initialization/CustomDensityProb.cpp | 12 - Source/Initialization/CustomMomentumProb.H | 27 + Source/Initialization/CustomMomentumProb.cpp | 14 - Source/Initialization/InjectorDensity.H | 175 +++++ Source/Initialization/InjectorDensity.cpp | 71 ++ Source/Initialization/InjectorMomentum.H | 195 ++++++ Source/Initialization/InjectorMomentum.cpp | 50 ++ Source/Initialization/InjectorPosition.H | 121 ++++ Source/Initialization/InjectorPosition.cpp | 19 + Source/Initialization/Make.package | 16 +- Source/Initialization/PlasmaInjector.H | 294 +-------- Source/Initialization/PlasmaInjector.cpp | 324 +++------ Source/Initialization/PlasmaProfiles.cpp | 41 -- Source/Parser/GpuParser.H | 61 ++ Source/Parser/GpuParser.cpp | 69 ++ Source/Parser/Make.package | 2 + Source/Parser/WarpXParser.H | 4 + Source/Parser/wp_parser_c.h | 122 +++- Source/Parser/wp_parser_y.c | 129 +++- Source/Parser/wp_parser_y.h | 22 +- Source/Particles/PhysicalParticleContainer.H | 15 +- Source/Particles/PhysicalParticleContainer.cpp | 870 ++++++++----------------- 23 files changed, 1487 insertions(+), 1209 deletions(-) create mode 100644 Source/Initialization/CustomDensityProb.H delete mode 100644 Source/Initialization/CustomDensityProb.cpp create mode 100644 Source/Initialization/CustomMomentumProb.H delete mode 100644 Source/Initialization/CustomMomentumProb.cpp create mode 100644 Source/Initialization/InjectorDensity.H create mode 100644 Source/Initialization/InjectorDensity.cpp create mode 100644 Source/Initialization/InjectorMomentum.H create mode 100644 Source/Initialization/InjectorMomentum.cpp create mode 100644 Source/Initialization/InjectorPosition.H create mode 100644 Source/Initialization/InjectorPosition.cpp delete mode 100644 Source/Initialization/PlasmaProfiles.cpp create mode 100644 Source/Parser/GpuParser.H create mode 100644 Source/Parser/GpuParser.cpp (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Initialization/CustomDensityProb.H b/Source/Initialization/CustomDensityProb.H new file mode 100644 index 000000000..44612c799 --- /dev/null +++ b/Source/Initialization/CustomDensityProb.H @@ -0,0 +1,43 @@ +#ifndef CUSTOM_DENSITY_PROB_H_ +#define CUSTOM_DENSITY_PROB_H_ + +#include +#include +#include +#include + +// An example of Custom Density Profile + +struct InjectorDensityCustom +{ + InjectorDensityCustom (std::string const& species_name) + : p(nullptr) + { + amrex::ParmParse pp(species_name); + std::vector v; + pp.getarr("custom_profile_params", v); + p = static_cast + (amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)*v.size())); + for (int i = 0; i < static_cast(v.size()); ++i) { + p[i] = v[i]; + } + } + + AMREX_GPU_HOST_DEVICE + amrex::Real + getDensity (amrex::Real, amrex::Real, amrex::Real) const noexcept + { + return p[0]; + } + + // Note that we are not allowed to have non-trivial destructor. + // So we rely on clear() to free memory. + void clear () { + amrex::The_Managed_Arena()->free(p); + } + +private: + amrex::Real* p; +}; + +#endif diff --git a/Source/Initialization/CustomDensityProb.cpp b/Source/Initialization/CustomDensityProb.cpp deleted file mode 100644 index 3efcb13c5..000000000 --- a/Source/Initialization/CustomDensityProb.cpp +++ /dev/null @@ -1,12 +0,0 @@ -#include - -#include - -using namespace amrex; - -/// -/// This "custom" density profile just does constant -/// -Real CustomDensityProfile::getDensity(Real x, Real y, Real z) const { - return params[0]; -} diff --git a/Source/Initialization/CustomMomentumProb.H b/Source/Initialization/CustomMomentumProb.H new file mode 100644 index 000000000..42090d0fa --- /dev/null +++ b/Source/Initialization/CustomMomentumProb.H @@ -0,0 +1,27 @@ +#ifndef CUSTOM_MOMENTUM_PROB_H +#define CUSTOM_MOMENTUM_PROB_H + +#include +#include +#include +#include + +// An example of Custom Momentum Profile + +struct InjectorMomentumCustom +{ + InjectorMomentumCustom (std::string const& /*a_species_name*/) {} + + AMREX_GPU_HOST_DEVICE + amrex::XDim3 + getMomentum (amrex::Real, amrex::Real, amrex::Real) const noexcept + { + return {0., 0., 0.}; + } + + // Note that we are not allowed to have non-trivial destructor. + // So we rely on clear() to free memory if needed. + void clear () { } +}; + +#endif diff --git a/Source/Initialization/CustomMomentumProb.cpp b/Source/Initialization/CustomMomentumProb.cpp deleted file mode 100644 index fa21252d0..000000000 --- a/Source/Initialization/CustomMomentumProb.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include - -#include - -using namespace amrex; - -/// -/// This "custom" momentum distribution just does 0 momentum -/// -void CustomMomentumDistribution::getMomentum(vec3& u, Real x, Real y, Real z) { - u[0] = 0; - u[1] = 0; - u[2] = 0; -} diff --git a/Source/Initialization/InjectorDensity.H b/Source/Initialization/InjectorDensity.H new file mode 100644 index 000000000..61471433f --- /dev/null +++ b/Source/Initialization/InjectorDensity.H @@ -0,0 +1,175 @@ +#ifndef INJECTOR_DENSITY_H_ +#define INJECTOR_DENSITY_H_ + +#include +#include +#include +#include +#include + +struct InjectorDensityConstant +{ + InjectorDensityConstant (amrex::Real a_rho) noexcept : m_rho(a_rho) {} + + AMREX_GPU_HOST_DEVICE + amrex::Real + getDensity (amrex::Real, amrex::Real, amrex::Real) const noexcept + { + return m_rho; + } + +private: + amrex::Real m_rho; +}; + +struct InjectorDensityParser +{ + InjectorDensityParser (WarpXParser const& a_parser) noexcept + : m_parser(a_parser) {} + + AMREX_GPU_HOST_DEVICE + amrex::Real + getDensity (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept + { + return m_parser(x,y,z); + } + + GpuParser m_parser; +}; + +struct InjectorDensityPredefined +{ + InjectorDensityPredefined (std::string const& a_species_name) noexcept; + + void clear (); + + AMREX_GPU_HOST_DEVICE + amrex::Real + getDensity (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept + { + switch (profile) + { + case Profile::parabolic_channel: + { + amrex::Real z_start = p[0]; + amrex::Real ramp_up = p[1]; + amrex::Real plateau = p[2]; + amrex::Real ramp_down = p[3]; + amrex::Real rc = p[4]; + amrex::Real n0 = p[5]; + amrex::Real n; + amrex::Real kp = PhysConst::q_e/PhysConst::c + *std::sqrt( n0/(PhysConst::m_e*PhysConst::ep0) ); + + if ((z-z_start)>=0 and + (z-z_start)=ramp_up and + (z-z_start)< ramp_up+plateau ) { + n = 1.; + } else if ((z-z_start)>=ramp_up+plateau and + (z-z_start)< ramp_up+plateau+ramp_down) { + n = 1.-((z-z_start)-ramp_up-plateau)/ramp_down; + } else { + n = 0.; + } + n *= n0*(1.+4.*(x*x+y*y)/(kp*kp*rc*rc*rc*rc)); + return n; + } + default: + amrex::Abort("InjectorDensityPredefined: how did we get here?"); + return 0.0; + } + } + +private: + enum struct Profile { null, parabolic_channel }; + Profile profile; + amrex::Real* p; +}; + +struct InjectorDensity + : public amrex::Gpu::Managed +{ + InjectorDensity (InjectorDensityConstant* t, amrex::Real a_rho) + : type(Type::constant), + object(t,a_rho) + { } + + InjectorDensity (InjectorDensityParser* t, WarpXParser const& a_parser) + : type(Type::parser), + object(t,a_parser) + { } + + InjectorDensity (InjectorDensityCustom* t, std::string const& a_species_name) + : type(Type::custom), + object(t,a_species_name) + { } + + InjectorDensity (InjectorDensityPredefined* t, std::string const& a_species_name) + : type(Type::predefined), + object(t,a_species_name) + { } + + InjectorDensity (InjectorDensity const&) = delete; + InjectorDensity (InjectorDensity&&) = delete; + void operator= (InjectorDensity const&) = delete; + void operator= (InjectorDensity &&) = delete; + + ~InjectorDensity (); + + std::size_t sharedMemoryNeeded () const noexcept; + bool useRandom () const noexcept { return false; } + + AMREX_GPU_HOST_DEVICE + amrex::Real + getDensity (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept + { + switch (type) + { + case Type::parser: + { + return object.parser.getDensity(x,y,z); + } + case Type::constant: + { + return object.constant.getDensity(x,y,z); + } + case Type::custom: + { + return object.custom.getDensity(x,y,z); + } + case Type::predefined: + { + return object.predefined.getDensity(x,y,z); + } + default: + { + amrex::Abort("InjectorDensity: unknown type"); + return 0.0; + } + } + } + +private: + enum struct Type { constant, custom, predefined, parser }; + Type type; + + union Object { + Object (InjectorDensityConstant*, amrex::Real a_rho) noexcept + : constant(a_rho) {} + Object (InjectorDensityParser*, WarpXParser const& a_parser) noexcept + : parser(a_parser) {} + Object (InjectorDensityCustom*, std::string const& a_species_name) noexcept + : custom(a_species_name) {} + Object (InjectorDensityPredefined*, std::string const& a_species_name) noexcept + : predefined(a_species_name) {} + InjectorDensityConstant constant; + InjectorDensityParser parser; + InjectorDensityCustom custom; + InjectorDensityPredefined predefined; + }; + Object object; +}; + +#endif diff --git a/Source/Initialization/InjectorDensity.cpp b/Source/Initialization/InjectorDensity.cpp new file mode 100644 index 000000000..f796180fd --- /dev/null +++ b/Source/Initialization/InjectorDensity.cpp @@ -0,0 +1,71 @@ +#include + +using namespace amrex; + +InjectorDensity::~InjectorDensity () +{ + switch (type) + { + case Type::parser: + { + object.parser.m_parser.clear(); + break; + } + case Type::custom: + { + object.custom.clear(); + break; + } + case Type::predefined: + { + object.predefined.clear(); + break; + } + } +} + +std::size_t +InjectorDensity::sharedMemoryNeeded () const noexcept +{ + switch (type) + { + case Type::parser: + { + return amrex::Gpu::numThreadsPerBlockParallelFor() * sizeof(double); + } + default: + return 0; + } +} + +InjectorDensityPredefined::InjectorDensityPredefined ( + std::string const& a_species_name) noexcept + : profile(Profile::null) +{ + ParmParse pp(a_species_name); + + std::vector v; + pp.getarr("predefined_profile_params", v); + p = static_cast + (amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)*v.size())); + for (int i = 0; i < static_cast(v.size()); ++i) { + p[i] = v[i]; + } + + std::string which_profile_s; + pp.query("predefined_profile_name", which_profile_s); + std::transform(which_profile_s.begin(), which_profile_s.end(), + which_profile_s.begin(), ::tolower); + if (which_profile_s == "parabolic_channel"){ + profile = Profile::parabolic_channel; + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(v.size() > 6, + "InjectorDensityPredefined::parabolic_channel: not enough parameters"); + } +} + +// Note that we are not allowed to have non-trivial destructor. +// So we rely on clear() to free memory. +void InjectorDensityPredefined::clear () +{ + amrex::The_Managed_Arena()->free(p); +} diff --git a/Source/Initialization/InjectorMomentum.H b/Source/Initialization/InjectorMomentum.H new file mode 100644 index 000000000..2434503cc --- /dev/null +++ b/Source/Initialization/InjectorMomentum.H @@ -0,0 +1,195 @@ +#ifndef INJECTOR_MOMENTUM_H_ +#define INJECTOR_MOMENTUM_H_ + +#include +#include +#include +#include + +struct InjectorMomentumConstant +{ + InjectorMomentumConstant (amrex::Real a_ux, amrex::Real a_uy, amrex::Real a_uz) noexcept + : m_ux(a_ux), m_uy(a_uy), m_uz(a_uz) {} + + AMREX_GPU_HOST_DEVICE + amrex::XDim3 + getMomentum (amrex::Real, amrex::Real, amrex::Real) const noexcept + { + return amrex::XDim3{m_ux,m_uy,m_uz}; + } +private: + amrex::Real m_ux, m_uy, m_uz; +}; + +struct InjectorMomentumGaussian +{ + InjectorMomentumGaussian (amrex::Real a_ux_m, amrex::Real a_uy_m, + amrex::Real a_uz_m, amrex::Real a_ux_th, + amrex::Real a_uy_th, amrex::Real a_uz_th) noexcept + : m_ux_m(a_ux_m), m_uy_m(a_uy_m), m_uz_m(a_uz_m), + m_ux_th(a_ux_th), m_uy_th(a_uy_th), m_uz_th(a_uz_th) + {} + + AMREX_GPU_HOST_DEVICE + amrex::XDim3 + getMomentum (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept + { + return amrex::XDim3{amrex::RandomNormal(m_ux_m, m_ux_th), + amrex::RandomNormal(m_uy_m, m_uy_th), + amrex::RandomNormal(m_uz_m, m_uz_th)}; + } +private: + amrex::Real m_ux_m, m_uy_m, m_uz_m; + amrex::Real m_ux_th, m_uy_th, m_uz_th; +}; + +struct InjectorMomentumRadialExpansion +{ + InjectorMomentumRadialExpansion (amrex::Real a_u_over_r) noexcept + : u_over_r(a_u_over_r) + {} + + AMREX_GPU_HOST_DEVICE + amrex::XDim3 + getMomentum (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept + { + return {x*u_over_r, y*u_over_r, z*u_over_r}; + } + +private: + amrex::Real u_over_r; +}; + +struct InjectorMomentumParser +{ + InjectorMomentumParser (WarpXParser const& a_ux_parser, + WarpXParser const& a_uy_parser, + WarpXParser const& a_uz_parser) noexcept + : m_ux_parser(a_ux_parser), m_uy_parser(a_uy_parser), + m_uz_parser(a_uz_parser) {} + + AMREX_GPU_HOST_DEVICE + amrex::XDim3 + getMomentum (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept + { + return amrex::XDim3{m_ux_parser(x,y,z),m_uy_parser(x,y,z),m_uz_parser(x,y,z)}; + } + + GpuParser m_ux_parser, m_uy_parser, m_uz_parser; +}; + +struct InjectorMomentum + : public amrex::Gpu::Managed +{ + InjectorMomentum (InjectorMomentumConstant* t, + amrex::Real a_ux, amrex::Real a_uy, amrex::Real a_uz) + : type(Type::constant), + object(t, a_ux, a_uy, a_uz) + { } + + InjectorMomentum (InjectorMomentumParser* t, + WarpXParser const& a_ux_parser, + WarpXParser const& a_uy_parser, + WarpXParser const& a_uz_parser) + : type(Type::parser), + object(t, a_ux_parser, a_uy_parser, a_uz_parser) + { } + + InjectorMomentum (InjectorMomentumGaussian* t, + amrex::Real a_ux_m, amrex::Real a_uy_m, amrex::Real a_uz_m, + amrex::Real a_ux_th, amrex::Real a_uy_th, amrex::Real a_uz_th) + : type(Type::gaussian), + object(t,a_ux_m,a_uy_m,a_uz_m,a_ux_th,a_uy_th,a_uz_th) + { } + + InjectorMomentum (InjectorMomentumCustom* t, + std::string const& a_species_name) + : type(Type::custom), + object(t, a_species_name) + { } + + InjectorMomentum (InjectorMomentumRadialExpansion* t, + amrex::Real u_over_r) + : type(Type::radial_expansion), + object(t, u_over_r) + { } + + InjectorMomentum (InjectorMomentum const&) = delete; + InjectorMomentum (InjectorMomentum&&) = delete; + void operator= (InjectorMomentum const&) = delete; + void operator= (InjectorMomentum &&) = delete; + + ~InjectorMomentum (); + + std::size_t sharedMemoryNeeded () const noexcept; + + bool useRandom () const noexcept; + + AMREX_GPU_HOST_DEVICE + amrex::XDim3 + getMomentum (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept + { + switch (type) + { + case Type::parser: + { + return object.parser.getMomentum(x,y,z); + } + case Type::gaussian: + { + return object.gaussian.getMomentum(x,y,z); + } + case Type::constant: + { + return object.constant.getMomentum(x,y,z); + } + case Type::radial_expansion: + { + return object.radial_expansion.getMomentum(x,y,z); + } + case Type::custom: + { + return object.custom.getMomentum(x,y,z); + } + default: + { + amrex::Abort("InjectorMomentum: unknown type"); + return {0.0,0.0,0.0}; + } + } + } + +private: + enum struct Type { constant, custom, gaussian, radial_expansion, parser }; + Type type; + + union Object { + Object (InjectorMomentumConstant*, + amrex::Real a_ux, amrex::Real a_uy, amrex::Real a_uz) noexcept + : constant(a_ux,a_uy,a_uz) {} + Object (InjectorMomentumCustom*, + std::string const& a_species_name) noexcept + : custom(a_species_name) {} + Object (InjectorMomentumGaussian*, + amrex::Real a_ux_m, amrex::Real a_uy_m, + amrex::Real a_uz_m, amrex::Real a_ux_th, + amrex::Real a_uy_th, amrex::Real a_uz_th) noexcept + : gaussian(a_ux_m,a_uy_m,a_uz_m,a_ux_th,a_uy_th,a_uz_th) {} + Object (InjectorMomentumRadialExpansion*, + amrex::Real u_over_r) noexcept + : radial_expansion(u_over_r) {} + Object (InjectorMomentumParser*, + WarpXParser const& a_ux_parser, + WarpXParser const& a_uy_parser, + WarpXParser const& a_uz_parser) noexcept + : parser(a_ux_parser, a_uy_parser, a_uz_parser) {} + InjectorMomentumConstant constant; + InjectorMomentumCustom custom; + InjectorMomentumGaussian gaussian; + InjectorMomentumRadialExpansion radial_expansion; + InjectorMomentumParser parser; + }; + Object object; +}; + +#endif diff --git a/Source/Initialization/InjectorMomentum.cpp b/Source/Initialization/InjectorMomentum.cpp new file mode 100644 index 000000000..4e483fc33 --- /dev/null +++ b/Source/Initialization/InjectorMomentum.cpp @@ -0,0 +1,50 @@ +#include + +using namespace amrex; + +InjectorMomentum::~InjectorMomentum () +{ + switch (type) + { + case Type::parser: + { + object.parser.m_ux_parser.clear(); + object.parser.m_uy_parser.clear(); + object.parser.m_uz_parser.clear(); + break; + } + case Type::custom: + { + object.custom.clear(); + break; + } + } +} + +std::size_t +InjectorMomentum::sharedMemoryNeeded () const noexcept +{ + switch (type) + { + case Type::parser: + { + return amrex::Gpu::numThreadsPerBlockParallelFor() * sizeof(double); + } + default: + return 0; + } +} + +bool +InjectorMomentum::useRandom () const noexcept +{ + switch (type) + { + case Type::gaussian: + { + return true; + } + default: + return false; + } +} diff --git a/Source/Initialization/InjectorPosition.H b/Source/Initialization/InjectorPosition.H new file mode 100644 index 000000000..e74345ae5 --- /dev/null +++ b/Source/Initialization/InjectorPosition.H @@ -0,0 +1,121 @@ +#ifndef INJECTOR_POSITION_H_ +#define INJECTOR_POSITION_H_ + +#include +#include +#include + +struct InjectorPositionRandom +{ + AMREX_GPU_HOST_DEVICE + amrex::XDim3 + getPositionUnitBox (int i_part, int ref_fac=1) const noexcept + { + return amrex::XDim3{amrex::Random(), amrex::Random(), amrex::Random()}; + } +}; + +struct InjectorPositionRegular +{ + InjectorPositionRegular (amrex::Dim3 const& a_ppc) noexcept : ppc(a_ppc) {} + + AMREX_GPU_HOST_DEVICE + amrex::XDim3 + getPositionUnitBox (int i_part, int ref_fac=1) const noexcept + { + int nx = ref_fac*ppc.x; + int ny = ref_fac*ppc.y; +#if (AMREX_SPACEDIM == 3) + int nz = ref_fac*ppc.z; +#else + int nz = 1; +#endif + int ix_part = i_part/(ny*nz); // written this way backward compatibility + int iz_part = (i_part-ix_part*(ny*nz)) / ny; + int iy_part = (i_part-ix_part*(ny*nz)) - ny*iz_part; + return amrex::XDim3{(0.5+ix_part)/nx, (0.5+iy_part)/ny, (0.5+iz_part) / nz}; + } +private: + amrex::Dim3 ppc; +}; + +struct InjectorPosition + : public amrex::Gpu::Managed +{ + InjectorPosition (InjectorPositionRandom* t, + amrex::Real a_xmin, amrex::Real a_xmax, + amrex::Real a_ymin, amrex::Real a_ymax, + amrex::Real a_zmin, amrex::Real a_zmax) + : type(Type::random), + object(t), + xmin(a_xmin), xmax(a_xmax), + ymin(a_ymin), ymax(a_ymax), + zmin(a_zmin), zmax(a_zmax) + { } + + InjectorPosition (InjectorPositionRegular* t, + amrex::Real a_xmin, amrex::Real a_xmax, + amrex::Real a_ymin, amrex::Real a_ymax, + amrex::Real a_zmin, amrex::Real a_zmax, + amrex::Dim3 const& a_ppc) + : type(Type::regular), + object(t, a_ppc), + xmin(a_xmin), xmax(a_xmax), + ymin(a_ymin), ymax(a_ymax), + zmin(a_zmin), zmax(a_zmax) + { } + + InjectorPosition (InjectorPosition const&) = delete; + InjectorPosition (InjectorPosition&&) = delete; + void operator= (InjectorPosition const&) = delete; + void operator= (InjectorPosition &&) = delete; + + std::size_t sharedMemoryNeeded () const noexcept { return 0; } + + bool useRandom () const noexcept; + + AMREX_GPU_HOST_DEVICE + amrex::XDim3 + getPositionUnitBox (int i_part, int ref_fac=1) const noexcept + { + switch (type) + { + case Type::regular: + { + return object.regular.getPositionUnitBox(i_part, ref_fac); + } + default: + { + return object.random.getPositionUnitBox(i_part, ref_fac); + } + }; + } + + AMREX_GPU_HOST_DEVICE + bool + insideBounds (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept + { + return (x < xmax and x >= xmin and + y < ymax and y >= ymin and + z < zmax and z >= zmin); + } + +private: + enum struct Type { random, regular }; + Type type; + + union Object { + Object (InjectorPositionRandom*) noexcept : random() {} + Object (InjectorPositionRegular*, amrex::Dim3 const& a_ppc) noexcept + : regular(a_ppc) {} + InjectorPositionRandom random; + InjectorPositionRegular regular; + }; + Object object; + + amrex::Real xmin, xmax; + amrex::Real ymin, ymax; + amrex::Real zmin, zmax; +}; + +#endif diff --git a/Source/Initialization/InjectorPosition.cpp b/Source/Initialization/InjectorPosition.cpp new file mode 100644 index 000000000..b77f26920 --- /dev/null +++ b/Source/Initialization/InjectorPosition.cpp @@ -0,0 +1,19 @@ +#include + +using namespace amrex; + +bool +InjectorPosition::useRandom () const noexcept +{ + switch (type) + { + case Type::random: + { + return true; + } + default: + { + return false; + } + }; +} diff --git a/Source/Initialization/Make.package b/Source/Initialization/Make.package index edcf402c9..3a924f207 100644 --- a/Source/Initialization/Make.package +++ b/Source/Initialization/Make.package @@ -1,9 +1,19 @@ -CEXE_sources += CustomDensityProb.cpp -CEXE_sources += PlasmaProfiles.cpp CEXE_sources += WarpXInitData.cpp -CEXE_sources += CustomMomentumProb.cpp + CEXE_sources += PlasmaInjector.cpp CEXE_headers += PlasmaInjector.H +CEXE_headers += InjectorPosition.H +CEXE_sources += InjectorPosition.cpp + +CEXE_headers += InjectorDensity.H +CEXE_sources += InjectorDensity.cpp + +CEXE_headers += InjectorMomentum.H +CEXE_sources += InjectorMomentum.cpp + +CEXE_headers += CustomDensityProb.H +CEXE_headers += CustomMomentumProb.H + INCLUDE_LOCATIONS += $(WARPX_HOME)/Source/Initialization VPATH_LOCATIONS += $(WARPX_HOME)/Source/Initialization diff --git a/Source/Initialization/PlasmaInjector.H b/Source/Initialization/PlasmaInjector.H index f998e217e..d5c064192 100644 --- a/Source/Initialization/PlasmaInjector.H +++ b/Source/Initialization/PlasmaInjector.H @@ -1,250 +1,16 @@ #ifndef PLASMA_INJECTOR_H_ #define PLASMA_INJECTOR_H_ -#include +#include +#include +#include -#include "AMReX_REAL.H" +#include #include #include #include -#include "AMReX_ParmParse.H" -#include "AMReX_Utility.H" - -enum class predefined_profile_flag { Null, parabolic_channel }; - -/// -/// PlasmaDensityProfile describes how the charge density -/// is set in particle initialization. Subclasses must define a -/// getDensity function that describes the charge density as a -/// function of x, y, and z. -/// -class PlasmaDensityProfile -{ -public: - virtual ~PlasmaDensityProfile() {}; - virtual amrex::Real getDensity(amrex::Real x, - amrex::Real y, - amrex::Real z) const = 0; -protected: - std::string _species_name; -}; - -/// -/// This describes a constant density distribution. -/// -class ConstantDensityProfile : public PlasmaDensityProfile -{ -public: - ConstantDensityProfile(amrex::Real _density); - virtual amrex::Real getDensity(amrex::Real x, - amrex::Real y, - amrex::Real z) const override; - -private: - amrex::Real _density; -}; - -/// -/// This describes a custom density distribution. Users can supply -/// in their problem directory. -/// -/// -class CustomDensityProfile : public PlasmaDensityProfile -{ -public: - CustomDensityProfile(const std::string& species_name); - virtual amrex::Real getDensity(amrex::Real x, - amrex::Real y, - amrex::Real z) const override; -private: - amrex::Vector params; -}; - -/// -/// This describes predefined density distributions. -/// -class PredefinedDensityProfile : public PlasmaDensityProfile -{ -public: - PredefinedDensityProfile(const std::string& species_name); - virtual amrex::Real getDensity(amrex::Real x, - amrex::Real y, - amrex::Real z) const override; - amrex::Real ParabolicChannel(amrex::Real x, - amrex::Real y, - amrex::Real z) const; -private: - predefined_profile_flag which_profile = predefined_profile_flag::Null; - amrex::Vector params; -}; - -/// -/// This describes a density function parsed in the input file. -/// -class ParseDensityProfile : public PlasmaDensityProfile -{ -public: - ParseDensityProfile(const std::string _parse_density_function); - virtual amrex::Real getDensity(amrex::Real x, - amrex::Real y, - amrex::Real z) const override; -private: - std::string _parse_density_function; - WarpXParser parser_density; -}; - -/// -/// PlasmaMomentumDistribution describes how the particle momenta -/// are set. Subclasses must define a getMomentum method that fills -/// a u with the 3 components of the particle momentum -/// -class PlasmaMomentumDistribution -{ -public: - using vec3 = std::array; - virtual ~PlasmaMomentumDistribution() {}; - virtual void getMomentum(vec3& u, amrex::Real x, amrex::Real y, amrex::Real z) = 0; -}; - -/// -/// This is a constant momentum distribution - all particles will -/// have the same ux, uy, and uz -/// -class ConstantMomentumDistribution : public PlasmaMomentumDistribution -{ -public: - ConstantMomentumDistribution(amrex::Real ux, - amrex::Real uy, - amrex::Real uz); - virtual void getMomentum(vec3& u, amrex::Real x, amrex::Real y, amrex::Real z) override; - -private: - amrex::Real _ux; - amrex::Real _uy; - amrex::Real _uz; -}; - -/// -/// This describes a custom momentum distribution. Users can supply -/// in their problem directory. -/// -/// -class CustomMomentumDistribution : public PlasmaMomentumDistribution -{ -public: - CustomMomentumDistribution(const std::string& species_name); - virtual void getMomentum(vec3& u, amrex::Real x, amrex::Real y, amrex::Real z) override; - -private: - amrex::Vector params; -}; - - -/// -/// This is a Gaussian Random momentum distribution. -/// Particles will get random momenta, drawn from a normal. -/// ux_m, ux_y, and ux_z describe the mean components in the x, y, and z -/// directions, while u_th is the standard deviation of the random -/// component. -/// -class GaussianRandomMomentumDistribution : public PlasmaMomentumDistribution -{ -public: - GaussianRandomMomentumDistribution(amrex::Real ux_m, - amrex::Real uy_m, - amrex::Real uz_m, - amrex::Real ux_th, - amrex::Real uy_th, - amrex::Real uz_th); - virtual void getMomentum(vec3& u, amrex::Real x, amrex::Real y, amrex::Real z) override; -private: - amrex::Real _ux_m; - amrex::Real _uy_m; - amrex::Real _uz_m; - amrex::Real _ux_th; - amrex::Real _uy_th; - amrex::Real _uz_th; -}; - -/// -/// This is a radially expanding momentum distribution -/// Particles will have a radial momentum proportional to their -/// radius, with proportionality constant u_over_r -class RadialExpansionMomentumDistribution : public PlasmaMomentumDistribution -{ -public: - RadialExpansionMomentumDistribution( amrex::Real u_over_r ); - virtual void getMomentum(vec3& u, amrex::Real x, amrex::Real y, amrex::Real z) override; -private: - amrex::Real _u_over_r; -}; - -/// -/// This describes a momentum distribution function parsed in the input file. -/// -class ParseMomentumFunction : public PlasmaMomentumDistribution -{ -public: - ParseMomentumFunction(const std::string _parse_momentum_function_ux, - const std::string _parse_momentum_function_uy, - const std::string _parse_momentum_function_uz); - virtual void getMomentum(vec3& u, - amrex::Real x, - amrex::Real y, - amrex::Real z) override; -private: - std::string _parse_momentum_function_ux; - std::string _parse_momentum_function_uy; - std::string _parse_momentum_function_uz; - WarpXParser parser_ux; - WarpXParser parser_uy; - WarpXParser parser_uz; -}; - - -/// -/// PlasmaParticlePosition describes how particles are initialized -/// into each cell box. Subclasses must define a -/// getPositionUnitBox function that returns the position of -/// particle number i_part in a unitary box. -/// -class PlasmaParticlePosition{ -public: - using vec3 = std::array; - virtual ~PlasmaParticlePosition() {}; - virtual void getPositionUnitBox(vec3& r, int i_part, int ref_fac=1) = 0; -}; - -/// -/// Particles are initialized with a random uniform -/// distribution inside each cell -/// -class RandomPosition : public PlasmaParticlePosition{ -public: - RandomPosition(int num_particles_per_cell); - virtual void getPositionUnitBox(vec3& r, int i_part, int ref_fac=1) override; -private: - amrex::Real _x; - amrex::Real _y; - amrex::Real _z; - int _num_particles_per_cell; -}; - -/// -/// Particles are regularly distributed inside each cell. The user provides -/// a 3d (resp. 2d) vector num_particles_per_cell_each_dim that contains -/// the number of particles per cell along each dimension. -/// -class RegularPosition : public PlasmaParticlePosition{ -public: - RegularPosition(const amrex::Vector& num_particles_per_cell_each_dim); - virtual void getPositionUnitBox(vec3& r, int i_part, int ref_fac=1) override; -private: - amrex::Real _x; - amrex::Real _y; - amrex::Real _z; - amrex::Vector _num_particles_per_cell_each_dim; -}; +#include +#include /// /// The PlasmaInjector class parses and stores information about the plasma @@ -256,28 +22,23 @@ class PlasmaInjector public: - using vec3 = std::array; - - PlasmaInjector(); + PlasmaInjector (); - PlasmaInjector(int ispecies, const std::string& name); + PlasmaInjector (int ispecies, const std::string& name); - amrex::Real getDensity(amrex::Real x, amrex::Real y, amrex::Real z); - - bool insideBounds(amrex::Real x, amrex::Real y, amrex::Real z); + bool insideBounds (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept; int num_particles_per_cell; amrex::Vector num_particles_per_cell_each_dim; - void getMomentum(vec3& u, amrex::Real x, amrex::Real y, amrex::Real z); - - void getPositionUnitBox(vec3& r, int i_part, int ref_fac=1); + // gamma * beta + amrex::XDim3 getMomentum (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept; - amrex::Real getCharge() {return charge;} - amrex::Real getMass() {return mass;} + amrex::Real getCharge () {return charge;} + amrex::Real getMass () {return mass;} - bool doInjection() { return part_pos != NULL;} + bool doInjection () const noexcept { return inj_pos != NULL;} bool add_single_particle = false; amrex::Vector single_particle_pos; @@ -306,6 +67,22 @@ public: amrex::Real ymin, ymax; amrex::Real zmin, zmax; + InjectorPosition* getInjectorPosition (); + InjectorDensity* getInjectorDensity (); + InjectorMomentum* getInjectorMomentum (); + + bool useRandom () const noexcept { + return inj_pos->useRandom() or inj_rho->useRandom() + or inj_mom->useRandom(); + } + + std::size_t + sharedMemoryNeeded () const noexcept { + return amrex::max(inj_pos->sharedMemoryNeeded(), + inj_rho->sharedMemoryNeeded(), + inj_mom->sharedMemoryNeeded()); + } + protected: amrex::Real mass, charge; @@ -315,13 +92,12 @@ protected: int species_id; std::string species_name; - std::unique_ptr rho_prof; - std::unique_ptr mom_dist; - std::unique_ptr part_pos; - - void parseDensity(amrex::ParmParse pp); - void parseMomentum(amrex::ParmParse pp); + std::unique_ptr inj_pos; + std::unique_ptr inj_rho; + std::unique_ptr inj_mom; + void parseDensity (amrex::ParmParse& pp); + void parseMomentum (amrex::ParmParse& pp); }; #endif diff --git a/Source/Initialization/PlasmaInjector.cpp b/Source/Initialization/PlasmaInjector.cpp index f9642d1b6..25d43f83e 100644 --- a/Source/Initialization/PlasmaInjector.cpp +++ b/Source/Initialization/PlasmaInjector.cpp @@ -55,192 +55,31 @@ namespace { } } -ConstantDensityProfile::ConstantDensityProfile(Real density) - : _density(density) -{} +PlasmaInjector::PlasmaInjector () {} -Real ConstantDensityProfile::getDensity(Real x, Real y, Real z) const -{ - return _density; -} - -CustomDensityProfile::CustomDensityProfile(const std::string& species_name) +PlasmaInjector::PlasmaInjector (int ispecies, const std::string& name) + : species_id(ispecies), species_name(name) { ParmParse pp(species_name); - pp.getarr("custom_profile_params", params); -} -PredefinedDensityProfile::PredefinedDensityProfile(const std::string& species_name) -{ - ParmParse pp(species_name); - std::string which_profile_s; - pp.getarr("predefined_profile_params", params); - pp.query("predefined_profile_name", which_profile_s); - if (which_profile_s == "parabolic_channel"){ - which_profile = predefined_profile_flag::parabolic_channel; - } -} - -ParseDensityProfile::ParseDensityProfile(std::string parse_density_function) - : _parse_density_function(parse_density_function) -{ - parser_density.define(parse_density_function); - parser_density.registerVariables({"x","y","z"}); - - ParmParse pp("my_constants"); - std::set symbols = parser_density.symbols(); - symbols.erase("x"); - symbols.erase("y"); - symbols.erase("z"); // after removing variables, we are left with constants - for (auto it = symbols.begin(); it != symbols.end(); ) { - Real v; - if (pp.query(it->c_str(), v)) { - parser_density.setConstant(*it, v); - it = symbols.erase(it); - } else { - ++it; - } - } - for (auto const& s : symbols) { // make sure there no unknown symbols - amrex::Abort("ParseDensityProfile: Unknown symbol "+s); - } -} - -Real ParseDensityProfile::getDensity(Real x, Real y, Real z) const -{ - return parser_density.eval(x,y,z); -} - -ConstantMomentumDistribution::ConstantMomentumDistribution(Real ux, - Real uy, - Real uz) - : _ux(ux), _uy(uy), _uz(uz) -{} - -void ConstantMomentumDistribution::getMomentum(vec3& u, Real x, Real y, Real z) { - u[0] = _ux; - u[1] = _uy; - u[2] = _uz; -} - -CustomMomentumDistribution::CustomMomentumDistribution(const std::string& species_name) -{ - ParmParse pp(species_name); - pp.getarr("custom_momentum_params", params); -} - -GaussianRandomMomentumDistribution::GaussianRandomMomentumDistribution(Real ux_m, - Real uy_m, - Real uz_m, - Real ux_th, - Real uy_th, - Real uz_th) - : _ux_m(ux_m), _uy_m(uy_m), _uz_m(uz_m), _ux_th(ux_th), _uy_th(uy_th), _uz_th(uz_th) -{ -} - -void GaussianRandomMomentumDistribution::getMomentum(vec3& u, Real x, Real y, Real z) { - Real ux_th = amrex::RandomNormal(0.0, _ux_th); - Real uy_th = amrex::RandomNormal(0.0, _uy_th); - Real uz_th = amrex::RandomNormal(0.0, _uz_th); - - u[0] = _ux_m + ux_th; - u[1] = _uy_m + uy_th; - u[2] = _uz_m + uz_th; -} -RadialExpansionMomentumDistribution::RadialExpansionMomentumDistribution(Real u_over_r) : _u_over_r( u_over_r ) -{ -} - -void RadialExpansionMomentumDistribution::getMomentum(vec3& u, Real x, Real y, Real z) { - u[0] = _u_over_r * x; - u[1] = _u_over_r * y; - u[2] = _u_over_r * z; -} - -ParseMomentumFunction::ParseMomentumFunction(std::string parse_momentum_function_ux, - std::string parse_momentum_function_uy, - std::string parse_momentum_function_uz) - : _parse_momentum_function_ux(parse_momentum_function_ux), - _parse_momentum_function_uy(parse_momentum_function_uy), - _parse_momentum_function_uz(parse_momentum_function_uz) -{ - parser_ux.define(parse_momentum_function_ux); - parser_uy.define(parse_momentum_function_uy); - parser_uz.define(parse_momentum_function_uz); - - amrex::Array,3> parsers{parser_ux, parser_uy, parser_uz}; - ParmParse pp("my_constants"); - for (auto& p : parsers) { - auto& parser = p.get(); - parser.registerVariables({"x","y","z"}); - std::set symbols = parser.symbols(); - symbols.erase("x"); - symbols.erase("y"); - symbols.erase("z"); // after removing variables, we are left with constants - for (auto it = symbols.begin(); it != symbols.end(); ) { - Real v; - if (pp.query(it->c_str(), v)) { - parser.setConstant(*it, v); - it = symbols.erase(it); - } else { - ++it; - } - } - for (auto const& s : symbols) { // make sure there no unknown symbols - amrex::Abort("ParseMomentumFunction: Unknown symbol "+s); - } - } -} - -void ParseMomentumFunction::getMomentum(vec3& u, Real x, Real y, Real z) -{ - u[0] = parser_ux.eval(x,y,z); - u[1] = parser_uy.eval(x,y,z); - u[2] = parser_uz.eval(x,y,z); -} - -RandomPosition::RandomPosition(int num_particles_per_cell): - _num_particles_per_cell(num_particles_per_cell) -{} - -void RandomPosition::getPositionUnitBox(vec3& r, int i_part, int ref_fac){ - r[0] = amrex::Random(); - r[1] = amrex::Random(); - r[2] = amrex::Random(); -} - -RegularPosition::RegularPosition(const amrex::Vector& num_particles_per_cell_each_dim) - : _num_particles_per_cell_each_dim(num_particles_per_cell_each_dim) -{} - -void RegularPosition::getPositionUnitBox(vec3& r, int i_part, int ref_fac) -{ - int nx = ref_fac*_num_particles_per_cell_each_dim[0]; - int ny = ref_fac*_num_particles_per_cell_each_dim[1]; -#if AMREX_SPACEDIM == 3 - int nz = ref_fac*_num_particles_per_cell_each_dim[2]; -#else - int nz = 1; -#endif - - int ix_part = i_part/(ny * nz); - int iy_part = (i_part % (ny * nz)) % ny; - int iz_part = (i_part % (ny * nz)) / ny; + pp.query("radially_weighted", radially_weighted); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(radially_weighted, "ERROR: Only radially_weighted=true is supported"); - r[0] = (0.5+ix_part)/nx; - r[1] = (0.5+iy_part)/ny; - r[2] = (0.5+iz_part)/nz; -} + // parse plasma boundaries + xmin = std::numeric_limits::lowest(); + ymin = std::numeric_limits::lowest(); + zmin = std::numeric_limits::lowest(); -PlasmaInjector::PlasmaInjector(){ - part_pos = NULL; -} + xmax = std::numeric_limits::max(); + ymax = std::numeric_limits::max(); + zmax = std::numeric_limits::max(); -PlasmaInjector::PlasmaInjector(int ispecies, const std::string& name) - : species_id(ispecies), species_name(name) -{ - ParmParse pp(species_name); + pp.query("xmin", xmin); + pp.query("ymin", ymin); + pp.query("zmin", zmin); + pp.query("xmax", xmax); + pp.query("ymax", ymax); + pp.query("zmax", zmax); // parse charge and mass std::string charge_s; @@ -292,7 +131,8 @@ PlasmaInjector::PlasmaInjector(int ispecies, const std::string& name) } else if (part_pos_s == "nrandompercell") { pp.query("num_particles_per_cell", num_particles_per_cell); - part_pos.reset(new RandomPosition(num_particles_per_cell)); + inj_pos.reset(new InjectorPosition((InjectorPositionRandom*)nullptr, + xmin, xmax, ymin, ymax, zmin, zmax)); parseDensity(pp); parseMomentum(pp); } else if (part_pos_s == "nuniformpercell") { @@ -301,7 +141,11 @@ PlasmaInjector::PlasmaInjector(int ispecies, const std::string& name) #if ( AMREX_SPACEDIM == 2 ) num_particles_per_cell_each_dim[2] = 1; #endif - part_pos.reset(new RegularPosition(num_particles_per_cell_each_dim)); + inj_pos.reset(new InjectorPosition((InjectorPositionRegular*)nullptr, + xmin, xmax, ymin, ymax, zmin, zmax, + Dim3{num_particles_per_cell_each_dim[0], + num_particles_per_cell_each_dim[1], + num_particles_per_cell_each_dim[2]})); num_particles_per_cell = num_particles_per_cell_each_dim[0] * num_particles_per_cell_each_dim[1] * num_particles_per_cell_each_dim[2]; @@ -310,52 +154,61 @@ PlasmaInjector::PlasmaInjector(int ispecies, const std::string& name) } else { StringParseAbortMessage("Injection style", part_pos_s); } +} - pp.query("radially_weighted", radially_weighted); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(radially_weighted, "ERROR: Only radially_weighted=true is supported"); - - // parse plasma boundaries - xmin = std::numeric_limits::lowest(); - ymin = std::numeric_limits::lowest(); - zmin = std::numeric_limits::lowest(); - - xmax = std::numeric_limits::max(); - ymax = std::numeric_limits::max(); - zmax = std::numeric_limits::max(); +namespace { +WarpXParser makeParser (std::string const& parse_function) +{ + WarpXParser parser(parse_function); + parser.registerVariables({"x","y","z"}); - pp.query("xmin", xmin); - pp.query("ymin", ymin); - pp.query("zmin", zmin); - pp.query("xmax", xmax); - pp.query("ymax", ymax); - pp.query("zmax", zmax); + ParmParse pp("my_constants"); + std::set symbols = parser.symbols(); + symbols.erase("x"); + symbols.erase("y"); + symbols.erase("z"); // after removing variables, we are left with constants + for (auto it = symbols.begin(); it != symbols.end(); ) { + Real v; + if (pp.query(it->c_str(), v)) { + parser.setConstant(*it, v); + it = symbols.erase(it); + } else { + ++it; + } + } + for (auto const& s : symbols) { // make sure there no unknown symbols + amrex::Abort("PlasmaInjector::makeParser: Unknown symbol "+s); + } + return parser; +} } -void PlasmaInjector::parseDensity(ParmParse pp){ +void PlasmaInjector::parseDensity (ParmParse& pp) +{ // parse density information std::string rho_prof_s; pp.get("profile", rho_prof_s); - std::transform(rho_prof_s.begin(), - rho_prof_s.end(), - rho_prof_s.begin(), - ::tolower); + std::transform(rho_prof_s.begin(), rho_prof_s.end(), + rho_prof_s.begin(), ::tolower); if (rho_prof_s == "constant") { pp.get("density", density); - rho_prof.reset(new ConstantDensityProfile(density)); + inj_rho.reset(new InjectorDensity((InjectorDensityConstant*)nullptr, density)); } else if (rho_prof_s == "custom") { - rho_prof.reset(new CustomDensityProfile(species_name)); + inj_rho.reset(new InjectorDensity((InjectorDensityCustom*)nullptr, species_name)); } else if (rho_prof_s == "predefined") { - rho_prof.reset(new PredefinedDensityProfile(species_name)); + inj_rho.reset(new InjectorDensity((InjectorDensityPredefined*)nullptr,species_name)); } else if (rho_prof_s == "parse_density_function") { pp.get("density_function(x,y,z)", str_density_function); - rho_prof.reset(new ParseDensityProfile(str_density_function)); + inj_rho.reset(new InjectorDensity((InjectorDensityParser*)nullptr, + makeParser(str_density_function))); } else { StringParseAbortMessage("Density profile type", rho_prof_s); } } -void PlasmaInjector::parseMomentum(ParmParse pp){ +void PlasmaInjector::parseMomentum (ParmParse& pp) +{ // parse momentum information std::string mom_dist_s; pp.get("momentum_distribution_type", mom_dist_s); @@ -370,9 +223,9 @@ void PlasmaInjector::parseMomentum(ParmParse pp){ pp.query("ux", ux); pp.query("uy", uy); pp.query("uz", uz); - mom_dist.reset(new ConstantMomentumDistribution(ux, uy, uz)); + inj_mom.reset(new InjectorMomentum((InjectorMomentumConstant*)nullptr, ux,uy, uz)); } else if (mom_dist_s == "custom") { - mom_dist.reset(new CustomMomentumDistribution(species_name)); + inj_mom.reset(new InjectorMomentum((InjectorMomentumCustom*)nullptr, species_name)); } else if (mom_dist_s == "gaussian") { Real ux_m = 0.; Real uy_m = 0.; @@ -386,42 +239,53 @@ void PlasmaInjector::parseMomentum(ParmParse pp){ pp.query("ux_th", ux_th); pp.query("uy_th", uy_th); pp.query("uz_th", uz_th); - mom_dist.reset(new GaussianRandomMomentumDistribution(ux_m, uy_m, uz_m, - ux_th, uy_th, uz_th)); + inj_mom.reset(new InjectorMomentum((InjectorMomentumGaussian*)nullptr, + ux_m, uy_m, uz_m, ux_th, uy_th, uz_th)); } else if (mom_dist_s == "radial_expansion") { Real u_over_r = 0.; pp.query("u_over_r", u_over_r); - mom_dist.reset(new RadialExpansionMomentumDistribution(u_over_r)); + inj_mom.reset(new InjectorMomentum + ((InjectorMomentumRadialExpansion*)nullptr, u_over_r)); } else if (mom_dist_s == "parse_momentum_function") { pp.get("momentum_function_ux(x,y,z)", str_momentum_function_ux); pp.get("momentum_function_uy(x,y,z)", str_momentum_function_uy); pp.get("momentum_function_uz(x,y,z)", str_momentum_function_uz); - mom_dist.reset(new ParseMomentumFunction(str_momentum_function_ux, - str_momentum_function_uy, - str_momentum_function_uz)); + inj_mom.reset(new InjectorMomentum((InjectorMomentumParser*)nullptr, + makeParser(str_momentum_function_ux), + makeParser(str_momentum_function_uy), + makeParser(str_momentum_function_uz))); } else { StringParseAbortMessage("Momentum distribution type", mom_dist_s); } } -void PlasmaInjector::getPositionUnitBox(vec3& r, int i_part, int ref_fac) { - return part_pos->getPositionUnitBox(r, i_part, ref_fac); +XDim3 PlasmaInjector::getMomentum (Real x, Real y, Real z) const noexcept +{ + return inj_mom->getMomentum(x, y, z); // gamma*beta +} + +bool PlasmaInjector::insideBounds (Real x, Real y, Real z) const noexcept +{ + return (x < xmax and x >= xmin and + y < ymax and y >= ymin and + z < zmax and z >= zmin); } -void PlasmaInjector::getMomentum(vec3& u, Real x, Real y, Real z) { - mom_dist->getMomentum(u, x, y, z); - u[0] *= PhysConst::c; - u[1] *= PhysConst::c; - u[2] *= PhysConst::c; +InjectorPosition* +PlasmaInjector::getInjectorPosition () +{ + return inj_pos.get(); } -bool PlasmaInjector::insideBounds(Real x, Real y, Real z) { - if (x >= xmax || x < xmin || - y >= ymax || y < ymin || - z >= zmax || z < zmin ) return false; - return true; +InjectorDensity* +PlasmaInjector::getInjectorDensity () +{ + return inj_rho.get(); } -Real PlasmaInjector::getDensity(Real x, Real y, Real z) { - return rho_prof->getDensity(x, y, z); +InjectorMomentum* +PlasmaInjector::getInjectorMomentum () +{ + return inj_mom.get(); } + diff --git a/Source/Initialization/PlasmaProfiles.cpp b/Source/Initialization/PlasmaProfiles.cpp deleted file mode 100644 index d9d207f7e..000000000 --- a/Source/Initialization/PlasmaProfiles.cpp +++ /dev/null @@ -1,41 +0,0 @@ -#include -#include -#include -#include - -using namespace amrex; - -Real PredefinedDensityProfile::getDensity(Real x, Real y, Real z) const { - Real n; - if ( which_profile == predefined_profile_flag::parabolic_channel ) { - n = ParabolicChannel(x,y,z); - } - return n; -} - -/// -/// plateau between linear upramp and downramp, and parab transverse profile -/// -Real PredefinedDensityProfile::ParabolicChannel(Real x, Real y, Real z) const { - // params = [z_start ramp_up plateau ramp_down rc n0] - Real z_start = params[0]; - Real ramp_up = params[1]; - Real plateau = params[2]; - Real ramp_down = params[3]; - Real rc = params[4]; - Real n0 = params[5]; - Real n; - Real kp = PhysConst::q_e/PhysConst::c*sqrt( n0/(PhysConst::m_e*PhysConst::ep0) ); - - if ((z-z_start)>=0 and (z-z_start)=ramp_up and (z-z_start)=ramp_up+plateau and (z-z_start) +#include + +class GpuParser +{ +public: + GpuParser (WarpXParser const& wp); + void clear (); + + AMREX_GPU_HOST_DEVICE + double + operator() (double x, double y, double z) const noexcept + { +#ifdef AMREX_USE_GPU + +#ifdef AMREX_DEVICE_COMPILE + amrex::Gpu::SharedMemory gsm; + double* p = gsm.dataPtr(); + int tid = threadIdx.x + threadIdx.y*blockDim.x + threadIdx.z*(blockDim.x*blockDim.y); + p[tid*3] = x; + p[tid*3+1] = y; + p[tid*3+2] = z; + return wp_ast_eval(m_gpu_parser.ast); +#else + m_var.x = x; + m_var.y = y; + m_var.z = z; + return wp_ast_eval(m_cpu_parser.ast); +#endif + +#else + +#ifdef _OPENMP + int tid = omp_get_thread_num(); +#else + int tid = 0; +#endif + m_var[tid].x = x; + m_var[tid].y = y; + m_var[tid].z = z; + return wp_ast_eval(m_parser[tid]->ast); +#endif + } + +private: + +#ifdef AMREX_USE_GPU + struct wp_parser m_gpu_parser; + struct wp_parser m_cpu_parser; + mutable amrex::XDim3 m_var; +#else + struct wp_parser** m_parser; + mutable amrex::XDim3* m_var; + int nthreads; +#endif +}; + +#endif diff --git a/Source/Parser/GpuParser.cpp b/Source/Parser/GpuParser.cpp new file mode 100644 index 000000000..ebba79498 --- /dev/null +++ b/Source/Parser/GpuParser.cpp @@ -0,0 +1,69 @@ +#include + +GpuParser::GpuParser (WarpXParser const& wp) +{ +#ifdef AMREX_USE_GPU + + struct wp_parser* a_wp = wp.m_parser; + m_gpu_parser.sz_mempool = wp_ast_size((struct wp_node*)a_wp); + m_gpu_parser.p_root = (struct wp_node*) + amrex::The_Managed_Arena()->alloc(m_gpu_parser.sz_mempool); + m_gpu_parser.p_free = m_gpu_parser.p_root; + // 0: don't free the source + m_gpu_parser.ast = wp_parser_ast_dup(&m_gpu_parser, a_wp->ast, 0); + wp_parser_regvar_gpu(&m_gpu_parser, "x", 0); + wp_parser_regvar_gpu(&m_gpu_parser, "y", 1); + wp_parser_regvar_gpu(&m_gpu_parser, "z", 2); + + m_cpu_parser.sz_mempool = wp_ast_size((struct wp_node*)a_wp); + m_cpu_parser.p_root = (struct wp_node*) + amrex::The_Managed_Arena()->alloc(m_cpu_parser.sz_mempool); + m_cpu_parser.p_free = m_cpu_parser.p_root; + // 0: don't free the source + m_cpu_parser.ast = wp_parser_ast_dup(&m_cpu_parser, a_wp->ast, 0); + wp_parser_regvar(&m_cpu_parser, "x", &(m_var.x)); + wp_parser_regvar(&m_cpu_parser, "y", &(m_var.y)); + wp_parser_regvar(&m_cpu_parser, "z", &(m_var.z)); + +#else + +#ifdef _OPENMP + nthreads = omp_get_max_threads(); +#else + nthreads = 1; +#endif + + m_parser = ::new struct wp_parser*[nthreads]; + m_var = ::new amrex::XDim3[nthreads]; + + for (int tid = 0; tid < nthreads; ++tid) + { +#ifdef _OPENMP + m_parser[tid] = wp_parser_dup(wp.m_parser[tid]); +#else + m_parser[tid] = wp_parser_dup(wp.m_parser); +#endif + wp_parser_regvar(m_parser[tid], "x", &(m_var[tid].x)); + wp_parser_regvar(m_parser[tid], "y", &(m_var[tid].y)); + wp_parser_regvar(m_parser[tid], "z", &(m_var[tid].z)); + } + +#endif +} + +void +GpuParser::clear () +{ +#ifdef AMREX_USE_GPU + amrex::The_Managed_Arena()->free(m_gpu_parser.ast); + amrex::The_Managed_Arena()->free(m_cpu_parser.ast); +#else + for (int tid = 0; tid < nthreads; ++tid) + { + wp_parser_delete(m_parser[tid]); + } + ::delete[] m_parser; + ::delete[] m_var; +#endif +} + diff --git a/Source/Parser/Make.package b/Source/Parser/Make.package index 26ef4fb43..5ce02cbda 100644 --- a/Source/Parser/Make.package +++ b/Source/Parser/Make.package @@ -3,6 +3,8 @@ cEXE_sources += wp_parser_y.c wp_parser.tab.c wp_parser.lex.c wp_parser_c.c cEXE_headers += wp_parser_y.h wp_parser.tab.h wp_parser.lex.h wp_parser_c.h CEXE_sources += WarpXParser.cpp CEXE_headers += WarpXParser.H +CEXE_headers += GpuParser.H +CEXE_sources += GpuParser.cpp INCLUDE_LOCATIONS += $(WARPX_HOME)/Source/Parser VPATH_LOCATIONS += $(WARPX_HOME)/Source/Parser diff --git a/Source/Parser/WarpXParser.H b/Source/Parser/WarpXParser.H index 046491e29..ffa61e457 100644 --- a/Source/Parser/WarpXParser.H +++ b/Source/Parser/WarpXParser.H @@ -13,6 +13,8 @@ #include #endif +class GpuParser; + class WarpXParser { public: @@ -46,6 +48,8 @@ public: std::set symbols () const; + friend class GpuParser; + private: void clear (); diff --git a/Source/Parser/wp_parser_c.h b/Source/Parser/wp_parser_c.h index d810bd685..3aafdec65 100644 --- a/Source/Parser/wp_parser_c.h +++ b/Source/Parser/wp_parser_c.h @@ -2,6 +2,8 @@ #define WP_PARSER_C_H_ #include "wp_parser_y.h" +#include +#include #ifdef __cplusplus extern "C" { @@ -18,71 +20,167 @@ extern "C" { #include #include -inline -double +AMREX_GPU_HOST_DEVICE +inline double wp_ast_eval (struct wp_node* node) { double result; +#ifdef AMREX_DEVICE_COMPILE + extern __shared__ double extern_xyz[]; + int tid = threadIdx.x + threadIdx.y*blockDim.x + threadIdx.z*(blockDim.x*blockDim.y); + double* x = extern_xyz + tid*3; +#endif + switch (node->type) { case WP_NUMBER: + { result = ((struct wp_number*)node)->value; break; + } case WP_SYMBOL: - result = *(((struct wp_symbol*)node)->pointer); + { +#ifdef AMREX_DEVICE_COMPILE + int i =((struct wp_symbol*)node)->ip.i; + result = x[i]; +#else + result = *(((struct wp_symbol*)node)->ip.p); +#endif break; + } case WP_ADD: + { result = wp_ast_eval(node->l) + wp_ast_eval(node->r); break; + } case WP_SUB: + { result = wp_ast_eval(node->l) - wp_ast_eval(node->r); break; + } case WP_MUL: + { result = wp_ast_eval(node->l) * wp_ast_eval(node->r); break; + } case WP_DIV: + { result = wp_ast_eval(node->l) / wp_ast_eval(node->r); break; + } case WP_NEG: + { result = -wp_ast_eval(node->l); break; + } case WP_F1: + { result = wp_call_f1(((struct wp_f1*)node)->ftype, wp_ast_eval(((struct wp_f1*)node)->l)); break; + } case WP_F2: + { result = wp_call_f2(((struct wp_f2*)node)->ftype, wp_ast_eval(((struct wp_f2*)node)->l), wp_ast_eval(((struct wp_f2*)node)->r)); break; + } case WP_ADD_VP: - result = node->lvp.v + *(node->rp); + { +#ifdef AMREX_DEVICE_COMPILE + int i = node->rip.i; + result = node->lvp.v + x[i]; +#else + result = node->lvp.v + *(node->rip.p); +#endif break; + } case WP_ADD_PP: - result = *(node->lvp.p) + *(node->rp); + { +#ifdef AMREX_DEVICE_COMPILE + int i = node->lvp.ip.i; + int j = node->rip.i; + result = x[i] + x[j]; +#else + result = *(node->lvp.ip.p) + *(node->rip.p); +#endif break; + } case WP_SUB_VP: - result = node->lvp.v - *(node->rp); + { +#ifdef AMREX_DEVICE_COMPILE + int i = node->rip.i; + result = node->lvp.v - x[i]; +#else + result = node->lvp.v - *(node->rip.p); +#endif break; + } case WP_SUB_PP: - result = *(node->lvp.p) - *(node->rp); + { +#ifdef AMREX_DEVICE_COMPILE + int i = node->lvp.ip.i; + int j = node->rip.i; + result = x[i] - x[j]; +#else + result = *(node->lvp.ip.p) - *(node->rip.p); +#endif break; + } case WP_MUL_VP: - result = node->lvp.v * *(node->rp); + { +#ifdef AMREX_DEVICE_COMPILE + int i = node->rip.i; + result = node->lvp.v * x[i]; +#else + result = node->lvp.v * *(node->rip.p); +#endif break; + } case WP_MUL_PP: - result = *(node->lvp.p) * *(node->rp); + { +#ifdef AMREX_DEVICE_COMPILE + int i = node->lvp.ip.i; + int j = node->rip.i; + result = x[i] * x[j]; +#else + result = *(node->lvp.ip.p) * *(node->rip.p); +#endif break; + } case WP_DIV_VP: - result = node->lvp.v / *(node->rp); + { +#ifdef AMREX_DEVICE_COMPILE + int i = node->rip.i; + result = node->lvp.v / x[i]; +#else + result = node->lvp.v / *(node->rip.p); +#endif break; + } case WP_DIV_PP: - result = *(node->lvp.p) / *(node->rp); + { +#ifdef AMREX_DEVICE_COMPILE + int i = node->lvp.ip.i; + int j = node->rip.i; + result = x[i] / x[j]; +#else + result = *(node->lvp.ip.p) / *(node->rip.p); +#endif break; + } case WP_NEG_P: - result = -*(node->lvp.p); + { +#ifdef AMREX_DEVICE_COMPILE + int i = node->rip.i; + result = -x[i]; +#else + result = -*(node->lvp.ip.p); +#endif break; + } default: yyerror("wp_ast_eval: unknown node type %d\n", node->type); } diff --git a/Source/Parser/wp_parser_y.c b/Source/Parser/wp_parser_y.c index 46cb199db..259f9368b 100644 --- a/Source/Parser/wp_parser_y.c +++ b/Source/Parser/wp_parser_y.c @@ -6,6 +6,8 @@ #include "wp_parser_y.h" #include "wp_parser.tab.h" +#include + static struct wp_node* wp_root = NULL; /* This is called by a bison rule to store the original AST in a @@ -33,7 +35,7 @@ wp_makesymbol (char* name) struct wp_symbol* symbol = (struct wp_symbol*) malloc(sizeof(struct wp_symbol)); symbol->type = WP_SYMBOL; symbol->name = strdup(name); - symbol->pointer = NULL; + symbol->ip.p = NULL; return symbol; } @@ -74,13 +76,19 @@ wp_newf2 (enum wp_f2_t ftype, struct wp_node* l, struct wp_node* r) return (struct wp_node*) tmp; } +AMREX_GPU_HOST_DEVICE void yyerror (char const *s, ...) { va_list vl; va_start(vl, s); +#ifdef AMREX_DEVICE_COMPILE + printf(s,"\n"); + assert(0); +#else vfprintf(stderr, s, vl); fprintf(stderr, "\n"); +#endif va_end(vl); } @@ -97,7 +105,7 @@ wp_parser_new (void) my_parser->ast = wp_parser_ast_dup(my_parser, wp_root,1); /* 1: free the source wp_root */ - if (my_parser->p_root + my_parser->sz_mempool != my_parser->p_free) { + if ((char*)my_parser->p_root + my_parser->sz_mempool != (char*)my_parser->p_free) { yyerror("wp_parser_new: error in memory size"); exit(1); } @@ -145,6 +153,7 @@ wp_parser_dup (struct wp_parser* source) return dest; } +AMREX_GPU_HOST_DEVICE double wp_call_f1 (enum wp_f1_t type, double a) { @@ -175,6 +184,7 @@ wp_call_f1 (enum wp_f1_t type, double a) } } +AMREX_GPU_HOST_DEVICE double wp_call_f2 (enum wp_f2_t type, double a, double b) { @@ -346,23 +356,23 @@ wp_parser_ast_dup (struct wp_parser* my_parser, struct wp_node* node, int move) #define WP_MOVEUP_R(node, v) \ struct wp_node* n = node->r->r; \ - double* p = node->r->rp; \ + double* p = node->r->rip.p; \ node->r = n; \ node->lvp.v = v; \ - node->rp = p; + node->rip.p = p; #define WP_MOVEUP_L(node, v) \ struct wp_node* n = node->l->r; \ - double* p = node->l->rp; \ + double* p = node->l->rip.p; \ node->r = n; \ node->lvp.v = v; \ - node->rp = p; + node->rip.p = p; #define WP_EVAL_R(node) node->r->lvp.v #define WP_EVAL_L(node) node->l->lvp.v #define WP_NEG_MOVEUP(node) \ node->r = node->l->r; \ node->lvp.v = -node->l->lvp.v; \ - node->rp = node->l->rp; + node->rip.p = node->l->rip.p; void wp_ast_optimize (struct wp_node* node) @@ -391,22 +401,22 @@ wp_ast_optimize (struct wp_node* node) node->r->type == WP_SYMBOL) { node->lvp.v = ((struct wp_number*)(node->l))->value; - node->rp = ((struct wp_symbol*)(node->r))->pointer; + node->rip.p = ((struct wp_symbol*)(node->r))->ip.p; node->type = WP_ADD_VP; } else if (node->l->type == WP_SYMBOL && node->r->type == WP_NUMBER) { node->lvp.v = ((struct wp_number*)(node->r))->value; - node->rp = ((struct wp_symbol*)(node->l))->pointer; + node->rip.p = ((struct wp_symbol*)(node->l))->ip.p; node->r = node->l; node->type = WP_ADD_VP; } else if (node->l->type == WP_SYMBOL && node->r->type == WP_SYMBOL) { - node->lvp.p = ((struct wp_symbol*)(node->l))->pointer; - node->rp = ((struct wp_symbol*)(node->r))->pointer; + node->lvp.ip.p = ((struct wp_symbol*)(node->l))->ip.p; + node->rip.p = ((struct wp_symbol*)(node->r))->ip.p; node->type = WP_ADD_PP; } else if (node->l->type == WP_NUMBER && @@ -454,22 +464,22 @@ wp_ast_optimize (struct wp_node* node) node->r->type == WP_SYMBOL) { node->lvp.v = ((struct wp_number*)(node->l))->value; - node->rp = ((struct wp_symbol*)(node->r))->pointer; + node->rip.p = ((struct wp_symbol*)(node->r))->ip.p; node->type = WP_SUB_VP; } else if (node->l->type == WP_SYMBOL && node->r->type == WP_NUMBER) { node->lvp.v = -((struct wp_number*)(node->r))->value; - node->rp = ((struct wp_symbol*)(node->l))->pointer; + node->rip.p = ((struct wp_symbol*)(node->l))->ip.p; node->r = node->l; node->type = WP_ADD_VP; } else if (node->l->type == WP_SYMBOL && node->r->type == WP_SYMBOL) { - node->lvp.p = ((struct wp_symbol*)(node->l))->pointer; - node->rp = ((struct wp_symbol*)(node->r))->pointer; + node->lvp.ip.p = ((struct wp_symbol*)(node->l))->ip.p; + node->rip.p = ((struct wp_symbol*)(node->r))->ip.p; node->type = WP_SUB_PP; } else if (node->l->type == WP_NUMBER && @@ -517,22 +527,22 @@ wp_ast_optimize (struct wp_node* node) node->r->type == WP_SYMBOL) { node->lvp.v = ((struct wp_number*)(node->l))->value; - node->rp = ((struct wp_symbol*)(node->r))->pointer; + node->rip.p = ((struct wp_symbol*)(node->r))->ip.p; node->type = WP_MUL_VP; } else if (node->l->type == WP_SYMBOL && node->r->type == WP_NUMBER) { node->lvp.v = ((struct wp_number*)(node->r))->value; - node->rp = ((struct wp_symbol*)(node->l))->pointer; + node->rip.p = ((struct wp_symbol*)(node->l))->ip.p; node->r = node->l; node->type = WP_MUL_VP; } else if (node->l->type == WP_SYMBOL && node->r->type == WP_SYMBOL) { - node->lvp.p = ((struct wp_symbol*)(node->l))->pointer; - node->rp = ((struct wp_symbol*)(node->r))->pointer; + node->lvp.ip.p = ((struct wp_symbol*)(node->l))->ip.p; + node->rip.p = ((struct wp_symbol*)(node->r))->ip.p; node->type = WP_MUL_PP; } else if (node->l->type == WP_NUMBER && @@ -580,22 +590,22 @@ wp_ast_optimize (struct wp_node* node) node->r->type == WP_SYMBOL) { node->lvp.v = ((struct wp_number*)(node->l))->value; - node->rp = ((struct wp_symbol*)(node->r))->pointer; + node->rip.p = ((struct wp_symbol*)(node->r))->ip.p; node->type = WP_DIV_VP; } else if (node->l->type == WP_SYMBOL && node->r->type == WP_NUMBER) { node->lvp.v = 1./((struct wp_number*)(node->r))->value; - node->rp = ((struct wp_symbol*)(node->l))->pointer; + node->rip.p = ((struct wp_symbol*)(node->l))->ip.p; node->r = node->l; node->type = WP_MUL_VP; } else if (node->l->type == WP_SYMBOL && node->r->type == WP_SYMBOL) { - node->lvp.p = ((struct wp_symbol*)(node->l))->pointer; - node->rp = ((struct wp_symbol*)(node->r))->pointer; + node->lvp.ip.p = ((struct wp_symbol*)(node->l))->ip.p; + node->rip.p = ((struct wp_symbol*)(node->r))->ip.p; node->type = WP_DIV_PP; } else if (node->l->type == WP_NUMBER && @@ -637,7 +647,7 @@ wp_ast_optimize (struct wp_node* node) } else if (node->l->type == WP_SYMBOL) { - node->lvp.p = ((struct wp_symbol*)(node->l))->pointer; + node->lvp.ip.p = ((struct wp_symbol*)(node->l))->ip.p; node->type = WP_NEG_P; } else if (node->l->type == WP_ADD_VP) @@ -936,7 +946,7 @@ wp_ast_regvar (struct wp_node* node, char const* name, double* p) break; case WP_SYMBOL: if (strcmp(name, ((struct wp_symbol*)node)->name) == 0) { - ((struct wp_symbol*)node)->pointer = p; + ((struct wp_symbol*)node)->ip.p = p; } break; case WP_ADD: @@ -961,11 +971,11 @@ wp_ast_regvar (struct wp_node* node, char const* name, double* p) case WP_MUL_VP: case WP_DIV_VP: wp_ast_regvar(node->r, name, p); - node->rp = ((struct wp_symbol*)(node->r))->pointer; + node->rip.p = ((struct wp_symbol*)(node->r))->ip.p; break; case WP_NEG_P: wp_ast_regvar(node->l, name, p); - node->lvp.p = ((struct wp_symbol*)(node->l))->pointer; + node->lvp.ip.p = ((struct wp_symbol*)(node->l))->ip.p; break; case WP_ADD_PP: case WP_SUB_PP: @@ -973,8 +983,8 @@ wp_ast_regvar (struct wp_node* node, char const* name, double* p) case WP_DIV_PP: wp_ast_regvar(node->l, name, p); wp_ast_regvar(node->r, name, p); - node->lvp.p = ((struct wp_symbol*)(node->l))->pointer; - node->rp = ((struct wp_symbol*)(node->r))->pointer; + node->lvp.ip.p = ((struct wp_symbol*)(node->l))->ip.p; + node->rip.p = ((struct wp_symbol*)(node->r))->ip.p; break; default: yyerror("wp_ast_regvar: unknown node type %d\n", node->type); @@ -982,6 +992,61 @@ wp_ast_regvar (struct wp_node* node, char const* name, double* p) } } +void +wp_ast_regvar_gpu (struct wp_node* node, char const* name, int i) +{ + switch (node->type) + { + case WP_NUMBER: + break; + case WP_SYMBOL: + if (strcmp(name, ((struct wp_symbol*)node)->name) == 0) { + ((struct wp_symbol*)node)->ip.i = i; + } + break; + case WP_ADD: + case WP_SUB: + case WP_MUL: + case WP_DIV: + wp_ast_regvar_gpu(node->l, name, i); + wp_ast_regvar_gpu(node->r, name, i); + break; + case WP_NEG: + wp_ast_regvar_gpu(node->l, name, i); + break; + case WP_F1: + wp_ast_regvar_gpu(node->l, name, i); + break; + case WP_F2: + wp_ast_regvar_gpu(node->l, name, i); + wp_ast_regvar_gpu(node->r, name, i); + break; + case WP_ADD_VP: + case WP_SUB_VP: + case WP_MUL_VP: + case WP_DIV_VP: + wp_ast_regvar_gpu(node->r, name, i); + node->rip.i = ((struct wp_symbol*)(node->r))->ip.i; + break; + case WP_NEG_P: + wp_ast_regvar_gpu(node->l, name, i); + node->lvp.ip.i = ((struct wp_symbol*)(node->l))->ip.i; + break; + case WP_ADD_PP: + case WP_SUB_PP: + case WP_MUL_PP: + case WP_DIV_PP: + wp_ast_regvar_gpu(node->l, name, i); + wp_ast_regvar_gpu(node->r, name, i); + node->lvp.ip.i = ((struct wp_symbol*)(node->l))->ip.i; + node->rip.i = ((struct wp_symbol*)(node->r))->ip.i; + break; + default: + yyerror("wp_ast_regvar_gpu: unknown node type %d\n", node->type); + exit(1); + } +} + void wp_ast_setconst (struct wp_node* node, char const* name, double c) { switch (node->type) @@ -1039,6 +1104,12 @@ wp_parser_regvar (struct wp_parser* parser, char const* name, double* p) wp_ast_regvar(parser->ast, name, p); } +void +wp_parser_regvar_gpu (struct wp_parser* parser, char const* name, int i) +{ + wp_ast_regvar_gpu(parser->ast, name, i); +} + void wp_parser_setconst (struct wp_parser* parser, char const* name, double c) { diff --git a/Source/Parser/wp_parser_y.h b/Source/Parser/wp_parser_y.h index 4a3aeda40..8c9f8e4e4 100644 --- a/Source/Parser/wp_parser_y.h +++ b/Source/Parser/wp_parser_y.h @@ -1,6 +1,8 @@ #ifndef WP_PARSER_Y_H_ #define WP_PARSER_Y_H_ +#include + #ifdef __cplusplus #include extern "C" { @@ -73,17 +75,22 @@ enum wp_node_t { * wp_node_t type can be safely checked to determine their real type. */ -union wp_vp { - double v; +union wp_ip { + int i; double* p; }; +union wp_vp { + double v; + union wp_ip ip; +}; + struct wp_node { enum wp_node_t type; struct wp_node* l; struct wp_node* r; union wp_vp lvp; // After optimization, this may store left value/pointer. - double* rp; // this may store right pointer. + union wp_ip rip; // this may store right pointer. }; struct wp_number { @@ -94,7 +101,7 @@ struct wp_number { struct wp_symbol { enum wp_node_t type; char* name; - double* pointer; + union wp_ip ip; }; struct wp_f1 { /* Builtin functions with one argument */ @@ -124,6 +131,7 @@ struct wp_node* wp_newf1 (enum wp_f1_t ftype, struct wp_node* l); struct wp_node* wp_newf2 (enum wp_f2_t ftype, struct wp_node* l, struct wp_node* r); +AMREX_GPU_HOST_DEVICE void yyerror (char const *s, ...); /*******************************************************************/ @@ -146,6 +154,7 @@ struct wp_parser* wp_parser_dup (struct wp_parser* source); struct wp_node* wp_parser_ast_dup (struct wp_parser* parser, struct wp_node* src, int move); void wp_parser_regvar (struct wp_parser* parser, char const* name, double* p); +void wp_parser_regvar_gpu (struct wp_parser* parser, char const* name, int i); void wp_parser_setconst (struct wp_parser* parser, char const* name, double c); /* We need to walk the tree in these functions */ @@ -153,10 +162,11 @@ void wp_ast_optimize (struct wp_node* node); size_t wp_ast_size (struct wp_node* node); void wp_ast_print (struct wp_node* node); void wp_ast_regvar (struct wp_node* node, char const* name, double* p); +void wp_ast_regvar_gpu (struct wp_node* node, char const* name, int i); void wp_ast_setconst (struct wp_node* node, char const* name, double c); -double wp_call_f1 (enum wp_f1_t type, double a); -double wp_call_f2 (enum wp_f2_t type, double a, double b); +AMREX_GPU_HOST_DEVICE double wp_call_f1 (enum wp_f1_t type, double a); +AMREX_GPU_HOST_DEVICE double wp_call_f2 (enum wp_f2_t type, double a, double b); #ifdef __cplusplus } diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index d55764682..416b1065c 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -87,11 +87,8 @@ public: // Inject particles in Box 'part_box' virtual void AddParticles (int lev); + void AddPlasma(int lev, amrex::RealBox part_realbox = amrex::RealBox()); - void AddPlasmaCPU (int lev, amrex::RealBox part_realbox); -#ifdef AMREX_USE_GPU - void AddPlasmaGPU (int lev, amrex::RealBox part_realbox); -#endif void MapParticletoBoostedFrame(amrex::Real& x, amrex::Real& y, amrex::Real& z, std::array& u); @@ -120,16 +117,8 @@ protected: bool boost_adjust_transverse_positions = false; bool do_backward_propagation = false; - long NumParticlesToAdd (const amrex::Box& overlap_box, - const amrex::RealBox& overlap_realbox, - const amrex::RealBox& tile_real_box, - const amrex::RealBox& particle_real_box); - - int GetRefineFac(const amrex::Real x, const amrex::Real y, const amrex::Real z); - std::unique_ptr m_refined_injection_mask = nullptr; - // Inject particles during the whole simulation - void ContinuousInjection(const amrex::RealBox& injection_box) override; + void ContinuousInjection (const amrex::RealBox& injection_box) override; }; diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index d47a7b220..31ffadddf 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -10,62 +10,6 @@ using namespace amrex; -long PhysicalParticleContainer:: -NumParticlesToAdd(const Box& overlap_box, const RealBox& overlap_realbox, - const RealBox& tile_realbox, const RealBox& particle_real_box) -{ - const int lev = 0; - const Geometry& geom = Geom(lev); - int num_ppc = plasma_injector->num_particles_per_cell; - const Real* dx = geom.CellSize(); - - long np = 0; - const auto& overlap_corner = overlap_realbox.lo(); - for (IntVect iv = overlap_box.smallEnd(); iv <= overlap_box.bigEnd(); overlap_box.next(iv)) - { - int fac; - if (do_continuous_injection) { -#if ( AMREX_SPACEDIM == 3 ) - Real x = overlap_corner[0] + (iv[0] + 0.5)*dx[0]; - Real y = overlap_corner[1] + (iv[1] + 0.5)*dx[1]; - Real z = overlap_corner[2] + (iv[2] + 0.5)*dx[2]; -#elif ( AMREX_SPACEDIM == 2 ) - Real x = overlap_corner[0] + (iv[0] + 0.5)*dx[0]; - Real y = 0; - Real z = overlap_corner[1] + (iv[1] + 0.5)*dx[1]; -#endif - fac = GetRefineFac(x, y, z); - } else { - fac = 1.0; - } - - int ref_num_ppc = num_ppc * AMREX_D_TERM(fac, *fac, *fac); - for (int i_part=0; i_part r; - plasma_injector->getPositionUnitBox(r, i_part, fac); -#if ( AMREX_SPACEDIM == 3 ) - Real x = overlap_corner[0] + (iv[0] + r[0])*dx[0]; - Real y = overlap_corner[1] + (iv[1] + r[1])*dx[1]; - Real z = overlap_corner[2] + (iv[2] + r[2])*dx[2]; -#elif ( AMREX_SPACEDIM == 2 ) - Real x = overlap_corner[0] + (iv[0] + r[0])*dx[0]; - Real y = 0; - Real z = overlap_corner[1] + (iv[1] + r[1])*dx[1]; -#endif - // If the new particle is not inside the tile box, - // go to the next generated particle. -#if ( AMREX_SPACEDIM == 3 ) - if(!tile_realbox.contains( RealVect{x, y, z} )) continue; -#elif ( AMREX_SPACEDIM == 2 ) - if(!tile_realbox.contains( RealVect{x, z} )) continue; -#endif - ++np; - } - } - - return np; -} - PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int ispecies, const std::string& name) : WarpXParticleContainer(amr_core, ispecies), @@ -127,9 +71,7 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core) void PhysicalParticleContainer::InitData() { AddParticles(0); // Note - add on level 0 - if (maxLevel() > 0) { - Redistribute(); // We then redistribute - } + Redistribute(); // We then redistribute } void PhysicalParticleContainer::MapParticletoBoostedFrame(Real& x, Real& y, Real& z, std::array& u) @@ -193,8 +135,6 @@ PhysicalParticleContainer::AddGaussianBeam(Real x_m, Real y_m, Real z_m, std::normal_distribution distz(z_m, z_rms); if (ParallelDescriptor::IOProcessor()) { - std::array u; - Real weight; // If do_symmetrize, create 4x fewer particles, and // Replicate each particle 4 times (x,y) (-x,y) (x,-y) (-x,-y) if (do_symmetrize){ @@ -202,36 +142,29 @@ PhysicalParticleContainer::AddGaussianBeam(Real x_m, Real y_m, Real z_m, } for (long i = 0; i < npart; ++i) { #if ( AMREX_SPACEDIM == 3 | WARPX_RZ) - weight = q_tot/npart/charge; + Real weight = q_tot/npart/charge; Real x = distx(mt); Real y = disty(mt); Real z = distz(mt); #elif ( AMREX_SPACEDIM == 2 ) - weight = q_tot/npart/charge/y_rms; + Real weight = q_tot/npart/charge/y_rms; Real x = distx(mt); Real y = 0.; Real z = distz(mt); #endif if (plasma_injector->insideBounds(x, y, z)) { - plasma_injector->getMomentum(u, x, y, z); + XDim3 u = plasma_injector->getMomentum(x, y, z); + u.x *= PhysConst::c; + u.y *= PhysConst::c; + u.z *= PhysConst::c; if (do_symmetrize){ - std::array u_tmp; - Real x_tmp, y_tmp; // Add four particles to the beam: - // (x,ux,y,uy) (-x,-ux,y,uy) (x,ux,-y,-uy) (-x,-ux,-y,-uy) - for (int ix=0; ix<2; ix++){ - for (int iy=0; iy<2; iy++){ - u_tmp = u; - x_tmp = x*std::pow(-1,ix); - u_tmp[0] *= std::pow(-1,ix); - y_tmp = y*std::pow(-1,iy); - u_tmp[1] *= std::pow(-1,iy); - CheckAndAddParticle(x_tmp, y_tmp, z, - u_tmp, weight/4); - } - } + CheckAndAddParticle( x, y, z, { u.x, u.y, u.z}, weight/4. ); + CheckAndAddParticle( x,-y, z, { u.x,-u.y, u.z}, weight/4. ); + CheckAndAddParticle(-x, y, z, {-u.x, u.y, u.z}, weight/4. ); + CheckAndAddParticle(-x,-y, z, {-u.x,-u.y, u.z}, weight/4. ); } else { - CheckAndAddParticle(x, y, z, u, weight); + CheckAndAddParticle(x, y, z, {u.x,u.y,u.z}, weight); } } } @@ -322,17 +255,7 @@ PhysicalParticleContainer::AddParticles (int lev) void PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) { -#ifdef AMREX_USE_GPU - AddPlasmaGPU(lev, part_realbox); -#else - AddPlasmaCPU(lev, part_realbox); -#endif -} - -void -PhysicalParticleContainer::AddPlasmaCPU (int lev, RealBox part_realbox) -{ - BL_PROFILE("PhysicalParticleContainer::AddPlasmaCPU"); + BL_PROFILE("PhysicalParticleContainer::AddPlasma"); // If no part_realbox is provided, initialize particles in the whole domain const Geometry& geom = Geom(lev); @@ -343,7 +266,8 @@ PhysicalParticleContainer::AddPlasmaCPU (int lev, RealBox part_realbox) Real rmax = std::min(plasma_injector->xmax, part_realbox.hi(0)); #endif - const Real* dx = geom.CellSize(); + const auto dx = geom.CellSizeArray(); + const auto problo = geom.ProbLoArray(); Real scale_fac; #if AMREX_SPACEDIM==3 @@ -358,490 +282,324 @@ PhysicalParticleContainer::AddPlasmaCPU (int lev, RealBox part_realbox) const int grid_id = mfi.index(); const int tile_id = mfi.LocalTileIndex(); GetParticles(lev)[std::make_pair(grid_id, tile_id)]; + if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) { + DefineAndReturnParticleTile(lev, grid_id, tile_id); + } } #endif MultiFab* cost = WarpX::getCosts(lev); - if ( (not m_refined_injection_mask) and WarpX::do_moving_window) + const int nlevs = numLevels(); + static bool refine_injection = false; + static Box fine_injection_box; + static int rrfac = 1; + // This does not work if the mesh is dynamic. But in that case, we should + // not use refined injected either. We also assume there is only one fine level. + if (WarpX::do_moving_window and WarpX::refine_plasma + and do_continuous_injection and nlevs == 2) { - Box mask_box = geom.Domain(); - mask_box.setSmall(WarpX::moving_window_dir, 0); - mask_box.setBig(WarpX::moving_window_dir, 0); - m_refined_injection_mask.reset( new IArrayBox(mask_box)); - m_refined_injection_mask->setVal(-1); + refine_injection = true; + fine_injection_box = ParticleBoxArray(1).minimalBox(); + fine_injection_box.setSmall(WarpX::moving_window_dir, std::numeric_limits::lowest()); + fine_injection_box.setBig(WarpX::moving_window_dir, std::numeric_limits::max()); + rrfac = m_gdb->refRatio(0)[0]; + fine_injection_box.coarsen(rrfac); } + InjectorPosition* inj_pos = plasma_injector->getInjectorPosition(); + InjectorDensity* inj_rho = plasma_injector->getInjectorDensity(); + InjectorMomentum* inj_mom = plasma_injector->getInjectorMomentum(); + Real gamma_boost = WarpX::gamma_boost; + Real beta_boost = WarpX::beta_boost; + Real t = WarpX::GetInstance().gett_new(lev); + +#ifdef WARPX_RZ + bool radially_weighted = plasma_injector->radially_weighted; +#endif + MFItInfo info; - if (do_tiling) { + if (do_tiling && Gpu::notInLaunchRegion()) { info.EnableTiling(tile_size); } - info.SetDynamic(true); - #ifdef _OPENMP + info.SetDynamic(true); #pragma omp parallel if (not WarpX::serialize_ics) #endif + for (MFIter mfi = MakeMFIter(lev, info); mfi.isValid(); ++mfi) { - std::array attribs; - attribs.fill(0.0); - - // Loop through the tiles - for (MFIter mfi = MakeMFIter(lev, info); mfi.isValid(); ++mfi) { - - Real wt = amrex::second(); - - const Box& tile_box = mfi.tilebox(); - const RealBox tile_realbox = WarpX::getRealBox(tile_box, lev); - - // Find the cells of part_box that overlap with tile_realbox - // If there is no overlap, just go to the next tile in the loop - RealBox overlap_realbox; - Box overlap_box; - Real ncells_adjust; - bool no_overlap = 0; - - for (int dir=0; dir= part_realbox.lo(dir) ) { - ncells_adjust = std::floor( (part_realbox.hi(dir) - tile_realbox.hi(dir))/dx[dir] ); - overlap_realbox.setHi( dir, part_realbox.hi(dir) - std::max(ncells_adjust, 0.) * dx[dir]); - } else { - no_overlap = 1; break; - } - // Count the number of cells in this direction in overlap_realbox - overlap_box.setSmall( dir, 0 ); - overlap_box.setBig( dir, - int( round((overlap_realbox.hi(dir)-overlap_realbox.lo(dir))/dx[dir] )) - 1); + Real wt = amrex::second(); + + const Box& tile_box = mfi.tilebox(); + const RealBox tile_realbox = WarpX::getRealBox(tile_box, lev); + + // Find the cells of part_box that overlap with tile_realbox + // If there is no overlap, just go to the next tile in the loop + RealBox overlap_realbox; + Box overlap_box; + IntVect shifted; + bool no_overlap = false; + + for (int dir=0; dir= part_realbox.lo(dir) ) { + Real ncells_adjust = std::floor( (part_realbox.hi(dir) - tile_realbox.hi(dir))/dx[dir] ); + overlap_realbox.setHi( dir, part_realbox.hi(dir) - std::max(ncells_adjust, 0.) * dx[dir]); + } else { + no_overlap = true; break; } + // Count the number of cells in this direction in overlap_realbox + overlap_box.setSmall( dir, 0 ); + overlap_box.setBig( dir, + int( std::round((overlap_realbox.hi(dir)-overlap_realbox.lo(dir)) + /dx[dir] )) - 1); + shifted[dir] = std::round((overlap_realbox.lo(dir)-problo[dir])/dx[dir]); + // shifted is exact in non-moving-window direction. That's all we care. + } + if (no_overlap == 1) { + continue; // Go to the next tile + } - const int grid_id = mfi.index(); - const int tile_id = mfi.LocalTileIndex(); - - // Loop through the cells of overlap_box and inject - // the corresponding particles - const auto& overlap_corner = overlap_realbox.lo(); - for (IntVect iv = overlap_box.smallEnd(); iv <= overlap_box.bigEnd(); overlap_box.next(iv)) - { - int fac; - if (do_continuous_injection) { -#if ( AMREX_SPACEDIM == 3 ) - Real x = overlap_corner[0] + (iv[0] + 0.5)*dx[0]; - Real y = overlap_corner[1] + (iv[1] + 0.5)*dx[1]; - Real z = overlap_corner[2] + (iv[2] + 0.5)*dx[2]; -#elif ( AMREX_SPACEDIM == 2 ) - Real x = overlap_corner[0] + (iv[0] + 0.5)*dx[0]; - Real y = 0; - Real z = overlap_corner[1] + (iv[1] + 0.5)*dx[1]; -#endif - fac = GetRefineFac(x, y, z); - } else { - fac = 1.0; - } - - int ref_num_ppc = num_ppc * AMREX_D_TERM(fac, *fac, *fac); - for (int i_part=0; i_part r; - plasma_injector->getPositionUnitBox(r, i_part, fac); -#if ( AMREX_SPACEDIM == 3 ) - Real x = overlap_corner[0] + (iv[0] + r[0])*dx[0]; - Real y = overlap_corner[1] + (iv[1] + r[1])*dx[1]; - Real z = overlap_corner[2] + (iv[2] + r[2])*dx[2]; -#elif ( AMREX_SPACEDIM == 2 ) - Real x = overlap_corner[0] + (iv[0] + r[0])*dx[0]; - Real y = 0; - Real z = overlap_corner[1] + (iv[1] + r[1])*dx[1]; -#endif - // If the new particle is not inside the tile box, - // go to the next generated particle. -#if ( AMREX_SPACEDIM == 3 ) - if(!tile_realbox.contains( RealVect{x, y, z} )) continue; -#elif ( AMREX_SPACEDIM == 2 ) - if(!tile_realbox.contains( RealVect{x, z} )) continue; -#endif - - // Save the x and y values to use in the insideBounds checks. - // This is needed with WARPX_RZ since x and y are modified. - Real xb = x; - Real yb = y; + const int grid_id = mfi.index(); + const int tile_id = mfi.LocalTileIndex(); -#ifdef WARPX_RZ - // Replace the x and y, choosing the angle randomly. - // These x and y are used to get the momentum and density - Real theta = 2.*MathConst::pi*amrex::Random(); - y = x*std::sin(theta); - x = x*std::cos(theta); -#endif + int max_new_particles = overlap_box.numPts() * num_ppc; - Real dens; - std::array u; - if (WarpX::gamma_boost == 1.){ - // Lab-frame simulation - // If the particle is not within the species's - // xmin, xmax, ymin, ymax, zmin, zmax, go to - // the next generated particle. - if (!plasma_injector->insideBounds(xb, yb, z)) continue; - plasma_injector->getMomentum(u, x, y, z); - dens = plasma_injector->getDensity(x, y, z); - } else { - // Boosted-frame simulation - Real c = PhysConst::c; - Real gamma_boost = WarpX::gamma_boost; - Real beta_boost = WarpX::beta_boost; - // Since the user provides the density distribution - // at t_lab=0 and in the lab-frame coordinates, - // we need to find the lab-frame position of this - // particle at t_lab=0, from its boosted-frame coordinates - // Assuming ballistic motion, this is given by: - // z0_lab = gamma*( z_boost*(1-beta*betaz_lab) - ct_boost*(betaz_lab-beta) ) - // where betaz_lab is the speed of the particle in the lab frame - // - // In order for this equation to be solvable, betaz_lab - // is explicitly assumed to have no dependency on z0_lab - plasma_injector->getMomentum(u, x, y, 0.); // No z0_lab dependency - // At this point u is the lab-frame momentum - // => Apply the above formula for z0_lab - Real gamma_lab = std::sqrt( 1 + (u[0]*u[0] + u[1]*u[1] + u[2]*u[2])/(c*c) ); - Real betaz_lab = u[2]/gamma_lab/c; - Real t = WarpX::GetInstance().gett_new(lev); - Real z0_lab = gamma_boost * ( z*(1-beta_boost*betaz_lab) - c*t*(betaz_lab-beta_boost) ); - // If the particle is not within the lab-frame zmin, zmax, etc. - // go to the next generated particle. - if (!plasma_injector->insideBounds(xb, yb, z0_lab)) continue; - // call `getDensity` with lab-frame parameters - dens = plasma_injector->getDensity(x, y, z0_lab); - // At this point u and dens are the lab-frame quantities - // => Perform Lorentz transform - dens = gamma_boost * dens * ( 1 - beta_boost*betaz_lab ); - u[2] = gamma_boost * ( u[2] -beta_boost*c*gamma_lab ); - } - Real weight = dens * scale_fac / (AMREX_D_TERM(fac, *fac, *fac)); -#ifdef WARPX_RZ - if (plasma_injector->radially_weighted) { - weight *= 2*MathConst::pi*xb; - } else { - // This is not correct since it might shift the particle - // out of the local grid - x = std::sqrt(xb*rmax); - weight *= dx[0]; - } -#endif - attribs[PIdx::w ] = weight; - attribs[PIdx::ux] = u[0]; - attribs[PIdx::uy] = u[1]; - attribs[PIdx::uz] = u[2]; - - if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) - { - auto& particle_tile = DefineAndReturnParticleTile(lev, grid_id, tile_id); - particle_tile.push_back_real(particle_comps["xold"], x); - particle_tile.push_back_real(particle_comps["yold"], y); - particle_tile.push_back_real(particle_comps["zold"], z); - - particle_tile.push_back_real(particle_comps["uxold"], u[0]); - particle_tile.push_back_real(particle_comps["uyold"], u[1]); - particle_tile.push_back_real(particle_comps["uzold"], u[2]); - } - - AddOneParticle(lev, grid_id, tile_id, x, y, z, attribs); + Vector cellid_v; + if (refine_injection and lev == 0) + { + // then how many new particles will be injected is not that simple + // We have to shift fine_injection_box because overlap_box has been shifted. + Box fine_overlap_box = overlap_box & amrex::shift(fine_injection_box,shifted); + max_new_particles += fine_overlap_box.numPts() * num_ppc + * (AMREX_D_TERM(rrfac,*rrfac,*rrfac)-1); + for (int icell = 0, ncells = overlap_box.numPts(); icell < ncells; ++icell) { + IntVect iv = overlap_box.atOffset(icell); + int r = (fine_overlap_box.contains(iv)) ? AMREX_D_TERM(rrfac,*rrfac,*rrfac) : 1; + for (int ipart = 0; ipart < r; ++ipart) { + cellid_v.push_back(icell); + cellid_v.push_back(ipart); } } + } + int const* hp_cellid = (cellid_v.empty()) ? nullptr : cellid_v.data(); + amrex::AsyncArray cellid_aa(hp_cellid, cellid_v.size()); + int const* dp_cellid = cellid_aa.data(); - if (cost) { - wt = (amrex::second() - wt) / tile_box.d_numPts(); - Array4 const& costarr = cost->array(mfi); - amrex::ParallelFor(tile_box, - [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - costarr(i,j,k) += wt; - }); - } + int pid; +#pragma omp critical (add_plasma_nextid) + { + pid = ParticleType::NextID(); + ParticleType::NextID(pid+max_new_particles); } - } -} + const int cpuid = ParallelDescriptor::MyProc(); -#ifdef AMREX_USE_GPU -void -PhysicalParticleContainer::AddPlasmaGPU (int lev, RealBox part_realbox) -{ - BL_PROFILE("PhysicalParticleContainer::AddPlasmaGPU"); + auto& particle_tile = GetParticles(lev)[std::make_pair(grid_id,tile_id)]; + bool do_boosted = false; + if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) { + do_boosted = true; + DefineAndReturnParticleTile(lev, grid_id, tile_id); + } + auto old_size = particle_tile.GetArrayOfStructs().size(); + auto new_size = old_size + max_new_particles; + particle_tile.resize(new_size); + + ParticleType* pp = particle_tile.GetArrayOfStructs()().data() + old_size; + auto& soa = particle_tile.GetStructOfArrays(); + GpuArray pa; + for (int ia = 0; ia < PIdx::nattribs; ++ia) { + pa[ia] = soa.GetRealData(ia).data() + old_size; + } + GpuArray pb; + if (do_boosted) { + pb[0] = soa.GetRealData(particle_comps[ "xold"]).data() + old_size; + pb[1] = soa.GetRealData(particle_comps[ "yold"]).data() + old_size; + pb[2] = soa.GetRealData(particle_comps[ "zold"]).data() + old_size; + pb[3] = soa.GetRealData(particle_comps["uxold"]).data() + old_size; + pb[4] = soa.GetRealData(particle_comps["uyold"]).data() + old_size; + pb[5] = soa.GetRealData(particle_comps["uzold"]).data() + old_size; + } - // If no part_realbox is provided, initialize particles in the whole domain - const Geometry& geom = Geom(lev); - if (!part_realbox.ok()) part_realbox = geom.ProbDomain(); + const GpuArray overlap_corner + {AMREX_D_DECL(overlap_realbox.lo(0), + overlap_realbox.lo(1), + overlap_realbox.lo(2))}; - int num_ppc = plasma_injector->num_particles_per_cell; #ifdef WARPX_RZ - Real rmax = std::min(plasma_injector->xmax, part_realbox.hi(0)); + { +#else + if (plasma_injector->useRandom()) { #endif + amrex::Gpu::streamSynchronize(); + amrex::CheckSeedArraySizeAndResize(max_new_particles); + } + + std::size_t shared_mem_bytes = plasma_injector->sharedMemoryNeeded(); + int lrrfac = rrfac; - const Real* dx = geom.CellSize(); + amrex::For(max_new_particles, [=] AMREX_GPU_DEVICE (int ip) noexcept + { + ParticleType& p = pp[ip]; + p.id() = pid+ip; + p.cpu() = cpuid; + + int cellid, i_part; + Real fac; + if (dp_cellid == nullptr) { + cellid = ip/num_ppc; + i_part = ip - cellid*num_ppc; + fac = 1.0; + } else { + cellid = dp_cellid[2*ip]; + i_part = dp_cellid[2*ip+1]; + fac = lrrfac; + } - Real scale_fac; -#if AMREX_SPACEDIM==3 - scale_fac = dx[0]*dx[1]*dx[2]/num_ppc; -#elif AMREX_SPACEDIM==2 - scale_fac = dx[0]*dx[1]/num_ppc; -#endif + IntVect iv = overlap_box.atOffset(cellid); -#ifdef _OPENMP - // First touch all tiles in the map in serial - for (MFIter mfi = MakeMFIter(lev); mfi.isValid(); ++mfi) { - const int grid_id = mfi.index(); - const int tile_id = mfi.LocalTileIndex(); - GetParticles(lev)[std::make_pair(grid_id, tile_id)]; - } + const XDim3 r = inj_pos->getPositionUnitBox(i_part, fac); +#if (AMREX_SPACEDIM == 3) + Real x = overlap_corner[0] + (iv[0]+r.x)*dx[0]; + Real y = overlap_corner[1] + (iv[1]+r.y)*dx[1]; + Real z = overlap_corner[2] + (iv[2]+r.z)*dx[2]; +#else + Real x = overlap_corner[0] + (iv[0]+r.x)*dx[0]; + Real y = 0.0; + Real z = overlap_corner[1] + (iv[1]+r.y)*dx[1]; #endif - MultiFab* cost = WarpX::getCosts(lev); - - if ( (not m_refined_injection_mask) and WarpX::do_moving_window) - { - Box mask_box = geom.Domain(); - mask_box.setSmall(WarpX::moving_window_dir, 0); - mask_box.setBig(WarpX::moving_window_dir, 0); - m_refined_injection_mask.reset( new IArrayBox(mask_box)); - m_refined_injection_mask->setVal(-1); - } - - MFItInfo info; - if (do_tiling) { - info.EnableTiling(tile_size); - } - info.SetDynamic(true); - -#ifdef _OPENMP -#pragma omp parallel if (not WarpX::serialize_ics) +#if (AMREX_SPACEDIM == 3) + if (!tile_realbox.contains(XDim3{x,y,z})) { + p.id() = -1; + return; + } +#else + if (!tile_realbox.contains(XDim3{x,z,0.0})) { + p.id() = -1; + return; + } #endif - { - std::array attribs; - attribs.fill(0.0); - - // Loop through the tiles - for (MFIter mfi = MakeMFIter(lev, info); mfi.isValid(); ++mfi) { - - Real wt = amrex::second(); - const Box& tile_box = mfi.tilebox(); - const RealBox tile_realbox = WarpX::getRealBox(tile_box, lev); + // Save the x and y values to use in the insideBounds checks. + // This is needed with WARPX_RZ since x and y are modified. + Real xb = x; + Real yb = y; - // Find the cells of part_box that overlap with tile_realbox - // If there is no overlap, just go to the next tile in the loop - RealBox overlap_realbox; - Box overlap_box; - Real ncells_adjust; - bool no_overlap = 0; +#ifdef WARPX_RZ + // Replace the x and y, choosing the angle randomly. + // These x and y are used to get the momentum and density + Real theta = 2.*MathConst::pi*amrex::Random(); + x = xb*std::cos(theta); + y = xb*std::sin(theta); +#endif - for (int dir=0; dirinsideBounds(xb, yb, z)) { + p.id() = -1; + return; } - if ( tile_realbox.hi(dir) >= part_realbox.lo(dir) ) { - ncells_adjust = std::floor( (part_realbox.hi(dir) - tile_realbox.hi(dir))/dx[dir] ); - overlap_realbox.setHi( dir, part_realbox.hi(dir) - std::max(ncells_adjust, 0.) * dx[dir]); - } else { - no_overlap = 1; break; + u = inj_mom->getMomentum(x, y, z); + dens = inj_rho->getDensity(x, y, z); + } else { + // Boosted-frame simulation + // Since the user provides the density distribution + // at t_lab=0 and in the lab-frame coordinates, + // we need to find the lab-frame position of this + // particle at t_lab=0, from its boosted-frame coordinates + // Assuming ballistic motion, this is given by: + // z0_lab = gamma*( z_boost*(1-beta*betaz_lab) - ct_boost*(betaz_lab-beta) ) + // where betaz_lab is the speed of the particle in the lab frame + // + // In order for this equation to be solvable, betaz_lab + // is explicitly assumed to have no dependency on z0_lab + u = inj_mom->getMomentum(x, y, 0.); // No z0_lab dependency + // At this point u is the lab-frame momentum + // => Apply the above formula for z0_lab + Real gamma_lab = std::sqrt( 1.+(u.x*u.x+u.y*u.y+u.z*u.z) ); + Real betaz_lab = u.z/(gamma_lab); + Real z0_lab = gamma_boost * ( z*(1-beta_boost*betaz_lab) + - PhysConst::c*t*(betaz_lab-beta_boost) ); + // If the particle is not within the lab-frame zmin, zmax, etc. + // go to the next generated particle. + if (!inj_pos->insideBounds(xb, yb, z0_lab)) { + p.id() = -1; + return; } - // Count the number of cells in this direction in overlap_realbox - overlap_box.setSmall( dir, 0 ); - overlap_box.setBig( dir, - int( round((overlap_realbox.hi(dir)-overlap_realbox.lo(dir))/dx[dir] )) - 1); + // call `getDensity` with lab-frame parameters + dens = inj_rho->getDensity(x, y, z0_lab); + // At this point u and dens are the lab-frame quantities + // => Perform Lorentz transform + dens = gamma_boost * dens * ( 1.0 - beta_boost*betaz_lab ); + u.z = gamma_boost * ( u.z -beta_boost*gamma_lab ); } - if (no_overlap == 1) { - continue; // Go to the next tile - } - - const int grid_id = mfi.index(); - const int tile_id = mfi.LocalTileIndex(); - - Cuda::HostVector host_particles; - std::array, PIdx::nattribs> host_attribs; - - // Loop through the cells of overlap_box and inject - // the corresponding particles - const auto& overlap_corner = overlap_realbox.lo(); - for (IntVect iv = overlap_box.smallEnd(); iv <= overlap_box.bigEnd(); overlap_box.next(iv)) - { - int fac; - if (do_continuous_injection) { -#if ( AMREX_SPACEDIM == 3 ) - Real x = overlap_corner[0] + (iv[0] + 0.5)*dx[0]; - Real y = overlap_corner[1] + (iv[1] + 0.5)*dx[1]; - Real z = overlap_corner[2] + (iv[2] + 0.5)*dx[2]; -#elif ( AMREX_SPACEDIM == 2 ) - Real x = overlap_corner[0] + (iv[0] + 0.5)*dx[0]; - Real y = 0; - Real z = overlap_corner[1] + (iv[1] + 0.5)*dx[1]; -#endif - fac = GetRefineFac(x, y, z); - } else { - fac = 1.0; - } - int ref_num_ppc = num_ppc * AMREX_D_TERM(fac, *fac, *fac); - for (int i_part=0; i_part r; - plasma_injector->getPositionUnitBox(r, i_part, fac); -#if ( AMREX_SPACEDIM == 3 ) - Real x = overlap_corner[0] + (iv[0] + r[0])*dx[0]; - Real y = overlap_corner[1] + (iv[1] + r[1])*dx[1]; - Real z = overlap_corner[2] + (iv[2] + r[2])*dx[2]; -#elif ( AMREX_SPACEDIM == 2 ) - Real x = overlap_corner[0] + (iv[0] + r[0])*dx[0]; - Real y = 0; - Real z = overlap_corner[1] + (iv[1] + r[1])*dx[1]; -#endif - // If the new particle is not inside the tile box, - // go to the next generated particle. -#if ( AMREX_SPACEDIM == 3 ) - if(!tile_realbox.contains( RealVect{x, y, z} )) continue; -#elif ( AMREX_SPACEDIM == 2 ) - if(!tile_realbox.contains( RealVect{x, z} )) continue; -#endif - - // Save the x and y values to use in the insideBounds checks. - // This is needed with WARPX_RZ since x and y are modified. - Real xb = x; - Real yb = y; - -#ifdef WARPX_RZ - // Replace the x and y, choosing the angle randomly. - // These x and y are used to get the momentum and density - Real theta = 2.*MathConst::pi*amrex::Random(); - x = xb*std::cos(theta); - y = xb*std::sin(theta); -#endif + u.x *= PhysConst::c; + u.y *= PhysConst::c; + u.z *= PhysConst::c; - Real dens; - std::array u; - if (WarpX::gamma_boost == 1.){ - // Lab-frame simulation - // If the particle is not within the species's - // xmin, xmax, ymin, ymax, zmin, zmax, go to - // the next generated particle. - if (!plasma_injector->insideBounds(xb, yb, z)) continue; - plasma_injector->getMomentum(u, x, y, z); - dens = plasma_injector->getDensity(x, y, z); - } else { - // Boosted-frame simulation - Real c = PhysConst::c; - Real gamma_boost = WarpX::gamma_boost; - Real beta_boost = WarpX::beta_boost; - // Since the user provides the density distribution - // at t_lab=0 and in the lab-frame coordinates, - // we need to find the lab-frame position of this - // particle at t_lab=0, from its boosted-frame coordinates - // Assuming ballistic motion, this is given by: - // z0_lab = gamma*( z_boost*(1-beta*betaz_lab) - ct_boost*(betaz_lab-beta) ) - // where betaz_lab is the speed of the particle in the lab frame - // - // In order for this equation to be solvable, betaz_lab - // is explicitly assumed to have no dependency on z0_lab - plasma_injector->getMomentum(u, x, y, 0.); // No z0_lab dependency - // At this point u is the lab-frame momentum - // => Apply the above formula for z0_lab - Real gamma_lab = std::sqrt( 1 + (u[0]*u[0] + u[1]*u[1] + u[2]*u[2])/(c*c) ); - Real betaz_lab = u[2]/gamma_lab/c; - Real t = WarpX::GetInstance().gett_new(lev); - Real z0_lab = gamma_boost * ( z*(1-beta_boost*betaz_lab) - c*t*(betaz_lab-beta_boost) ); - // If the particle is not within the lab-frame zmin, zmax, etc. - // go to the next generated particle. - if (!plasma_injector->insideBounds(xb, yb, z0_lab)) continue; - // call `getDensity` with lab-frame parameters - dens = plasma_injector->getDensity(x, y, z0_lab); - // At this point u and dens are the lab-frame quantities - // => Perform Lorentz transform - dens = gamma_boost * dens * ( 1 - beta_boost*betaz_lab ); - u[2] = gamma_boost * ( u[2] -beta_boost*c*gamma_lab ); - } - Real weight = dens * scale_fac / (AMREX_D_TERM(fac, *fac, *fac)); + // Real weight = dens * scale_fac / (AMREX_D_TERM(fac, *fac, *fac)); + Real weight = dens * scale_fac; #ifdef WARPX_RZ - if (plasma_injector->radially_weighted) { - weight *= 2*MathConst::pi*xb; - } else { - // This is not correct since it might shift the particle - // out of the local grid - x = std::sqrt(xb*rmax); - weight *= dx[0]; - } + if (radially_weighted) { + weight *= 2.*MathConst::pi*xb; + } else { + // This is not correct since it might shift the particle + // out of the local grid + x = std::sqrt(xb*rmax); + weight *= dx[0]; + } #endif - attribs[PIdx::w ] = weight; - attribs[PIdx::ux] = u[0]; - attribs[PIdx::uy] = u[1]; - attribs[PIdx::uz] = u[2]; - - // note - this will be slow on the GPU, need to revisit - if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) - { - auto& particle_tile = DefineAndReturnParticleTile(lev, grid_id, tile_id); - particle_tile.push_back_real(particle_comps["xold"], x); - particle_tile.push_back_real(particle_comps["yold"], y); - particle_tile.push_back_real(particle_comps["zold"], z); - - particle_tile.push_back_real(particle_comps["uxold"], u[0]); - particle_tile.push_back_real(particle_comps["uyold"], u[1]); - particle_tile.push_back_real(particle_comps["uzold"], u[2]); - } + pa[PIdx::w ][ip] = weight; + pa[PIdx::ux][ip] = u.x; + pa[PIdx::uy][ip] = u.y; + pa[PIdx::uz][ip] = u.z; + + if (do_boosted) { + pb[0][ip] = x; + pb[1][ip] = y; + pb[2][ip] = z; + pb[3][ip] = u.x; + pb[4][ip] = u.y; + pb[5][ip] = u.z; + } - ParticleType p; - p.id() = ParticleType::NextID(); - p.cpu() = ParallelDescriptor::MyProc(); #if (AMREX_SPACEDIM == 3) - p.pos(0) = x; - p.pos(1) = y; - p.pos(2) = z; + p.pos(0) = x; + p.pos(1) = y; + p.pos(2) = z; #elif (AMREX_SPACEDIM == 2) #ifdef WARPX_RZ - attribs[PIdx::theta] = theta; + pa[PIdx::theta][ip] = theta; #endif - p.pos(0) = xb; - p.pos(1) = z; + p.pos(0) = xb; + p.pos(1) = z; #endif - - host_particles.push_back(p); - for (int kk = 0; kk < PIdx::nattribs; ++kk) - host_attribs[kk].push_back(attribs[kk]); - } - } - - auto& particle_tile = GetParticles(lev)[std::make_pair(grid_id,tile_id)]; - auto old_size = particle_tile.GetArrayOfStructs().size(); - auto new_size = old_size + host_particles.size(); - particle_tile.resize(new_size); - - Cuda::thrust_copy(host_particles.begin(), - host_particles.end(), - particle_tile.GetArrayOfStructs().begin() + old_size); - - for (int kk = 0; kk < PIdx::nattribs; ++kk) { - Cuda::thrust_copy(host_attribs[kk].begin(), - host_attribs[kk].end(), - particle_tile.GetStructOfArrays().GetRealData(kk).begin() + old_size); - } - - if (cost) { - wt = (amrex::second() - wt) / tile_box.d_numPts(); - Array4 const& costarr = cost->array(mfi); - amrex::ParallelFor(tile_box, - [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - costarr(i,j,k) += wt; - }); - } - } + }, shared_mem_bytes); + + if (cost) { + wt = (amrex::second() - wt) / tile_box.d_numPts(); + Array4 const& costarr = cost->array(mfi); + amrex::ParallelFor(tile_box, + [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + costarr(i,j,k) += wt; + }); + } } + + // The function that calls this is responsible for redistributing particles. } -#endif #ifdef WARPX_DO_ELECTROSTATIC void @@ -2034,74 +1792,6 @@ void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real } } -int PhysicalParticleContainer::GetRefineFac(const Real x, const Real y, const Real z) -{ - if (finestLevel() == 0) return 1; - if (not WarpX::refine_plasma) return 1; - - IntVect iv; - const Geometry& geom = Geom(0); - - std::array offset; - -#if ( AMREX_SPACEDIM == 3) - offset[0] = geom.ProbLo(0); - offset[1] = geom.ProbLo(1); - offset[2] = geom.ProbLo(2); -#elif ( AMREX_SPACEDIM == 2 ) - offset[0] = geom.ProbLo(0); - offset[1] = 0.0; - offset[2] = geom.ProbLo(1); -#endif - - AMREX_D_TERM(iv[0]=static_cast(floor((x-offset[0])*geom.InvCellSize(0)));, - iv[1]=static_cast(floor((y-offset[1])*geom.InvCellSize(1)));, - iv[2]=static_cast(floor((z-offset[2])*geom.InvCellSize(2)));); - - iv += geom.Domain().smallEnd(); - - const int dir = WarpX::moving_window_dir; - - IntVect iv2 = iv; - iv2[dir] = 0; - - if ( (*m_refined_injection_mask)(iv2) != -1) return (*m_refined_injection_mask)(iv2); - - int ref_fac = 1; - for (int lev = 0; lev < finestLevel(); ++lev) - { - const IntVect rr = m_gdb->refRatio(lev); - const BoxArray& fine_ba = this->ParticleBoxArray(lev+1); - const int num_boxes = fine_ba.size(); - Vector stretched_boxes; - const int safety_factor = 4; - for (int i = 0; i < num_boxes; ++i) - { - Box bx = fine_ba[i]; - bx.coarsen(ref_fac*rr[dir]); - bx.setSmall(dir, std::numeric_limits::min()/safety_factor); - bx.setBig(dir, std::numeric_limits::max()/safety_factor); - stretched_boxes.push_back(bx); - } - - BoxArray stretched_ba(stretched_boxes.dataPtr(), stretched_boxes.size()); - - const int num_ghost = 0; - if ( stretched_ba.intersects(Box(iv, iv), num_ghost) ) - { - ref_fac *= rr[dir]; - } - else - { - break; - } - } - - (*m_refined_injection_mask)(iv2) = ref_fac; - - return ref_fac; -} - /* \brief Inject particles during the simulation * \param injection_box: domain where particles should be injected. */ -- cgit v1.2.3 From 6296cdccbe4c9d64105579cc649caa603425f7ba Mon Sep 17 00:00:00 2001 From: Dave Grote Date: Fri, 26 Jul 2019 11:24:47 -0700 Subject: Fixed reference to picsar in Evolve profiling --- Source/Particles/PhysicalParticleContainer.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index b913ed5cd..9d229d36b 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1170,7 +1170,7 @@ PhysicalParticleContainer::Evolve (int lev, BL_PROFILE("PPC::Evolve()"); BL_PROFILE_VAR_NS("PPC::Evolve::Copy", blp_copy); BL_PROFILE_VAR_NS("PICSAR::FieldGather", blp_pxr_fg); - BL_PROFILE_VAR_NS("PICSAR::ParticlePush", blp_pxr_pp); + BL_PROFILE_VAR_NS("PICSAR::ParticlePush", blp_ppc_pp); BL_PROFILE_VAR_NS("PPC::Evolve::partition", blp_partition); const std::array& dx = WarpX::CellSize(lev); @@ -1527,10 +1527,10 @@ PhysicalParticleContainer::Evolve (int lev, // // Particle Push // - BL_PROFILE_VAR_START(blp_pxr_pp); + BL_PROFILE_VAR_START(blp_ppc_pp); PushPX(pti, m_xp[thread_num], m_yp[thread_num], m_zp[thread_num], m_giv[thread_num], dt); - BL_PROFILE_VAR_STOP(blp_pxr_pp); + BL_PROFILE_VAR_STOP(blp_ppc_pp); // // Current Deposition -- cgit v1.2.3 From 272f963511d71a3f9edf95a1d5108df2c2f70f37 Mon Sep 17 00:00:00 2001 From: Dave Grote Date: Fri, 26 Jul 2019 11:27:50 -0700 Subject: Another fix of picsar reference in Evolve --- Source/Particles/PhysicalParticleContainer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 9d229d36b..7f8118b44 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1170,7 +1170,7 @@ PhysicalParticleContainer::Evolve (int lev, BL_PROFILE("PPC::Evolve()"); BL_PROFILE_VAR_NS("PPC::Evolve::Copy", blp_copy); BL_PROFILE_VAR_NS("PICSAR::FieldGather", blp_pxr_fg); - BL_PROFILE_VAR_NS("PICSAR::ParticlePush", blp_ppc_pp); + BL_PROFILE_VAR_NS("PPC::ParticlePush", blp_ppc_pp); BL_PROFILE_VAR_NS("PPC::Evolve::partition", blp_partition); const std::array& dx = WarpX::CellSize(lev); -- cgit v1.2.3 From 1b67c15a76868332487c203562ab3e824d2f2e65 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Fri, 26 Jul 2019 14:05:14 -0700 Subject: get rid of Fortran field gather except for RZ --- Source/Evolve/WarpXEvolveEM.cpp | 16 ++++++++-------- Source/Particles/MultiParticleContainer.H | 7 ++++--- Source/Particles/MultiParticleContainer.cpp | 9 +++++---- Source/Particles/PhysicalParticleContainer.H | 14 +++++++------- Source/Particles/PhysicalParticleContainer.cpp | 24 +++++++++++++++++------- Source/Particles/WarpXParticleContainer.H | 7 ++++--- 6 files changed, 45 insertions(+), 32 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index f1f7c698c..32a4747db 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -189,9 +189,9 @@ WarpX::EvolveEM (int numsteps) UpdateAuxilaryData(); for (int lev = 0; lev <= finest_level; ++lev) { - mypc->FieldGatherFortran(lev, - *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); + mypc->FieldGather(lev, + *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], + *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); } last_plot_file_step = step+1; @@ -241,11 +241,11 @@ WarpX::EvolveEM (int numsteps) UpdateAuxilaryData(); for (int lev = 0; lev <= finest_level; ++lev) { - mypc->FieldGatherFortran(lev, - *Efield_aux[lev][0],*Efield_aux[lev][1], - *Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1], - *Bfield_aux[lev][2]); + mypc->FieldGather(lev, + *Efield_aux[lev][0],*Efield_aux[lev][1], + *Efield_aux[lev][2], + *Bfield_aux[lev][0],*Bfield_aux[lev][1], + *Bfield_aux[lev][2]); } if (write_plot_file) diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 76e3c44bc..7c9ede411 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -84,9 +84,10 @@ public: /// Performs the field gather operation using the input fields E and B, for all the species /// in the MultiParticleContainer. This is the electromagnetic version of the field gather. /// - void FieldGatherFortran (int lev, - const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz); + void FieldGather (int lev, + const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, + const amrex::MultiFab& Ez, const amrex::MultiFab& Bx, + const amrex::MultiFab& By, const amrex::MultiFab& Bz); /// /// This evolves all the particles by one PIC time step, including current deposition, the diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index f9d29e254..beb9de207 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -239,12 +239,13 @@ MultiParticleContainer::sumParticleCharge (bool local) #endif // WARPX_DO_ELECTROSTATIC void -MultiParticleContainer::FieldGatherFortran (int lev, - const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, - const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz) +MultiParticleContainer::FieldGather (int lev, + const MultiFab& Ex, const MultiFab& Ey, + const MultiFab& Ez, const MultiFab& Bx, + const MultiFab& By, const MultiFab& Bz) { for (auto& pc : allcontainers) { - pc->FieldGatherFortran(lev, Ex, Ey, Ez, Bx, By, Bz); + pc->FieldGather(lev, Ex, Ey, Ez, Bx, By, Bz); } } diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index eebc9b55d..33572c87d 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -31,13 +31,13 @@ public: amrex::Real t, amrex::Real dt) override; #endif // WARPX_DO_ELECTROSTATIC - virtual void FieldGatherFortran (int lev, - const amrex::MultiFab& Ex, - const amrex::MultiFab& Ey, - const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, - const amrex::MultiFab& By, - const amrex::MultiFab& Bz) final; + virtual void FieldGather (int lev, + const amrex::MultiFab& Ex, + const amrex::MultiFab& Ey, + const amrex::MultiFab& Ez, + const amrex::MultiFab& Bx, + const amrex::MultiFab& By, + const amrex::MultiFab& Bz) final; void FieldGather (WarpXParIter& pti, RealVector& Exp, diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index b98797b56..91ad03e9b 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1055,9 +1055,9 @@ PhysicalParticleContainer::EvolveES (const Vector& dx = WarpX::CellSize(lev); @@ -1066,11 +1066,14 @@ PhysicalParticleContainer::FieldGatherFortran (int lev, MultiFab* cost = WarpX::getCosts(lev); #ifdef _OPENMP -#pragma omp parallel +#pragma omp parallel #endif { - Cuda::ManagedDeviceVector xp, yp, zp; - +#ifdef _OPENMP + int thread_num = omp_get_thread_num(); +#else + int thread_num = 0; +#endif for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { Real wt = amrex::second(); @@ -1106,7 +1109,7 @@ PhysicalParticleContainer::FieldGatherFortran (int lev, // // copy data from particle container to temp arrays // - pti.GetPosition(xp, yp, zp); + pti.GetPosition(m_xp[thread_num], m_yp[thread_num], m_zp[thread_num]); const std::array& xyzmin = WarpX::LowerCorner(box, lev); const int* ixyzmin = box.loVect(); @@ -1116,6 +1119,7 @@ PhysicalParticleContainer::FieldGatherFortran (int lev, // const int ll4symtry = false; long lvect_fieldgathe = 64; +#ifdef WARPX_RZ warpx_geteb_energy_conserving( &np, xp.dataPtr(), @@ -1135,6 +1139,12 @@ PhysicalParticleContainer::FieldGatherFortran (int lev, BL_TO_FORTRAN_ANYD(bzfab), &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, &lvect_fieldgathe, &WarpX::field_gathering_algo); +#else + int e_is_nodal = Ex.is_nodal() and Ey.is_nodal() and Ez.is_nodal(); + FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + &exfab, &eyfab, &ezfab, &bxfab, &byfab, &bzfab, + Ex.nGrow(), e_is_nodal, 0, np, thread_num, lev, lev); +#endif if (cost) { const Box& tbx = pti.tilebox(); diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 1b2a06171..7cf53260a 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -103,9 +103,10 @@ public: virtual void FieldGatherES (const amrex::Vector, 3> >& E, const amrex::Vector > > >& masks) {} - virtual void FieldGatherFortran (int lev, - const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz) {} + virtual void FieldGather (int lev, + const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, + const amrex::MultiFab& Ez, const amrex::MultiFab& Bx, + const amrex::MultiFab& By, const amrex::MultiFab& Bz) {} #ifdef WARPX_DO_ELECTROSTATIC virtual void EvolveES (const amrex::Vector, 3> >& E, -- cgit v1.2.3 From 6f351e2a6162bf57d0201978ff03ef740293e3af Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Fri, 26 Jul 2019 14:14:14 -0700 Subject: fix RZ. Now both 2D and RZ give same result as before --- Source/Particles/PhysicalParticleContainer.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 91ad03e9b..6d47373f9 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1122,9 +1122,9 @@ PhysicalParticleContainer::FieldGather (int lev, #ifdef WARPX_RZ warpx_geteb_energy_conserving( &np, - xp.dataPtr(), - yp.dataPtr(), - zp.dataPtr(), + m_xp[thread_num].dataPtr(), + m_yp[thread_num].dataPtr(), + m_zp[thread_num].dataPtr(), Exp.dataPtr(),Eyp.dataPtr(),Ezp.dataPtr(), Bxp.dataPtr(),Byp.dataPtr(),Bzp.dataPtr(), ixyzmin, -- cgit v1.2.3 From a0bb6f3ee7863c97292c99715854821ede3bcdb5 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Sun, 28 Jul 2019 18:21:34 -0700 Subject: No need to resize random number seeds anymore --- Source/Initialization/InjectorDensity.H | 1 - Source/Initialization/InjectorMomentum.H | 2 -- Source/Initialization/InjectorMomentum.cpp | 13 ------------- Source/Initialization/InjectorPosition.H | 2 -- Source/Initialization/InjectorPosition.cpp | 19 ------------------- Source/Initialization/Make.package | 1 - Source/Initialization/PlasmaInjector.H | 5 ----- Source/Particles/PhysicalParticleContainer.cpp | 9 --------- 8 files changed, 52 deletions(-) delete mode 100644 Source/Initialization/InjectorPosition.cpp (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Initialization/InjectorDensity.H b/Source/Initialization/InjectorDensity.H index 61471433f..a27affd95 100644 --- a/Source/Initialization/InjectorDensity.H +++ b/Source/Initialization/InjectorDensity.H @@ -119,7 +119,6 @@ struct InjectorDensity ~InjectorDensity (); std::size_t sharedMemoryNeeded () const noexcept; - bool useRandom () const noexcept { return false; } AMREX_GPU_HOST_DEVICE amrex::Real diff --git a/Source/Initialization/InjectorMomentum.H b/Source/Initialization/InjectorMomentum.H index 2434503cc..baf1e0e05 100644 --- a/Source/Initialization/InjectorMomentum.H +++ b/Source/Initialization/InjectorMomentum.H @@ -123,8 +123,6 @@ struct InjectorMomentum std::size_t sharedMemoryNeeded () const noexcept; - bool useRandom () const noexcept; - AMREX_GPU_HOST_DEVICE amrex::XDim3 getMomentum (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept diff --git a/Source/Initialization/InjectorMomentum.cpp b/Source/Initialization/InjectorMomentum.cpp index 4e483fc33..f9070b29c 100644 --- a/Source/Initialization/InjectorMomentum.cpp +++ b/Source/Initialization/InjectorMomentum.cpp @@ -35,16 +35,3 @@ InjectorMomentum::sharedMemoryNeeded () const noexcept } } -bool -InjectorMomentum::useRandom () const noexcept -{ - switch (type) - { - case Type::gaussian: - { - return true; - } - default: - return false; - } -} diff --git a/Source/Initialization/InjectorPosition.H b/Source/Initialization/InjectorPosition.H index e74345ae5..83ea8aaf2 100644 --- a/Source/Initialization/InjectorPosition.H +++ b/Source/Initialization/InjectorPosition.H @@ -72,8 +72,6 @@ struct InjectorPosition std::size_t sharedMemoryNeeded () const noexcept { return 0; } - bool useRandom () const noexcept; - AMREX_GPU_HOST_DEVICE amrex::XDim3 getPositionUnitBox (int i_part, int ref_fac=1) const noexcept diff --git a/Source/Initialization/InjectorPosition.cpp b/Source/Initialization/InjectorPosition.cpp deleted file mode 100644 index b77f26920..000000000 --- a/Source/Initialization/InjectorPosition.cpp +++ /dev/null @@ -1,19 +0,0 @@ -#include - -using namespace amrex; - -bool -InjectorPosition::useRandom () const noexcept -{ - switch (type) - { - case Type::random: - { - return true; - } - default: - { - return false; - } - }; -} diff --git a/Source/Initialization/Make.package b/Source/Initialization/Make.package index 3a924f207..2c6458b6d 100644 --- a/Source/Initialization/Make.package +++ b/Source/Initialization/Make.package @@ -4,7 +4,6 @@ CEXE_sources += PlasmaInjector.cpp CEXE_headers += PlasmaInjector.H CEXE_headers += InjectorPosition.H -CEXE_sources += InjectorPosition.cpp CEXE_headers += InjectorDensity.H CEXE_sources += InjectorDensity.cpp diff --git a/Source/Initialization/PlasmaInjector.H b/Source/Initialization/PlasmaInjector.H index d5c064192..c0d74ab0c 100644 --- a/Source/Initialization/PlasmaInjector.H +++ b/Source/Initialization/PlasmaInjector.H @@ -71,11 +71,6 @@ public: InjectorDensity* getInjectorDensity (); InjectorMomentum* getInjectorMomentum (); - bool useRandom () const noexcept { - return inj_pos->useRandom() or inj_rho->useRandom() - or inj_mom->useRandom(); - } - std::size_t sharedMemoryNeeded () const noexcept { return amrex::max(inj_pos->sharedMemoryNeeded(), diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 31ffadddf..b205ba274 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -430,15 +430,6 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) overlap_realbox.lo(1), overlap_realbox.lo(2))}; -#ifdef WARPX_RZ - { -#else - if (plasma_injector->useRandom()) { -#endif - amrex::Gpu::streamSynchronize(); - amrex::CheckSeedArraySizeAndResize(max_new_particles); - } - std::size_t shared_mem_bytes = plasma_injector->sharedMemoryNeeded(); int lrrfac = rrfac; -- cgit v1.2.3 From a85bb92ffd79ac3bd04970dd2af94b74fd194985 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Wed, 31 Jul 2019 12:22:20 -0700 Subject: comments in PPC --- Source/Particles/PhysicalParticleContainer.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index b205ba274..61902af5d 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -368,8 +368,13 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) const int grid_id = mfi.index(); const int tile_id = mfi.LocalTileIndex(); + // Max number of new particles, if particles are created in the whole + // overlap_box. All of them are created, and invalid ones are then + // discaded int max_new_particles = overlap_box.numPts() * num_ppc; + // If refine injection, build pointer dp_cellid that holds pointer to + // array of refined cell IDs. Vector cellid_v; if (refine_injection and lev == 0) { @@ -391,6 +396,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) amrex::AsyncArray cellid_aa(hp_cellid, cellid_v.size()); int const* dp_cellid = cellid_aa.data(); + // Update NextID to include particles created in this function int pid; #pragma omp critical (add_plasma_nextid) { @@ -433,6 +439,10 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) std::size_t shared_mem_bytes = plasma_injector->sharedMemoryNeeded(); int lrrfac = rrfac; + // Loop over all new particles and inject them (creates too many + // particles, in particular does not consider xmin, xmax etc.). + // The invalid ones are given negative ID and are deleted during the + // next redistribute. amrex::For(max_new_particles, [=] AMREX_GPU_DEVICE (int ip) noexcept { ParticleType& p = pp[ip]; -- cgit v1.2.3 From 254e048de10275870a92d2862c77d237679a3d11 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Wed, 31 Jul 2019 15:27:38 -0700 Subject: cleaning suggested in Remi's review --- Source/Particles/Gather/FieldGather.H | 20 ++++++++++---------- Source/Particles/PhysicalParticleContainer.cpp | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/Gather/FieldGather.H b/Source/Particles/Gather/FieldGather.H index a1d3baed7..be96dd393 100644 --- a/Source/Particles/Gather/FieldGather.H +++ b/Source/Particles/Gather/FieldGather.H @@ -52,33 +52,33 @@ void doGatherShapeN(const amrex::Real * const xp, [=] AMREX_GPU_DEVICE (long ip) { // --- Compute shape factors // x direction - // Get particle position after 1/2 push back in position - const amrex::Real xmid = (xp[ip]-xmin)*dxi; + // Get particle position + const amrex::Real x = (xp[ip]-xmin)*dxi; // Compute shape factors for node-centered quantities amrex::Real AMREX_RESTRICT sx [depos_order + 1]; // j: leftmost grid point (node-centered) that particle touches - const int j = compute_shape_factor(sx, xmid); + const int j = compute_shape_factor(sx, x); // Compute shape factors for cell-centered quantities amrex::Real AMREX_RESTRICT sx0[depos_order + 1 - lower_in_v]; // j0: leftmost grid point (cell-centered) that particle touches const int j0 = compute_shape_factor( - sx0, xmid-stagger_shift); + sx0, x-stagger_shift); #if (AMREX_SPACEDIM == 3) // y direction - const amrex::Real ymid= (yp[ip]-ymin)*dyi; + const amrex::Real y = (yp[ip]-ymin)*dyi; amrex::Real AMREX_RESTRICT sy [depos_order + 1]; - const int k = compute_shape_factor(sy, ymid); + const int k = compute_shape_factor(sy, y); amrex::Real AMREX_RESTRICT sy0[depos_order + 1 - lower_in_v]; const int k0 = compute_shape_factor( - sy0, ymid-stagger_shift); + sy0, y-stagger_shift); #endif // z direction - const amrex::Real zmid= (zp[ip]-zmin)*dzi; + const amrex::Real z = (zp[ip]-zmin)*dzi; amrex::Real AMREX_RESTRICT sz [depos_order + 1]; - const int l = compute_shape_factor(sz, zmid); + const int l = compute_shape_factor(sz, z); amrex::Real AMREX_RESTRICT sz0[depos_order + 1 - lower_in_v]; const int l0 = compute_shape_factor( - sz0, zmid-stagger_shift); + sz0, z-stagger_shift); // Set fields on particle to zero Exp[ip] = 0; diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 0fe5831a8..d1f399f9a 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1124,9 +1124,9 @@ PhysicalParticleContainer::FieldGather (int lev, // // Field Gather // - const int ll4symtry = false; - long lvect_fieldgathe = 64; #ifdef WARPX_RZ + const int ll4symtry = false; + long lvect_fieldgathe = 64; warpx_geteb_energy_conserving( &np, m_xp[thread_num].dataPtr(), @@ -1421,7 +1421,7 @@ PhysicalParticleContainer::Evolve (int lev, // BL_PROFILE_VAR_START(blp_pxr_fg); #ifdef WARPX_RZ - const int ll4symtry = false; + const int ll4symtry = false; long lvect_fieldgathe = 64; const std::array& xyzmin_grid = WarpX::LowerCorner(box, lev); const int* ixyzmin_grid = box.loVect(); -- cgit v1.2.3 From ae96f412292262174bd89a2fe4e559b517f5b0d1 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Fri, 2 Aug 2019 09:24:01 -0700 Subject: remove particles below or above density thresholds --- Source/Initialization/PlasmaInjector.H | 2 ++ Source/Initialization/PlasmaInjector.cpp | 3 +++ Source/Particles/PhysicalParticleContainer.cpp | 12 ++++++++++++ 3 files changed, 17 insertions(+) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Initialization/PlasmaInjector.H b/Source/Initialization/PlasmaInjector.H index 5ae2a967e..f7e86bff5 100644 --- a/Source/Initialization/PlasmaInjector.H +++ b/Source/Initialization/PlasmaInjector.H @@ -66,6 +66,8 @@ public: amrex::Real xmin, xmax; amrex::Real ymin, ymax; amrex::Real zmin, zmax; + amrex::Real density_min = 0; + amrex::Real density_max = std::numeric_limits::max(); InjectorPosition* getInjectorPosition (); InjectorDensity* getInjectorDensity (); diff --git a/Source/Initialization/PlasmaInjector.cpp b/Source/Initialization/PlasmaInjector.cpp index a94a253a3..280f9e58a 100644 --- a/Source/Initialization/PlasmaInjector.cpp +++ b/Source/Initialization/PlasmaInjector.cpp @@ -81,6 +81,9 @@ PlasmaInjector::PlasmaInjector (int ispecies, const std::string& name) pp.query("ymax", ymax); pp.query("zmax", zmax); + pp.query("density_min", density_min); + pp.query("density_max", density_max); + // parse charge and mass std::string charge_s; pp.get("charge", charge_s); diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index a34fb69e2..2d15c51b1 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -320,6 +320,8 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) Real gamma_boost = WarpX::gamma_boost; Real beta_boost = WarpX::beta_boost; Real t = WarpX::GetInstance().gett_new(lev); + Real density_min = plasma_injector->density_min; + Real density_max = plasma_injector->density_max; #ifdef WARPX_RZ bool radially_weighted = plasma_injector->radially_weighted; @@ -519,6 +521,11 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) } u = inj_mom->getMomentum(x, y, z); dens = inj_rho->getDensity(x, y, z); + // Remove particle if density below threshold + if ( (dens < density_min) || (dens > density_max) ){ + p.id() = -1; + return; + } } else { // Boosted-frame simulation // Since the user provides the density distribution @@ -546,6 +553,11 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) } // call `getDensity` with lab-frame parameters dens = inj_rho->getDensity(x, y, z0_lab); + // Remove particle if density below threshold + if ( (dens < density_min) || (dens > density_max) ){ + p.id() = -1; + return; + } // At this point u and dens are the lab-frame quantities // => Perform Lorentz transform dens = gamma_boost * dens * ( 1.0 - beta_boost*betaz_lab ); -- cgit v1.2.3 From 0c6441b735a4226bd9e1b51a7efdf48111745c1b Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Fri, 2 Aug 2019 11:31:39 -0700 Subject: add max plasma density --- Source/Particles/PhysicalParticleContainer.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 2d15c51b1..4ece22580 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -522,10 +522,12 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) u = inj_mom->getMomentum(x, y, z); dens = inj_rho->getDensity(x, y, z); // Remove particle if density below threshold - if ( (dens < density_min) || (dens > density_max) ){ + if ( dens < density_min ){ p.id() = -1; return; } + // Cut density if above threshold + dens = amrex::min(dens, density_max); } else { // Boosted-frame simulation // Since the user provides the density distribution @@ -554,10 +556,12 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) // call `getDensity` with lab-frame parameters dens = inj_rho->getDensity(x, y, z0_lab); // Remove particle if density below threshold - if ( (dens < density_min) || (dens > density_max) ){ + if ( dens < density_min ){ p.id() = -1; return; } + // Cut density if above threshold + dens = amrex::min(dens, density_max); // At this point u and dens are the lab-frame quantities // => Perform Lorentz transform dens = gamma_boost * dens * ( 1.0 - beta_boost*betaz_lab ); -- cgit v1.2.3 From 43ef6e29a1eeba61b9fdb95efb1bb1e301ba1b6d Mon Sep 17 00:00:00 2001 From: Dave Grote Date: Fri, 2 Aug 2019 17:25:35 -0700 Subject: PushP now uses FIeldGather --- Source/Particles/PhysicalParticleContainer.cpp | 7 +++++++ Source/Particles/RigidInjectedParticleContainer.cpp | 7 +++++++ 2 files changed, 14 insertions(+) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index a34fb69e2..89ea3c9f3 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1647,6 +1647,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, const int ll4symtry = false; long lvect_fieldgathe = 64; +#ifdef WARPX_RZ warpx_geteb_energy_conserving( &np, m_xp[thread_num].dataPtr(), @@ -1666,6 +1667,12 @@ PhysicalParticleContainer::PushP (int lev, Real dt, BL_TO_FORTRAN_ANYD(bzfab), &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, &lvect_fieldgathe, &WarpX::field_gathering_algo); +#else + int e_is_nodal = Ex.is_nodal() and Ey.is_nodal() and Ez.is_nodal(); + FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + &exfab, &eyfab, &ezfab, &bxfab, &byfab, &bzfab, + Ex.nGrow(), e_is_nodal, 0, np, thread_num, lev, lev); +#endif // This wraps the momentum advance so that inheritors can modify the call. // Extract pointers to the different particle quantities diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index 84c14dd7c..df809a5f0 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -427,6 +427,7 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, const int ll4symtry = false; long lvect_fieldgathe = 64; +#ifdef WARPX_RZ warpx_geteb_energy_conserving( &np, m_xp[thread_num].dataPtr(), @@ -446,6 +447,12 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, BL_TO_FORTRAN_ANYD(bzfab), &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, &lvect_fieldgathe, &WarpX::field_gathering_algo); +#else + int e_is_nodal = Ex.is_nodal() and Ey.is_nodal() and Ez.is_nodal(); + FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + &exfab, &eyfab, &ezfab, &bxfab, &byfab, &bzfab, + Ex.nGrow(), e_is_nodal, 0, np, thread_num, lev, lev); +#endif // Save the position and momenta, making copies auto uxp_save = uxp; -- cgit v1.2.3 From 1f54f88a93486d189c96333916edeb289631ec8f Mon Sep 17 00:00:00 2001 From: Dave Grote Date: Fri, 2 Aug 2019 17:46:46 -0700 Subject: Minor fix in PhysicalParticleContainer::PushP --- Source/Particles/PhysicalParticleContainer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 89ea3c9f3..0e6072287 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1641,13 +1641,13 @@ PhysicalParticleContainer::PushP (int lev, Real dt, // pti.GetPosition(m_xp[thread_num], m_yp[thread_num], m_zp[thread_num]); +#ifdef WARPX_RZ const std::array& xyzmin_grid = WarpX::LowerCorner(box, lev); const int* ixyzmin_grid = box.loVect(); const int ll4symtry = false; long lvect_fieldgathe = 64; -#ifdef WARPX_RZ warpx_geteb_energy_conserving( &np, m_xp[thread_num].dataPtr(), -- cgit v1.2.3 From 7b9aa6dd3108f10ce088614abccb239645a5dfe0 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Sat, 3 Aug 2019 17:27:40 -0700 Subject: add ionization_level component --- Source/Diagnostics/ParticleIO.cpp | 6 ++- Source/Evolve/WarpXEvolveEM.cpp | 3 ++ Source/Laser/LaserParticleContainer.cpp | 5 ++- Source/Particles/Deposition/CurrentDeposition.H | 14 ++++++- Source/Particles/PhysicalParticleContainer.cpp | 56 +++++++++++++++++++------ Source/Particles/WarpXParticleContainer.H | 2 + Source/Particles/WarpXParticleContainer.cpp | 41 ++++++++++-------- 7 files changed, 91 insertions(+), 36 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index f2a543ed5..acb8a934a 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -112,7 +112,11 @@ MultiParticleContainer::WritePlotFile (const std::string& dir) const real_names.push_back("uyold"); real_names.push_back("uzold"); } - + + if(pc->do_field_ionization){ + real_names.push_back("ionization_level"); + } + // Convert momentum to SI pc->ConvertUnits(ConvertDirection::WarpX_to_SI); // real_names contains a list of all particle attributes. diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index 32a4747db..aa180cc95 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -276,6 +276,9 @@ WarpX::EvolveEM (int numsteps) void WarpX::OneStep_nosub (Real cur_time) { + // Loop over species. For each ionizable species, create particles in + // product species. + // mypc->doFieldIonization(); // Push particle from x^{n} to x^{n+1} // from p^{n-1/2} to p^{n+1/2} // Deposit current j^{n+1/2} diff --git a/Source/Laser/LaserParticleContainer.cpp b/Source/Laser/LaserParticleContainer.cpp index 515aa1f5d..37e8f8bf2 100644 --- a/Source/Laser/LaserParticleContainer.cpp +++ b/Source/Laser/LaserParticleContainer.cpp @@ -504,13 +504,14 @@ LaserParticleContainer::Evolve (int lev, // Current Deposition // // Deposit inside domains - DepositCurrent(pti, wp, uxp, uyp, uzp, &jx, &jy, &jz, + Real* ion_lev = nullptr; + DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, &jx, &jy, &jz, 0, np_current, thread_num, lev, lev, dt); bool has_buffer = cjx; if (has_buffer){ // Deposit in buffers - DepositCurrent(pti, wp, uxp, uyp, uzp, cjx, cjy, cjz, + DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, cjx, cjy, cjz, np_current, np-np_current, thread_num, lev, lev-1, dt); } diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 9b9853902..b84f5a5bf 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -26,6 +26,7 @@ void doDepositionShapeN(const amrex::Real * const xp, const amrex::Real * const uxp, const amrex::Real * const uyp, const amrex::Real * const uzp, + const amrex::Real * const ion_lev, const amrex::Array4& jx_arr, const amrex::Array4& jy_arr, const amrex::Array4& jz_arr, @@ -36,6 +37,7 @@ void doDepositionShapeN(const amrex::Real * const xp, const amrex::Real stagger_shift, const amrex::Real q) { + const bool do_ionization = ion_lev; const amrex::Real dxi = 1.0/dx[0]; const amrex::Real dzi = 1.0/dx[2]; const amrex::Real dts2dx = 0.5*dt*dxi; @@ -61,7 +63,10 @@ void doDepositionShapeN(const amrex::Real * const xp, const amrex::Real gaminv = 1.0/std::sqrt(1.0 + uxp[ip]*uxp[ip]*clightsq + uyp[ip]*uyp[ip]*clightsq + uzp[ip]*uzp[ip]*clightsq); - const amrex::Real wq = q*wp[ip]; + amrex::Real wq = q*wp[ip]; + if (do_ionization){ + wq *= ion_lev[ip]; + } const amrex::Real vx = uxp[ip]*gaminv; const amrex::Real vy = uyp[ip]*gaminv; const amrex::Real vz = uzp[ip]*gaminv; @@ -156,6 +161,7 @@ void doEsirkepovDepositionShapeN (const amrex::Real * const xp, const amrex::Real * const uxp, const amrex::Real * const uyp, const amrex::Real * const uzp, + const amrex::Real * ion_lev, const amrex::Array4& Jx_arr, const amrex::Array4& Jy_arr, const amrex::Array4& Jz_arr, @@ -166,6 +172,7 @@ void doEsirkepovDepositionShapeN (const amrex::Real * const xp, const amrex::Dim3 lo, const amrex::Real q) { + const bool do_ionization = ion_lev; const amrex::Real dxi = 1.0/dx[0]; const amrex::Real dtsdx0 = dt*dxi; const amrex::Real xmin = xyzmin[0]; @@ -201,7 +208,10 @@ void doEsirkepovDepositionShapeN (const amrex::Real * const xp, + uzp[ip]*uzp[ip]*clightsq); // wqx, wqy wqz are particle current in each direction - const amrex::Real wq = q*wp[ip]; + amrex::Real wq = q*wp[ip]; + if (do_ionization){ + wq *= ion_lev[ip]; + } const amrex::Real wqx = wq*invdtdx; #if (AMREX_SPACEDIM == 3) const amrex::Real wqy = wq*invdtdy; diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 7d63bd8e7..d0db0fbdc 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -37,6 +37,12 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp // for this species. pp.query("do_boosted_frame_diags", do_boosted_frame_diags); + pp.query("do_field_ionization", do_field_ionization); + if (do_field_ionization){ + AddRealComp("ionization_level"); + plot_flags.resize(PIdx::nattribs + 1, 1); + } + pp.query("plot_species", plot_species); int do_user_plot_vars; do_user_plot_vars = pp.queryarr("plot_vars", plot_vars); @@ -73,6 +79,10 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core) : WarpXParticleContainer(amr_core, 0) { plasma_injector.reset(new PlasmaInjector()); + if (do_field_ionization){ + Print()<<"AddRealComp\n"; + AddRealComp("ionization_level"); + } } void PhysicalParticleContainer::InitData() @@ -196,17 +206,20 @@ PhysicalParticleContainer::CheckAndAddParticle(Real x, Real y, Real z, attribs[PIdx::uz] = u[2]; attribs[PIdx::w ] = weight; - if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) + if ( (NumRuntimeRealComps()>0) || (NumRuntimeIntComps()>0) ) { // need to create old values auto& particle_tile = DefineAndReturnParticleTile(0, 0, 0); - particle_tile.push_back_real(particle_comps["xold"], x); - particle_tile.push_back_real(particle_comps["yold"], y); - particle_tile.push_back_real(particle_comps["zold"], z); + if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) + { + particle_tile.push_back_real(particle_comps["xold"], x); + particle_tile.push_back_real(particle_comps["yold"], y); + particle_tile.push_back_real(particle_comps["zold"], z); - particle_tile.push_back_real(particle_comps["uxold"], u[0]); - particle_tile.push_back_real(particle_comps["uyold"], u[1]); - particle_tile.push_back_real(particle_comps["uzold"], u[2]); + particle_tile.push_back_real(particle_comps["uxold"], u[0]); + particle_tile.push_back_real(particle_comps["uyold"], u[1]); + particle_tile.push_back_real(particle_comps["uzold"], u[2]); + } } // add particle AddOneParticle(0, 0, 0, x, y, z, attribs); @@ -289,7 +302,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) const int grid_id = mfi.index(); const int tile_id = mfi.LocalTileIndex(); GetParticles(lev)[std::make_pair(grid_id, tile_id)]; - if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) { + if ( (NumRuntimeRealComps()>0) || (NumRuntimeIntComps()>0) ) { DefineAndReturnParticleTile(lev, grid_id, tile_id); } } @@ -416,10 +429,12 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) auto& particle_tile = GetParticles(lev)[std::make_pair(grid_id,tile_id)]; bool do_boosted = false; - if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) { - do_boosted = true; + if ( (NumRuntimeRealComps()>0) || (NumRuntimeIntComps()>0) ) { DefineAndReturnParticleTile(lev, grid_id, tile_id); } + do_boosted = WarpX::do_boosted_frame_diagnostic + && do_boosted_frame_diags; + auto old_size = particle_tile.GetArrayOfStructs().size(); auto new_size = old_size + max_new_particles; particle_tile.resize(new_size); @@ -439,7 +454,11 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) pb[4] = soa.GetRealData(particle_comps["uyold"]).data() + old_size; pb[5] = soa.GetRealData(particle_comps["uzold"]).data() + old_size; } - + Real* pi; + if (do_field_ionization) { + pi = soa.GetRealData(particle_comps[ "ionization_level"]).data() + old_size; + } + const GpuArray overlap_corner {AMREX_D_DECL(overlap_realbox.lo(0), overlap_realbox.lo(1), @@ -568,6 +587,10 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) u.z = gamma_boost * ( u.z -beta_boost*gamma_lab ); } + if (do_field_ionization) { + pi[ip] = 2.; // species_ionization_level; + } + u.x *= PhysConst::c; u.y *= PhysConst::c; u.z *= PhysConst::c; @@ -1351,13 +1374,20 @@ PhysicalParticleContainer::Evolve (int lev, lev, lev-1, dt); } } else { + Real* AMREX_RESTRICT ion_lev; + if (do_field_ionization){ + ion_lev = pti.GetAttribs(particle_comps["ionization_level"]).dataPtr(); + } else { + ion_lev = nullptr; + } + // Deposit inside domains - DepositCurrent(pti, wp, uxp, uyp, uzp, &jx, &jy, &jz, + DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, &jx, &jy, &jz, 0, np_current, thread_num, lev, lev, dt); if (has_buffer){ // Deposit in buffers - DepositCurrent(pti, wp, uxp, uyp, uzp, cjx, cjy, cjz, + DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, cjx, cjy, cjz, np_current, np-np_current, thread_num, lev, lev-1, dt); } diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 7cf53260a..a7b80ce8d 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -180,6 +180,7 @@ public: RealVector& uxp, RealVector& uyp, RealVector& uzp, + amrex::Real* ion_lev, amrex::MultiFab* jx, amrex::MultiFab* jy, amrex::MultiFab* jz, @@ -289,6 +290,7 @@ protected: // This is currently required because continuous injection does not // support all features allowed by direct injection. int do_continuous_injection = 0; + int do_field_ionization = 0; int do_boosted_frame_diags = 1; diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 89f233b2c..bce86a925 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -237,12 +237,14 @@ WarpXParticleContainer::AddNParticles (int lev, p.pos(1) = z[i]; #endif - if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) - { - auto& ptile = DefineAndReturnParticleTile(0, 0, 0); - ptile.push_back_real(particle_comps["xold"], x[i]); - ptile.push_back_real(particle_comps["yold"], y[i]); - ptile.push_back_real(particle_comps["zold"], z[i]); + if ( (NumRuntimeRealComps()>0) || (NumRuntimeIntComps()>0) ){ + auto& ptile = DefineAndReturnParticleTile(0, 0, 0); + if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) + { + ptile.push_back_real(particle_comps["xold"], x[i]); + ptile.push_back_real(particle_comps["yold"], y[i]); + ptile.push_back_real(particle_comps["zold"], z[i]); + } } particle_tile.push_back(p); @@ -255,12 +257,14 @@ WarpXParticleContainer::AddNParticles (int lev, particle_tile.push_back_real(PIdx::uy, vy + ibegin, vy + iend); particle_tile.push_back_real(PIdx::uz, vz + ibegin, vz + iend); - if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) - { - auto& ptile = DefineAndReturnParticleTile(0, 0, 0); - ptile.push_back_real(particle_comps["uxold"], vx + ibegin, vx + iend); - ptile.push_back_real(particle_comps["uyold"], vy + ibegin, vy + iend); - ptile.push_back_real(particle_comps["uzold"], vz + ibegin, vz + iend); + if ( (NumRuntimeRealComps()>0) || (NumRuntimeIntComps()>0) ){ + auto& ptile = DefineAndReturnParticleTile(0, 0, 0); + if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) + { + ptile.push_back_real(particle_comps["uxold"], vx + ibegin, vx + iend); + ptile.push_back_real(particle_comps["uyold"], vy + ibegin, vy + iend); + ptile.push_back_real(particle_comps["uzold"], vz + ibegin, vz + iend); + } } for (int comp = PIdx::uz+1; comp < PIdx::nattribs; ++comp) @@ -432,6 +436,7 @@ void WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, RealVector& wp, RealVector& uxp, RealVector& uyp, RealVector& uzp, + Real* ion_lev, MultiFab* jx, MultiFab* jy, MultiFab* jz, const long offset, const long np_to_depose, int thread_num, int lev, int depos_lev, @@ -514,34 +519,34 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { if (WarpX::nox == 1){ doEsirkepovDepositionShapeN<1>(xp, yp, zp, wp.dataPtr(), uxp.dataPtr(), - uyp.dataPtr(), uzp.dataPtr(), jx_arr, jy_arr, + uyp.dataPtr(), uzp.dataPtr(), ion_lev, jx_arr, jy_arr, jz_arr, np_to_depose, dt, dx, xyzmin, lo, q); } else if (WarpX::nox == 2){ doEsirkepovDepositionShapeN<2>(xp, yp, zp, wp.dataPtr(), uxp.dataPtr(), - uyp.dataPtr(), uzp.dataPtr(), jx_arr, jy_arr, + uyp.dataPtr(), uzp.dataPtr(), ion_lev, jx_arr, jy_arr, jz_arr, np_to_depose, dt, dx, xyzmin, lo, q); } else if (WarpX::nox == 3){ doEsirkepovDepositionShapeN<3>(xp, yp, zp, wp.dataPtr(), uxp.dataPtr(), - uyp.dataPtr(), uzp.dataPtr(), jx_arr, jy_arr, + uyp.dataPtr(), uzp.dataPtr(), ion_lev, jx_arr, jy_arr, jz_arr, np_to_depose, dt, dx, xyzmin, lo, q); } } else { if (WarpX::nox == 1){ doDepositionShapeN<1>(xp, yp, zp, wp.dataPtr(), uxp.dataPtr(), - uyp.dataPtr(), uzp.dataPtr(), jx_arr, jy_arr, + uyp.dataPtr(), uzp.dataPtr(), ion_lev, jx_arr, jy_arr, jz_arr, np_to_depose, dt, dx, xyzmin, lo, stagger_shift, q); } else if (WarpX::nox == 2){ doDepositionShapeN<2>(xp, yp, zp, wp.dataPtr(), uxp.dataPtr(), - uyp.dataPtr(), uzp.dataPtr(), jx_arr, jy_arr, + uyp.dataPtr(), uzp.dataPtr(), ion_lev, jx_arr, jy_arr, jz_arr, np_to_depose, dt, dx, xyzmin, lo, stagger_shift, q); } else if (WarpX::nox == 3){ doDepositionShapeN<3>(xp, yp, zp, wp.dataPtr(), uxp.dataPtr(), - uyp.dataPtr(), uzp.dataPtr(), jx_arr, jy_arr, + uyp.dataPtr(), uzp.dataPtr(), ion_lev, jx_arr, jy_arr, jz_arr, np_to_depose, dt, dx, xyzmin, lo, stagger_shift, q); } -- cgit v1.2.3 From 58d728b3160c69ce8e9f117b255e7b057cc419fe Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Sat, 3 Aug 2019 17:45:23 -0700 Subject: add ionization initialization --- Source/Evolve/WarpXEvolveEM.cpp | 2 +- Source/Particles/MultiParticleContainer.H | 5 ++ Source/Particles/MultiParticleContainer.cpp | 51 ++++++++++++++++++++ Source/Particles/PhysicalParticleContainer.H | 22 +++++++++ Source/Particles/PhysicalParticleContainer.cpp | 66 ++++++++++++++++++++++---- Source/Particles/WarpXParticleContainer.H | 14 +++++- 6 files changed, 149 insertions(+), 11 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index aa180cc95..4bd9d4566 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -278,7 +278,7 @@ WarpX::OneStep_nosub (Real cur_time) { // Loop over species. For each ionizable species, create particles in // product species. - // mypc->doFieldIonization(); + mypc->doFieldIonization(); // Push particle from x^{n} to x^{n+1} // from p^{n-1/2} to p^{n+1/2} // Deposit current j^{n+1/2} diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 7c9ede411..4ea9c2763 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -128,6 +128,8 @@ public: /// std::unique_ptr GetChargeDensity(int lev, bool local = false); + void doFieldIonization(); + void Checkpoint (const std::string& dir) const; void WritePlotFile (const std::string& dir) const; @@ -207,6 +209,9 @@ private: void ReadParameters (); + void mapSpeciesProduct (); + int getSpeciesID (std::string product_str); + // Number of species dumped in BoostedFrameDiagnostics int nspecies_boosted_frame_diags = 0; // map_species_boosted_frame_diags[i] is the species ID in diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 982e04e39..16c9b9762 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -1,10 +1,12 @@ #include #include #include +#include #include #include #include +#include using namespace amrex; @@ -29,6 +31,10 @@ MultiParticleContainer::MultiParticleContainer (AmrCore* amr_core) pc_tmp.reset(new PhysicalParticleContainer(amr_core)); + // For each species, get the ID of its product species. + // This is used for ionization and pair creation processes. + mapSpeciesProduct(); + // Compute the number of species for which lab-frame data is dumped // nspecies_lab_frame_diags, and map their ID to MultiParticleContainer // particle IDs in map_species_lab_diags. @@ -461,3 +467,48 @@ MultiParticleContainer::doContinuousInjection() const } return warpx_do_continuous_injection; } + +/* \brief Get ID of product species of each species. + * The users specifies the name of the product species, + * this routine get its ID. + */ +void +MultiParticleContainer::mapSpeciesProduct() +{ + for (int i=0; iionization_product_name and store its ID into + // pc->ionization_product. + if (pc->do_field_ionization){ + int i_product = getSpeciesID(pc->ionization_product_name); + pc->ionization_product = i_product; + } + } +} + +/* \brief Given a species name, return its ID. + */ +int +MultiParticleContainer::getSpeciesID(std::string product_str) +{ + int i_product; + bool found = 0; + // Loop over species + for (int i=0; i, 3> >& E, const amrex::Vector > > >& masks) override; @@ -105,6 +109,24 @@ public: void SplitParticles(int lev); + virtual void copyParticles(int lev) override; + + virtual int doFieldIonization( + const int lev, + amrex::Cuda::ManagedDeviceVector& x_buf, + amrex::Cuda::ManagedDeviceVector& y_buf, + amrex::Cuda::ManagedDeviceVector& z_buf, + amrex::Cuda::ManagedDeviceVector& ux_buf, + amrex::Cuda::ManagedDeviceVector& uy_buf, + amrex::Cuda::ManagedDeviceVector& uz_buf, + amrex::Cuda::ManagedDeviceVector& ex_buf, + amrex::Cuda::ManagedDeviceVector& ey_buf, + amrex::Cuda::ManagedDeviceVector& ez_buf, + amrex::Cuda::ManagedDeviceVector& bx_buf, + amrex::Cuda::ManagedDeviceVector& by_buf, + amrex::Cuda::ManagedDeviceVector& bz_buf, + amrex::Cuda::ManagedDeviceVector& w_buf) override; + // Inject particles in Box 'part_box' virtual void AddParticles (int lev); diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index d0db0fbdc..d2fcc93e0 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -38,10 +39,10 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp pp.query("do_boosted_frame_diags", do_boosted_frame_diags); pp.query("do_field_ionization", do_field_ionization); - if (do_field_ionization){ - AddRealComp("ionization_level"); - plot_flags.resize(PIdx::nattribs + 1, 1); - } + // If do_field_ionization, read initialization data from input file and + // read ionization energies from table. + if (do_field_ionization) + InitIonizationModule(); pp.query("plot_species", plot_species); int do_user_plot_vars; @@ -79,10 +80,12 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core) : WarpXParticleContainer(amr_core, 0) { plasma_injector.reset(new PlasmaInjector()); +/* if (do_field_ionization){ Print()<<"AddRealComp\n"; AddRealComp("ionization_level"); } +*/ } void PhysicalParticleContainer::InitData() @@ -1604,25 +1607,38 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti, copy_attribs(pti, x, y, z); } + Real* AMREX_RESTRICT ion_lev = nullptr; + if (do_field_ionization){ + ion_lev = pti.GetAttribs(particle_comps["ionization_level"]).dataPtr(); + } + // Loop over the particles and update their momentum const Real q = this->charge; const Real m = this-> mass; if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Boris){ - amrex::ParallelFor( pti.numParticles(), + amrex::ParallelFor( + pti.numParticles(), [=] AMREX_GPU_DEVICE (long i) { + Real qp = q; + if (ion_lev){ qp *= ion_lev[i]; } UpdateMomentumBoris( ux[i], uy[i], uz[i], gi[i], - Ex[i], Ey[i], Ez[i], Bx[i], By[i], Bz[i], q, m, dt); + Ex[i], Ey[i], Ez[i], Bx[i], + By[i], Bz[i], qp, m, dt); UpdatePosition( x[i], y[i], z[i], ux[i], uy[i], uz[i], dt ); } ); } else if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Vay) { - amrex::ParallelFor( pti.numParticles(), + amrex::ParallelFor( + pti.numParticles(), [=] AMREX_GPU_DEVICE (long i) { + Real qp = q; + if (ion_lev){ qp *= ion_lev[i]; } UpdateMomentumVay( ux[i], uy[i], uz[i], gi[i], - Ex[i], Ey[i], Ez[i], Bx[i], By[i], Bz[i], q, m, dt); + Ex[i], Ey[i], Ez[i], Bx[i], + By[i], Bz[i], qp, m, dt); UpdatePosition( x[i], y[i], z[i], - ux[i], uy[i], uz[i], dt ); + ux[i], uy[i], uz[i], dt ); } ); } else { @@ -2058,3 +2074,35 @@ PhysicalParticleContainer::FieldGather (WarpXParIter& pti, } } } + + +void PhysicalParticleContainer::InitIonizationModule() +{ + ParmParse pp(species_name); + pp.query("species_ionization_level", species_ionization_level); + pp.get("ionization_product", ionization_product_name); + pp.get("physical_element", physical_element); + // Add Real component for ionization level + AddRealComp("ionization_level"); + plot_flags.resize(PIdx::nattribs + 1, 1); + // Get atomic number and ionization energies from file + int ion_element_id = ion_map_ids[physical_element]; + ion_atomic_number = ion_atomic_numbers[ion_element_id]; + ionization_energies.resize(ion_atomic_number); + int offset = ion_energy_offsets[ion_element_id]; + for(int i=0; i particle_comps; @@ -290,8 +292,18 @@ protected: // This is currently required because continuous injection does not // support all features allowed by direct injection. int do_continuous_injection = 0; - int do_field_ionization = 0; + int do_field_ionization = 0; + int ionization_product; + std::string ionization_product_name; + int ion_atomic_number; + int species_ionization_level = 0; + amrex::Gpu::ManagedVector ionization_energies; + amrex::Gpu::ManagedVector adk_power; + amrex::Gpu::ManagedVector adk_prefactor; + amrex::Gpu::ManagedVector adk_exp_prefactor; + std::string physical_element; + int do_boosted_frame_diags = 1; amrex::Vector local_rho; -- cgit v1.2.3 From 43c06604530ac2c532d1c731fd60bbb5570446e1 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Sat, 3 Aug 2019 17:56:59 -0700 Subject: add ionization table data --- Source/Particles/PhysicalParticleContainer.H | 18 -- Source/Particles/PhysicalParticleContainer.cpp | 2 +- Source/Utils/IonizationEnergiesTable.H | 136 ++++++++ Source/Utils/Make.package | 1 + Source/Utils/WarpXConst.H | 25 +- Source/Utils/atomic_data.txt | 427 +++++++++++++++++++++++++ Source/Utils/write_atomic_data_cpp.py | 67 ++++ 7 files changed, 647 insertions(+), 29 deletions(-) create mode 100644 Source/Utils/IonizationEnergiesTable.H create mode 100644 Source/Utils/atomic_data.txt create mode 100644 Source/Utils/write_atomic_data_cpp.py (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index ddf7d1253..a6cf9d81f 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -108,25 +108,7 @@ public: virtual void PostRestart () final {} void SplitParticles(int lev); - - virtual void copyParticles(int lev) override; - virtual int doFieldIonization( - const int lev, - amrex::Cuda::ManagedDeviceVector& x_buf, - amrex::Cuda::ManagedDeviceVector& y_buf, - amrex::Cuda::ManagedDeviceVector& z_buf, - amrex::Cuda::ManagedDeviceVector& ux_buf, - amrex::Cuda::ManagedDeviceVector& uy_buf, - amrex::Cuda::ManagedDeviceVector& uz_buf, - amrex::Cuda::ManagedDeviceVector& ex_buf, - amrex::Cuda::ManagedDeviceVector& ey_buf, - amrex::Cuda::ManagedDeviceVector& ez_buf, - amrex::Cuda::ManagedDeviceVector& bx_buf, - amrex::Cuda::ManagedDeviceVector& by_buf, - amrex::Cuda::ManagedDeviceVector& bz_buf, - amrex::Cuda::ManagedDeviceVector& w_buf) override; - // Inject particles in Box 'part_box' virtual void AddParticles (int lev); diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index d2fcc93e0..0183a44b9 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -2079,7 +2079,7 @@ PhysicalParticleContainer::FieldGather (WarpXParIter& pti, void PhysicalParticleContainer::InitIonizationModule() { ParmParse pp(species_name); - pp.query("species_ionization_level", species_ionization_level); + pp.query("ionization_level", species_ionization_level); pp.get("ionization_product", ionization_product_name); pp.get("physical_element", physical_element); // Add Real component for ionization level diff --git a/Source/Utils/IonizationEnergiesTable.H b/Source/Utils/IonizationEnergiesTable.H new file mode 100644 index 000000000..475236abd --- /dev/null +++ b/Source/Utils/IonizationEnergiesTable.H @@ -0,0 +1,136 @@ +#include +#include + +#ifndef WARPX_IONIZATION_TABLE_H_ +#define WARPX_IONIZATION_TABLE_H_ + +std::map ion_map_ids = { + {"H", 0}, + {"He", 1}, + {"Li", 2}, + {"Be", 3}, + {"B", 4}, + {"C", 5}, + {"N", 6}, + {"O", 7}, + {"F", 8}, + {"Ne", 9}, + {"Na", 10}, + {"Mg", 11}, + {"Al", 12}, + {"Si", 13}, + {"P", 14}, + {"S", 15}, + {"Cl", 16}, + {"Ar", 17}, + {"Kr", 18}, + {"Rb", 19}, + {"Xe", 20}, + {"Rn", 21} }; + +const int nelements = 22; + +const int ion_atomic_numbers[nelements] = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, 36, 37, + 54, 86}; + +const int ion_energy_offsets[nelements] = { + 0, 1, 3, 6, 10, 15, 21, 28, 36, 45, + 55, 66, 78, 91, 105, 120, 136, 153, 171, 207, + 244, 298}; + +const int energies_tab_length = 384; + +const amrex::Real table_ionization_energies[energies_tab_length]{ + // H + 13.59843449, + // He + 24.58738880, 54.4177650, + // Li + 5.39171495, 75.6400964, 122.4543581, + // Be + 9.322699, 18.21115, 153.896203, 217.7185843, + // B + 8.298019, 25.15483, 37.93058, 259.3715, 340.226020, + // C + 11.2602880, 24.383154, 47.88778, 64.49352, 392.090515, 489.993194, + // N + 14.53413, 29.60125, 47.4453, 77.4735, 97.8901, 552.06732, 667.046116, + // O + 13.618055, 35.12112, 54.93554, 77.41350, 113.8990, 138.1189, 739.32682, + 871.40988, + // F + 17.42282, 34.97081, 62.70798, 87.175, 114.249, 157.16311, 185.1868, + 953.89804, 1103.11747, + // Ne + 21.564540, 40.96297, 63.4233, 97.1900, 126.247, 157.934, 207.271, + 239.0970, 1195.80783, 1362.19915, + // Na + 5.1390769, 47.28636, 71.6200, 98.936, 138.404, 172.23, 208.504, + 264.192, 299.856, 1465.13449, 1648.70218, + // Mg + 7.646236, 15.035271, 80.1436, 109.2654, 141.33, 186.76, 225.02, + 265.924, 327.99, 367.489, 1761.80487, 1962.66365, + // Al + 5.985769, 18.82855, 28.447642, 119.9924, 153.8252, 190.49, 241.76, + 284.64, 330.21, 398.65, 442.005, 2085.97700, 2304.14005, + // Si + 8.15168, 16.34585, 33.49300, 45.14179, 166.767, 205.279, 246.57, + 303.59, 351.28, 401.38, 476.273, 523.415, 2437.65813, 2673.17753, + // P + 10.486686, 19.76949, 30.20264, 51.44387, 65.02511, 220.430, 263.57, + 309.60, 372.31, 424.40, 479.44, 560.62, 611.741, 2816.90876, + 3069.8415, + // S + 10.36001, 23.33788, 34.86, 47.222, 72.5945, 88.0529, 280.954, + 328.794, 379.84, 447.7, 504.55, 564.41, 651.96, 706.994, + 3223.7807, 3494.1879, + // Cl + 12.967632, 23.81364, 39.80, 53.24, 67.68, 96.94, 114.2013, + 348.306, 400.851, 456.7, 530.0, 591.58, 656.30, 750.23, + 809.198, 3658.3437, 3946.2909, + // Ar + 15.7596117, 27.62967, 40.735, 59.58, 74.84, 91.290, 124.41, + 143.4567, 422.60, 479.76, 540.4, 619.0, 685.5, 755.13, + 855.5, 918.375, 4120.6656, 4426.2228, + // Kr + 13.9996053, 24.35984, 35.838, 50.85, 64.69, 78.49, 109.13, + 125.802, 233.0, 268, 308, 350, 391, 446, + 492, 540, 591, 640, 785, 831.6, 882.8, + 945, 999.0, 1042, 1155.0, 1205.23, 2928.9, 3072, + 3228, 3380, 3584, 3752.0, 3971, 4109.083, 17296.420, + 17936.209, + // Rb + 4.1771280, 27.28954, 39.247, 52.20, 68.44, 82.9, 98.67, + 132.79, 150.628, 277.12, 313.1, 356.0, 400, 443, + 502, 550, 601, 654, 706.0, 857, 905.3, + 958.9, 1024, 1080, 1125, 1242.5, 1294.57, 3133.3, + 3281, 3443, 3600, 3815, 3988, 4214, 4356.865, + 18305.884, 18965.516, + // Xe + 12.1298436, 20.975, 31.05, 42.20, 54.1, 66.703, 91.6, + 105.9778, 179.84, 202.0, 229.02, 255.0, 281, 314, + 343, 374, 404, 434, 549, 582, 616, + 650, 700, 736, 818, 857.0, 1493, 1571, + 1653, 1742, 1826, 1919, 2023, 2113, 2209, + 2300, 2556, 2637, 2726, 2811, 2975, 3068, + 3243, 3333.8, 7660, 7889, 8144, 8382, 8971, + 9243, 9581, 9810.37, 40271.724, 41299.71, + // Rn + 10.74850, 21.4, 29.4, 36.9, 52.9, 64.0, 88.0, + 102.0, 154.0, 173.9, 195.0, 218.0, 240, 264, + 293, 317, 342, 367, 488, 520, 550, + 580, 640, 680, 760, 800, 850, 920, + 980, 1050, 1110, 1180, 1250, 1310, 1390, + 1460, 1520, 1590, 1660, 1720, 2033, 2094, + 2158, 2227, 2293, 2357, 2467, 2535, 2606, + 2674, 2944, 3010, 3082, 3149, 3433, 3510, + 3699, 3777, 6169, 6318, 6476, 6646, 6807, + 6964, 7283, 7450, 7630, 7800, 8260, 8410, + 8570, 8710, 9610, 9780, 10120, 10290, 21770, + 22160, 22600, 22990, 26310, 26830, 27490, 27903.1, + 110842.0, 112843.7 }; + +#endif // #ifndef WARPX_IONIZATION_TABLE_H_ + diff --git a/Source/Utils/Make.package b/Source/Utils/Make.package index cd335dbcf..389b3670a 100644 --- a/Source/Utils/Make.package +++ b/Source/Utils/Make.package @@ -7,6 +7,7 @@ CEXE_headers += WarpXUtil.H CEXE_headers += WarpXAlgorithmSelection.H CEXE_sources += WarpXAlgorithmSelection.cpp CEXE_headers += NCIGodfreyTables.H +CEXE_headers += IonizationEnergiesTable.H INCLUDE_LOCATIONS += $(WARPX_HOME)/Source/Utils VPATH_LOCATIONS += $(WARPX_HOME)/Source/Utils diff --git a/Source/Utils/WarpXConst.H b/Source/Utils/WarpXConst.H index 4e226e55f..bfb67b3d5 100644 --- a/Source/Utils/WarpXConst.H +++ b/Source/Utils/WarpXConst.H @@ -3,20 +3,25 @@ #include -// Physical constants -namespace PhysConst +// Math constants +namespace MathConst { - static constexpr amrex::Real c = 299792458.; - static constexpr amrex::Real ep0 = 8.854187817e-12; - static constexpr amrex::Real mu0 = 1.2566370614359173e-06; - static constexpr amrex::Real q_e = 1.6021764620000001e-19; - static constexpr amrex::Real m_e = 9.10938291e-31; - static constexpr amrex::Real m_p = 1.6726231000000001e-27; + static constexpr amrex::Real pi = 3.14159265358979323846; } -namespace MathConst +// Physical constants +namespace PhysConst { - static constexpr amrex::Real pi = 3.14159265358979323846; + static constexpr amrex::Real c = 299792458.; + static constexpr amrex::Real ep0 = 8.854187817e-12; + static constexpr amrex::Real mu0 = 1.2566370614359173e-06; + static constexpr amrex::Real q_e = 1.6021764620000001e-19; + static constexpr amrex::Real m_e = 9.10938291e-31; + static constexpr amrex::Real m_p = 1.6726231000000001e-27; + static constexpr amrex::Real hbar = 1.05457180010e-34; + static constexpr amrex::Real alpha = mu0/(4*MathConst::pi)*q_e*q_e*c/hbar; + static constexpr amrex::Real r_e = 1./(4*MathConst::pi*ep0) * q_e*q_e/(m_e*c*c); + // static constexpr amrex::Real alpha = 0.0072973525693; } #endif diff --git a/Source/Utils/atomic_data.txt b/Source/Utils/atomic_data.txt new file mode 100644 index 000000000..140f5a26a --- /dev/null +++ b/Source/Utils/atomic_data.txt @@ -0,0 +1,427 @@ +# Reference: +# Kramida, A., Ralchenko, Yu., Reader, J., and NIST ASD Team (2014). +# NIST Atomic Spectra Database (ver. 5.2), [Online]. +# Available: http://physics.nist.gov/asd [2017, March 3]. + +# License (from https://www.nist.gov/director/licensing): +# This data was developed by employees of the National Institute of Standards +# and Technology (NIST), an agency of the Federal Government. Pursuant to title +# 17 United States Code Section 105, works of NIST employees are not subject to +# copyright protection in the United States and are considered to be in the +# public domain. +# The data is provided by NIST as a public service and is expressly provided +# "AS IS." NIST MAKES NO WARRANTY OF ANY KIND, EXPRESS, IMPLIED OR STATUTORY, +# INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTY OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT AND DATA ACCURACY. NIST +# does not warrant or make any representations regarding the use of the data or +# the results thereof, including but not limited to the correctness, accuracy, +# reliability or usefulness of the data. NIST SHALL NOT BE LIABLE AND YOU +# HEREBY RELEASE NIST FROM LIABILITY FOR ANY INDIRECT, CONSEQUENTIAL, SPECIAL, +# OR INCIDENTAL DAMAGES (INCLUDING DAMAGES FOR LOSS OF BUSINESS PROFITS, +# BUSINESS INTERRUPTION, LOSS OF BUSINESS INFORMATION, AND THE LIKE), WHETHER +# ARISING IN TORT, CONTRACT, OR OTHERWISE, ARISING FROM OR RELATING TO THE DATA +# (OR THE USE OF OR INABILITY TO USE THIS DATA), EVEN IF NIST HAS BEEN ADVISED +# OF THE POSSIBILITY OF SUCH DAMAGES. +# To the extent that NIST may hold copyright in countries other than the United +# States, you are hereby granted the non-exclusive irrevocable and unconditional +# right to print, publish, prepare derivative works and distribute the NIST +# data, in any medium, or authorize others to do so on your behalf, on a +# royalty-free basis throughout the world. +# You may improve, modify, and create derivative works of the data or any +# portion of the data, and you may copy and distribute such modifications or +# works. Modified works should carry a notice stating that you changed the data +# and should note the date and nature of any such change. Please explicitly +# acknowledge the National Institute of Standards and Technology as the source +# of the data: Data citation recommendations are provided below. +# Permission to use this data is contingent upon your acceptance of the terms +# of this agreement and upon your providing appropriate acknowledgments of +# NIST's creation of the data. + +----------------------------------------------------------------------------- +At. num | Sp. Name | Ion Charge | Ionization Energy (eV) | +--------|---------------|------------|--------------------------------------| + 1 | H I | 0 | (13.59843449) | + 2 | He I | 0 | 24.58738880 | + 2 | He II | +1 | (54.4177650) | + 3 | Li I | 0 | 5.39171495 | + 3 | Li II | +1 | [75.6400964] | + 3 | Li III | +2 | (122.4543581) | + 4 | Be I | 0 | 9.322699 | + 4 | Be II | +1 | 18.21115 | + 4 | Be III | +2 | [153.896203] | + 4 | Be IV | +3 | (217.7185843) | + 5 | B I | 0 | 8.298019 | + 5 | B II | +1 | 25.15483 | + 5 | B III | +2 | 37.93058 | + 5 | B IV | +3 | 259.3715 | + 5 | B V | +4 | (340.226020) | + 6 | C I | 0 | 11.2602880 | + 6 | C II | +1 | 24.383154 | + 6 | C III | +2 | 47.88778 | + 6 | C IV | +3 | 64.49352 | + 6 | C V | +4 | [392.090515] | + 6 | C VI | +5 | (489.993194) | + 7 | N I | 0 | 14.53413 | + 7 | N II | +1 | [29.60125] | + 7 | N III | +2 | [47.4453] | + 7 | N IV | +3 | 77.4735 | + 7 | N V | +4 | 97.8901 | + 7 | N VI | +5 | [552.06732] | + 7 | N VII | +6 | (667.046116) | + 8 | O I | 0 | 13.618055 | + 8 | O II | +1 | 35.12112 | + 8 | O III | +2 | [54.93554] | + 8 | O IV | +3 | 77.41350 | + 8 | O V | +4 | 113.8990 | + 8 | O VI | +5 | [138.1189] | + 8 | O VII | +6 | [739.32682] | + 8 | O VIII | +7 | (871.40988) | + 9 | F I | 0 | 17.42282 | + 9 | F II | +1 | 34.97081 | + 9 | F III | +2 | [62.70798] | + 9 | F IV | +3 | [87.175] | + 9 | F V | +4 | [114.249] | + 9 | F VI | +5 | 157.16311 | + 9 | F VII | +6 | [185.1868] | + 9 | F VIII | +7 | [953.89804] | + 9 | F IX | +8 | (1103.11747) | + 10 | Ne I | 0 | 21.564540 | + 10 | Ne II | +1 | 40.96297 | + 10 | Ne III | +2 | 63.4233 | + 10 | Ne IV | +3 | [97.1900] | + 10 | Ne V | +4 | 126.247 | + 10 | Ne VI | +5 | 157.934 | + 10 | Ne VII | +6 | 207.271 | + 10 | Ne VIII | +7 | [239.0970] | + 10 | Ne IX | +8 | [1195.80783] | + 10 | Ne X | +9 | (1362.19915) | + 11 | Na I | 0 | 5.1390769 | + 11 | Na II | +1 | 47.28636 | + 11 | Na III | +2 | [71.6200] | + 11 | Na IV | +3 | [98.936] | + 11 | Na V | +4 | 138.404 | + 11 | Na VI | +5 | [172.23] | + 11 | Na VII | +6 | [208.504] | + 11 | Na VIII | +7 | [264.192] | + 11 | Na IX | +8 | (299.856) | + 11 | Na X | +9 | (1465.13449) | + 11 | Na XI | +10 | (1648.70218) | + 12 | Mg I | 0 | 7.646236 | + 12 | Mg II | +1 | 15.035271 | + 12 | Mg III | +2 | 80.1436 | + 12 | Mg IV | +3 | [109.2654] | + 12 | Mg V | +4 | [141.33] | + 12 | Mg VI | +5 | [186.76] | + 12 | Mg VII | +6 | [225.02] | + 12 | Mg VIII | +7 | [265.924] | + 12 | Mg IX | +8 | [327.99] | + 12 | Mg X | +9 | (367.489) | + 12 | Mg XI | +10 | (1761.80487) | + 12 | Mg XII | +11 | (1962.66365) | + 13 | Al I | 0 | 5.985769 | + 13 | Al II | +1 | 18.82855 | + 13 | Al III | +2 | 28.447642 | + 13 | Al IV | +3 | 119.9924 | + 13 | Al V | +4 | [153.8252] | + 13 | Al VI | +5 | [190.49] | + 13 | Al VII | +6 | [241.76] | + 13 | Al VIII | +7 | [284.64] | + 13 | Al IX | +8 | [330.21] | + 13 | Al X | +9 | [398.65] | + 13 | Al XI | +10 | (442.005) | + 13 | Al XII | +11 | (2085.97700) | + 13 | Al XIII | +12 | (2304.14005) | + 14 | Si I | 0 | 8.15168 | + 14 | Si II | +1 | 16.34585 | + 14 | Si III | +2 | 33.49300 | + 14 | Si IV | +3 | 45.14179 | + 14 | Si V | +4 | 166.767 | + 14 | Si VI | +5 | [205.279] | + 14 | Si VII | +6 | [246.57] | + 14 | Si VIII | +7 | [303.59] | + 14 | Si IX | +8 | [351.28] | + 14 | Si X | +9 | [401.38] | + 14 | Si XI | +10 | [476.273] | + 14 | Si XII | +11 | (523.415) | + 14 | Si XIII | +12 | (2437.65813) | + 14 | Si XIV | +13 | (2673.17753) | + 15 | P I | 0 | 10.486686 | + 15 | P II | +1 | [19.76949] | + 15 | P III | +2 | 30.20264 | + 15 | P IV | +3 | 51.44387 | + 15 | P V | +4 | 65.02511 | + 15 | P VI | +5 | [220.430] | + 15 | P VII | +6 | [263.57] | + 15 | P VIII | +7 | [309.60] | + 15 | P IX | +8 | [372.31] | + 15 | P X | +9 | [424.40] | + 15 | P XI | +10 | [479.44] | + 15 | P XII | +11 | [560.62] | + 15 | P XIII | +12 | (611.741) | + 15 | P XIV | +13 | (2816.90876) | + 15 | P XV | +14 | (3069.8415) | + 16 | S I | 0 | 10.36001 | + 16 | S II | +1 | 23.33788 | + 16 | S III | +2 | [34.86] | + 16 | S IV | +3 | [47.222] | + 16 | S V | +4 | 72.5945 | + 16 | S VI | +5 | 88.0529 | + 16 | S VII | +6 | 280.954 | + 16 | S VIII | +7 | 328.794 | + 16 | S IX | +8 | [379.84] | + 16 | S X | +9 | [447.7] | + 16 | S XI | +10 | [504.55] | + 16 | S XII | +11 | [564.41] | + 16 | S XIII | +12 | [651.96] | + 16 | S XIV | +13 | (706.994) | + 16 | S XV | +14 | (3223.7807) | + 16 | S XVI | +15 | (3494.1879) | + 17 | Cl I | 0 | 12.967632 | + 17 | Cl II | +1 | 23.81364 | + 17 | Cl III | +2 | [39.80] | + 17 | Cl IV | +3 | [53.24] | + 17 | Cl V | +4 | [67.68] | + 17 | Cl VI | +5 | [96.94] | + 17 | Cl VII | +6 | 114.2013 | + 17 | Cl VIII | +7 | 348.306 | + 17 | Cl IX | +8 | 400.851 | + 17 | Cl X | +9 | [456.7] | + 17 | Cl XI | +10 | [530.0] | + 17 | Cl XII | +11 | [591.58] | + 17 | Cl XIII | +12 | [656.30] | + 17 | Cl XIV | +13 | [750.23] | + 17 | Cl XV | +14 | (809.198) | + 17 | Cl XVI | +15 | (3658.3437) | + 17 | Cl XVII | +16 | (3946.2909) | + 18 | Ar I | 0 | 15.7596117 | + 18 | Ar II | +1 | 27.62967 | + 18 | Ar III | +2 | [40.735] | + 18 | Ar IV | +3 | [59.58] | + 18 | Ar V | +4 | [74.84] | + 18 | Ar VI | +5 | 91.290 | + 18 | Ar VII | +6 | [124.41] | + 18 | Ar VIII | +7 | [143.4567] | + 18 | Ar IX | +8 | [422.60] | + 18 | Ar X | +9 | [479.76] | + 18 | Ar XI | +10 | [540.4] | + 18 | Ar XII | +11 | [619.0] | + 18 | Ar XIII | +12 | [685.5] | + 18 | Ar XIV | +13 | [755.13] | + 18 | Ar XV | +14 | [855.5] | + 18 | Ar XVI | +15 | (918.375) | + 18 | Ar XVII | +16 | (4120.6656) | + 18 | Ar XVIII | +17 | (4426.2228) | + 36 | Kr I | 0 | 13.9996053 | + 36 | Kr II | +1 | 24.35984 | + 36 | Kr III | +2 | 35.838 | + 36 | Kr IV | +3 | 50.85 | + 36 | Kr V | +4 | [64.69] | + 36 | Kr VI | +5 | [78.49] | + 36 | Kr VII | +6 | (109.13) | + 36 | Kr VIII | +7 | 125.802 | + 36 | Kr IX | +8 | [233.0] | + 36 | Kr X | +9 | (268) | + 36 | Kr XI | +10 | (308) | + 36 | Kr XII | +11 | (350) | + 36 | Kr XIII | +12 | (391) | + 36 | Kr XIV | +13 | (446) | + 36 | Kr XV | +14 | (492) | + 36 | Kr XVI | +15 | (540) | + 36 | Kr XVII | +16 | (591) | + 36 | Kr XVIII | +17 | (640) | + 36 | Kr XIX | +18 | [785] | + 36 | Kr XX | +19 | [831.6] | + 36 | Kr XXI | +20 | [882.8] | + 36 | Kr XXII | +21 | [945] | + 36 | Kr XXIII | +22 | [999.0] | + 36 | Kr XXIV | +23 | [1042] | + 36 | Kr XXV | +24 | [1155.0] | + 36 | Kr XXVI | +25 | [1205.23] | + 36 | Kr XXVII | +26 | [2928.9] | + 36 | Kr XXVIII | +27 | [3072] | + 36 | Kr XXIX | +28 | [3228] | + 36 | Kr XXX | +29 | [3380] | + 36 | Kr XXXI | +30 | [3584] | + 36 | Kr XXXII | +31 | [3752.0] | + 36 | Kr XXXIII | +32 | [3971] | + 36 | Kr XXXIV | +33 | (4109.083) | + 36 | Kr XXXV | +34 | (17296.420) | + 36 | Kr XXXVI | +35 | (17936.209) | + 37 | Rb I | 0 | 4.1771280 | + 37 | Rb II | +1 | 27.28954 | + 37 | Rb III | +2 | [39.247] | + 37 | Rb IV | +3 | [52.20] | + 37 | Rb V | +4 | (68.44) | + 37 | Rb VI | +5 | (82.9) | + 37 | Rb VII | +6 | [98.67] | + 37 | Rb VIII | +7 | [132.79] | + 37 | Rb IX | +8 | 150.628 | + 37 | Rb X | +9 | 277.12 | + 37 | Rb XI | +10 | (313.1) | + 37 | Rb XII | +11 | (356.0) | + 37 | Rb XIII | +12 | (400) | + 37 | Rb XIV | +13 | (443) | + 37 | Rb XV | +14 | (502) | + 37 | Rb XVI | +15 | (550) | + 37 | Rb XVII | +16 | (601) | + 37 | Rb XVIII | +17 | (654) | + 37 | Rb XIX | +18 | (706.0) | + 37 | Rb XX | +19 | [857] | + 37 | Rb XXI | +20 | [905.3] | + 37 | Rb XXII | +21 | [958.9] | + 37 | Rb XXIII | +22 | [1024] | + 37 | Rb XXIV | +23 | [1080] | + 37 | Rb XXV | +24 | [1125] | + 37 | Rb XXVI | +25 | [1242.5] | + 37 | Rb XXVII | +26 | [1294.57] | + 37 | Rb XXVIII | +27 | [3133.3] | + 37 | Rb XXIX | +28 | [3281] | + 37 | Rb XXX | +29 | [3443] | + 37 | Rb XXXI | +30 | [3600] | + 37 | Rb XXXII | +31 | [3815] | + 37 | Rb XXXIII | +32 | [3988] | + 37 | Rb XXXIV | +33 | [4214] | + 37 | Rb XXXV | +34 | (4356.865) | + 37 | Rb XXXVI | +35 | (18305.884) | + 37 | Rb XXXVII | +36 | (18965.516) | + 54 | Xe I | 0 | 12.1298436 | + 54 | Xe II | +1 | [20.975] | + 54 | Xe III | +2 | [31.05] | + 54 | Xe IV | +3 | 42.20 | + 54 | Xe V | +4 | [54.1] | + 54 | Xe VI | +5 | 66.703 | + 54 | Xe VII | +6 | [91.6] | + 54 | Xe VIII | +7 | 105.9778 | + 54 | Xe IX | +8 | 179.84 | + 54 | Xe X | +9 | (202.0) | + 54 | Xe XI | +10 | [229.02] | + 54 | Xe XII | +11 | (255.0) | + 54 | Xe XIII | +12 | (281) | + 54 | Xe XIV | +13 | (314) | + 54 | Xe XV | +14 | (343) | + 54 | Xe XVI | +15 | (374) | + 54 | Xe XVII | +16 | (404) | + 54 | Xe XVIII | +17 | (434) | + 54 | Xe XIX | +18 | (549) | + 54 | Xe XX | +19 | (582) | + 54 | Xe XXI | +20 | (616) | + 54 | Xe XXII | +21 | (650) | + 54 | Xe XXIII | +22 | (700) | + 54 | Xe XXIV | +23 | (736) | + 54 | Xe XXV | +24 | (818) | + 54 | Xe XXVI | +25 | [857.0] | + 54 | Xe XXVII | +26 | (1493) | + 54 | Xe XXVIII | +27 | (1571) | + 54 | Xe XXIX | +28 | (1653) | + 54 | Xe XXX | +29 | (1742) | + 54 | Xe XXXI | +30 | (1826) | + 54 | Xe XXXII | +31 | (1919) | + 54 | Xe XXXIII | +32 | (2023) | + 54 | Xe XXXIV | +33 | (2113) | + 54 | Xe XXXV | +34 | (2209) | + 54 | Xe XXXVI | +35 | (2300) | + 54 | Xe XXXVII | +36 | (2556) | + 54 | Xe XXXVIII | +37 | (2637) | + 54 | Xe XXXIX | +38 | (2726) | + 54 | Xe XL | +39 | (2811) | + 54 | Xe XLI | +40 | (2975) | + 54 | Xe XLII | +41 | (3068) | + 54 | Xe XLIII | +42 | (3243) | + 54 | Xe XLIV | +43 | [3333.8] | + 54 | Xe XLV | +44 | (7660) | + 54 | Xe XLVI | +45 | (7889) | + 54 | Xe XLVII | +46 | (8144) | + 54 | Xe XLVIII | +47 | (8382) | + 54 | Xe XLIX | +48 | (8971) | + 54 | Xe L | +49 | (9243) | + 54 | Xe LI | +50 | (9581) | + 54 | Xe LII | +51 | (9810.37) | + 54 | Xe LIII | +52 | (40271.724) | + 54 | Xe LIV | +53 | (41299.71) | + 86 | Rn I | 0 | 10.74850 | + 86 | Rn II | +1 | [21.4] | + 86 | Rn III | +2 | [29.4] | + 86 | Rn IV | +3 | (36.9) | + 86 | Rn V | +4 | (52.9) | + 86 | Rn VI | +5 | (64.0) | + 86 | Rn VII | +6 | (88.0) | + 86 | Rn VIII | +7 | (102.0) | + 86 | Rn IX | +8 | (154.0) | + 86 | Rn X | +9 | (173.9) | + 86 | Rn XI | +10 | (195.0) | + 86 | Rn XII | +11 | (218.0) | + 86 | Rn XIII | +12 | (240) | + 86 | Rn XIV | +13 | (264) | + 86 | Rn XV | +14 | (293) | + 86 | Rn XVI | +15 | (317) | + 86 | Rn XVII | +16 | (342) | + 86 | Rn XVIII | +17 | (367) | + 86 | Rn XIX | +18 | (488) | + 86 | Rn XX | +19 | (520) | + 86 | Rn XXI | +20 | (550) | + 86 | Rn XXII | +21 | (580) | + 86 | Rn XXIII | +22 | (640) | + 86 | Rn XXIV | +23 | (680) | + 86 | Rn XXV | +24 | (760) | + 86 | Rn XXVI | +25 | (800) | + 86 | Rn XXVII | +26 | (850) | + 86 | Rn XXVIII | +27 | (920) | + 86 | Rn XXIX | +28 | (980) | + 86 | Rn XXX | +29 | (1050) | + 86 | Rn XXXI | +30 | (1110) | + 86 | Rn XXXII | +31 | (1180) | + 86 | Rn XXXIII | +32 | (1250) | + 86 | Rn XXXIV | +33 | (1310) | + 86 | Rn XXXV | +34 | (1390) | + 86 | Rn XXXVI | +35 | (1460) | + 86 | Rn XXXVII | +36 | (1520) | + 86 | Rn XXXVIII | +37 | (1590) | + 86 | Rn XXXIX | +38 | (1660) | + 86 | Rn XL | +39 | (1720) | + 86 | Rn XLI | +40 | (2033) | + 86 | Rn XLII | +41 | (2094) | + 86 | Rn XLIII | +42 | (2158) | + 86 | Rn XLIV | +43 | (2227) | + 86 | Rn XLV | +44 | (2293) | + 86 | Rn XLVI | +45 | (2357) | + 86 | Rn XLVII | +46 | (2467) | + 86 | Rn XLVIII | +47 | (2535) | + 86 | Rn XLIX | +48 | (2606) | + 86 | Rn L | +49 | (2674) | + 86 | Rn LI | +50 | (2944) | + 86 | Rn LII | +51 | (3010) | + 86 | Rn LIII | +52 | (3082) | + 86 | Rn LIV | +53 | (3149) | + 86 | Rn LV | +54 | (3433) | + 86 | Rn LVI | +55 | (3510) | + 86 | Rn LVII | +56 | (3699) | + 86 | Rn LVIII | +57 | [3777] | + 86 | Rn LIX | +58 | (6169) | + 86 | Rn LX | +59 | (6318) | + 86 | Rn LXI | +60 | (6476) | + 86 | Rn LXII | +61 | (6646) | + 86 | Rn LXIII | +62 | (6807) | + 86 | Rn LXIV | +63 | (6964) | + 86 | Rn LXV | +64 | (7283) | + 86 | Rn LXVI | +65 | (7450) | + 86 | Rn LXVII | +66 | (7630) | + 86 | Rn LXVIII | +67 | (7800) | + 86 | Rn LXIX | +68 | (8260) | + 86 | Rn LXX | +69 | (8410) | + 86 | Rn LXXI | +70 | (8570) | + 86 | Rn LXXII | +71 | (8710) | + 86 | Rn LXXIII | +72 | (9610) | + 86 | Rn LXXIV | +73 | (9780) | + 86 | Rn LXXV | +74 | (10120) | + 86 | Rn LXXVI | +75 | (10290) | + 86 | Rn LXXVII | +76 | (21770) | + 86 | Rn LXXVIII | +77 | (22160) | + 86 | Rn LXXIX | +78 | (22600) | + 86 | Rn LXXX | +79 | (22990) | + 86 | Rn LXXXI | +80 | (26310) | + 86 | Rn LXXXII | +81 | (26830) | + 86 | Rn LXXXIII | +82 | (27490) | + 86 | Rn LXXXIV | +83 | (27903.1) | + 86 | Rn LXXXV | +84 | (110842.0) | + 86 | Rn LXXXVI | +85 | (112843.7) | +----------------------------------------------------------------------------- diff --git a/Source/Utils/write_atomic_data_cpp.py b/Source/Utils/write_atomic_data_cpp.py new file mode 100644 index 000000000..21d61a075 --- /dev/null +++ b/Source/Utils/write_atomic_data_cpp.py @@ -0,0 +1,67 @@ +import re, os +import numpy as np +from scipy.constants import e + +filename = os.path.join( '.', 'atomic_data.txt' ) +with open(filename) as f: + text_data = f.read() + +# Read full table from file and get names, atomic numbers and offsets +# position in table of ionization energies for all species +regex_command = '\n\s+(\d+)\s+\|\s+([A-Z]+[a-z]*)\s+\w+\s+\|\s+\+*(\d+)\s+\|\s+\(*\[*(\d+\.*\d*)' +list_of_tuples = re.findall( regex_command, text_data ) +ion_atom_numbers = [int(i) for i in list(dict.fromkeys( [x[0] for x in list_of_tuples] ))] +ion_names = list(dict.fromkeys( [x[1] for x in list_of_tuples] )) +ion_offsets = np.concatenate(([0], np.cumsum(np.array(ion_atom_numbers)[:-1])), axis=0) + +# Head of CPP file +cpp_string = '' +cpp_string += '#include \n' +cpp_string += '#include \n\n' +cpp_string += '#ifndef WARPX_IONIZATION_TABLE_H_\n' +cpp_string += '#define WARPX_IONIZATION_TABLE_H_\n\n' + +# Map each element to ID in table +cpp_string += 'std::map ion_map_ids = {' +for count, name in enumerate(ion_names): + cpp_string += '\n {"' + name + '", ' + str(count) + '},' +cpp_string = cpp_string[:-1] +cpp_string += ' };\n\n' + +# Atomic number of each species +cpp_string += 'const int nelements = ' + str(len(ion_names)) + ';\n\n' +cpp_string += 'const int ion_atomic_numbers[nelements] = {' +for count, atom_num in enumerate(ion_atom_numbers): + if count%10==0: cpp_string += '\n ' + cpp_string += str(atom_num) + ', ' +cpp_string = cpp_string[:-2] +cpp_string += '};\n\n' + +# Offset of each element in table of ionization energies +cpp_string += 'const int ion_energy_offsets[nelements] = {' +for count, offset in enumerate(ion_offsets): + if count%10==0: cpp_string += '\n ' + cpp_string += str(offset) + ', ' +cpp_string = cpp_string[:-2] +cpp_string += '};\n\n' + +# Table of ionization energies +cpp_string += 'const int energies_tab_length = ' + str(len(list_of_tuples)) + ';\n\n' +cpp_string += 'const amrex::Real table_ionization_energies[energies_tab_length]{' +for element in ion_names: + cpp_string += '\n // ' + element + regex_command = \ + '\n\s+(\d+)\s+\|\s+%s\s+\w+\s+\|\s+\+*(\d+)\s+\|\s+\(*\[*(\d+\.*\d*)' \ + %element + list_of_tuples = re.findall( regex_command, text_data ) + for count, energy in enumerate([x[2] for x in list_of_tuples]): + if count%7==0: cpp_string += '\n ' + cpp_string += energy + ', ' +cpp_string = cpp_string[:-2] +cpp_string += ' };\n\n' + +# Write the string to file +cpp_string += '#endif // #ifndef WARPX_IONIZATION_TABLE_H_\n' +f= open("IonizationEnergiesTable.H","w") +f.write(cpp_string) +f.close() -- cgit v1.2.3 From a0f579f27969c3f86e54874b5a3c6a732dff9f53 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Sun, 4 Aug 2019 15:47:12 -0700 Subject: add target species handling --- Source/Particles/MultiParticleContainer.H | 2 +- Source/Particles/MultiParticleContainer.cpp | 155 ++++++++++++++++++++++++- Source/Particles/PhysicalParticleContainer.H | 2 + Source/Particles/PhysicalParticleContainer.cpp | 21 +++- Source/Particles/WarpXParticleContainer.H | 5 +- 5 files changed, 179 insertions(+), 6 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 4ea9c2763..8f7fd56e3 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -7,7 +7,7 @@ #include #include - +#include #include #include #include diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 16c9b9762..736b906df 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -3,6 +3,9 @@ #include #include +#include +#include +#include #include #include #include @@ -508,7 +511,155 @@ MultiParticleContainer::getSpeciesID(std::string product_str) return i_product; } - void MultiParticleContainer::doFieldIonization() -{} +{ + /* + amrex::Gpu::SharedMemory grid_ids; + amrex::Gpu::SharedMemory tile_ids; + amrex::Gpu::SharedMemory is_ionized; + */ + + for (auto& pc : allcontainers){ + if (!pc->do_field_ionization){ continue; } + const Real * const AMREX_RESTRICT p_ionization_energies = pc->ionization_energies.dataPtr(); + const Real * const AMREX_RESTRICT p_adk_prefactor = pc->adk_prefactor.dataPtr(); + const Real * const AMREX_RESTRICT p_adk_exp_prefactor = pc->adk_exp_prefactor.dataPtr(); + const Real * const AMREX_RESTRICT p_adk_power = pc->adk_power.dataPtr(); + for (int lev = 0; lev <= pc->finestLevel(); ++lev){ + +#ifdef _OPENMP + // First touch all tiles in the map in serial + for (MFIter mfi = pc->MakeMFIter(lev); mfi.isValid(); ++mfi) { + const int grid_id = mfi.index(); + const int tile_id = mfi.LocalTileIndex(); + pc->GetParticles(lev)[std::make_pair(grid_id,tile_id)]; + if ( (pc->NumRuntimeRealComps()>0) || (pc->NumRuntimeIntComps()>0) ) { + pc->DefineAndReturnParticleTile(lev, grid_id, tile_id); + } + } +#endif + MFItInfo info; + if (pc->do_tiling && Gpu::notInLaunchRegion()) { + info.EnableTiling(pc->tile_size); + } +#ifdef _OPENMP + info.SetDynamic(true); +#pragma omp parallel +#endif + for (MFIter mfi = pc->MakeMFIter(lev, info); mfi.isValid(); ++mfi) + { + const int grid_id = mfi.index(); + const int tile_id = mfi.LocalTileIndex(); + auto& particle_tile = pc->GetParticles(lev)[std::make_pair(grid_id,tile_id)]; + auto& soa = particle_tile.GetStructOfArrays(); + const int np = particle_tile.GetArrayOfStructs().size(); + if (np == 0) break; + amrex::Gpu::ManagedVector is_ionized; + // const int np_lev = pc->NumberOfParticlesAtLevel(lev); + is_ionized.resize(np); + int * const AMREX_RESTRICT p_is_ionized = is_ionized.dataPtr(); + const Real * const AMREX_RESTRICT ux = soa.GetRealData(PIdx::ux).data(); + const Real * const AMREX_RESTRICT uy = soa.GetRealData(PIdx::uy).data(); + const Real * const AMREX_RESTRICT uz = soa.GetRealData(PIdx::uz).data(); + const Real * const AMREX_RESTRICT ex = soa.GetRealData(PIdx::Ex).data(); + const Real * const AMREX_RESTRICT ey = soa.GetRealData(PIdx::Ey).data(); + const Real * const AMREX_RESTRICT ez = soa.GetRealData(PIdx::Ez).data(); + const Real * const AMREX_RESTRICT bx = soa.GetRealData(PIdx::Bx).data(); + const Real * const AMREX_RESTRICT by = soa.GetRealData(PIdx::By).data(); + const Real * const AMREX_RESTRICT bz = soa.GetRealData(PIdx::Bz).data(); + Real * const AMREX_RESTRICT ilev_real = soa.GetRealData(pc->particle_comps["ionization_level"]).data(); + Real c = PhysConst::c; + Real c2_inv = 1./c/c; + + ParallelFor( + np, + [=] AMREX_GPU_DEVICE (long i) { + Real random_draw = Random(); + Real ga = std::sqrt(1. + + (ux[i]*ux[i] + + uy[i]*uy[i] + + uz[i]*uz[i]) * c2_inv); + Real E = std::sqrt( + - ( ux[i]*ex[i] + uy[i]*ey[i] + uz[i]*ez[i] ) * ( ux[i]*ex[i] + uy[i]*ey[i] + uz[i]*ez[i] ) * c2_inv + + ( ga *ex[i] + uy[i]*bz[i] - uz[i]*by[i] ) * ( ga *ex[i] + uy[i]*bz[i] - uz[i]*by[i] ) + + ( ga *ey[i] + uz[i]*bx[i] - ux[i]*bz[i] ) * ( ga *ey[i] + uz[i]*bx[i] - ux[i]*bz[i] ) + + ( ga *ez[i] + ux[i]*by[i] - uy[i]*bx[i] ) * ( ga *ez[i] + ux[i]*by[i] - uy[i]*bx[i] ) + ); + int ilev = (int) round(ilev_real[i]); + // int ilev = static_cast(round(ilev_real[i])); + Real p; + Real w_dtau; + Print()< is_ionized_cumsum_vector; + is_ionized_cumsum_vector.resize(np); + int np_ionized = p_is_ionized[0]; + for(int i=1; iionization_product]; + auto& prod_particle_tile = prod_pc->GetParticles(lev)[std::make_pair(grid_id,tile_id)]; + const int np_old = prod_particle_tile.GetArrayOfStructs().size(); + const int np_new = np_old + np_ionized; + prod_particle_tile.resize(np_new); + auto& prod_soa = prod_particle_tile.GetStructOfArrays(); + + GpuArray prod_pa; + for (int ia = 0; ia < PIdx::nattribs; ++ia) { + prod_pa[ia] = prod_soa.GetRealData(ia).data() + np_old; + } + WarpXParticleContainer::ParticleType* prod_pp = prod_particle_tile.GetArrayOfStructs()().data() + np_old; + WarpXParticleContainer::ParticleType* pp = particle_tile.GetArrayOfStructs()().data(); + + GpuArray pa; + for (int ia = 0; ia < PIdx::nattribs; ++ia) { + pa[ia] = soa.GetRealData(ia).data(); + } + + const int cpuid = ParallelDescriptor::MyProc(); + int* AMREX_RESTRICT is_ionized_cumsum = is_ionized_cumsum_vector.dataPtr(); + amrex::For( + np, [=] AMREX_GPU_DEVICE (int ip) noexcept + { + if(is_ionized[ip]){ + int i = is_ionized_cumsum[ip]; + WarpXParticleContainer::ParticleType& prod_p = prod_pp[i]; + WarpXParticleContainer::ParticleType& p = pp[ip]; + prod_p.id() = 9999; + prod_p.cpu() = cpuid; + prod_p.pos(0) = p.pos(0); + prod_p.pos(1) = p.pos(1); +#if (AMREX_SPACEDIM == 3) + prod_p.pos(20 = p.pos(2); +#endif + for (int ia = 0; ia < PIdx::nattribs; ++ia) { + prod_pa[ia][i] = pa[ia][ip]; + } + } + } + ); + } // MFIter + } + } +} diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index a6cf9d81f..6f9fdc485 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -108,6 +108,8 @@ public: virtual void PostRestart () final {} void SplitParticles(int lev); + + virtual int doFieldIonization (const int lev) override; // Inject particles in Box 'part_box' virtual void AddParticles (int lev); diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 0183a44b9..8232b621f 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -215,7 +215,7 @@ PhysicalParticleContainer::CheckAndAddParticle(Real x, Real y, Real z, auto& particle_tile = DefineAndReturnParticleTile(0, 0, 0); if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) { - particle_tile.push_back_real(particle_comps["xold"], x); + particle_tile.push_back_real(particle_comps["xold"], x); particle_tile.push_back_real(particle_comps["yold"], y); particle_tile.push_back_real(particle_comps["zold"], z); @@ -591,7 +591,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) } if (do_field_ionization) { - pi[ip] = 2.; // species_ionization_level; + pi[ip] = species_ionization_level; } u.x *= PhysConst::c; @@ -2105,4 +2105,21 @@ void PhysicalParticleContainer::InitIonizationModule() adk_power.resize(ion_atomic_number); adk_prefactor.resize(ion_atomic_number); adk_exp_prefactor.resize(ion_atomic_number); + for (int i=0; i Date: Mon, 5 Aug 2019 09:23:16 -0700 Subject: Removed warpx_geteb_energy_conserving --- Source/FortranInterface/WarpX_f.H | 28 ----- Source/FortranInterface/WarpX_picsar.F90 | 81 --------------- Source/Particles/PhysicalParticleContainer.cpp | 113 +-------------------- .../Particles/RigidInjectedParticleContainer.cpp | 28 ----- 4 files changed, 4 insertions(+), 246 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/FortranInterface/WarpX_f.H b/Source/FortranInterface/WarpX_f.H index 0440148eb..689029059 100644 --- a/Source/FortranInterface/WarpX_f.H +++ b/Source/FortranInterface/WarpX_f.H @@ -106,34 +106,6 @@ extern "C" const long* nox, const long* noy,const long* noz, const int* l_nodal, const long* lvect, const long* current_depo_algo); - // Current deposition finalize for RZ - void warpx_current_deposition_rz_volume_scaling( - amrex::Real* jx, const long* jx_ng, const int* jx_ntot, - amrex::Real* jy, const long* jy_ng, const int* jy_ntot, - amrex::Real* jz, const long* jz_ng, const int* jz_ntot, - const amrex::Real* rmin, - const amrex::Real* dr); - - // Field gathering - - void warpx_geteb_energy_conserving(const long* np, - const amrex::Real* xp, const amrex::Real* yp, const amrex::Real* zp, - amrex::Real* exp, amrex::Real* eyp, amrex::Real* ezp, - amrex::Real* bxp, amrex::Real* byp, amrex::Real* bzp, - const int* ixyzmin, - const amrex::Real* xmin, const amrex::Real* ymin, const amrex::Real* zmin, - const amrex::Real* dx, const amrex::Real* dy, const amrex::Real* dz, - const long* nox, const long* noy, const long* noz, - const amrex::Real* exg, const int* exg_lo, const int* exg_hi, - const amrex::Real* eyg, const int* eyg_lo, const int* eyg_hi, - const amrex::Real* ezg, const int* ezg_lo, const int* ezg_hi, - const amrex::Real* bxg, const int* bxg_lo, const int* bxg_hi, - const amrex::Real* byg, const int* byg_lo, const int* byg_hi, - const amrex::Real* bzg, const int* bzg_lo, const int* bzg_hi, - const int* ll4symtry, const int* l_lower_order_in_v, - const int* l_nodal, const long* lvect, - const long* field_gathe_algo); - // Particle pusher (velocity and position) void warpx_particle_pusher(const long* np, diff --git a/Source/FortranInterface/WarpX_picsar.F90 b/Source/FortranInterface/WarpX_picsar.F90 index 14bd79ad4..e65c30e42 100644 --- a/Source/FortranInterface/WarpX_picsar.F90 +++ b/Source/FortranInterface/WarpX_picsar.F90 @@ -1,19 +1,16 @@ #if (AMREX_SPACEDIM == 3) -#define WRPX_PXR_GETEB_ENERGY_CONSERVING geteb3d_energy_conserving_generic #define WRPX_PXR_CURRENT_DEPOSITION depose_jxjyjz_generic #elif (AMREX_SPACEDIM == 2) #ifdef WARPX_RZ -#define WRPX_PXR_GETEB_ENERGY_CONSERVING geteb2drz_energy_conserving_generic #define WRPX_PXR_CURRENT_DEPOSITION depose_jrjtjz_generic_rz #define WRPX_PXR_RZ_VOLUME_SCALING_RHO apply_rz_volume_scaling_rho #else -#define WRPX_PXR_GETEB_ENERGY_CONSERVING geteb2dxz_energy_conserving_generic #define WRPX_PXR_CURRENT_DEPOSITION depose_jxjyjz_generic_2d #endif @@ -52,84 +49,6 @@ module warpx_to_pxr_module contains - ! _________________________________________________________________ - !> - !> @brief - !> Main subroutine for the field gathering process - !> - !> @param[in] np number of particles - !> @param[in] xp,yp,zp particle position arrays - !> @param[in] ex,ey,ez particle electric fields in each direction - !> @param[in] bx,by,bz particle magnetic fields in each direction - !> @param[in] ixyzmin tile grid minimum index - !> @param[in] xmin,ymin,zmin tile grid minimum position - !> @param[in] dx,dy,dz space discretization steps - !> @param[in] xyzmin grid minimum position - !> @param[in] dxyz space discretization steps - !> @param[in] nox,noy,noz interpolation order - !> @param[in] exg,eyg,ezg electric field grid arrays - !> @param[in] bxg,byg,bzg electric field grid arrays - !> @param[in] lvect vector length - !> - subroutine warpx_geteb_energy_conserving(np,xp,yp,zp, & - ex,ey,ez,bx,by,bz,ixyzmin,xmin,ymin,zmin,dx,dy,dz,nox,noy,noz, & - exg,exg_lo,exg_hi,eyg,eyg_lo,eyg_hi,ezg,ezg_lo,ezg_hi, & - bxg,bxg_lo,bxg_hi,byg,byg_lo,byg_hi,bzg,bzg_lo,bzg_hi, & - ll4symtry,l_lower_order_in_v, l_nodal,& - lvect,field_gathe_algo) & - bind(C, name="warpx_geteb_energy_conserving") - - integer, intent(in) :: exg_lo(AMREX_SPACEDIM), eyg_lo(AMREX_SPACEDIM), ezg_lo(AMREX_SPACEDIM), & - bxg_lo(AMREX_SPACEDIM), byg_lo(AMREX_SPACEDIM), bzg_lo(AMREX_SPACEDIM) - integer, intent(in) :: exg_hi(AMREX_SPACEDIM), eyg_hi(AMREX_SPACEDIM), ezg_hi(AMREX_SPACEDIM), & - bxg_hi(AMREX_SPACEDIM), byg_hi(AMREX_SPACEDIM), bzg_hi(AMREX_SPACEDIM) - integer, intent(in) :: ixyzmin(AMREX_SPACEDIM) - real(amrex_real), intent(in) :: xmin,ymin,zmin,dx,dy,dz - integer(c_long), intent(in) :: field_gathe_algo - integer(c_long), intent(in) :: np,nox,noy,noz - integer(c_int), intent(in) :: ll4symtry,l_lower_order_in_v, l_nodal - integer(c_long),intent(in) :: lvect - real(amrex_real), intent(in), dimension(np) :: xp,yp,zp - real(amrex_real), intent(out), dimension(np) :: ex,ey,ez,bx,by,bz - real(amrex_real),intent(in):: exg(*), eyg(*), ezg(*), bxg(*), byg(*), bzg(*) - logical(pxr_logical) :: pxr_ll4symtry, pxr_l_lower_order_in_v, pxr_l_nodal - - ! Compute the number of valid cells and guard cells - integer(c_long) :: exg_nvalid(AMREX_SPACEDIM), eyg_nvalid(AMREX_SPACEDIM), ezg_nvalid(AMREX_SPACEDIM), & - bxg_nvalid(AMREX_SPACEDIM), byg_nvalid(AMREX_SPACEDIM), bzg_nvalid(AMREX_SPACEDIM), & - exg_nguards(AMREX_SPACEDIM), eyg_nguards(AMREX_SPACEDIM), ezg_nguards(AMREX_SPACEDIM), & - bxg_nguards(AMREX_SPACEDIM), byg_nguards(AMREX_SPACEDIM), bzg_nguards(AMREX_SPACEDIM) - - pxr_ll4symtry = ll4symtry .eq. 1 - pxr_l_lower_order_in_v = l_lower_order_in_v .eq. 1 - pxr_l_nodal = l_nodal .eq. 1 - - exg_nguards = ixyzmin - exg_lo - eyg_nguards = ixyzmin - eyg_lo - ezg_nguards = ixyzmin - ezg_lo - bxg_nguards = ixyzmin - bxg_lo - byg_nguards = ixyzmin - byg_lo - bzg_nguards = ixyzmin - bzg_lo - exg_nvalid = exg_lo + exg_hi - 2_c_long*ixyzmin + 1_c_long - eyg_nvalid = eyg_lo + eyg_hi - 2_c_long*ixyzmin + 1_c_long - ezg_nvalid = ezg_lo + ezg_hi - 2_c_long*ixyzmin + 1_c_long - bxg_nvalid = bxg_lo + bxg_hi - 2_c_long*ixyzmin + 1_c_long - byg_nvalid = byg_lo + byg_hi - 2_c_long*ixyzmin + 1_c_long - bzg_nvalid = bzg_lo + bzg_hi - 2_c_long*ixyzmin + 1_c_long - - CALL WRPX_PXR_GETEB_ENERGY_CONSERVING(np,xp,yp,zp, & - ex,ey,ez,bx,by,bz,xmin,ymin,zmin,dx,dy,dz,nox,noy,noz, & - exg,exg_nguards,exg_nvalid,& - eyg,eyg_nguards,eyg_nvalid,& - ezg,ezg_nguards,ezg_nvalid,& - bxg,bxg_nguards,bxg_nvalid,& - byg,byg_nguards,byg_nvalid,& - bzg,bzg_nguards,bzg_nvalid,& - pxr_ll4symtry, pxr_l_lower_order_in_v, pxr_l_nodal, & - lvect, field_gathe_algo ) - - end subroutine warpx_geteb_energy_conserving - ! _________________________________________________________________ !> !> @brief diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 7d63bd8e7..08f8a77b4 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -893,40 +893,13 @@ PhysicalParticleContainer::FieldGather (int lev, // pti.GetPosition(m_xp[thread_num], m_yp[thread_num], m_zp[thread_num]); - const std::array& xyzmin = WarpX::LowerCorner(box, lev); - const int* ixyzmin = box.loVect(); - // // Field Gather // -#ifdef WARPX_RZ - const int ll4symtry = false; - long lvect_fieldgathe = 64; - warpx_geteb_energy_conserving( - &np, - m_xp[thread_num].dataPtr(), - m_yp[thread_num].dataPtr(), - m_zp[thread_num].dataPtr(), - Exp.dataPtr(),Eyp.dataPtr(),Ezp.dataPtr(), - Bxp.dataPtr(),Byp.dataPtr(),Bzp.dataPtr(), - ixyzmin, - &xyzmin[0], &xyzmin[1], &xyzmin[2], - &dx[0], &dx[1], &dx[2], - &WarpX::nox, &WarpX::noy, &WarpX::noz, - BL_TO_FORTRAN_ANYD(exfab), - BL_TO_FORTRAN_ANYD(eyfab), - BL_TO_FORTRAN_ANYD(ezfab), - BL_TO_FORTRAN_ANYD(bxfab), - BL_TO_FORTRAN_ANYD(byfab), - BL_TO_FORTRAN_ANYD(bzfab), - &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, - &lvect_fieldgathe, &WarpX::field_gathering_algo); -#else int e_is_nodal = Ex.is_nodal() and Ey.is_nodal() and Ez.is_nodal(); FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, &exfab, &eyfab, &ezfab, &bxfab, &byfab, &bzfab, Ex.nGrow(), e_is_nodal, 0, np, thread_num, lev, lev); -#endif if (cost) { const Box& tbx = pti.tilebox(); @@ -1195,35 +1168,9 @@ PhysicalParticleContainer::Evolve (int lev, // Field Gather of Aux Data (i.e., the full solution) // BL_PROFILE_VAR_START(blp_pxr_fg); -#ifdef WARPX_RZ - const int ll4symtry = false; - long lvect_fieldgathe = 64; - const std::array& xyzmin_grid = WarpX::LowerCorner(box, lev); - const int* ixyzmin_grid = box.loVect(); - warpx_geteb_energy_conserving( - &np_gather, - m_xp[thread_num].dataPtr(), - m_yp[thread_num].dataPtr(), - m_zp[thread_num].dataPtr(), - Exp.dataPtr(),Eyp.dataPtr(),Ezp.dataPtr(), - Bxp.dataPtr(),Byp.dataPtr(),Bzp.dataPtr(), - ixyzmin_grid, - &xyzmin_grid[0], &xyzmin_grid[1], &xyzmin_grid[2], - &dx[0], &dx[1], &dx[2], - &WarpX::nox, &WarpX::noy, &WarpX::noz, - BL_TO_FORTRAN_ANYD(*exfab), - BL_TO_FORTRAN_ANYD(*eyfab), - BL_TO_FORTRAN_ANYD(*ezfab), - BL_TO_FORTRAN_ANYD(*bxfab), - BL_TO_FORTRAN_ANYD(*byfab), - BL_TO_FORTRAN_ANYD(*bzfab), - &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, - &lvect_fieldgathe, &WarpX::field_gathering_algo); -#else FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, exfab, eyfab, ezfab, bxfab, byfab, bzfab, Ex.nGrow(), e_is_nodal, 0, np_gather, thread_num, lev, lev); -#endif if (np_gather < np) { @@ -1293,29 +1240,6 @@ PhysicalParticleContainer::Evolve (int lev, } // Field gather for particles in gather buffers -#ifdef WARPX_RZ - - long ncrse = np - nfine_gather; - warpx_geteb_energy_conserving( - &ncrse, - m_xp[thread_num].dataPtr()+nfine_gather, - m_yp[thread_num].dataPtr()+nfine_gather, - m_zp[thread_num].dataPtr()+nfine_gather, - Exp.dataPtr()+nfine_gather, Eyp.dataPtr()+nfine_gather, Ezp.dataPtr()+nfine_gather, - Bxp.dataPtr()+nfine_gather, Byp.dataPtr()+nfine_gather, Bzp.dataPtr()+nfine_gather, - cixyzmin_grid, - &cxyzmin_grid[0], &cxyzmin_grid[1], &cxyzmin_grid[2], - &cdx[0], &cdx[1], &cdx[2], - &WarpX::nox, &WarpX::noy, &WarpX::noz, - BL_TO_FORTRAN_ANYD(*cexfab), - BL_TO_FORTRAN_ANYD(*ceyfab), - BL_TO_FORTRAN_ANYD(*cezfab), - BL_TO_FORTRAN_ANYD(*cbxfab), - BL_TO_FORTRAN_ANYD(*cbyfab), - BL_TO_FORTRAN_ANYD(*cbzfab), - &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, - &lvect_fieldgathe, &WarpX::field_gathering_algo); -#else e_is_nodal = cEx->is_nodal() and cEy->is_nodal() and cEz->is_nodal(); FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, cexfab, ceyfab, cezfab, @@ -1323,7 +1247,6 @@ PhysicalParticleContainer::Evolve (int lev, cEx->nGrow(), e_is_nodal, nfine_gather, np-nfine_gather, thread_num, lev, lev-1); -#endif } BL_PROFILE_VAR_STOP(blp_pxr_fg); @@ -1657,38 +1580,10 @@ PhysicalParticleContainer::PushP (int lev, Real dt, // pti.GetPosition(m_xp[thread_num], m_yp[thread_num], m_zp[thread_num]); -#ifdef WARPX_RZ - const std::array& xyzmin_grid = WarpX::LowerCorner(box, lev); - const int* ixyzmin_grid = box.loVect(); - - const int ll4symtry = false; - long lvect_fieldgathe = 64; - - warpx_geteb_energy_conserving( - &np, - m_xp[thread_num].dataPtr(), - m_yp[thread_num].dataPtr(), - m_zp[thread_num].dataPtr(), - Exp.dataPtr(),Eyp.dataPtr(),Ezp.dataPtr(), - Bxp.dataPtr(),Byp.dataPtr(),Bzp.dataPtr(), - ixyzmin_grid, - &xyzmin_grid[0], &xyzmin_grid[1], &xyzmin_grid[2], - &dx[0], &dx[1], &dx[2], - &WarpX::nox, &WarpX::noy, &WarpX::noz, - BL_TO_FORTRAN_ANYD(exfab), - BL_TO_FORTRAN_ANYD(eyfab), - BL_TO_FORTRAN_ANYD(ezfab), - BL_TO_FORTRAN_ANYD(bxfab), - BL_TO_FORTRAN_ANYD(byfab), - BL_TO_FORTRAN_ANYD(bzfab), - &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, - &lvect_fieldgathe, &WarpX::field_gathering_algo); -#else - int e_is_nodal = Ex.is_nodal() and Ey.is_nodal() and Ez.is_nodal(); - FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, - &exfab, &eyfab, &ezfab, &bxfab, &byfab, &bzfab, - Ex.nGrow(), e_is_nodal, 0, np, thread_num, lev, lev); -#endif + int e_is_nodal = Ex.is_nodal() and Ey.is_nodal() and Ez.is_nodal(); + FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + &exfab, &eyfab, &ezfab, &bxfab, &byfab, &bzfab, + Ex.nGrow(), e_is_nodal, 0, np, thread_num, lev, lev); // This wraps the momentum advance so that inheritors can modify the call. // Extract pointers to the different particle quantities diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index 83556e69f..f049fdb7c 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -421,38 +421,10 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, // pti.GetPosition(m_xp[thread_num], m_yp[thread_num], m_zp[thread_num]); -#ifdef WARPX_RZ - const std::array& xyzmin_grid = WarpX::LowerCorner(box, lev); - const int* ixyzmin_grid = box.loVect(); - - const int ll4symtry = false; - long lvect_fieldgathe = 64; - - warpx_geteb_energy_conserving( - &np, - m_xp[thread_num].dataPtr(), - m_yp[thread_num].dataPtr(), - m_zp[thread_num].dataPtr(), - Exp.dataPtr(),Eyp.dataPtr(),Ezp.dataPtr(), - Bxp.dataPtr(),Byp.dataPtr(),Bzp.dataPtr(), - ixyzmin_grid, - &xyzmin_grid[0], &xyzmin_grid[1], &xyzmin_grid[2], - &dx[0], &dx[1], &dx[2], - &WarpX::nox, &WarpX::noy, &WarpX::noz, - BL_TO_FORTRAN_ANYD(exfab), - BL_TO_FORTRAN_ANYD(eyfab), - BL_TO_FORTRAN_ANYD(ezfab), - BL_TO_FORTRAN_ANYD(bxfab), - BL_TO_FORTRAN_ANYD(byfab), - BL_TO_FORTRAN_ANYD(bzfab), - &ll4symtry, &WarpX::l_lower_order_in_v, &WarpX::do_nodal, - &lvect_fieldgathe, &WarpX::field_gathering_algo); -#else int e_is_nodal = Ex.is_nodal() and Ey.is_nodal() and Ez.is_nodal(); FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, &exfab, &eyfab, &ezfab, &bxfab, &byfab, &bzfab, Ex.nGrow(), e_is_nodal, 0, np, thread_num, lev, lev); -#endif // Save the position and momenta, making copies auto uxp_save = uxp; -- cgit v1.2.3 From ce63dce8b7d71c12cafe6309f6afc4c5cfc3d5c5 Mon Sep 17 00:00:00 2001 From: Dave Grote Date: Mon, 5 Aug 2019 09:43:57 -0700 Subject: Replaced WARPX_RZ with WARPX_DIM_RZ --- Source/Diagnostics/ParticleIO.cpp | 2 +- Source/Evolve/WarpXEvolveEM.cpp | 2 +- Source/FortranInterface/WarpX_f.H | 4 ++-- Source/FortranInterface/WarpX_picsar.F90 | 8 ++++---- Source/Make.WarpX | 1 - Source/Particles/PhysicalParticleContainer.cpp | 14 +++++++------- Source/Particles/Pusher/GetAndSetPosition.H | 6 +++--- Source/Particles/Pusher/UpdatePosition.H | 2 +- Source/Particles/WarpXParticleContainer.H | 2 +- Source/Particles/WarpXParticleContainer.cpp | 26 +++++++++++++------------- Source/Utils/WarpXAlgorithmSelection.cpp | 2 +- Source/WarpX.cpp | 16 ++++++++-------- 12 files changed, 42 insertions(+), 43 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index f2a543ed5..f159e5302 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -98,7 +98,7 @@ MultiParticleContainer::WritePlotFile (const std::string& dir) const real_names.push_back("By"); real_names.push_back("Bz"); -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ real_names.push_back("theta"); #endif diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index 57a0c44c0..e9bf98f81 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -498,7 +498,7 @@ WarpX::ComputeDt () if (maxwell_fdtd_solver_id == 0) { // CFL time step Yee solver -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ // Derived semi-analytically by R. Lehe deltat = cfl * 1./( std::sqrt((1+0.2105)/(dx[0]*dx[0]) + 1./(dx[1]*dx[1])) * PhysConst::c ); #else diff --git a/Source/FortranInterface/WarpX_f.H b/Source/FortranInterface/WarpX_f.H index 689029059..07cfcb42e 100644 --- a/Source/FortranInterface/WarpX_f.H +++ b/Source/FortranInterface/WarpX_f.H @@ -62,7 +62,7 @@ #define WRPX_PUSH_LEAPFROG warpx_push_leapfrog_2d #define WRPX_PUSH_LEAPFROG_POSITIONS warpx_push_leapfrog_positions_2d -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ #define WRPX_COMPUTE_DIVE warpx_compute_dive_rz #else #define WRPX_COMPUTE_DIVE warpx_compute_dive_2d @@ -314,7 +314,7 @@ extern "C" const BL_FORT_FAB_ARG_ANYD(ey), const BL_FORT_FAB_ARG_ANYD(ez), const amrex::Real* dx -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ ,const amrex::Real* rmin #endif ); diff --git a/Source/FortranInterface/WarpX_picsar.F90 b/Source/FortranInterface/WarpX_picsar.F90 index e65c30e42..0625d40f9 100644 --- a/Source/FortranInterface/WarpX_picsar.F90 +++ b/Source/FortranInterface/WarpX_picsar.F90 @@ -4,7 +4,7 @@ #elif (AMREX_SPACEDIM == 2) -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ #define WRPX_PXR_CURRENT_DEPOSITION depose_jrjtjz_generic_rz #define WRPX_PXR_RZ_VOLUME_SCALING_RHO apply_rz_volume_scaling_rho @@ -138,7 +138,7 @@ subroutine warpx_charge_deposition(rho,np,xp,yp,zp,w,q,xmin,ymin,zmin,dx,dy,dz,n ! Dimension 2 #elif (AMREX_SPACEDIM==2) -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ logical(pxr_logical) :: l_2drz = .TRUE._c_long #else logical(pxr_logical) :: l_2drz = .FALSE._c_long @@ -175,7 +175,7 @@ subroutine warpx_charge_deposition(rho,np,xp,yp,zp,w,q,xmin,ymin,zmin,dx,dy,dz,n real(amrex_real), intent(IN OUT):: rho(*) real(amrex_real), intent(IN) :: rmin, dr -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ integer(c_long) :: type_rz_depose = 1 #endif @@ -184,7 +184,7 @@ subroutine warpx_charge_deposition(rho,np,xp,yp,zp,w,q,xmin,ymin,zmin,dx,dy,dz,n rho_nvalid = rho_ntot - 2*rho_ng rho_nguards = rho_ng -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ CALL WRPX_PXR_RZ_VOLUME_SCALING_RHO( & rho,rho_nguards,rho_nvalid, & rmin,dr,type_rz_depose) diff --git a/Source/Make.WarpX b/Source/Make.WarpX index d2551e059..069623b50 100644 --- a/Source/Make.WarpX +++ b/Source/Make.WarpX @@ -141,7 +141,6 @@ endif ifeq ($(USE_RZ),TRUE) USERSuffix := $(USERSuffix).RZ - DEFINES += -DWARPX_RZ endif ifeq ($(DO_ELECTROSTATIC),TRUE) diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 08f8a77b4..7d90857b4 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -148,7 +148,7 @@ PhysicalParticleContainer::AddGaussianBeam(Real x_m, Real y_m, Real z_m, npart /= 4; } for (long i = 0; i < npart; ++i) { -#if ( AMREX_SPACEDIM == 3 | WARPX_RZ) +#if ( AMREX_SPACEDIM == 3 | WARPX_DIM_RZ) Real weight = q_tot/npart/charge; Real x = distx(mt); Real y = disty(mt); @@ -269,7 +269,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) if (!part_realbox.ok()) part_realbox = geom.ProbDomain(); int num_ppc = plasma_injector->num_particles_per_cell; -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ Real rmax = std::min(plasma_injector->xmax, part_realbox.hi(0)); #endif @@ -323,7 +323,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) Real density_min = plasma_injector->density_min; Real density_max = plasma_injector->density_max; -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ bool radially_weighted = plasma_injector->radially_weighted; #endif @@ -496,11 +496,11 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) #endif // Save the x and y values to use in the insideBounds checks. - // This is needed with WARPX_RZ since x and y are modified. + // This is needed with WARPX_DIM_RZ since x and y are modified. Real xb = x; Real yb = y; -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ // Replace the x and y, choosing the angle randomly. // These x and y are used to get the momentum and density Real theta = 2.*MathConst::pi*amrex::Random(); @@ -574,7 +574,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) // Real weight = dens * scale_fac / (AMREX_D_TERM(fac, *fac, *fac)); Real weight = dens * scale_fac; -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ if (radially_weighted) { weight *= 2.*MathConst::pi*xb; } else { @@ -603,7 +603,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) p.pos(1) = y; p.pos(2) = z; #elif (AMREX_SPACEDIM == 2) -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ pa[PIdx::theta][ip] = theta; #endif p.pos(0) = xb; diff --git a/Source/Particles/Pusher/GetAndSetPosition.H b/Source/Particles/Pusher/GetAndSetPosition.H index 42c61343e..3c74baeb2 100644 --- a/Source/Particles/Pusher/GetAndSetPosition.H +++ b/Source/Particles/Pusher/GetAndSetPosition.H @@ -5,7 +5,7 @@ #include #include -#ifndef WARPX_RZ +#ifndef WARPX_DIM_RZ /* \brief Extract the particle's coordinates from the ParticleType struct `p`, * and stores them in the variables `x`, `y`, `z`. */ @@ -42,7 +42,7 @@ void SetPosition( #endif } -# else // if WARPX_RZ is True +# elif defined WARPX_DIM_RZ /* \brief Extract the particle's coordinates from `theta` and the attributes * of the ParticleType struct `p` (which contains the radius), @@ -71,6 +71,6 @@ void SetCylindricalPositionFromCartesian( p.pos(1) = z; } -#endif // WARPX_RZ +#endif // WARPX_DIM_RZ #endif // WARPX_PARTICLES_PUSHER_GETANDSETPOSITION_H_ diff --git a/Source/Particles/Pusher/UpdatePosition.H b/Source/Particles/Pusher/UpdatePosition.H index 0a4f579f4..a9df63a30 100644 --- a/Source/Particles/Pusher/UpdatePosition.H +++ b/Source/Particles/Pusher/UpdatePosition.H @@ -20,7 +20,7 @@ void UpdatePosition( const amrex::Real inv_gamma = 1./std::sqrt(1. + (ux*ux + uy*uy + uz*uz)*inv_c2); // Update positions over one time step x += ux * inv_gamma * dt; -#if (AMREX_SPACEDIM == 3) || (defined WARPX_RZ) // RZ pushes particles in 3D +#if (AMREX_SPACEDIM == 3) || (defined WARPX_DIM_RZ) // RZ pushes particles in 3D y += uy * inv_gamma * dt; #endif z += uz * inv_gamma * dt; diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 7cf53260a..a609b4cb3 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -13,7 +13,7 @@ struct PIdx enum { // Particle Attributes stored in amrex::ParticleContainer's struct of array w = 0, // weight ux, uy, uz, Ex, Ey, Ez, Bx, By, Bz, -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ theta, // RZ needs all three position components #endif nattribs diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index f6c7afed5..c5c6afc19 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -27,7 +27,7 @@ void WarpXParIter::GetPosition (Cuda::ManagedDeviceVector& x, Cuda::ManagedDeviceVector& y, Cuda::ManagedDeviceVector& z) const { amrex::ParIter<0,0,PIdx::nattribs>::GetPosition(x, z); -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ const auto& attribs = GetAttribs(); const auto& theta = attribs[PIdx::theta]; y.resize(x.size()); @@ -44,7 +44,7 @@ WarpXParIter::GetPosition (Cuda::ManagedDeviceVector& x, Cuda::ManagedDevi void WarpXParIter::SetPosition (const Cuda::ManagedDeviceVector& x, const Cuda::ManagedDeviceVector& y, const Cuda::ManagedDeviceVector& z) { -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ auto& attribs = GetAttribs(); auto& theta = attribs[PIdx::theta]; Cuda::ManagedDeviceVector r(x.size()); @@ -80,7 +80,7 @@ WarpXParticleContainer::WarpXParticleContainer (AmrCore* amr_core, int ispecies) particle_comps["Bx"] = PIdx::Bx; particle_comps["By"] = PIdx::By; particle_comps["Bz"] = PIdx::Bz; -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ particle_comps["theta"] = PIdx::theta; #endif @@ -163,7 +163,7 @@ WarpXParticleContainer::AddOneParticle (ParticleTileType& particle_tile, p.pos(1) = y; p.pos(2) = z; #elif (AMREX_SPACEDIM == 2) -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ attribs[PIdx::theta] = std::atan2(y, x); x = std::sqrt(x*x + y*y); #endif @@ -209,7 +209,7 @@ WarpXParticleContainer::AddNParticles (int lev, std::size_t np = iend-ibegin; -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ Vector theta(np); #endif @@ -228,7 +228,7 @@ WarpXParticleContainer::AddNParticles (int lev, p.pos(1) = y[i]; p.pos(2) = z[i]; #elif (AMREX_SPACEDIM == 2) -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ theta[i-ibegin] = std::atan2(y[i], x[i]); p.pos(0) = std::sqrt(x[i]*x[i] + y[i]*y[i]); #else @@ -265,7 +265,7 @@ WarpXParticleContainer::AddNParticles (int lev, for (int comp = PIdx::uz+1; comp < PIdx::nattribs; ++comp) { -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ if (comp == PIdx::theta) { particle_tile.push_back_real(comp, theta.front(), theta.back()); } @@ -610,7 +610,7 @@ WarpXParticleContainer::DepositCharge ( WarpXParIter& pti, RealVector& wp, &ngRho, &ngRho, &ngRho, &WarpX::nox,&WarpX::noy,&WarpX::noz, &lvect, &WarpX::charge_deposition_algo); -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ warpx_charge_deposition_rz_volume_scaling( data_ptr, &ngRho, rholen.getVect(), &xyzmin[0], &dx[0]); @@ -670,7 +670,7 @@ WarpXParticleContainer::DepositCharge ( WarpXParIter& pti, RealVector& wp, &ngRho, &ngRho, &ngRho, &WarpX::nox,&WarpX::noy,&WarpX::noz, &lvect, &WarpX::charge_deposition_algo); -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ warpx_charge_deposition_rz_volume_scaling( data_ptr, &ngRho, rholen.getVect(), &cxyzmin_tile[0], &cdx[0]); @@ -830,7 +830,7 @@ WarpXParticleContainer::GetChargeDensity (int lev, bool local) &dx[0], &dx[1], &dx[2], &nx, &ny, &nz, &nxg, &nyg, &nzg, &WarpX::nox,&WarpX::noy,&WarpX::noz, &lvect, &WarpX::charge_deposition_algo); -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ long ngRho = WarpX::nox; warpx_charge_deposition_rz_volume_scaling( data_ptr, &ngRho, rholen.getVect(), @@ -1015,7 +1015,7 @@ WarpXParticleContainer::PushX (int lev, Real dt) Real* AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); Real* AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); Real* AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ Real* AMREX_RESTRICT theta = attribs[PIdx::theta].dataPtr(); #endif // Loop over the particles and update their position @@ -1023,12 +1023,12 @@ WarpXParticleContainer::PushX (int lev, Real dt) [=] AMREX_GPU_DEVICE (long i) { ParticleType& p = pstructs[i]; // Particle object that gets updated Real x, y, z; // Temporary variables -#ifndef WARPX_RZ +#ifndef WARPX_DIM_RZ GetPosition( x, y, z, p ); // Initialize x, y, z UpdatePosition( x, y, z, ux[i], uy[i], uz[i], dt); SetPosition( p, x, y, z ); // Update the object p #else - // For WARPX_RZ, the particles are still pushed in 3D Cartesian + // For WARPX_DIM_RZ, the particles are still pushed in 3D Cartesian GetCartesianPositionFromCylindrical( x, y, z, p, theta[i] ); UpdatePosition( x, y, z, ux[i], uy[i], uz[i], dt); SetCylindricalPositionFromCartesian( p, theta[i], x, y, z ); diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index 2c8038ccd..3aa4eb7b7 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -8,7 +8,7 @@ const std::map maxwell_solver_algo_to_int = { {"yee", MaxwellSolverAlgo::Yee }, -#ifndef WARPX_RZ // Not available in RZ +#ifndef WARPX_DIM_RZ // Not available in RZ {"ckc", MaxwellSolverAlgo::CKC }, #endif {"default", MaxwellSolverAlgo::Yee } diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 18252cb64..d45dd3a71 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -993,7 +993,7 @@ WarpX::ComputeDivB (MultiFab& divB, int dcomp, { Real dxinv = 1./dx[0], dyinv = 1./dx[1], dzinv = 1./dx[2]; -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ const Real rmin = GetInstance().Geom(0).ProbLo(0); #endif @@ -1012,7 +1012,7 @@ WarpX::ComputeDivB (MultiFab& divB, int dcomp, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { warpx_computedivb(i, j, k, dcomp, divBfab, Bxfab, Byfab, Bzfab, dxinv, dyinv, dzinv -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ ,rmin #endif ); @@ -1027,7 +1027,7 @@ WarpX::ComputeDivB (MultiFab& divB, int dcomp, { Real dxinv = 1./dx[0], dyinv = 1./dx[1], dzinv = 1./dx[2]; -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ const Real rmin = GetInstance().Geom(0).ProbLo(0); #endif @@ -1046,7 +1046,7 @@ WarpX::ComputeDivB (MultiFab& divB, int dcomp, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { warpx_computedivb(i, j, k, dcomp, divBfab, Bxfab, Byfab, Bzfab, dxinv, dyinv, dzinv -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ ,rmin #endif ); @@ -1061,7 +1061,7 @@ WarpX::ComputeDivE (MultiFab& divE, int dcomp, { Real dxinv = 1./dx[0], dyinv = 1./dx[1], dzinv = 1./dx[2]; -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ const Real rmin = GetInstance().Geom(0).ProbLo(0); #endif @@ -1080,7 +1080,7 @@ WarpX::ComputeDivE (MultiFab& divE, int dcomp, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { warpx_computedive(i, j, k, dcomp, divEfab, Exfab, Eyfab, Ezfab, dxinv, dyinv, dzinv -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ ,rmin #endif ); @@ -1095,7 +1095,7 @@ WarpX::ComputeDivE (MultiFab& divE, int dcomp, { Real dxinv = 1./dx[0], dyinv = 1./dx[1], dzinv = 1./dx[2]; -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ const Real rmin = GetInstance().Geom(0).ProbLo(0); #endif @@ -1114,7 +1114,7 @@ WarpX::ComputeDivE (MultiFab& divE, int dcomp, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { warpx_computedive(i, j, k, dcomp, divEfab, Exfab, Eyfab, Ezfab, dxinv, dyinv, dzinv -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ ,rmin #endif ); -- cgit v1.2.3 From dd12326a323019abda4cb20d1d5d49e332084fad Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Mon, 5 Aug 2019 17:48:59 -0700 Subject: get dt from WarpX instance, to initialize ionization prefactors --- Source/Particles/MultiParticleContainer.cpp | 45 ++++++++++++++------------ Source/Particles/PhysicalParticleContainer.cpp | 18 ++++++++--- 2 files changed, 38 insertions(+), 25 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 329a43ca7..a79670582 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -34,10 +34,6 @@ MultiParticleContainer::MultiParticleContainer (AmrCore* amr_core) pc_tmp.reset(new PhysicalParticleContainer(amr_core)); - // For each species, get the ID of its product species. - // This is used for ionization and pair creation processes. - mapSpeciesProduct(); - // Compute the number of species for which lab-frame data is dumped // nspecies_lab_frame_diags, and map their ID to MultiParticleContainer // particle IDs in map_species_lab_diags. @@ -144,6 +140,9 @@ MultiParticleContainer::InitData () pc->InitData(); } pc_tmp->InitData(); + // For each species, get the ID of its product species. + // This is used for ionization and pair creation processes. + mapSpeciesProduct(); } @@ -511,21 +510,23 @@ MultiParticleContainer::getSpeciesID(std::string product_str) return i_product; } +//void +//MultiParticleContainer::InitIonizationModule() + void MultiParticleContainer::doFieldIonization() { - /* - amrex::Gpu::SharedMemory grid_ids; - amrex::Gpu::SharedMemory tile_ids; - amrex::Gpu::SharedMemory is_ionized; - */ - for (auto& pc : allcontainers){ + if (!pc->do_field_ionization){ continue; } + + auto& prod_pc = allcontainers[pc->ionization_product]; + const Real * const AMREX_RESTRICT p_ionization_energies = pc->ionization_energies.dataPtr(); const Real * const AMREX_RESTRICT p_adk_prefactor = pc->adk_prefactor.dataPtr(); const Real * const AMREX_RESTRICT p_adk_exp_prefactor = pc->adk_exp_prefactor.dataPtr(); const Real * const AMREX_RESTRICT p_adk_power = pc->adk_power.dataPtr(); + for (int lev = 0; lev <= pc->finestLevel(); ++lev){ #ifdef _OPENMP @@ -537,6 +538,10 @@ MultiParticleContainer::doFieldIonization() if ( (pc->NumRuntimeRealComps()>0) || (pc->NumRuntimeIntComps()>0) ) { pc->DefineAndReturnParticleTile(lev, grid_id, tile_id); } + prod_pc->GetParticles(lev)[std::make_pair(grid_id,tile_id)]; + if ( (prod_pc->NumRuntimeRealComps()>0) || (prod_pc->NumRuntimeIntComps()>0) ) { + prod_pc->DefineAndReturnParticleTile(lev, grid_id, tile_id); + } } #endif MFItInfo info; @@ -575,7 +580,7 @@ MultiParticleContainer::doFieldIonization() ParallelFor( np, [=] AMREX_GPU_DEVICE (long i) { - Real random_draw = Random(); + Real random_draw = amrex::Random(); Real ga = std::sqrt(1. + (ux[i]*ux[i] + uy[i]*uy[i] + @@ -589,15 +594,14 @@ MultiParticleContainer::doFieldIonization() int ilev = (int) round(ilev_real[i]); // int ilev = static_cast(round(ilev_real[i])); Real p; - Real w_dtau; - if (E<1.e-100*(p_ionization_energies[0])){ - p = 0.; - } else { - w_dtau = 1./ ga * p_adk_prefactor[ilev] * - std::pow(E,p_adk_power[ilev]) * - std::exp( p_adk_exp_prefactor[ilev]/E ); - p = 1. - std::exp( - w_dtau ); - } + //if (E<1.e-20*(p_ionization_energies[0])){ + // p = 0.; + //} else { + Real w_dtau = 1./ ga * p_adk_prefactor[ilev] * + std::pow(E,p_adk_power[ilev]) * + std::exp( p_adk_exp_prefactor[ilev]/E ); + p = 1. - std::exp( - w_dtau ); + //} p_is_ionized[i] = 0; if (random_draw < p){ ilev_real[i] += 1.; @@ -617,7 +621,6 @@ MultiParticleContainer::doFieldIonization() break; } - auto& prod_pc = allcontainers[pc->ionization_product]; auto& prod_particle_tile = prod_pc->GetParticles(lev)[std::make_pair(grid_id,tile_id)]; const int np_old = prod_particle_tile.GetArrayOfStructs().size(); const int np_new = np_old + np_ionized; diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 8232b621f..fa01f472d 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -41,8 +41,8 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp pp.query("do_field_ionization", do_field_ionization); // If do_field_ionization, read initialization data from input file and // read ionization energies from table. - if (do_field_ionization) - InitIonizationModule(); + //if (do_field_ionization) + // InitIonizationModule(); pp.query("plot_species", plot_species); int do_user_plot_vars; @@ -90,6 +90,7 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core) void PhysicalParticleContainer::InitData() { + if (do_field_ionization) {InitIonizationModule();} AddParticles(0); // Note - add on level 0 Redistribute(); // We then redistribute } @@ -2078,6 +2079,7 @@ PhysicalParticleContainer::FieldGather (WarpXParIter& pti, void PhysicalParticleContainer::InitIonizationModule() { + if (!do_field_ionization) return; ParmParse pp(species_name); pp.query("ionization_level", species_ionization_level); pp.get("ionization_product", ionization_product_name); @@ -2092,16 +2094,22 @@ void PhysicalParticleContainer::InitIonizationModule() int offset = ion_energy_offsets[ion_element_id]; for(int i=0; i Date: Tue, 6 Aug 2019 13:07:57 -0700 Subject: cleaning, and add capability for boosted frame runtime particle components --- Source/Particles/MultiParticleContainer.cpp | 318 +++++++++++++++---------- Source/Particles/PhysicalParticleContainer.H | 3 + Source/Particles/PhysicalParticleContainer.cpp | 82 +++++++ Source/Particles/WarpXParticleContainer.H | 8 + 4 files changed, 283 insertions(+), 128 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index a79670582..6c6a03231 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -513,155 +513,217 @@ MultiParticleContainer::getSpeciesID(std::string product_str) //void //MultiParticleContainer::InitIonizationModule() +namespace +{ + static void createIonizedParticles( + int lev, const MFIter& mfi, + std::unique_ptr< WarpXParticleContainer>& pc_source, + std::unique_ptr< WarpXParticleContainer>& pc_product, + const int * const p_is_ionized) + { + + const int grid_id = mfi.index(); + const int tile_id = mfi.LocalTileIndex(); + + // Get source particle data + auto& ptile_source = pc_source->GetParticles(lev)[std::make_pair(grid_id,tile_id)]; + const int np_source = ptile_source.GetArrayOfStructs().size(); + // --- source AoS particle data + WarpXParticleContainer::ParticleType* particles_source = ptile_source.GetArrayOfStructs()().data(); + // --- source SoA particle data + auto& soa_source = ptile_source.GetStructOfArrays(); + GpuArray attribs_source; + for (int ia = 0; ia < PIdx::nattribs; ++ia) { + attribs_source[ia] = soa_source.GetRealData(ia).data(); + } + // --- source runtime attribs + bool do_boosted_product = WarpX::do_boosted_frame_diagnostic + && pc_product->DoBoostedFrameDiags(); + bool do_boosted_source = WarpX::do_boosted_frame_diagnostic + && pc_source->DoBoostedFrameDiags(); + GpuArray runtime_attribs_source; + // + if (do_boosted_product && do_boosted_source) { + // If boosted frame diagnostics for source species, store them + std::map comps_source = pc_source->getParticleComps(); + runtime_attribs_source[0] = soa_source.GetRealData(comps_source[ "xold"]).data(); + runtime_attribs_source[1] = soa_source.GetRealData(comps_source[ "yold"]).data(); + runtime_attribs_source[2] = soa_source.GetRealData(comps_source[ "zold"]).data(); + runtime_attribs_source[3] = soa_source.GetRealData(comps_source["uxold"]).data(); + runtime_attribs_source[4] = soa_source.GetRealData(comps_source["uyold"]).data(); + runtime_attribs_source[5] = soa_source.GetRealData(comps_source["uzold"]).data(); + } else if (do_boosted_product && !do_boosted_source){ + // Otherwise, store current particle momenta. + // Positions are copied from AoS data. + runtime_attribs_source[3] = soa_source.GetRealData(PIdx::ux).data(); + runtime_attribs_source[4] = soa_source.GetRealData(PIdx::uy).data(); + runtime_attribs_source[5] = soa_source.GetRealData(PIdx::uz).data(); + } + + // Indices of product particle for each ionized source particle. + // i_product[i] is the location in product tile of product particle + // from source particle i. + amrex::Gpu::ManagedVector i_product; + i_product.resize(np_source); + // 0GetParticles(lev)[std::make_pair(grid_id,tile_id)]; + // old and new (i.e., including ionized particles) number of particles + // for product species + const int np_product_old = ptile_product.GetArrayOfStructs().size(); + const int np_product_new = np_product_old + np_ionized; + // Allocate extra space in product species for ionized particles. + ptile_product.resize(np_product_new); + // --- product AoS particle data + // First element is the first newly-created product particle + WarpXParticleContainer::ParticleType* particles_product = ptile_product.GetArrayOfStructs()().data() + np_product_old; + // --- product SoA particle data + auto& soa_product = ptile_product.GetStructOfArrays(); + GpuArray attribs_product; + for (int ia = 0; ia < PIdx::nattribs; ++ia) { + // First element is the first newly-created product particle + attribs_product[ia] = soa_product.GetRealData(ia).data() + np_product_old; + } + // --- product runtime attribs + GpuArray runtime_attribs_product; + if (do_boosted_product) { + std::map comps_product = pc_product->getParticleComps(); + runtime_attribs_product[0] = soa_product.GetRealData(comps_product[ "xold"]).data() + np_product_old; + runtime_attribs_product[1] = soa_product.GetRealData(comps_product[ "yold"]).data() + np_product_old; + runtime_attribs_product[2] = soa_product.GetRealData(comps_product[ "zold"]).data() + np_product_old; + runtime_attribs_product[3] = soa_product.GetRealData(comps_product["uxold"]).data() + np_product_old; + runtime_attribs_product[4] = soa_product.GetRealData(comps_product["uyold"]).data() + np_product_old; + runtime_attribs_product[5] = soa_product.GetRealData(comps_product["uzold"]).data() + np_product_old; + } + + int pid_product; +#pragma omp critical (doFieldIonization_nextid) + { + // ID of first newly-created product particle + pid_product = pc_product->NextID(); + // Update NextID to include particles created in this function + pc_product->setNextID(pid_product+np_ionized); + } + const int cpuid = ParallelDescriptor::MyProc(); + + // Loop over all source particles. If is_ionized, copy particle data + // to corresponding product particle. + amrex::For( + np_source, [=] AMREX_GPU_DEVICE (int is) noexcept + { + if(p_is_ionized[is]){ + int ip = p_i_product[is]; + // is: index of ionized particle in source species + // ip: index of corresponding created particle in product species + WarpXParticleContainer::ParticleType& p_product = particles_product[ip]; + WarpXParticleContainer::ParticleType& p_source = particles_source[is]; + // Copy particle from source to product: AoS + p_product.id() = pid_product + ip; + p_product.cpu() = cpuid; + p_product.pos(0) = p_source.pos(0); + p_product.pos(1) = p_source.pos(1); +#if (AMREX_SPACEDIM == 3) + p_product.pos(2) = p_source.pos(2); +#endif + // Copy particle from source to product: SoA + for (int ia = 0; ia < PIdx::nattribs; ++ia) { + attribs_product[ia][ip] = attribs_source[ia][is]; + } + // Update xold etc. if boosted frame diagnostics required + // for product species. For position, we need a different + // handling depending on do_boosted_source. For momentum, + // runtime_attribs_source[3-5] contains appropriate data. + if (do_boosted_product) { + if (do_boosted_source) { + runtime_attribs_product[0][ip] = runtime_attribs_source[0][ip]; + runtime_attribs_product[1][ip] = runtime_attribs_source[1][ip]; + runtime_attribs_product[2][ip] = runtime_attribs_source[2][ip]; + } else { + runtime_attribs_product[0][ip] = p_source.pos(0); + runtime_attribs_product[1][ip] = p_source.pos(1); + runtime_attribs_product[2][ip] = p_source.pos(2); + } + runtime_attribs_product[3][ip] = runtime_attribs_source[3][ip]; + runtime_attribs_product[4][ip] = runtime_attribs_source[4][ip]; + runtime_attribs_product[5][ip] = runtime_attribs_source[5][ip]; + } + } + } + ); + } +} + void MultiParticleContainer::doFieldIonization() { - for (auto& pc : allcontainers){ - - if (!pc->do_field_ionization){ continue; } - auto& prod_pc = allcontainers[pc->ionization_product]; + // Loop over all species. + // Ionized particles in pc_source create particles in pc_product + for (auto& pc_source : allcontainers){ + + // Skip if not ionizable + if (!pc_source->do_field_ionization){ continue; } - const Real * const AMREX_RESTRICT p_ionization_energies = pc->ionization_energies.dataPtr(); - const Real * const AMREX_RESTRICT p_adk_prefactor = pc->adk_prefactor.dataPtr(); - const Real * const AMREX_RESTRICT p_adk_exp_prefactor = pc->adk_exp_prefactor.dataPtr(); - const Real * const AMREX_RESTRICT p_adk_power = pc->adk_power.dataPtr(); + // Get product species + auto& pc_product = allcontainers[pc_source->ionization_product]; - for (int lev = 0; lev <= pc->finestLevel(); ++lev){ + for (int lev = 0; lev <= pc_source->finestLevel(); ++lev){ #ifdef _OPENMP - // First touch all tiles in the map in serial - for (MFIter mfi = pc->MakeMFIter(lev); mfi.isValid(); ++mfi) { + // Touch all tiles of source species in serial if additional arguments + for (MFIter mfi = pc_source->MakeMFIter(lev); mfi.isValid(); ++mfi) { const int grid_id = mfi.index(); const int tile_id = mfi.LocalTileIndex(); - pc->GetParticles(lev)[std::make_pair(grid_id,tile_id)]; - if ( (pc->NumRuntimeRealComps()>0) || (pc->NumRuntimeIntComps()>0) ) { - pc->DefineAndReturnParticleTile(lev, grid_id, tile_id); - } - prod_pc->GetParticles(lev)[std::make_pair(grid_id,tile_id)]; - if ( (prod_pc->NumRuntimeRealComps()>0) || (prod_pc->NumRuntimeIntComps()>0) ) { - prod_pc->DefineAndReturnParticleTile(lev, grid_id, tile_id); + pc_source->GetParticles(lev)[std::make_pair(grid_id,tile_id)]; + if ( (pc_source->NumRuntimeRealComps()>0) || (pc_source->NumRuntimeIntComps()>0) ) { + pc_source->DefineAndReturnParticleTile(lev, grid_id, tile_id); } } #endif + // Touch all tiles of product species in serial + for (MFIter mfi = pc_source->MakeMFIter(lev); mfi.isValid(); ++mfi) { + const int grid_id = mfi.index(); + const int tile_id = mfi.LocalTileIndex(); + pc_product->GetParticles(lev)[std::make_pair(grid_id,tile_id)]; + pc_product->DefineAndReturnParticleTile(lev, grid_id, tile_id); + } + + // Enable tiling MFItInfo info; - if (pc->do_tiling && Gpu::notInLaunchRegion()) { - info.EnableTiling(pc->tile_size); + if (pc_source->do_tiling && Gpu::notInLaunchRegion()) { + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + pc_product->do_tiling, + "For ionization, either all or none of the " + "particle species must use tiling."); + info.EnableTiling(pc_source->tile_size); } + #ifdef _OPENMP info.SetDynamic(true); #pragma omp parallel #endif - for (MFIter mfi = pc->MakeMFIter(lev, info); mfi.isValid(); ++mfi) + // Loop over all grids (if not tiling) or grids and tiles (if tiling) + for (MFIter mfi = pc_source->MakeMFIter(lev, info); mfi.isValid(); ++mfi) { - const int grid_id = mfi.index(); - const int tile_id = mfi.LocalTileIndex(); - auto& particle_tile = pc->GetParticles(lev)[std::make_pair(grid_id,tile_id)]; - auto& soa = particle_tile.GetStructOfArrays(); - const int np = particle_tile.GetArrayOfStructs().size(); - if (np == 0) break; + // Ionization mask: one element per source particles. + // 0 if not ionized, 1 if ionized. amrex::Gpu::ManagedVector is_ionized; - // const int np_lev = pc->NumberOfParticlesAtLevel(lev); - is_ionized.resize(np); - int * const AMREX_RESTRICT p_is_ionized = is_ionized.dataPtr(); - const Real * const AMREX_RESTRICT ux = soa.GetRealData(PIdx::ux).data(); - const Real * const AMREX_RESTRICT uy = soa.GetRealData(PIdx::uy).data(); - const Real * const AMREX_RESTRICT uz = soa.GetRealData(PIdx::uz).data(); - const Real * const AMREX_RESTRICT ex = soa.GetRealData(PIdx::Ex).data(); - const Real * const AMREX_RESTRICT ey = soa.GetRealData(PIdx::Ey).data(); - const Real * const AMREX_RESTRICT ez = soa.GetRealData(PIdx::Ez).data(); - const Real * const AMREX_RESTRICT bx = soa.GetRealData(PIdx::Bx).data(); - const Real * const AMREX_RESTRICT by = soa.GetRealData(PIdx::By).data(); - const Real * const AMREX_RESTRICT bz = soa.GetRealData(PIdx::Bz).data(); - Real * const AMREX_RESTRICT ilev_real = soa.GetRealData(pc->particle_comps["ionization_level"]).data(); - Real c = PhysConst::c; - Real c2_inv = 1./c/c; - - ParallelFor( - np, - [=] AMREX_GPU_DEVICE (long i) { - Real random_draw = amrex::Random(); - Real ga = std::sqrt(1. + - (ux[i]*ux[i] + - uy[i]*uy[i] + - uz[i]*uz[i]) * c2_inv); - Real E = std::sqrt( - - ( ux[i]*ex[i] + uy[i]*ey[i] + uz[i]*ez[i] ) * ( ux[i]*ex[i] + uy[i]*ey[i] + uz[i]*ez[i] ) * c2_inv - + ( ga *ex[i] + uy[i]*bz[i] - uz[i]*by[i] ) * ( ga *ex[i] + uy[i]*bz[i] - uz[i]*by[i] ) - + ( ga *ey[i] + uz[i]*bx[i] - ux[i]*bz[i] ) * ( ga *ey[i] + uz[i]*bx[i] - ux[i]*bz[i] ) - + ( ga *ez[i] + ux[i]*by[i] - uy[i]*bx[i] ) * ( ga *ez[i] + ux[i]*by[i] - uy[i]*bx[i] ) - ); - int ilev = (int) round(ilev_real[i]); - // int ilev = static_cast(round(ilev_real[i])); - Real p; - //if (E<1.e-20*(p_ionization_energies[0])){ - // p = 0.; - //} else { - Real w_dtau = 1./ ga * p_adk_prefactor[ilev] * - std::pow(E,p_adk_power[ilev]) * - std::exp( p_adk_exp_prefactor[ilev]/E ); - p = 1. - std::exp( - w_dtau ); - //} - p_is_ionized[i] = 0; - if (random_draw < p){ - ilev_real[i] += 1.; - p_is_ionized[i] = 1; - } - } - ); // ParallelFor - - amrex::Gpu::ManagedVector is_ionized_cumsum_vector; - is_ionized_cumsum_vector.resize(np); - int np_ionized = p_is_ionized[0]; - for(int i=1; iGetParticles(lev)[std::make_pair(grid_id,tile_id)]; - const int np_old = prod_particle_tile.GetArrayOfStructs().size(); - const int np_new = np_old + np_ionized; - prod_particle_tile.resize(np_new); - auto& prod_soa = prod_particle_tile.GetStructOfArrays(); - - GpuArray prod_pa; - for (int ia = 0; ia < PIdx::nattribs; ++ia) { - prod_pa[ia] = prod_soa.GetRealData(ia).data() + np_old; - } - WarpXParticleContainer::ParticleType* prod_pp = prod_particle_tile.GetArrayOfStructs()().data() + np_old; - WarpXParticleContainer::ParticleType* pp = particle_tile.GetArrayOfStructs()().data(); - - GpuArray pa; - for (int ia = 0; ia < PIdx::nattribs; ++ia) { - pa[ia] = soa.GetRealData(ia).data(); - } + pc_source->buildIonizationMask(mfi, lev, is_ionized); + const int * const AMREX_RESTRICT p_is_ionized = is_ionized.dataPtr(); - const int cpuid = ParallelDescriptor::MyProc(); - int* AMREX_RESTRICT is_ionized_cumsum = is_ionized_cumsum_vector.dataPtr(); - amrex::For( - np, [=] AMREX_GPU_DEVICE (int ip) noexcept - { - if(is_ionized[ip]){ - int i = is_ionized_cumsum[ip]; - WarpXParticleContainer::ParticleType& prod_p = prod_pp[i]; - WarpXParticleContainer::ParticleType& p = pp[ip]; - prod_p.id() = 9999; - prod_p.cpu() = cpuid; - prod_p.pos(0) = p.pos(0); - prod_p.pos(1) = p.pos(1); -#if (AMREX_SPACEDIM == 3) - prod_p.pos(2) = p.pos(2); -#endif - for (int ia = 0; ia < PIdx::nattribs; ++ia) { - prod_pa[ia][i] = pa[ia][ip]; - } - } - } - ); + createIonizedParticles(lev, mfi, pc_source, pc_product, p_is_ionized); } // MFIter - } - } + } // lev + } // pc_source } diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index 6f9fdc485..8dab1a2f7 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -111,6 +111,9 @@ public: virtual int doFieldIonization (const int lev) override; + virtual void buildIonizationMask (const amrex::MFIter& mfi, const int lev, + amrex::Gpu::ManagedVector& ionization_mask) override; + // Inject particles in Box 'part_box' virtual void AddParticles (int lev); diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index fa01f472d..7870e683c 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -2133,3 +2133,85 @@ PhysicalParticleContainer::doFieldIonization(const int lev) Print()<<"in PhysicalParticleContainer::doFieldIonization\n"; return 0; } + +/* \brief create mask of ionized particles (1 if ionized, 0 otherwise) + * + * \param mfi: tile or grid + * \param lev: MR level + * \param ionization_mask: Array with as many elements as particles in mfi. + * This function initialized the array, and set each element to 1 or 0 + * depending on whether the particle is ionized or not. + */ +void +PhysicalParticleContainer::buildIonizationMask(const amrex::MFIter& mfi, const int lev, + amrex::Gpu::ManagedVector& ionization_mask) +{ + // Get pointers to ionization data from pc_source + const Real * const AMREX_RESTRICT p_ionization_energies = ionization_energies.dataPtr(); + const Real * const AMREX_RESTRICT p_adk_prefactor = adk_prefactor.dataPtr(); + const Real * const AMREX_RESTRICT p_adk_exp_prefactor = adk_exp_prefactor.dataPtr(); + const Real * const AMREX_RESTRICT p_adk_power = adk_power.dataPtr(); + + // Current tile info + const int grid_id = mfi.index(); + const int tile_id = mfi.LocalTileIndex(); + + // Get GPU-friendly arrays of particle data + auto& ptile = GetParticles(lev)[std::make_pair(grid_id,tile_id)]; + // Only need attribs (i.e., SoA data) + auto& soa = ptile.GetStructOfArrays(); + const int np = ptile.GetArrayOfStructs().size(); + + // If no particle, nothing to do. + if (np == 0) return; + // Otherwise, resize ionization_mask, and get poiters to attribs arrays. + ionization_mask.resize(np); + int * const AMREX_RESTRICT p_ionization_mask = ionization_mask.data(); + const Real * const AMREX_RESTRICT ux = soa.GetRealData(PIdx::ux).data(); + const Real * const AMREX_RESTRICT uy = soa.GetRealData(PIdx::uy).data(); + const Real * const AMREX_RESTRICT uz = soa.GetRealData(PIdx::uz).data(); + const Real * const AMREX_RESTRICT ex = soa.GetRealData(PIdx::Ex).data(); + const Real * const AMREX_RESTRICT ey = soa.GetRealData(PIdx::Ey).data(); + const Real * const AMREX_RESTRICT ez = soa.GetRealData(PIdx::Ez).data(); + const Real * const AMREX_RESTRICT bx = soa.GetRealData(PIdx::Bx).data(); + const Real * const AMREX_RESTRICT by = soa.GetRealData(PIdx::By).data(); + const Real * const AMREX_RESTRICT bz = soa.GetRealData(PIdx::Bz).data(); + Real* ilev_real = soa.GetRealData(particle_comps["ionization_level"]).data(); + + Real c = PhysConst::c; + Real c2_inv = 1./c/c; + + // Loop over all particles in grid/tile. If ionized, set mask to 1 + // and increment ionization level. + ParallelFor( + np, + [=] AMREX_GPU_DEVICE (long i) { + Real random_draw = amrex::Random(); + // Compute electric field amplitude in the particle's frame of + // reference (particularly important when in boosted frame). + Real ga = std::sqrt(1. + (ux[i]*ux[i] + uy[i]*uy[i] + uz[i]*uz[i]) * c2_inv); + Real E = std::sqrt( + - ( ux[i]*ex[i] + uy[i]*ey[i] + uz[i]*ez[i] ) * ( ux[i]*ex[i] + uy[i]*ey[i] + uz[i]*ez[i] ) * c2_inv + + ( ga *ex[i] + uy[i]*bz[i] - uz[i]*by[i] ) * ( ga *ex[i] + uy[i]*bz[i] - uz[i]*by[i] ) + + ( ga *ey[i] + uz[i]*bx[i] - ux[i]*bz[i] ) * ( ga *ey[i] + uz[i]*bx[i] - ux[i]*bz[i] ) + + ( ga *ez[i] + ux[i]*by[i] - uy[i]*bx[i] ) * ( ga *ez[i] + ux[i]*by[i] - uy[i]*bx[i] ) + ); + // Get index of ionization_level + int ilev = (int) round(ilev_real[i]); + // Compute probability of ionization p + Real p; + Real w_dtau = 1./ ga * p_adk_prefactor[ilev] * + std::pow(E,p_adk_power[ilev]) * + std::exp( p_adk_exp_prefactor[ilev]/E ); + p = 1. - std::exp( - w_dtau ); + p_ionization_mask[i] = 0; + + if (random_draw < p){ + // increment particle's ionization level + ilev_real[i] += 1.; + // update mask + p_ionization_mask[i] = 1; + } + } + ); +} diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 1c1b3fd22..017a857a7 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -250,6 +250,8 @@ public: static int NextID () { return ParticleType::NextID(); } + void setNextID(int next_id) { ParticleType::NextID(next_id); } + bool do_splitting = false; // split along axes (0) or diagonals (1) @@ -277,6 +279,12 @@ public: virtual int doFieldIonization ( const int lev) { return 0; }; + virtual void buildIonizationMask (const amrex::MFIter& mfi, const int lev, + amrex::Gpu::ManagedVector& ionization_mask) + {}; + + std::map getParticleComps() { return particle_comps;} + protected: std::map particle_comps; -- cgit v1.2.3 From dab4a0081eac6929db26d16b79ef57c5882df0b4 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Wed, 7 Aug 2019 15:06:49 -0700 Subject: use c++ deposition by default. includes old attribs --- Source/Particles/MultiParticleContainer.cpp | 28 +++----------- Source/Particles/PhysicalParticleContainer.cpp | 53 ++++++++++++-------------- Source/WarpX.cpp | 2 +- 3 files changed, 31 insertions(+), 52 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 84cd107f8..36b41e0aa 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -516,12 +516,11 @@ MultiParticleContainer::getSpeciesID(std::string product_str) namespace { static void createIonizedParticles( - int lev, const MFIter& mfi, + int lev, const MFIter& mfi, std::unique_ptr< WarpXParticleContainer>& pc_source, std::unique_ptr< WarpXParticleContainer>& pc_product, const int * const p_is_ionized) { - // Print()<<"in createIonizedParticles\n"; const int grid_id = mfi.index(); const int tile_id = mfi.LocalTileIndex(); @@ -530,7 +529,6 @@ namespace const int np_source = ptile_source.GetArrayOfStructs().size(); if (np_source == 0) return; // --- source AoS particle data - //Print()<<"la 1\n"; WarpXParticleContainer::ParticleType* particles_source = ptile_source.GetArrayOfStructs()().data(); // --- source SoA particle data auto& soa_source = ptile_source.GetStructOfArrays(); @@ -544,10 +542,10 @@ namespace bool do_boosted_source = WarpX::do_boosted_frame_diagnostic && pc_source->DoBoostedFrameDiags(); GpuArray runtime_attribs_source; - // - //Print()<<"la 2\n"; + // Prepare arrays for boosted frame diagnostics. + // If do_boosted_product, need different treatment + // depending on do_boosted_source if (do_boosted_product && do_boosted_source) { - //Print()<<"do_boosted_product && do_boosted_source\n"; // If boosted frame diagnostics for source species, store them std::map comps_source = pc_source->getParticleComps(); runtime_attribs_source[0] = soa_source.GetRealData(comps_source[ "xold"]).data(); @@ -557,7 +555,6 @@ namespace runtime_attribs_source[4] = soa_source.GetRealData(comps_source["uyold"]).data(); runtime_attribs_source[5] = soa_source.GetRealData(comps_source["uzold"]).data(); } else if (do_boosted_product && !do_boosted_source){ - //Print()<<"do_boosted_product && !do_boosted_source\n"; // Otherwise, store current particle momenta. // Positions are copied from AoS data. runtime_attribs_source[3] = soa_source.GetRealData(PIdx::ux).data(); @@ -568,27 +565,20 @@ namespace // Indices of product particle for each ionized source particle. // i_product[i] is the location in product tile of product particle // from source particle i. - //Print()<<"la 3\n"; amrex::Gpu::ManagedVector i_product; i_product.resize(np_source); // 0GetParticles(lev)[std::make_pair(grid_id,tile_id)]; // old and new (i.e., including ionized particles) number of particles @@ -601,18 +591,15 @@ namespace // First element is the first newly-created product particle WarpXParticleContainer::ParticleType* particles_product = ptile_product.GetArrayOfStructs()().data() + np_product_old; // --- product SoA particle data - //Print()<<"la 8\n"; auto& soa_product = ptile_product.GetStructOfArrays(); GpuArray attribs_product; for (int ia = 0; ia < PIdx::nattribs; ++ia) { // First element is the first newly-created product particle attribs_product[ia] = soa_product.GetRealData(ia).data() + np_product_old; } - //Print()<<"la 9\n"; // --- product runtime attribs GpuArray runtime_attribs_product; if (do_boosted_product) { - //Print()<<"do_boosted_product 1\n"; std::map comps_product = pc_product->getParticleComps(); runtime_attribs_product[0] = soa_product.GetRealData(comps_product[ "xold"]).data() + np_product_old; runtime_attribs_product[1] = soa_product.GetRealData(comps_product[ "yold"]).data() + np_product_old; @@ -640,7 +627,7 @@ namespace if(p_is_ionized[is]){ int ip = p_i_product[is]; // is: index of ionized particle in source species - // ip: index of corresponding created particle in product species + // ip: index of corresponding new particle in product species WarpXParticleContainer::ParticleType& p_product = particles_product[ip]; WarpXParticleContainer::ParticleType& p_source = particles_source[is]; // Copy particle from source to product: AoS @@ -660,23 +647,18 @@ namespace // handling depending on do_boosted_source. For momentum, // runtime_attribs_source[3-5] contains appropriate data. if (do_boosted_product) { - //Print()<<"do_boosted_product\n"; if (do_boosted_source) { - //Print()<<"do_boosted_product do_boosted_source\n"; runtime_attribs_product[0][ip] = runtime_attribs_source[0][ip]; runtime_attribs_product[1][ip] = runtime_attribs_source[1][ip]; runtime_attribs_product[2][ip] = runtime_attribs_source[2][ip]; } else { - //Print()<<"do_boosted_product NO do_boosted_source\n"; runtime_attribs_product[0][ip] = p_source.pos(0); runtime_attribs_product[1][ip] = p_source.pos(1); runtime_attribs_product[2][ip] = p_source.pos(2); } - //Print()<<"momenta\n"; runtime_attribs_product[3][ip] = runtime_attribs_source[3][ip]; runtime_attribs_product[4][ip] = runtime_attribs_source[4][ip]; runtime_attribs_product[5][ip] = runtime_attribs_source[5][ip]; - //Print()<<"done\n"; } } } diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 7870e683c..75e975131 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -2094,14 +2094,11 @@ void PhysicalParticleContainer::InitIonizationModule() int offset = ion_energy_offsets[ion_element_id]; for(int i=0; i WarpX::boost_direction = {0,0,0}; int WarpX::do_compute_max_step_from_zmax = 0; Real WarpX::zmax_plasma_to_compute_max_step = 0.; -long WarpX::use_picsar_deposition = 1; +long WarpX::use_picsar_deposition = 0; long WarpX::current_deposition_algo; long WarpX::charge_deposition_algo; long WarpX::field_gathering_algo; -- cgit v1.2.3 From 4be45ed13bcd1dfab7455e09b2a9b96cd2064fca Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Wed, 7 Aug 2019 15:18:36 -0700 Subject: cleaning, remote outdated functions --- Source/Particles/MultiParticleContainer.cpp | 9 +-------- Source/Particles/PhysicalParticleContainer.H | 2 -- Source/Particles/PhysicalParticleContainer.cpp | 13 ------------- Source/Particles/WarpXParticleContainer.H | 3 --- 4 files changed, 1 insertion(+), 26 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 136d23f6c..48f685e8f 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -646,8 +646,6 @@ void MultiParticleContainer::doFieldIonization() { - Print()<<"in MultiParticleContainer::doFieldIonization\n"; - // Loop over all species. // Ionized particles in pc_source create particles in pc_product for (auto& pc_source : allcontainers){ @@ -698,16 +696,11 @@ MultiParticleContainer::doFieldIonization() { // Ionization mask: one element per source particles. // 0 if not ionized, 1 if ionized. - // Print()<<"here 1\n"; amrex::Gpu::ManagedVector is_ionized; - // Print()<<"here 2\n"; pc_source->buildIonizationMask(mfi, lev, is_ionized); - // Print()<<"here 3\n"; const int * const AMREX_RESTRICT p_is_ionized = is_ionized.dataPtr(); - // Print()<<"here 4\n"; createIonizedParticles(lev, mfi, pc_source, pc_product, p_is_ionized); - // Print()<<"here 5\n"; - } // MFIter + } } // lev } // pc_source } diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index 8dab1a2f7..c10dcc61b 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -108,8 +108,6 @@ public: virtual void PostRestart () final {} void SplitParticles(int lev); - - virtual int doFieldIonization (const int lev) override; virtual void buildIonizationMask (const amrex::MFIter& mfi, const int lev, amrex::Gpu::ManagedVector& ionization_mask) override; diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 75e975131..02602b844 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -80,12 +80,6 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core) : WarpXParticleContainer(amr_core, 0) { plasma_injector.reset(new PlasmaInjector()); -/* - if (do_field_ionization){ - Print()<<"AddRealComp\n"; - AddRealComp("ionization_level"); - } -*/ } void PhysicalParticleContainer::InitData() @@ -2121,13 +2115,6 @@ void PhysicalParticleContainer::InitIonizationModule() } } -int -PhysicalParticleContainer::doFieldIonization(const int lev) -{ - Print()<<"in PhysicalParticleContainer::doFieldIonization\n"; - return 0; -} - /* \brief create mask of ionized particles (1 if ionized, 0 otherwise) * * \param mfi: tile or grid diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 017a857a7..82190b307 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -276,9 +276,6 @@ public: virtual void setIonizationProduct (int i_product) {}; - virtual int doFieldIonization ( - const int lev) { return 0; }; - virtual void buildIonizationMask (const amrex::MFIter& mfi, const int lev, amrex::Gpu::ManagedVector& ionization_mask) {}; -- cgit v1.2.3 From ef45547cd84301628d456486a32f980af60c4b63 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 7 Aug 2019 15:52:31 -0700 Subject: Remove unused variables --- Source/Particles/PhysicalParticleContainer.cpp | 2 -- 1 file changed, 2 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 7d90857b4..59d3ce76f 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1176,8 +1176,6 @@ PhysicalParticleContainer::Evolve (int lev, { const IntVect& ref_ratio = WarpX::RefRatio(lev-1); const Box& cbox = amrex::coarsen(box,ref_ratio); - const std::array& cxyzmin_grid = WarpX::LowerCorner(cbox, lev-1); - const int* cixyzmin_grid = cbox.loVect(); // Data on the grid FArrayBox const* cexfab = &(*cEx)[pti]; -- cgit v1.2.3 From 30fa1002b9d106acbbbbfd0f01c7e80f47c72747 Mon Sep 17 00:00:00 2001 From: Maxence Thevenet Date: Wed, 7 Aug 2019 18:57:59 -0400 Subject: add profile data, remove typo in example --- Examples/Modules/ionization/inputs.2d | 1 - Source/Particles/MultiParticleContainer.cpp | 4 +++- Source/Particles/PhysicalParticleContainer.cpp | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Examples/Modules/ionization/inputs.2d b/Examples/Modules/ionization/inputs.2d index 1f2b77a7e..ddf30763a 100644 --- a/Examples/Modules/ionization/inputs.2d +++ b/Examples/Modules/ionization/inputs.2d @@ -4,7 +4,6 @@ max_step = 500 warpx.verbose = 1 warpx.cfl = 1 -warpx.plot_rho = 1 warpx.do_pml = 1 warpx.use_filter = 1 diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 48f685e8f..dafe8fe93 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -521,6 +521,8 @@ namespace std::unique_ptr< WarpXParticleContainer>& pc_product, const int * const p_is_ionized) { + BL_PROFILE("createIonizedParticles"); + const int grid_id = mfi.index(); const int tile_id = mfi.LocalTileIndex(); @@ -645,7 +647,7 @@ namespace void MultiParticleContainer::doFieldIonization() { - + BL_PROFILE("MPC::doFieldIonization"); // Loop over all species. // Ionized particles in pc_source create particles in pc_product for (auto& pc_source : allcontainers){ diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 02602b844..22b7df88f 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -2127,6 +2127,7 @@ void PhysicalParticleContainer::buildIonizationMask(const amrex::MFIter& mfi, const int lev, amrex::Gpu::ManagedVector& ionization_mask) { + BL_PROFILE("PPC::buildIonizationMask"); // Get pointers to ionization data from pc_source const Real * const AMREX_RESTRICT p_ionization_energies = ionization_energies.dataPtr(); const Real * const AMREX_RESTRICT p_adk_prefactor = adk_prefactor.dataPtr(); -- cgit v1.2.3 From b2754b63e9a25c550cc849f84078503bc94a417f Mon Sep 17 00:00:00 2001 From: Maxence Thevenet Date: Wed, 7 Aug 2019 21:17:42 -0400 Subject: exclusive_scan requires ManagedDeviceVector --- Source/Particles/MultiParticleContainer.cpp | 6 +++--- Source/Particles/PhysicalParticleContainer.H | 2 +- Source/Particles/PhysicalParticleContainer.cpp | 2 +- Source/Particles/WarpXParticleContainer.H | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 9eb978d05..6d977e16c 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -519,7 +519,7 @@ namespace int lev, const MFIter& mfi, std::unique_ptr< WarpXParticleContainer>& pc_source, std::unique_ptr< WarpXParticleContainer>& pc_product, - amrex::Gpu::ManagedVector& is_ionized) + amrex::Gpu::ManagedDeviceVector& is_ionized) { BL_PROFILE("createIonizedParticles"); @@ -550,7 +550,7 @@ namespace // Indices of product particle for each ionized source particle. // i_product[i]-1 is the location in product tile of product particle // from source particle i. - amrex::Gpu::ManagedVector i_product; + amrex::Gpu::ManagedDeviceVector i_product; i_product.resize(np_source); // 0 is_ionized; + amrex::Gpu::ManagedDeviceVector is_ionized; pc_source->buildIonizationMask(mfi, lev, is_ionized); // Create particles in pc_product createIonizedParticles(lev, mfi, pc_source, pc_product, is_ionized); diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index c10dcc61b..44c5030fa 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -110,7 +110,7 @@ public: void SplitParticles(int lev); virtual void buildIonizationMask (const amrex::MFIter& mfi, const int lev, - amrex::Gpu::ManagedVector& ionization_mask) override; + amrex::Gpu::ManagedDeviceVector& ionization_mask) override; // Inject particles in Box 'part_box' virtual void AddParticles (int lev); diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 22b7df88f..f1544d90c 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -2125,7 +2125,7 @@ void PhysicalParticleContainer::InitIonizationModule() */ void PhysicalParticleContainer::buildIonizationMask(const amrex::MFIter& mfi, const int lev, - amrex::Gpu::ManagedVector& ionization_mask) + amrex::Gpu::ManagedDeviceVector& ionization_mask) { BL_PROFILE("PPC::buildIonizationMask"); // Get pointers to ionization data from pc_source diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 82190b307..334eb3282 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -277,7 +277,7 @@ public: virtual void setIonizationProduct (int i_product) {}; virtual void buildIonizationMask (const amrex::MFIter& mfi, const int lev, - amrex::Gpu::ManagedVector& ionization_mask) + amrex::Gpu::ManagedDeviceVector& ionization_mask) {}; std::map getParticleComps() { return particle_comps;} -- cgit v1.2.3 From c7a56a07ee33f5d1cce29719a946036e36b5897c Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Thu, 8 Aug 2019 11:00:08 -0700 Subject: change ionization level from Real to Int --- Source/Diagnostics/ParticleIO.cpp | 3 ++- Source/Laser/LaserParticleContainer.cpp | 2 +- Source/Particles/Deposition/CurrentDeposition.H | 4 ++-- Source/Particles/PhysicalParticleContainer.cpp | 29 +++++++++++++------------ Source/Particles/WarpXParticleContainer.H | 9 ++++++-- Source/Particles/WarpXParticleContainer.cpp | 2 +- 6 files changed, 28 insertions(+), 21 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index 2b70c9379..618ecc6c5 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -114,7 +114,8 @@ MultiParticleContainer::WritePlotFile (const std::string& dir) const } if(pc->do_field_ionization){ - real_names.push_back("ionization_level"); + int_names.push_back("ionization_level"); + int_flags.resize(1, 1); } // Convert momentum to SI diff --git a/Source/Laser/LaserParticleContainer.cpp b/Source/Laser/LaserParticleContainer.cpp index 37e8f8bf2..e2aebdd5e 100644 --- a/Source/Laser/LaserParticleContainer.cpp +++ b/Source/Laser/LaserParticleContainer.cpp @@ -504,7 +504,7 @@ LaserParticleContainer::Evolve (int lev, // Current Deposition // // Deposit inside domains - Real* ion_lev = nullptr; + int* ion_lev = nullptr; DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, &jx, &jy, &jz, 0, np_current, thread_num, lev, lev, dt); diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index deff65424..86cf89cd3 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -26,7 +26,7 @@ void doDepositionShapeN(const amrex::Real * const xp, const amrex::Real * const uxp, const amrex::Real * const uyp, const amrex::Real * const uzp, - const amrex::Real * const ion_lev, + const int * const ion_lev, const amrex::Array4& jx_arr, const amrex::Array4& jy_arr, const amrex::Array4& jz_arr, @@ -184,7 +184,7 @@ void doEsirkepovDepositionShapeN (const amrex::Real * const xp, const amrex::Real * const uxp, const amrex::Real * const uyp, const amrex::Real * const uzp, - const amrex::Real * ion_lev, + const int * ion_lev, const amrex::Array4& Jx_arr, const amrex::Array4& Jy_arr, const amrex::Array4& Jz_arr, diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 7fdb783b7..98d9e5d18 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -452,9 +452,9 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) pb[4] = soa.GetRealData(particle_comps["uyold"]).data() + old_size; pb[5] = soa.GetRealData(particle_comps["uzold"]).data() + old_size; } - Real* pi; + int* pi; if (do_field_ionization) { - pi = soa.GetRealData(particle_comps[ "ionization_level"]).data() + old_size; + pi = soa.GetIntData(particle_icomps["ionization_level"]).data() + old_size; } const GpuArray overlap_corner @@ -1293,9 +1293,9 @@ PhysicalParticleContainer::Evolve (int lev, lev, lev-1, dt); } } else { - Real* AMREX_RESTRICT ion_lev; + int* AMREX_RESTRICT ion_lev; if (do_field_ionization){ - ion_lev = pti.GetAttribs(particle_comps["ionization_level"]).dataPtr(); + ion_lev = pti.GetiAttribs(particle_icomps["ionization_level"]).dataPtr(); } else { ion_lev = nullptr; } @@ -1523,9 +1523,9 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti, copy_attribs(pti, x, y, z); } - Real* AMREX_RESTRICT ion_lev = nullptr; + int* AMREX_RESTRICT ion_lev = nullptr; if (do_field_ionization){ - ion_lev = pti.GetAttribs(particle_comps["ionization_level"]).dataPtr(); + ion_lev = pti.GetiAttribs(particle_icomps["ionization_level"]).dataPtr(); } // Loop over the particles and update their momentum @@ -1972,7 +1972,7 @@ void PhysicalParticleContainer::InitIonizationModule() pp.get("ionization_product", ionization_product_name); pp.get("physical_element", physical_element); // Add Real component for ionization level - AddRealComp("ionization_level"); + AddIntComp("ionization_level"); plot_flags.resize(PIdx::nattribs + 1, 1); // Get atomic number and ionization energies from file int ion_element_id = ion_map_ids[physical_element]; @@ -2051,7 +2051,8 @@ PhysicalParticleContainer::buildIonizationMask(const amrex::MFIter& mfi, const i const Real * const AMREX_RESTRICT bx = soa.GetRealData(PIdx::Bx).data(); const Real * const AMREX_RESTRICT by = soa.GetRealData(PIdx::By).data(); const Real * const AMREX_RESTRICT bz = soa.GetRealData(PIdx::Bz).data(); - Real* ilev_real = soa.GetRealData(particle_comps["ionization_level"]).data(); + // Real* ilev_real = soa.GetIntData(particle_icomps["ionization_level"]).data(); + int* ion_lev = soa.GetIntData(particle_icomps["ionization_level"]).data(); Real c = PhysConst::c; Real c2_inv = 1./c/c; @@ -2063,9 +2064,9 @@ PhysicalParticleContainer::buildIonizationMask(const amrex::MFIter& mfi, const i np, [=] AMREX_GPU_DEVICE (long i) { // Get index of ionization_level - int ilev = (int) round(ilev_real[i]); + // int ilev = (int) round(ilev_real[i]); p_ionization_mask[i] = 0; - if ( ilev particle_comps; + std::map particle_icomps; int species_id; diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 2e418aae0..a7c30764d 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -428,7 +428,7 @@ void WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, RealVector& wp, RealVector& uxp, RealVector& uyp, RealVector& uzp, - Real* ion_lev, + int* ion_lev, MultiFab* jx, MultiFab* jy, MultiFab* jz, const long offset, const long np_to_depose, int thread_num, int lev, int depos_lev, -- cgit v1.2.3 From 6123a53ecc4a75b617cd63b5ebde50ada940a78e Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Thu, 8 Aug 2019 14:21:07 -0700 Subject: minor cleaning, remove commented lines --- Source/Particles/PhysicalParticleContainer.cpp | 2 -- 1 file changed, 2 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 98d9e5d18..a9b57aef6 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -2051,7 +2051,6 @@ PhysicalParticleContainer::buildIonizationMask(const amrex::MFIter& mfi, const i const Real * const AMREX_RESTRICT bx = soa.GetRealData(PIdx::Bx).data(); const Real * const AMREX_RESTRICT by = soa.GetRealData(PIdx::By).data(); const Real * const AMREX_RESTRICT bz = soa.GetRealData(PIdx::Bz).data(); - // Real* ilev_real = soa.GetIntData(particle_icomps["ionization_level"]).data(); int* ion_lev = soa.GetIntData(particle_icomps["ionization_level"]).data(); Real c = PhysConst::c; @@ -2064,7 +2063,6 @@ PhysicalParticleContainer::buildIonizationMask(const amrex::MFIter& mfi, const i np, [=] AMREX_GPU_DEVICE (long i) { // Get index of ionization_level - // int ilev = (int) round(ilev_real[i]); p_ionization_mask[i] = 0; if ( ion_lev[i] Date: Thu, 8 Aug 2019 14:34:40 -0700 Subject: use Weiqun's convention: space before brackets for function declaration and definition --- Source/Particles/MultiParticleContainer.H | 2 +- Source/Particles/MultiParticleContainer.cpp | 10 +++++----- Source/Particles/PhysicalParticleContainer.H | 4 +--- Source/Particles/PhysicalParticleContainer.cpp | 12 +++++------- Source/Particles/WarpXParticleContainer.H | 2 +- 5 files changed, 13 insertions(+), 17 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index fd9a33e86..aaed96423 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -127,7 +127,7 @@ public: /// std::unique_ptr GetChargeDensity(int lev, bool local = false); - void doFieldIonization(); + void doFieldIonization (); void Checkpoint (const std::string& dir) const; diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 24f9f2f11..04f052086 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -453,7 +453,7 @@ MultiParticleContainer::UpdateContinuousInjectionPosition(Real dt) const } int -MultiParticleContainer::doContinuousInjection() const +MultiParticleContainer::doContinuousInjection () const { int warpx_do_continuous_injection = 0; for (int i=0; i& pc_source, std::unique_ptr< WarpXParticleContainer>& pc_product, @@ -647,7 +647,7 @@ namespace } void -MultiParticleContainer::doFieldIonization() +MultiParticleContainer::doFieldIonization () { BL_PROFILE("MPC::doFieldIonization"); // Loop over all species. diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index 44c5030fa..c7494fbdf 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -22,9 +22,7 @@ public: virtual void InitData () override; - void InitIonizationLevel (); - - void InitIonizationModule(); + void InitIonizationModule (); #ifdef WARPX_DO_ELECTROSTATIC virtual void FieldGatherES(const amrex::Vector, 3> >& E, diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index a9b57aef6..b20ea8090 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -39,10 +39,6 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp pp.query("do_boosted_frame_diags", do_boosted_frame_diags); pp.query("do_field_ionization", do_field_ionization); - // If do_field_ionization, read initialization data from input file and - // read ionization energies from table. - //if (do_field_ionization) - // InitIonizationModule(); pp.query("plot_species", plot_species); int do_user_plot_vars; @@ -84,6 +80,8 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core) void PhysicalParticleContainer::InitData() { + // Init ionization module here instead of in the PhysicalParticleContainer + // constructor to have access to dt if (do_field_ionization) {InitIonizationModule();} AddParticles(0); // Note - add on level 0 Redistribute(); // We then redistribute @@ -1964,7 +1962,7 @@ PhysicalParticleContainer::FieldGather (WarpXParIter& pti, } -void PhysicalParticleContainer::InitIonizationModule() +void PhysicalParticleContainer::InitIonizationModule () { if (!do_field_ionization) return; ParmParse pp(species_name); @@ -2017,8 +2015,8 @@ void PhysicalParticleContainer::InitIonizationModule() * depending on whether the particle is ionized or not. */ void -PhysicalParticleContainer::buildIonizationMask(const amrex::MFIter& mfi, const int lev, - amrex::Gpu::ManagedDeviceVector& ionization_mask) +PhysicalParticleContainer::buildIonizationMask (const amrex::MFIter& mfi, const int lev, + amrex::Gpu::ManagedDeviceVector& ionization_mask) { BL_PROFILE("PPC::buildIonizationMask"); // Get pointers to ionization data from pc_source diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index dd5a150c4..efa299c43 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -284,7 +284,7 @@ public: amrex::Gpu::ManagedDeviceVector& ionization_mask) {}; - std::map getParticleComps() { return particle_comps;} + std::map getParticleComps () { return particle_comps;} protected: -- cgit v1.2.3 From c87ee4b5e5d62f8e90a719a42ff164f99983a62c Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Thu, 8 Aug 2019 14:44:06 -0700 Subject: final minor cleaning --- Source/Particles/MultiParticleContainer.cpp | 6 +++--- Source/Particles/PhysicalParticleContainer.cpp | 6 +++--- Source/Utils/WarpXConst.H | 1 - 3 files changed, 6 insertions(+), 7 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 04f052086..e39930216 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -511,8 +511,8 @@ MultiParticleContainer::getSpeciesID (std::string product_str) namespace { - // For particle i in mfi, if is_ionized[i]=1, copy particle from - // particle i container pc_source into pc_product + // For particle i in mfi, if is_ionized[i]=1, copy particle + // particle i from container pc_source into pc_product static void createIonizedParticles ( int lev, const MFIter& mfi, std::unique_ptr< WarpXParticleContainer>& pc_source, @@ -663,7 +663,7 @@ MultiParticleContainer::doFieldIonization () for (int lev = 0; lev <= pc_source->finestLevel(); ++lev){ #ifdef _OPENMP - // Touch all tiles of source species in serial if additional arguments + // Touch all tiles of source species in serial if runtime attribs for (MFIter mfi = pc_source->MakeMFIter(lev); mfi.isValid(); ++mfi) { const int grid_id = mfi.index(); const int tile_id = mfi.LocalTileIndex(); diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index b20ea8090..466446157 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -81,7 +81,7 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core) void PhysicalParticleContainer::InitData() { // Init ionization module here instead of in the PhysicalParticleContainer - // constructor to have access to dt + // constructor because dt is required if (do_field_ionization) {InitIonizationModule();} AddParticles(0); // Note - add on level 0 Redistribute(); // We then redistribute @@ -208,10 +208,10 @@ PhysicalParticleContainer::CheckAndAddParticle(Real x, Real y, Real z, auto& particle_tile = DefineAndReturnParticleTile(0, 0, 0); if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) { - particle_tile.push_back_real(particle_comps["xold"], x); + particle_tile.push_back_real(particle_comps["xold"], x); particle_tile.push_back_real(particle_comps["yold"], y); particle_tile.push_back_real(particle_comps["zold"], z); - + particle_tile.push_back_real(particle_comps["uxold"], u[0]); particle_tile.push_back_real(particle_comps["uyold"], u[1]); particle_tile.push_back_real(particle_comps["uzold"], u[2]); diff --git a/Source/Utils/WarpXConst.H b/Source/Utils/WarpXConst.H index bfb67b3d5..6901d4629 100644 --- a/Source/Utils/WarpXConst.H +++ b/Source/Utils/WarpXConst.H @@ -21,7 +21,6 @@ namespace PhysConst static constexpr amrex::Real hbar = 1.05457180010e-34; static constexpr amrex::Real alpha = mu0/(4*MathConst::pi)*q_e*q_e*c/hbar; static constexpr amrex::Real r_e = 1./(4*MathConst::pi*ep0) * q_e*q_e/(m_e*c*c); - // static constexpr amrex::Real alpha = 0.0072973525693; } #endif -- cgit v1.2.3 From 5cc13f04eaf1a41c22245129bf083f38e7c596ef Mon Sep 17 00:00:00 2001 From: Dave Grote Date: Thu, 8 Aug 2019 15:48:08 -0700 Subject: Converted DepositCharge to c++ (and removed duplication of code) --- Source/Laser/LaserParticleContainer.cpp | 14 +- Source/Particles/Deposition/Make.package | 1 + Source/Particles/PhysicalParticleContainer.cpp | 14 +- Source/Particles/WarpXParticleContainer.H | 10 +- Source/Particles/WarpXParticleContainer.cpp | 243 +++++++------------------ 5 files changed, 100 insertions(+), 182 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Laser/LaserParticleContainer.cpp b/Source/Laser/LaserParticleContainer.cpp index 515aa1f5d..786ebc622 100644 --- a/Source/Laser/LaserParticleContainer.cpp +++ b/Source/Laser/LaserParticleContainer.cpp @@ -453,7 +453,12 @@ LaserParticleContainer::Evolve (int lev, pti.GetPosition(m_xp[thread_num], m_yp[thread_num], m_zp[thread_num]); BL_PROFILE_VAR_STOP(blp_copy); - if (rho) DepositCharge(pti, wp, rho, crho, 0, np_current, np, thread_num, lev); + if (rho) { + DepositCharge(pti, wp, rho, 0, 0, np_current, thread_num, lev, lev); + if (crho) { + DepositCharge(pti, wp, crho, 0, np_current, np-np_current, thread_num, lev, lev-1); + } + } // // Particle Push @@ -522,7 +527,12 @@ LaserParticleContainer::Evolve (int lev, pti.SetPosition(m_xp[thread_num], m_yp[thread_num], m_zp[thread_num]); BL_PROFILE_VAR_STOP(blp_copy); - if (rho) DepositCharge(pti, wp, rho, crho, 1, np_current, np, thread_num, lev); + if (rho) { + DepositCharge(pti, wp, rho, 1, 0, np_current, thread_num, lev, lev); + if (crho) { + DepositCharge(pti, wp, crho, 1, np_current, np-np_current, thread_num, lev, lev-1); + } + } if (cost) { const Box& tbx = pti.tilebox(); diff --git a/Source/Particles/Deposition/Make.package b/Source/Particles/Deposition/Make.package index 0d5ebe2a7..e1aace998 100644 --- a/Source/Particles/Deposition/Make.package +++ b/Source/Particles/Deposition/Make.package @@ -1,3 +1,4 @@ CEXE_headers += CurrentDeposition.H +CEXE_headers += ChargeDeposition.H INCLUDE_LOCATIONS += $(WARPX_HOME)/Source/Particles/Deposition VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/Deposition diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 59d3ce76f..d10390204 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1156,7 +1156,12 @@ PhysicalParticleContainer::Evolve (int lev, pti.GetPosition(m_xp[thread_num], m_yp[thread_num], m_zp[thread_num]); BL_PROFILE_VAR_STOP(blp_copy); - if (rho) DepositCharge(pti, wp, rho, crho, 0, np_current, np, thread_num, lev); + if (rho) { + DepositCharge(pti, wp, rho, 0, 0, np_current, thread_num, lev, lev); + if (has_buffer){ + DepositCharge(pti, wp, crho, 0, np_current, np-np_current, thread_num, lev, lev-1); + } + } if (! do_not_push) { @@ -1292,7 +1297,12 @@ PhysicalParticleContainer::Evolve (int lev, BL_PROFILE_VAR_STOP(blp_copy); } - if (rho) DepositCharge(pti, wp, rho, crho, 1, np_current, np, thread_num, lev); + if (rho) { + DepositCharge(pti, wp, rho, 1, 0, np_current, thread_num, lev, lev); + if (has_buffer){ + DepositCharge(pti, wp, crho, 1, np_current, np-np_current, thread_num, lev, lev-1); + } + } if (cost) { const Box& tbx = pti.tilebox(); diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index a609b4cb3..ac5b47ada 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -167,13 +167,13 @@ public: virtual void DepositCharge(WarpXParIter& pti, RealVector& wp, - amrex::MultiFab* rhomf, - amrex::MultiFab* crhomf, + amrex::MultiFab* rho, int icomp, - const long np_current, - const long np, + const long offset, + const long np_to_depose, int thread_num, - int lev ); + int lev, + int depos_lev); virtual void DepositCurrent(WarpXParIter& pti, RealVector& wp, diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index c5c6afc19..ec0f78564 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -12,6 +12,7 @@ #include #include #include +#include using namespace amrex; @@ -552,140 +553,91 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, } void -WarpXParticleContainer::DepositCharge ( WarpXParIter& pti, RealVector& wp, - MultiFab* rhomf, MultiFab* crhomf, int icomp, - const long np_current, - const long np, int thread_num, int lev ) +WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector& wp, + MultiFab* rho, int icomp, + const long offset, const long np_to_depose, + int thread_num, int lev, int depos_lev) { + AMREX_ALWAYS_ASSERT_WITH_MESSAGE((depos_lev==(lev-1)) || + (depos_lev==(lev )), + "Deposition buffers only work for lev-1"); - BL_PROFILE_VAR_NS("PICSAR::ChargeDeposition", blp_pxr_chd); - BL_PROFILE_VAR_NS("PPC::Evolve::Accumulate", blp_accumulate); - - const std::array& xyzmin_tile = WarpX::LowerCorner(pti.tilebox(), lev); - const long lvect = 8; + // If no particles, do not do anything + if (np_to_depose == 0) return; - long ngRho = rhomf->nGrow(); - Real* data_ptr; - Box tile_box = convert(pti.tilebox(), IntVect::TheUnitVector()); + const long ngRho = rho->nGrow(); + const std::array& dx = WarpX::CellSize(std::max(depos_lev,0)); + Real q = this->charge; - const std::array& dx = WarpX::CellSize(lev); - const std::array& cdx = WarpX::CellSize(std::max(lev-1,0)); + BL_PROFILE_VAR_NS("PPC::ChargeDeposition", blp_ppc_chd); + BL_PROFILE_VAR_NS("PPC::Evolve::Accumulate", blp_accumulate); - // Deposit charge for particles that are not in the current buffers - if (np_current > 0) - { - const std::array& xyzmin = xyzmin_tile; + // Get tile box where charge is deposited. + // The tile box is different when depositing in the buffers (depos_lev const& rho_arr = rho->array(pti); #else - tile_box.grow(ngRho); - local_rho[thread_num].resize(tile_box); + // Tiling is on: rho_ptr points to local_rho[thread_num] + Box tb = amrex::convert(tilebox, IntVect::TheUnitVector()); - data_ptr = local_rho[thread_num].dataPtr(); - auto rholen = local_rho[thread_num].length(); + local_rho[thread_num].resize(tb); - local_rho[thread_num].setVal(0.0); -#endif + // local_rho[thread_num] is set to zero + local_rho[thread_num].setVal(0.0); -#if (AMREX_SPACEDIM == 3) - const long nx = rholen[0]-1-2*ngRho; - const long ny = rholen[1]-1-2*ngRho; - const long nz = rholen[2]-1-2*ngRho; -#else - const long nx = rholen[0]-1-2*ngRho; - const long ny = 0; - const long nz = rholen[1]-1-2*ngRho; -#endif - BL_PROFILE_VAR_START(blp_pxr_chd); - warpx_charge_deposition(data_ptr, &np_current, - m_xp[thread_num].dataPtr(), - m_yp[thread_num].dataPtr(), - m_zp[thread_num].dataPtr(), - wp.dataPtr(), - &this->charge, - &xyzmin[0], &xyzmin[1], &xyzmin[2], - &dx[0], &dx[1], &dx[2], &nx, &ny, &nz, - &ngRho, &ngRho, &ngRho, - &WarpX::nox,&WarpX::noy,&WarpX::noz, - &lvect, &WarpX::charge_deposition_algo); -#ifdef WARPX_DIM_RZ - warpx_charge_deposition_rz_volume_scaling( - data_ptr, &ngRho, rholen.getVect(), - &xyzmin[0], &dx[0]); + Array4 const& rho_arr = local_rho[thread_num].array(); #endif - BL_PROFILE_VAR_STOP(blp_pxr_chd); - -#ifndef AMREX_USE_GPU - BL_PROFILE_VAR_START(blp_accumulate); - - (*rhomf)[pti].atomicAdd(local_rho[thread_num], tile_box, tile_box, 0, icomp, 1); - - BL_PROFILE_VAR_STOP(blp_accumulate); -#endif - } - - // Deposit charge for particles that are in the current buffers - if (np_current < np) - { - const IntVect& ref_ratio = WarpX::RefRatio(lev-1); - const Box& ctilebox = amrex::coarsen(pti.tilebox(), ref_ratio); - const std::array& cxyzmin_tile = WarpX::LowerCorner(ctilebox, lev-1); - -#ifdef AMREX_USE_GPU - data_ptr = (*crhomf)[pti].dataPtr(icomp); - auto rholen = (*crhomf)[pti].length(); -#else - tile_box = amrex::convert(ctilebox, IntVect::TheUnitVector()); - tile_box.grow(ngRho); - local_rho[thread_num].resize(tile_box); + // GPU, no tiling: deposit directly in rho + // CPU, tiling: deposit into local_rho - data_ptr = local_rho[thread_num].dataPtr(); - auto rholen = local_rho[thread_num].length(); - - local_rho[thread_num].setVal(0.0); -#endif + Real* AMREX_RESTRICT xp = m_xp[thread_num].dataPtr() + offset; + Real* AMREX_RESTRICT zp = m_zp[thread_num].dataPtr() + offset; + Real* AMREX_RESTRICT yp = m_yp[thread_num].dataPtr() + offset; -#if (AMREX_SPACEDIM == 3) - const long nx = rholen[0]-1-2*ngRho; - const long ny = rholen[1]-1-2*ngRho; - const long nz = rholen[2]-1-2*ngRho; -#else - const long nx = rholen[0]-1-2*ngRho; - const long ny = 0; - const long nz = rholen[1]-1-2*ngRho; -#endif + // Lower corner of tile box physical domain + // Note that this includes guard cells since it is after tilebox.ngrow + const std::array& xyzmin = WarpX::LowerCorner(tilebox, depos_lev); + // Indices of the lower bound + const Dim3 lo = lbound(tilebox); - long ncrse = np - np_current; - BL_PROFILE_VAR_START(blp_pxr_chd); - warpx_charge_deposition(data_ptr, &ncrse, - m_xp[thread_num].dataPtr() + np_current, - m_yp[thread_num].dataPtr() + np_current, - m_zp[thread_num].dataPtr() + np_current, - wp.dataPtr() + np_current, - &this->charge, - &cxyzmin_tile[0], &cxyzmin_tile[1], &cxyzmin_tile[2], - &cdx[0], &cdx[1], &cdx[2], &nx, &ny, &nz, - &ngRho, &ngRho, &ngRho, - &WarpX::nox,&WarpX::noy,&WarpX::noz, - &lvect, &WarpX::charge_deposition_algo); + BL_PROFILE_VAR_START(blp_ppc_chd); + if (WarpX::nox == 1){ + doChargeDepositionShapeN<1>(xp, yp, zp, wp.dataPtr()+offset, rho_arr, + np_to_depose, dx, xyzmin, lo, q); + } else if (WarpX::nox == 2){ + doChargeDepositionShapeN<2>(xp, yp, zp, wp.dataPtr()+offset, rho_arr, + np_to_depose, dx, xyzmin, lo, q); + } else if (WarpX::nox == 3){ + doChargeDepositionShapeN<3>(xp, yp, zp, wp.dataPtr()+offset, rho_arr, + np_to_depose, dx, xyzmin, lo, q); + } #ifdef WARPX_DIM_RZ - warpx_charge_deposition_rz_volume_scaling( - data_ptr, &ngRho, rholen.getVect(), - &cxyzmin_tile[0], &cdx[0]); + warpx_charge_deposition_rz_volume_scaling( + data_ptr, &ngRho, rholen.getVect(), + &xyzmin[0], &dx[0]); #endif - BL_PROFILE_VAR_STOP(blp_pxr_chd); + BL_PROFILE_VAR_STOP(blp_ppc_chd); #ifndef AMREX_USE_GPU - BL_PROFILE_VAR_START(blp_accumulate); + BL_PROFILE_VAR_START(blp_accumulate); - (*crhomf)[pti].atomicAdd(local_rho[thread_num], tile_box, tile_box, 0, icomp, 1); + (*rho)[pti].atomicAdd(local_rho[thread_num], tb, tb, 0, icomp, 1); - BL_PROFILE_VAR_STOP(blp_accumulate); + BL_PROFILE_VAR_STOP(blp_accumulate); #endif - } -}; +} void WarpXParticleContainer::DepositCharge (Vector >& rho, bool local) @@ -762,8 +714,6 @@ WarpXParticleContainer::GetChargeDensity (int lev, bool local) BoxArray nba = ba; nba.surroundingNodes(); - const std::array& dx = WarpX::CellSize(lev); - const int ng = WarpX::nox; auto rho = std::unique_ptr(new MultiFab(nba,dm,1,ng)); @@ -773,74 +723,21 @@ WarpXParticleContainer::GetChargeDensity (int lev, bool local) #pragma omp parallel { #endif - Cuda::ManagedDeviceVector xp, yp, zp; #ifdef _OPENMP - FArrayBox rho_loc; + int thread_num = omp_get_thread_num(); +#else + int thread_num = 0; #endif for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { + const long np = pti.numParticles(); auto& wp = pti.GetAttribs(PIdx::w); - const long np = pti.numParticles(); + pti.GetPosition(m_xp[thread_num], m_yp[thread_num], m_zp[thread_num]); - pti.GetPosition(xp, yp, zp); - - // Data on the grid - Real* data_ptr; - FArrayBox& rhofab = (*rho)[pti]; -#ifdef _OPENMP - const std::array& xyzmin_tile = WarpX::LowerCorner(pti.tilebox(), lev); - Box tile_box = convert(pti.tilebox(), IntVect::TheUnitVector()); - const std::array& xyzmin = xyzmin_tile; - tile_box.grow(ng); - rho_loc.resize(tile_box); - rho_loc = 0.0; - data_ptr = rho_loc.dataPtr(); - auto rholen = rho_loc.length(); -#else - const Box& box = pti.validbox(); - const std::array& xyzmin_grid = WarpX::LowerCorner(box, lev); - const std::array& xyzmin = xyzmin_grid; - data_ptr = rhofab.dataPtr(); - auto rholen = rhofab.length(); -#endif - -#if (AMREX_SPACEDIM == 3) - const long nx = rholen[0]-1-2*ng; - const long ny = rholen[1]-1-2*ng; - const long nz = rholen[2]-1-2*ng; -#else - const long nx = rholen[0]-1-2*ng; - const long ny = 0; - const long nz = rholen[1]-1-2*ng; -#endif - - long nxg = ng; - long nyg = ng; - long nzg = ng; - long lvect = 8; - - warpx_charge_deposition(data_ptr, - &np, - xp.dataPtr(), - yp.dataPtr(), - zp.dataPtr(), wp.dataPtr(), - &this->charge, &xyzmin[0], &xyzmin[1], &xyzmin[2], - &dx[0], &dx[1], &dx[2], &nx, &ny, &nz, - &nxg, &nyg, &nzg, &WarpX::nox,&WarpX::noy,&WarpX::noz, - &lvect, &WarpX::charge_deposition_algo); -#ifdef WARPX_DIM_RZ - long ngRho = WarpX::nox; - warpx_charge_deposition_rz_volume_scaling( - data_ptr, &ngRho, rholen.getVect(), - &xyzmin[0], &dx[0]); -#endif - -#ifdef _OPENMP - rhofab.atomicAdd(rho_loc); + DepositCharge(pti, wp, rho.get(), 0, 0, np, thread_num, lev, lev); } -#endif } if (!local) rho->SumBoundary(gm.periodicity()); -- cgit v1.2.3 From 8b50555b58cb88a61f806193d34aead9ffbf1612 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Thu, 8 Aug 2019 19:17:00 -0700 Subject: hidden typo '-.- --- Source/Particles/PhysicalParticleContainer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 466446157..2c6deefef 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -2001,7 +2001,7 @@ void PhysicalParticleContainer::InitIonizationModule () adk_power[i] = -(2*n_eff - 1); Real Uion = ionization_energies[i]; adk_prefactor[i] = dt * wa * C2 * ( Uion/(2*UH) ) - * std::pow(std::pow(2*(Uion/UH),3./2)*Ea,2*n_eff - 1); + * std::pow(2*std::pow((Uion/UH),3./2)*Ea,2*n_eff - 1); adk_exp_prefactor[i] = -2./3 * std::pow( Uion/UH,3./2) * Ea; } } -- cgit v1.2.3 From 1b9682b5d2705369a8a3c7da6ea1a00b9eb824d2 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Mon, 12 Aug 2019 09:52:36 -0700 Subject: typos in doc, and rename ionization_product --- Docs/source/running_cpp/parameters.rst | 8 ++++---- Source/Particles/PhysicalParticleContainer.cpp | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Docs/source/running_cpp/parameters.rst b/Docs/source/running_cpp/parameters.rst index 6b567d5b1..6d7000cd8 100644 --- a/Docs/source/running_cpp/parameters.rst +++ b/Docs/source/running_cpp/parameters.rst @@ -340,16 +340,16 @@ Particle initialization Do field ionization for this species (using the ADK theory). * ``.physical_element`` (`string`) - Only read of `do_field_ionization = 1`. Symbol of chemical element for + Only read if `do_field_ionization = 1`. Symbol of chemical element for this species. Example: for Helium, use ``physical_element = He``. -* ``.ionization_product`` (`string`) - Only read of `do_field_ionization = 1`. Name of species in which ionized +* ``.ionization_product_species`` (`string`) + Only read if `do_field_ionization = 1`. Name of species in which ionized electrons are stored. This species must be created as a regular species in the input file (in particular, it must be in `particles.species_names`). * ``.ionization_level`` (`int`) optional (default `0`) - Only read of `do_field_ionization = 1`. Initial ionization level of the + Only read if `do_field_ionization = 1`. Initial ionization level of the species (must be smaller than the atomic number of chemical element given in `physical_element`). diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 2c6deefef..69d7fa686 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1967,7 +1967,7 @@ void PhysicalParticleContainer::InitIonizationModule () if (!do_field_ionization) return; ParmParse pp(species_name); pp.query("ionization_level", species_ionization_level); - pp.get("ionization_product", ionization_product_name); + pp.get("ionization_product_species", ionization_product_name); pp.get("physical_element", physical_element); // Add Real component for ionization level AddIntComp("ionization_level"); -- cgit v1.2.3 From 1cc426d477fd160c0b59bf777a8be715527ccfb8 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Mon, 12 Aug 2019 09:56:33 -0700 Subject: use ionization_initial_level --- Docs/source/running_cpp/parameters.rst | 2 +- Source/Particles/PhysicalParticleContainer.cpp | 4 ++-- Source/Particles/WarpXParticleContainer.H | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Docs/source/running_cpp/parameters.rst b/Docs/source/running_cpp/parameters.rst index 6d7000cd8..59628366b 100644 --- a/Docs/source/running_cpp/parameters.rst +++ b/Docs/source/running_cpp/parameters.rst @@ -348,7 +348,7 @@ Particle initialization electrons are stored. This species must be created as a regular species in the input file (in particular, it must be in `particles.species_names`). -* ``.ionization_level`` (`int`) optional (default `0`) +* ``.ionization_initial_level`` (`int`) optional (default `0`) Only read if `do_field_ionization = 1`. Initial ionization level of the species (must be smaller than the atomic number of chemical element given in `physical_element`). diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 69d7fa686..a92f36a3c 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -584,7 +584,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) } if (do_field_ionization) { - pi[ip] = species_ionization_level; + pi[ip] = ionization_initial_level; } u.x *= PhysConst::c; @@ -1966,7 +1966,7 @@ void PhysicalParticleContainer::InitIonizationModule () { if (!do_field_ionization) return; ParmParse pp(species_name); - pp.query("ionization_level", species_ionization_level); + pp.query("ionization_level", ionization_initial_level); pp.get("ionization_product_species", ionization_product_name); pp.get("physical_element", physical_element); // Add Real component for ionization level diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 2a625b4e1..b89406b3e 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -308,7 +308,7 @@ protected: int ionization_product; std::string ionization_product_name; int ion_atomic_number; - int species_ionization_level = 0; + int ionization_initial_level = 0; amrex::Gpu::ManagedVector ionization_energies; amrex::Gpu::ManagedVector adk_power; amrex::Gpu::ManagedVector adk_prefactor; -- cgit v1.2.3 From fab45a507406dfd764dd793b0b4d612e86297fbb Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Mon, 12 Aug 2019 10:57:02 -0700 Subject: typo --- Source/Particles/PhysicalParticleContainer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index a92f36a3c..a221be01c 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1966,7 +1966,7 @@ void PhysicalParticleContainer::InitIonizationModule () { if (!do_field_ionization) return; ParmParse pp(species_name); - pp.query("ionization_level", ionization_initial_level); + pp.query("ionization_initial_level", ionization_initial_level); pp.get("ionization_product_species", ionization_product_name); pp.get("physical_element", physical_element); // Add Real component for ionization level -- cgit v1.2.3 From 69be3657b7410db305e102e89c366ebbc8be59d1 Mon Sep 17 00:00:00 2001 From: Dave Grote Date: Fri, 16 Aug 2019 16:37:23 -0700 Subject: Converted field gather for RZ multimode to C++ --- Source/Particles/Gather/FieldGather.H | 80 ++++++++++++++++++---- Source/Particles/PhysicalParticleContainer.H | 1 - Source/Particles/PhysicalParticleContainer.cpp | 20 +++--- .../Particles/RigidInjectedParticleContainer.cpp | 2 +- 4 files changed, 76 insertions(+), 27 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/Gather/FieldGather.H b/Source/Particles/Gather/FieldGather.H index 8f5e8d4cf..219a9d055 100644 --- a/Source/Particles/Gather/FieldGather.H +++ b/Source/Particles/Gather/FieldGather.H @@ -2,9 +2,10 @@ #define FIELDGATHER_H_ #include "ShapeFactors.H" +#include /* \brief Field gather for particles handled by thread thread_num - * /param xp, yp, zp : Pointer to arrays of particle positions. + * \param xp, yp, zp : Pointer to arrays of particle positions. * \param Exp, Eyp, Ezp: Pointer to array of electric field on particles. * \param Bxp, Byp, Bzp: Pointer to array of magnetic field on particles. * \param ex_arr ey_arr: Array4 of current density, either full array or tile. @@ -15,6 +16,7 @@ * \param xyzmin : Physical lower bounds of domain. * \param lo : Index lower bounds of domain. * \param stagger_shift: 0 if nodal, 0.5 if staggered. + * \param n_rz_azimuthal_modes: Number of azimuthal modes when using RZ geometry */ template void doGatherShapeN(const amrex::Real * const xp, @@ -33,7 +35,8 @@ void doGatherShapeN(const amrex::Real * const xp, const std::array& dx, const std::array xyzmin, const amrex::Dim3 lo, - const amrex::Real stagger_shift) + const amrex::Real stagger_shift, + const long n_rz_azimuthal_modes) { const amrex::Real dxi = 1.0/dx[0]; const amrex::Real dzi = 1.0/dx[2]; @@ -56,8 +59,8 @@ void doGatherShapeN(const amrex::Real * const xp, // x direction // Get particle position #ifdef WARPX_DIM_RZ - const amrex::Real r = std::sqrt(xp[ip]*xp[ip] + yp[ip]*yp[ip]); - const amrex::Real x = (r - xmin)*dxi; + const amrex::Real rp = std::sqrt(xp[ip]*xp[ip] + yp[ip]*yp[ip]); + const amrex::Real x = (rp - xmin)*dxi; #else const amrex::Real x = (xp[ip]-xmin)*dxi; #endif @@ -103,7 +106,7 @@ void doGatherShapeN(const amrex::Real * const xp, for (int iz=0; iz<=depos_order; iz++){ for (int ix=0; ix<=depos_order; ix++){ Eyp[ip] += sx[ix]*sz[iz]* - ey_arr(lo.x+j+ix, lo.y+l+iz, 0); + ey_arr(lo.x+j+ix, lo.y+l+iz, 0, 0); } } // Gather field on particle Exp[i] from field on grid ex_arr @@ -111,9 +114,9 @@ void doGatherShapeN(const amrex::Real * const xp, for (int iz=0; iz<=depos_order; iz++){ for (int ix=0; ix<=depos_order-lower_in_v; ix++){ Exp[ip] += sx0[ix]*sz[iz]* - ex_arr(lo.x+j0+ix, lo.y+l +iz, 0); + ex_arr(lo.x+j0+ix, lo.y+l +iz, 0, 0); Bzp[ip] += sx0[ix]*sz[iz]* - bz_arr(lo.x+j0+ix, lo.y+l +iz, 0); + bz_arr(lo.x+j0+ix, lo.y+l +iz, 0, 0); } } // Gather field on particle Ezp[i] from field on grid ez_arr @@ -121,30 +124,79 @@ void doGatherShapeN(const amrex::Real * const xp, for (int iz=0; iz<=depos_order-lower_in_v; iz++){ for (int ix=0; ix<=depos_order; ix++){ Ezp[ip] += sx[ix]*sz0[iz]* - ez_arr(lo.x+j+ix, lo.y+l0 +iz, 0); + ez_arr(lo.x+j+ix, lo.y+l0 +iz, 0, 0); Bxp[ip] += sx[ix]*sz0[iz]* - bx_arr(lo.x+j+ix, lo.y+l0 +iz, 0); + bx_arr(lo.x+j+ix, lo.y+l0 +iz, 0, 0); } } // Gather field on particle Byp[i] from field on grid by_arr for (int iz=0; iz<=depos_order-lower_in_v; iz++){ for (int ix=0; ix<=depos_order-lower_in_v; ix++){ Byp[ip] += sx0[ix]*sz0[iz]* - by_arr(lo.x+j0+ix, lo.y+l0+iz, 0); + by_arr(lo.x+j0+ix, lo.y+l0+iz, 0, 0); } } #ifdef WARPX_DIM_RZ - // Convert Exp and Eyp (which are actually Er and Etheta) to Ex and Ey + amrex::Real costheta; amrex::Real sintheta; - if (r > 0.) { - costheta = xp[ip]/r; - sintheta = yp[ip]/r; + if (rp > 0.) { + costheta = xp[ip]/rp; + sintheta = yp[ip]/rp; } else { costheta = 1.; sintheta = 0.; } + const Complex xy0 = Complex{costheta, -sintheta}; + Complex xy = xy0; + + for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { + + // Gather field on particle Eyp[i] from field on grid ey_arr + for (int iz=0; iz<=depos_order; iz++){ + for (int ix=0; ix<=depos_order; ix++){ + const amrex::Real dEy = (+ ey_arr(lo.x+j+ix, lo.y+l+iz, 0, 2*imode)*xy.real() + - ey_arr(lo.x+j+ix, lo.y+l+iz, 0, 2*imode+1)*xy.imag()); + Eyp[ip] += sx[ix]*sz[iz]*dEy; + } + } + // Gather field on particle Exp[i] from field on grid ex_arr + // Gather field on particle Bzp[i] from field on grid bz_arr + for (int iz=0; iz<=depos_order; iz++){ + for (int ix=0; ix<=depos_order-lower_in_v; ix++){ + const amrex::Real dEx = (+ ex_arr(lo.x+j0+ix, lo.y+l +iz, 0, 2*imode)*xy.real() + - ex_arr(lo.x+j0+ix, lo.y+l +iz, 0, 2*imode+1)*xy.imag()); + Exp[ip] += sx0[ix]*sz[iz]*dEx; + const amrex::Real dBz = (+ bz_arr(lo.x+j0+ix, lo.y+l +iz, 0, 2*imode)*xy.real() + - bz_arr(lo.x+j0+ix, lo.y+l +iz, 0, 2*imode+1)*xy.imag()); + Bzp[ip] += sx0[ix]*sz[iz]*dBz; + } + } + // Gather field on particle Ezp[i] from field on grid ez_arr + // Gather field on particle Bxp[i] from field on grid bx_arr + for (int iz=0; iz<=depos_order-lower_in_v; iz++){ + for (int ix=0; ix<=depos_order; ix++){ + const amrex::Real dEz = (+ ez_arr(lo.x+j+ix, lo.y+l0 +iz, 0, 2*imode)*xy.real() + - ez_arr(lo.x+j+ix, lo.y+l0 +iz, 0, 2*imode+1)*xy.imag()); + Ezp[ip] += sx[ix]*sz0[iz]*dEz; + const amrex::Real dBx = (+ bx_arr(lo.x+j+ix, lo.y+l0 +iz, 0, 2*imode)*xy.real() + - bx_arr(lo.x+j+ix, lo.y+l0 +iz, 0, 2*imode+1)*xy.imag()); + Bxp[ip] += sx[ix]*sz0[iz]*dBx; + } + } + // Gather field on particle Byp[i] from field on grid by_arr + for (int iz=0; iz<=depos_order-lower_in_v; iz++){ + for (int ix=0; ix<=depos_order-lower_in_v; ix++){ + const amrex::Real dBy = (+ by_arr(lo.x+j0+ix, lo.y+l0+iz, 0, 2*imode)*xy.real() + - by_arr(lo.x+j0+ix, lo.y+l0+iz, 0, 2*imode+1)*xy.imag()); + Byp[ip] += sx0[ix]*sz0[iz]*dBy; + } + } + xy = xy*xy0; + } + + // Convert Exp and Eyp (which are actually Er and Etheta) to Ex and Ey const amrex::Real Exp_save = Exp[ip]; Exp[ip] = costheta*Exp[ip] - sintheta*Eyp[ip]; Eyp[ip] = costheta*Eyp[ip] + sintheta*Exp_save; diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index ebd4eac2b..b80619733 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -53,7 +53,6 @@ public: amrex::FArrayBox const * byfab, amrex::FArrayBox const * bzfab, const int ngE, const int e_is_nodal, - const long n_rz_azimuthal_modes, const long offset, const long np_to_gather, int thread_num, diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index c44a9a8b7..ec360f2c4 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -899,7 +899,7 @@ PhysicalParticleContainer::FieldGather (int lev, int e_is_nodal = Ex.is_nodal() and Ey.is_nodal() and Ez.is_nodal(); FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, &exfab, &eyfab, &ezfab, &bxfab, &byfab, &bzfab, - Ex.nGrow(), e_is_nodal, WarpX::n_rz_azimuthal_modes, + Ex.nGrow(), e_is_nodal, 0, np, thread_num, lev, lev); if (cost) { @@ -1176,7 +1176,7 @@ PhysicalParticleContainer::Evolve (int lev, BL_PROFILE_VAR_START(blp_pxr_fg); FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, exfab, eyfab, ezfab, bxfab, byfab, bzfab, - Ex.nGrow(), e_is_nodal, WarpX::n_rz_azimuthal_modes, + Ex.nGrow(), e_is_nodal, 0, np_gather, thread_num, lev, lev); if (np_gather < np) @@ -1250,7 +1250,6 @@ PhysicalParticleContainer::Evolve (int lev, cexfab, ceyfab, cezfab, cbxfab, cbyfab, cbzfab, cEx->nGrow(), e_is_nodal, - WarpX::n_rz_azimuthal_modes, nfine_gather, np-nfine_gather, thread_num, lev, lev-1); } @@ -1594,7 +1593,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, int e_is_nodal = Ex.is_nodal() and Ey.is_nodal() and Ez.is_nodal(); FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, &exfab, &eyfab, &ezfab, &bxfab, &byfab, &bzfab, - Ex.nGrow(), e_is_nodal, WarpX::n_rz_azimuthal_modes, + Ex.nGrow(), e_is_nodal, 0, np, thread_num, lev, lev); // This wraps the momentum advance so that inheritors can modify the call. @@ -1835,7 +1834,6 @@ PhysicalParticleContainer::FieldGather (WarpXParIter& pti, const int ngE, const int e_is_nodal, const long offset, const long np_to_gather, - const long n_rz_azimuthal_modes, int thread_num, int lev, int gather_lev) @@ -1890,7 +1888,7 @@ PhysicalParticleContainer::FieldGather (WarpXParIter& pti, Byp.dataPtr() + offset, Bzp.dataPtr() + offset, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, np_to_gather, dx, - xyzmin, lo, stagger_shift); + xyzmin, lo, stagger_shift, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doGatherShapeN<2,1>(xp, yp, zp, Exp.dataPtr() + offset, Eyp.dataPtr() + offset, @@ -1898,7 +1896,7 @@ PhysicalParticleContainer::FieldGather (WarpXParIter& pti, Byp.dataPtr() + offset, Bzp.dataPtr() + offset, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, np_to_gather, dx, - xyzmin, lo, stagger_shift); + xyzmin, lo, stagger_shift, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doGatherShapeN<3,1>(xp, yp, zp, Exp.dataPtr() + offset, Eyp.dataPtr() + offset, @@ -1906,7 +1904,7 @@ PhysicalParticleContainer::FieldGather (WarpXParIter& pti, Byp.dataPtr() + offset, Bzp.dataPtr() + offset, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, np_to_gather, dx, - xyzmin, lo, stagger_shift); + xyzmin, lo, stagger_shift, WarpX::n_rz_azimuthal_modes); } } else { if (WarpX::nox == 1){ @@ -1916,7 +1914,7 @@ PhysicalParticleContainer::FieldGather (WarpXParIter& pti, Byp.dataPtr() + offset, Bzp.dataPtr() + offset, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, np_to_gather, dx, - xyzmin, lo, stagger_shift); + xyzmin, lo, stagger_shift, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doGatherShapeN<2,0>(xp, yp, zp, Exp.dataPtr() + offset, Eyp.dataPtr() + offset, @@ -1924,7 +1922,7 @@ PhysicalParticleContainer::FieldGather (WarpXParIter& pti, Byp.dataPtr() + offset, Bzp.dataPtr() + offset, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, np_to_gather, dx, - xyzmin, lo, stagger_shift); + xyzmin, lo, stagger_shift, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doGatherShapeN<3,0>(xp, yp, zp, Exp.dataPtr() + offset, Eyp.dataPtr() + offset, @@ -1932,7 +1930,7 @@ PhysicalParticleContainer::FieldGather (WarpXParIter& pti, Byp.dataPtr() + offset, Bzp.dataPtr() + offset, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, np_to_gather, dx, - xyzmin, lo, stagger_shift); + xyzmin, lo, stagger_shift, WarpX::n_rz_azimuthal_modes); } } } diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index 695955e15..a87168a75 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -426,7 +426,7 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, int e_is_nodal = Ex.is_nodal() and Ey.is_nodal() and Ez.is_nodal(); FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, &exfab, &eyfab, &ezfab, &bxfab, &byfab, &bzfab, - Ex.nGrow(), e_is_nodal, WarpX::n_rz_azimuthal_modes, + Ex.nGrow(), e_is_nodal, 0, np, thread_num, lev, lev); // Save the position and momenta, making copies -- cgit v1.2.3 From 25690770d463071483bc78c99f29bbe056ec9355 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Mon, 19 Aug 2019 11:04:28 -0700 Subject: must check the overlap box is not empty --- Source/Particles/PhysicalParticleContainer.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index d10390204..3d321c495 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -390,14 +390,16 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) // then how many new particles will be injected is not that simple // We have to shift fine_injection_box because overlap_box has been shifted. Box fine_overlap_box = overlap_box & amrex::shift(fine_injection_box,shifted); - max_new_particles += fine_overlap_box.numPts() * num_ppc - * (AMREX_D_TERM(rrfac,*rrfac,*rrfac)-1); - for (int icell = 0, ncells = overlap_box.numPts(); icell < ncells; ++icell) { - IntVect iv = overlap_box.atOffset(icell); - int r = (fine_overlap_box.contains(iv)) ? AMREX_D_TERM(rrfac,*rrfac,*rrfac) : 1; - for (int ipart = 0; ipart < r; ++ipart) { - cellid_v.push_back(icell); - cellid_v.push_back(ipart); + if (fine_overlap_box.ok()) { + max_new_particles += fine_overlap_box.numPts() * num_ppc + * (AMREX_D_TERM(rrfac,*rrfac,*rrfac)-1); + for (int icell = 0, ncells = overlap_box.numPts(); icell < ncells; ++icell) { + IntVect iv = overlap_box.atOffset(icell); + int r = (fine_overlap_box.contains(iv)) ? AMREX_D_TERM(rrfac,*rrfac,*rrfac) : 1; + for (int ipart = 0; ipart < r; ++ipart) { + cellid_v.push_back(icell); + cellid_v.push_back(ipart); + } } } } -- cgit v1.2.3 From 83f9298a244c57749e4c4a1a159c779466996278 Mon Sep 17 00:00:00 2001 From: Maxence Thevenet Date: Tue, 20 Aug 2019 14:25:51 -0400 Subject: copy member variables to tmp local variables --- Source/Particles/PhysicalParticleContainer.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 27d8cc5ae..4096d1911 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -463,6 +463,9 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) std::size_t shared_mem_bytes = plasma_injector->sharedMemoryNeeded(); int lrrfac = rrfac; + bool loc_do_field_ionization = do_field_ionization; + int loc_ionization_initial_level = ionization_initial_level; + // Loop over all new particles and inject them (creates too many // particles, in particular does not consider xmin, xmax etc.). // The invalid ones are given negative ID and are deleted during the @@ -583,8 +586,8 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) u.z = gamma_boost * ( u.z -beta_boost*gamma_lab ); } - if (do_field_ionization) { - pi[ip] = ionization_initial_level; + if (loc_do_field_ionization) { + pi[ip] = loc_ionization_initial_level; } u.x *= PhysConst::c; @@ -2084,11 +2087,10 @@ PhysicalParticleContainer::buildIonizationMask (const amrex::MFIter& mfi, const + ( ga *ez[i] + ux[i]*by[i] - uy[i]*bx[i] ) * ( ga *ez[i] + ux[i]*by[i] - uy[i]*bx[i] ) ); // Compute probability of ionization p - Real p; Real w_dtau = 1./ ga * p_adk_prefactor[ion_lev[i]] * std::pow(E,p_adk_power[ion_lev[i]]) * std::exp( p_adk_exp_prefactor[ion_lev[i]]/E ); - p = 1. - std::exp( - w_dtau ); + Real p = 1. - std::exp( - w_dtau ); if (random_draw < p){ // increment particle's ionization level -- cgit v1.2.3 From 5bd07149d39ed3727e81d63d03924953594a2362 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 21 Aug 2019 08:03:56 -0700 Subject: Cleanup PICSAR reference and fortran files --- Docs/source/running_cpp/parameters.rst | 16 ---- Examples/Tests/Langmuir/inputs.multi.rz.rt | 1 - Source/FieldSolver/WarpXPushFieldsEM.cpp | 4 +- Source/FortranInterface/Make.package | 1 - Source/FortranInterface/WarpX_f.H | 19 ---- Source/Laser/LaserParticleContainer.cpp | 8 +- Source/Particles/MultiParticleContainer.H | 1 - Source/Particles/PhysicalParticleContainer.cpp | 37 +++----- Source/Particles/WarpXParticleContainer.H | 15 --- Source/Particles/WarpXParticleContainer.cpp | 126 ------------------------- Source/Utils/WarpXAlgorithmSelection.H | 18 +--- Source/Utils/WarpXAlgorithmSelection.cpp | 8 -- Source/WarpX.H | 1 - Source/WarpX.cpp | 5 - 14 files changed, 22 insertions(+), 238 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Docs/source/running_cpp/parameters.rst b/Docs/source/running_cpp/parameters.rst index d411cc0a1..e13d98ff4 100644 --- a/Docs/source/running_cpp/parameters.rst +++ b/Docs/source/running_cpp/parameters.rst @@ -541,10 +541,6 @@ Numerics and algorithms (see `Esirkepov, Comp. Phys. Comm. (2001) `__) - ``direct``: simpler current deposition algorithm, described in the section :doc:`../theory/picsar_theory`. Note that this algorithm is not strictly charge-conserving. - - ``direct-vectorized`` (only available in 3D, and when running on CPU/KNL - as opposed to GPU): - mathematically equivalent to ``direct``, but uses an optimized algorithm - for vectorization on CPU/KNL (see `Vincenti, Comp. Phys. Comm. (2017) - `__) If ``algo.current_deposition`` is not specified, the default is ``esirkepov``. @@ -553,24 +549,12 @@ Numerics and algorithms - ``standard``: standard charge deposition algorithm, described in the section :doc:`../theory/picsar_theory`. - - ``vectorized`` (only available in 3D, and when running on CPU/KNL - as opposed to GPU): - mathematically equivalent to ``standard``, but uses an optimized algorithm - for vectorization on CPU/KNL (see `Vincenti, Comp. Phys. Comm. (2017) - `__) - - If ``algo.charge_deposition`` is not specified, ``vectorized`` is the default - whenever it is available ; ``standard`` is the default otherwise. * ``algo.field_gathering`` (`string`, optional) The algorithm for field gathering. Available options are: - ``standard``: gathers directly from the grid points (either staggered or nodal gridpoints depending on ``warpx.do_nodal``). - - ``vectorized`` (not available when running on GPU): mathematically - equivalent to ``standard``, but uses optimized vector instructions for CPU/KNL. - - If ``algo.field_gathering`` is not specified, ``vectorized`` is the default - on CPU/KNL ; ``standard`` is the default on GPU. * ``algo.particle_pusher`` (`string`, optional) The algorithm for the particle pusher. Available options are: diff --git a/Examples/Tests/Langmuir/inputs.multi.rz.rt b/Examples/Tests/Langmuir/inputs.multi.rz.rt index 873efb9fc..b602e8da5 100644 --- a/Examples/Tests/Langmuir/inputs.multi.rz.rt +++ b/Examples/Tests/Langmuir/inputs.multi.rz.rt @@ -28,7 +28,6 @@ warpx.verbose = 1 # Algorithms algo.field_gathering = standard algo.current_deposition = esirkepov -algo.use_picsar_deposition = 0 # Interpolation interpolation.nox = 1 diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 1df05bc0f..9c71e3542 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -136,7 +136,7 @@ WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real a_dt) MultiFab* cost = costs[lev].get(); const IntVect& rr = (lev > 0) ? refRatio(lev-1) : IntVect::TheUnitVector(); - // xmin is only used by the picsar kernel with cylindrical geometry, + // xmin is only used by the kernel for cylindrical geometry, // in which case it is actually rmin. const Real xmin = Geom(0).ProbLo(0); @@ -329,7 +329,7 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) MultiFab* cost = costs[lev].get(); const IntVect& rr = (lev > 0) ? refRatio(lev-1) : IntVect::TheUnitVector(); - // xmin is only used by the picsar kernel with cylindrical geometry, + // xmin is only used by the kernel for cylindrical geometry, // in which case it is actually rmin. const Real xmin = Geom(0).ProbLo(0); diff --git a/Source/FortranInterface/Make.package b/Source/FortranInterface/Make.package index fe0c2ba3d..d7b17fa56 100644 --- a/Source/FortranInterface/Make.package +++ b/Source/FortranInterface/Make.package @@ -1,6 +1,5 @@ CEXE_headers += WarpX_f.H F90EXE_sources += WarpX_f.F90 -F90EXE_sources += WarpX_picsar.F90 INCLUDE_LOCATIONS += $(WARPX_HOME)/Source/FortranInterface VPATH_LOCATIONS += $(WARPX_HOME)/Source/FortranInterface diff --git a/Source/FortranInterface/WarpX_f.H b/Source/FortranInterface/WarpX_f.H index 8bf3dc425..9db54ba18 100644 --- a/Source/FortranInterface/WarpX_f.H +++ b/Source/FortranInterface/WarpX_f.H @@ -26,7 +26,6 @@ #define WRPX_PUSH_PML_F warpx_push_pml_f_3d #define WRPX_DAMP_PML warpx_damp_pml_3d #define WRPX_DAMP_PML_F warpx_damp_pml_f_3d - #define WRPX_SUM_FINE_TO_CRSE_NODAL warpx_sum_fine_to_crse_nodal_3d #define WRPX_ZERO_OUT_BNDRY warpx_zero_out_bndry_3d #define WRPX_BUILD_MASK warpx_build_mask_3d @@ -65,25 +64,7 @@ extern "C" { #endif - - // Current deposition - void warpx_current_deposition( - amrex::Real* jx, const long* jx_ng, const int* jx_ntot, - amrex::Real* jy, const long* jy_ng, const int* jy_ntot, - amrex::Real* jz, const long* jz_ng, const int* jz_ntot, - const long* np, - const amrex::Real* xp, const amrex::Real* yp, const amrex::Real* zp, - const amrex::Real* uxp, const amrex::Real* uyp,const amrex::Real* uzp, - const amrex::Real* gip, const amrex::Real* w, const amrex::Real* q, - const amrex::Real* xmin, const amrex::Real* ymin, const amrex::Real* zmin, - const amrex::Real* dt, - const amrex::Real* dx, const amrex::Real* dy, const amrex::Real* dz, - const long* nox, const long* noy,const long* noz, - const int* l_nodal, const long* lvect, const long* current_depo_algo); - - // Laser pusher - void warpx_gaussian_laser( const long* np, amrex::Real* Xp, amrex::Real* Yp, amrex::Real* t, amrex::Real* wavelength, amrex::Real* e_max, amrex::Real* waist, amrex::Real* duration, amrex::Real* t_peak, amrex::Real* f, amrex::Real* amplitude, diff --git a/Source/Laser/LaserParticleContainer.cpp b/Source/Laser/LaserParticleContainer.cpp index 786ebc622..fe44e16ed 100644 --- a/Source/Laser/LaserParticleContainer.cpp +++ b/Source/Laser/LaserParticleContainer.cpp @@ -397,8 +397,8 @@ LaserParticleContainer::Evolve (int lev, { BL_PROFILE("Laser::Evolve()"); BL_PROFILE_VAR_NS("Laser::Evolve::Copy", blp_copy); - BL_PROFILE_VAR_NS("PICSAR::LaserParticlePush", blp_pxr_pp); - BL_PROFILE_VAR_NS("PICSAR::LaserCurrentDepo", blp_pxr_cd); + BL_PROFILE_VAR_NS("Laser::ParticlePush", blp_pp); + BL_PROFILE_VAR_NS("Laser::CurrentDepo", blp_cd); BL_PROFILE_VAR_NS("Laser::Evolve::Accumulate", blp_accumulate); Real t_lab = t; @@ -463,7 +463,7 @@ LaserParticleContainer::Evolve (int lev, // // Particle Push // - BL_PROFILE_VAR_START(blp_pxr_pp); + BL_PROFILE_VAR_START(blp_pp); // Find the coordinates of the particles in the emission plane calculate_laser_plane_coordinates( &np, m_xp[thread_num].dataPtr(), @@ -503,7 +503,7 @@ LaserParticleContainer::Evolve (int lev, wp.dataPtr(), amplitude_E.dataPtr(), &p_X[0], &p_X[1], &p_X[2], &nvec[0], &nvec[1], &nvec[2], &mobility, &dt, &PhysConst::c, &WarpX::beta_boost, &WarpX::gamma_boost ); - BL_PROFILE_VAR_STOP(blp_pxr_pp); + BL_PROFILE_VAR_STOP(blp_pp); // // Current Deposition diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 7c9ede411..311fb6cdb 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -124,7 +124,6 @@ public: /// /// This deposits the particle charge onto a node-centered MultiFab and returns a unique ptr /// to it. The charge density is accumulated over all the particles in the MultiParticleContainer - /// This version uses PICSAR routines to perform the deposition. /// std::unique_ptr GetChargeDensity(int lev, bool local = false); diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index d10390204..2cc698f11 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -928,7 +928,7 @@ PhysicalParticleContainer::Evolve (int lev, { BL_PROFILE("PPC::Evolve()"); BL_PROFILE_VAR_NS("PPC::Evolve::Copy", blp_copy); - BL_PROFILE_VAR_NS("PICSAR::FieldGather", blp_pxr_fg); + BL_PROFILE_VAR_NS("PPC::FieldGather", blp_fg); BL_PROFILE_VAR_NS("PPC::ParticlePush", blp_ppc_pp); BL_PROFILE_VAR_NS("PPC::Evolve::partition", blp_partition); @@ -1172,7 +1172,7 @@ PhysicalParticleContainer::Evolve (int lev, // // Field Gather of Aux Data (i.e., the full solution) // - BL_PROFILE_VAR_START(blp_pxr_fg); + BL_PROFILE_VAR_START(blp_fg); FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, exfab, eyfab, ezfab, bxfab, byfab, bzfab, Ex.nGrow(), e_is_nodal, 0, np_gather, thread_num, lev, lev); @@ -1252,7 +1252,7 @@ PhysicalParticleContainer::Evolve (int lev, thread_num, lev, lev-1); } - BL_PROFILE_VAR_STOP(blp_pxr_fg); + BL_PROFILE_VAR_STOP(blp_fg); // // Particle Push @@ -1265,28 +1265,15 @@ PhysicalParticleContainer::Evolve (int lev, // // Current Deposition // - if (WarpX::use_picsar_deposition) { - // Deposit inside domains - DepositCurrentFortran(pti, wp, uxp, uyp, uzp, &jx, &jy, &jz, - 0, np_current, thread_num, - lev, lev, dt); - if (has_buffer){ - // Deposit in buffers - DepositCurrentFortran(pti, wp, uxp, uyp, uzp, cjx, cjy, cjz, - np_current, np-np_current, thread_num, - lev, lev-1, dt); - } - } else { - // Deposit inside domains - DepositCurrent(pti, wp, uxp, uyp, uzp, &jx, &jy, &jz, - 0, np_current, thread_num, - lev, lev, dt); - if (has_buffer){ - // Deposit in buffers - DepositCurrent(pti, wp, uxp, uyp, uzp, cjx, cjy, cjz, - np_current, np-np_current, thread_num, - lev, lev-1, dt); - } + // Deposit inside domains + DepositCurrent(pti, wp, uxp, uyp, uzp, &jx, &jy, &jz, + 0, np_current, thread_num, + lev, lev, dt); + if (has_buffer){ + // Deposit in buffers + DepositCurrent(pti, wp, uxp, uyp, uzp, cjx, cjy, cjz, + np_current, np-np_current, thread_num, + lev, lev-1, dt); } // diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index ac5b47ada..7caef1814 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -190,21 +190,6 @@ public: int depos_lev, amrex::Real dt); - virtual void DepositCurrentFortran(WarpXParIter& pti, - RealVector& wp, - RealVector& uxp, - RealVector& uyp, - RealVector& uzp, - amrex::MultiFab* jx, - amrex::MultiFab* jy, - amrex::MultiFab* jz, - const long offset, - const long np_to_depose, - int thread_num, - int lev, - int depos_lev, - amrex::Real dt); - // If particles start outside of the domain, ContinuousInjection // makes sure that they are initialized when they enter the domain, and // NOT before. Virtual function, overriden by derived classes. diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index befa5cfed..e82f43ce3 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -282,132 +282,6 @@ WarpXParticleContainer::AddNParticles (int lev, Redistribute(); } -/* \brief Current Deposition for thread thread_num using PICSAR - * \param pti : Particle iterator - * \param wp : Array of particle weights - * \param uxp uyp uzp : Array of particle - * \param jx jy jz : Full array of current density - * \param offset : Index of first particle for which current is deposited - * \param np_to_depose: Number of particles for which current is deposited. - Particles [offset,offset+np_tp_depose] deposit their - current - * \param thread_num : Thread number (if tiling) - * \param lev : Level of box that contains particles - * \param depos_lev : Level on which particles deposit (if buffers are used) - * \param dt : Time step for particle level - */ -void -WarpXParticleContainer::DepositCurrentFortran(WarpXParIter& pti, - RealVector& wp, RealVector& uxp, - RealVector& uyp, RealVector& uzp, - MultiFab* jx, MultiFab* jy, MultiFab* jz, - const long offset, const long np_to_depose, - int thread_num, int lev, int depos_lev, - Real dt) -{ - AMREX_ALWAYS_ASSERT_WITH_MESSAGE((depos_lev==(lev-1)) || - (depos_lev==(lev )), - "Deposition buffers only work for lev-1"); - // If no particles, do not do anything - if (np_to_depose == 0) return; - - const long ngJ = jx->nGrow(); - const std::array& dx = WarpX::CellSize(std::max(depos_lev,0)); - const long lvect = 8; - int j_is_nodal = jx->is_nodal() and jy->is_nodal() and jz->is_nodal(); - - BL_PROFILE_VAR_NS("PICSAR::CurrentDeposition", blp_pxr_cd); - BL_PROFILE_VAR_NS("PPC::Evolve::Accumulate", blp_accumulate); - - // Get tile box where current is deposited. - // The tile box is different when depositing in the buffers (depos_lev& xyzmin_tile = WarpX::LowerCorner(tilebox, depos_lev); - const std::array& xyzmin = xyzmin_tile; - -#ifdef AMREX_USE_GPU - // No tiling on GPU: jx_ptr points to the full - // jx array (same for jy_ptr and jz_ptr). - Real* jx_ptr = (*jx)[pti].dataPtr(); - Real* jy_ptr = (*jy)[pti].dataPtr(); - Real* jz_ptr = (*jz)[pti].dataPtr(); - - auto jxntot = (*jx)[pti].length(); - auto jyntot = (*jy)[pti].length(); - auto jzntot = (*jz)[pti].length(); -#else - // Tiling is on: jx_ptr points to local_jx[thread_num] - // (same for jy_ptr and jz_ptr) - tbx.grow(ngJ); - tby.grow(ngJ); - tbz.grow(ngJ); - - local_jx[thread_num].resize(tbx); - local_jy[thread_num].resize(tby); - local_jz[thread_num].resize(tbz); - - Real* jx_ptr = local_jx[thread_num].dataPtr(); - Real* jy_ptr = local_jy[thread_num].dataPtr(); - Real* jz_ptr = local_jz[thread_num].dataPtr(); - - // local_jx[thread_num] is set to zero - local_jx[thread_num].setVal(0.0); - local_jy[thread_num].setVal(0.0); - local_jz[thread_num].setVal(0.0); - - auto jxntot = local_jx[thread_num].length(); - auto jyntot = local_jy[thread_num].length(); - auto jzntot = local_jz[thread_num].length(); -#endif - // GPU, no tiling: deposit directly in jx - // CPU, tiling: deposit into local_jx - // (same for jx and jz) - BL_PROFILE_VAR_START(blp_pxr_cd); - warpx_current_deposition( - jx_ptr, &ngJ, jxntot.getVect(), - jy_ptr, &ngJ, jyntot.getVect(), - jz_ptr, &ngJ, jzntot.getVect(), - &np_to_depose, - m_xp[thread_num].dataPtr() + offset, - m_yp[thread_num].dataPtr() + offset, - m_zp[thread_num].dataPtr() + offset, - uxp.dataPtr() + offset, - uyp.dataPtr() + offset, - uzp.dataPtr() + offset, - m_giv[thread_num].dataPtr() + offset, - wp.dataPtr() + offset, &this->charge, - &xyzmin[0], &xyzmin[1], &xyzmin[2], - &dt, &dx[0], &dx[1], &dx[2], - &WarpX::nox,&WarpX::noy,&WarpX::noz, &j_is_nodal, - &lvect,&WarpX::current_deposition_algo); - - BL_PROFILE_VAR_STOP(blp_pxr_cd); - -#ifndef AMREX_USE_GPU - BL_PROFILE_VAR_START(blp_accumulate); - // CPU, tiling: atomicAdd local_jx into jx - // (same for jx and jz) - (*jx)[pti].atomicAdd(local_jx[thread_num], tbx, tbx, 0, 0, 1); - (*jy)[pti].atomicAdd(local_jy[thread_num], tby, tby, 0, 0, 1); - (*jz)[pti].atomicAdd(local_jz[thread_num], tbz, tbz, 0, 0, 1); - BL_PROFILE_VAR_STOP(blp_accumulate); -#endif -} - /* \brief Current Deposition for thread thread_num * \param pti : Particle iterator * \param wp : Array of particle weights diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index 6a32513b7..54c721abf 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -5,8 +5,6 @@ #include struct MaxwellSolverAlgo { - // These numbers corresponds to the algorithm code in WarpX's - // `warpx_push_bvec` and `warpx_push_evec_f` enum { Yee = 0, CKC = 1 @@ -14,8 +12,6 @@ struct MaxwellSolverAlgo { }; struct ParticlePusherAlgo { - // These numbers correspond to the algorithm code in WarpX's - // `warpx_particle_pusher` enum { Boris = 0, Vay = 1 @@ -24,12 +20,8 @@ struct ParticlePusherAlgo { struct CurrentDepositionAlgo { enum { - // These numbers corresponds to the algorithm code in PICSAR's - // `depose_jxjyjz_generic` and `depose_jxjyjz_generic_2d` - Direct = 3, - DirectVectorized = 2, - EsirkepovNonOptimized = 1, - Esirkepov = 0 + Esirkepov = 0, + Direct = 1 }; }; @@ -41,11 +33,9 @@ struct ChargeDepositionAlgo { }; struct GatheringAlgo { - // These numbers corresponds to the algorithm code in PICSAR's - // `geteb3d_energy_conserving_generic` function + // Only the Standard algorithm is implemented enum { - Vectorized = 0, - Standard = 1 + Standard = 0 }; }; diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index 842085a36..24e55a77f 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -23,9 +23,6 @@ const std::map particle_pusher_algo_to_int = { const std::map current_deposition_algo_to_int = { {"esirkepov", CurrentDepositionAlgo::Esirkepov }, {"direct", CurrentDepositionAlgo::Direct }, -#if (!defined AMREX_USE_GPU)&&(AMREX_SPACEDIM == 3) // Only available on CPU and 3D - {"direct-vectorized", CurrentDepositionAlgo::DirectVectorized }, -#endif {"default", CurrentDepositionAlgo::Esirkepov } }; @@ -36,12 +33,7 @@ const std::map charge_deposition_algo_to_int = { const std::map gathering_algo_to_int = { {"standard", GatheringAlgo::Standard }, -#ifndef AMREX_USE_GPU // Only available on CPU - {"vectorized", GatheringAlgo::Vectorized }, - {"default", GatheringAlgo::Vectorized } -#else {"default", GatheringAlgo::Standard } -#endif }; diff --git a/Source/WarpX.H b/Source/WarpX.H index 927cc1f32..0f78f22f3 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -83,7 +83,6 @@ public: static amrex::Vector B_external; // Algorithms - static long use_picsar_deposition; static long current_deposition_algo; static long charge_deposition_algo; static long field_gathering_algo; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 7b90225dc..b3cb2fbc9 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -38,7 +38,6 @@ Vector WarpX::boost_direction = {0,0,0}; int WarpX::do_compute_max_step_from_zmax = 0; Real WarpX::zmax_plasma_to_compute_max_step = 0.; -long WarpX::use_picsar_deposition = 0; long WarpX::current_deposition_algo; long WarpX::charge_deposition_algo; long WarpX::field_gathering_algo; @@ -525,10 +524,6 @@ WarpX::ReadParameters () { ParmParse pp("algo"); - // If not in RZ mode, read use_picsar_deposition - // In RZ mode, use_picsar_deposition is on, as the C++ version - // of the deposition does not support RZ - pp.query("use_picsar_deposition", use_picsar_deposition); current_deposition_algo = GetAlgorithmInteger(pp, "current_deposition"); charge_deposition_algo = GetAlgorithmInteger(pp, "charge_deposition"); field_gathering_algo = GetAlgorithmInteger(pp, "field_gathering"); -- cgit v1.2.3 From 9795e23c12b3f3553c748e2e335b716f3df62e22 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Wed, 21 Aug 2019 08:45:25 -0700 Subject: add comments and make sure charge=q_e for ionizable species --- Source/Particles/Deposition/CurrentDeposition.H | 12 ++++++++++++ Source/Particles/MultiParticleContainer.cpp | 9 ++++++++- Source/Particles/PhysicalParticleContainer.cpp | 9 +++++++-- Source/Utils/write_atomic_data_cpp.py | 6 ++++++ 4 files changed, 33 insertions(+), 3 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 86cf89cd3..7dde90e96 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -7,6 +7,10 @@ * /param xp, yp, zp : Pointer to arrays of particle positions. * \param wp : Pointer to array of particle weights. * \param uxp uyp uzp : Pointer to arrays of particle momentum. + * \param ion_lev : Pointer to array of particle ionization level. This is + required to have the charge of each macroparticle + since q is a scalar. For non-ionizable species, + ion_lev is a null pointer. * \param jx_arr : Array4 of current density, either full array or tile. * \param jy_arr : Array4 of current density, either full array or tile. * \param jz_arr : Array4 of current density, either full array or tile. @@ -37,6 +41,8 @@ void doDepositionShapeN(const amrex::Real * const xp, const amrex::Real stagger_shift, const amrex::Real q) { + // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer + // (do_ionization=1) const bool do_ionization = ion_lev; const amrex::Real dxi = 1.0/dx[0]; const amrex::Real dzi = 1.0/dx[2]; @@ -166,6 +172,10 @@ void doDepositionShapeN(const amrex::Real * const xp, * /param xp, yp, zp : Pointer to arrays of particle positions. * \param wp : Pointer to array of particle weights. * \param uxp uyp uzp : Pointer to arrays of particle momentum. + * \param ion_lev : Pointer to array of particle ionization level. This is + required to have the charge of each macroparticle + since q is a scalar. For non-ionizable species, + ion_lev is a null pointer. * \param Jx_arr : Array4 of current density, either full array or tile. * \param Jy_arr : Array4 of current density, either full array or tile. * \param Jz_arr : Array4 of current density, either full array or tile. @@ -195,6 +205,8 @@ void doEsirkepovDepositionShapeN (const amrex::Real * const xp, const amrex::Dim3 lo, const amrex::Real q) { + // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer + // (do_ionization=1) const bool do_ionization = ion_lev; const amrex::Real dxi = 1.0/dx[0]; const amrex::Real dtsdx0 = dt*dxi; diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index e39930216..9e2cd097f 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -513,7 +513,7 @@ namespace { // For particle i in mfi, if is_ionized[i]=1, copy particle // particle i from container pc_source into pc_product - static void createIonizedParticles ( + void createIonizedParticles ( int lev, const MFIter& mfi, std::unique_ptr< WarpXParticleContainer>& pc_source, std::unique_ptr< WarpXParticleContainer>& pc_product, @@ -558,6 +558,9 @@ namespace // The advantage of inclusive_scan is that the sum of is_ionized // is in the last element, so no other reduction is required to get // number of particles. + // Gpu::inclusive_scan runs on the current GPU stream, and synchronizes + // with the CPU, so that the next line (executed by the CPU) has the + // updated values of i_product amrex::Gpu::inclusive_scan(is_ionized.begin(), is_ionized.end(), i_product.begin()); int np_ionized = i_product[np_source-1]; if (np_ionized == 0) return; @@ -662,6 +665,10 @@ MultiParticleContainer::doFieldIonization () for (int lev = 0; lev <= pc_source->finestLevel(); ++lev){ + // When using runtime components, AMReX requires to touch all tiles + // in serial and create particles tiles with runtime components if + // they do not exist (or if they were defined by default, i.e., + // without runtime component). #ifdef _OPENMP // Touch all tiles of source species in serial if runtime attribs for (MFIter mfi = pc_source->MakeMFIter(lev); mfi.isValid(); ++mfi) { diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 4096d1911..4562b5c0a 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1979,12 +1979,17 @@ void PhysicalParticleContainer::InitIonizationModule () { if (!do_field_ionization) return; ParmParse pp(species_name); + if (charge != PhysConst::q_e){ + amrex::Warning( + "charge != q_e for ionizable species: overriding user value and setting charge = q_e."); + charge = PhysConst::q_e; + } pp.query("ionization_initial_level", ionization_initial_level); pp.get("ionization_product_species", ionization_product_name); pp.get("physical_element", physical_element); - // Add Real component for ionization level + // Add runtime integer component for ionization level AddIntComp("ionization_level"); - plot_flags.resize(PIdx::nattribs + 1, 1); + // plot_flags.resize(PIdx::nattribs + 1, 1); // Get atomic number and ionization energies from file int ion_element_id = ion_map_ids[physical_element]; ion_atomic_number = ion_atomic_numbers[ion_element_id]; diff --git a/Source/Utils/write_atomic_data_cpp.py b/Source/Utils/write_atomic_data_cpp.py index 21d61a075..b085d50eb 100644 --- a/Source/Utils/write_atomic_data_cpp.py +++ b/Source/Utils/write_atomic_data_cpp.py @@ -1,3 +1,9 @@ +''' +This python script reads ionization tables in atomic_data.txt (generated from +the NIST website) and extracts ionization levels into C++ file +IonizationEnergiesTable.H, which contains tables + metadata. +''' + import re, os import numpy as np from scipy.constants import e -- cgit v1.2.3 From 16dd4d0491b32fb669a1ee07ed2866f68aafd6da Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Wed, 21 Aug 2019 09:14:21 -0700 Subject: charge deposition depends on ionization level --- Source/Laser/LaserParticleContainer.cpp | 12 ++++++++---- Source/Particles/Deposition/ChargeDeposition.H | 13 ++++++++++++- Source/Particles/PhysicalParticleContainer.cpp | 25 +++++++++++++++++++++---- Source/Particles/WarpXParticleContainer.H | 1 + Source/Particles/WarpXParticleContainer.cpp | 23 ++++++++++++++++------- 5 files changed, 58 insertions(+), 16 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Laser/LaserParticleContainer.cpp b/Source/Laser/LaserParticleContainer.cpp index e9a2d42de..9fac71894 100644 --- a/Source/Laser/LaserParticleContainer.cpp +++ b/Source/Laser/LaserParticleContainer.cpp @@ -454,9 +454,10 @@ LaserParticleContainer::Evolve (int lev, BL_PROFILE_VAR_STOP(blp_copy); if (rho) { - DepositCharge(pti, wp, rho, 0, 0, np_current, thread_num, lev, lev); + int* AMREX_RESTRICT ion_lev = nullptr; if (crho) { - DepositCharge(pti, wp, crho, 0, np_current, np-np_current, thread_num, lev, lev-1); + DepositCharge(pti, wp, ion_lev, crho, 0, np_current, + np-np_current, thread_num, lev, lev-1); } } @@ -529,9 +530,12 @@ LaserParticleContainer::Evolve (int lev, BL_PROFILE_VAR_STOP(blp_copy); if (rho) { - DepositCharge(pti, wp, rho, 1, 0, np_current, thread_num, lev, lev); + int* AMREX_RESTRICT ion_lev = nullptr; + DepositCharge(pti, wp, ion_lev, rho, 1, 0, + np_current, thread_num, lev, lev); if (crho) { - DepositCharge(pti, wp, crho, 1, np_current, np-np_current, thread_num, lev, lev-1); + DepositCharge(pti, wp, ion_lev, crho, 1, np_current, + np-np_current, thread_num, lev, lev-1); } } diff --git a/Source/Particles/Deposition/ChargeDeposition.H b/Source/Particles/Deposition/ChargeDeposition.H index 26d42370e..4253f5e76 100755 --- a/Source/Particles/Deposition/ChargeDeposition.H +++ b/Source/Particles/Deposition/ChargeDeposition.H @@ -6,6 +6,10 @@ /* \brief Charge Deposition for thread thread_num * /param xp, yp, zp : Pointer to arrays of particle positions. * \param wp : Pointer to array of particle weights. + * \param ion_lev : Pointer to array of particle ionization level. This is + required to have the charge of each macroparticle + since q is a scalar. For non-ionizable species, + ion_lev is a null pointer. * \param rho_arr : Array4 of charge density, either full array or tile. * \param np_to_depose : Number of particles for which current is deposited. * \param dx : 3D cell size @@ -18,6 +22,7 @@ void doChargeDepositionShapeN(const amrex::Real * const xp, const amrex::Real * const yp, const amrex::Real * const zp, const amrex::Real * const wp, + const int * const ion_lev, const amrex::Array4& rho_arr, const long np_to_depose, const std::array& dx, @@ -25,6 +30,9 @@ void doChargeDepositionShapeN(const amrex::Real * const xp, const amrex::Dim3 lo, const amrex::Real q) { + // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer + // (do_ionization=1) + const bool do_ionization = ion_lev; const amrex::Real dxi = 1.0/dx[0]; const amrex::Real dzi = 1.0/dx[2]; #if (AMREX_SPACEDIM == 2) @@ -43,7 +51,10 @@ void doChargeDepositionShapeN(const amrex::Real * const xp, np_to_depose, [=] AMREX_GPU_DEVICE (long ip) { // --- Get particle quantities - const amrex::Real wq = q*wp[ip]*invvol; + amrex::Real wq = q*wp[ip]*invvol; + if (do_ionization){ + wq *= ion_lev[ip]; + } // --- Compute shape factors // x direction diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 4562b5c0a..3db413104 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1179,9 +1179,18 @@ PhysicalParticleContainer::Evolve (int lev, BL_PROFILE_VAR_STOP(blp_copy); if (rho) { - DepositCharge(pti, wp, rho, 0, 0, np_current, thread_num, lev, lev); + + int* AMREX_RESTRICT ion_lev; + if (do_field_ionization){ + ion_lev = pti.GetiAttribs(particle_icomps["ionization_level"]).dataPtr(); + } else { + ion_lev = nullptr; + } + DepositCharge(pti, wp, ion_lev, rho, 0, 0, + np_current, thread_num, lev, lev); if (has_buffer){ - DepositCharge(pti, wp, crho, 0, np_current, np-np_current, thread_num, lev, lev-1); + DepositCharge(pti, wp, ion_lev, crho, 0, np_current, + np-np_current, thread_num, lev, lev-1); } } @@ -1327,9 +1336,17 @@ PhysicalParticleContainer::Evolve (int lev, } if (rho) { - DepositCharge(pti, wp, rho, 1, 0, np_current, thread_num, lev, lev); + int* AMREX_RESTRICT ion_lev; + if (do_field_ionization){ + ion_lev = pti.GetiAttribs(particle_icomps["ionization_level"]).dataPtr(); + } else { + ion_lev = nullptr; + } + DepositCharge(pti, wp, ion_lev, rho, 1, 0, + np_current, thread_num, lev, lev); if (has_buffer){ - DepositCharge(pti, wp, crho, 1, np_current, np-np_current, thread_num, lev, lev-1); + DepositCharge(pti, wp, ion_lev, crho, 1, np_current, + np-np_current, thread_num, lev, lev-1); } } diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 65eec3c2d..c252eee6b 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -171,6 +171,7 @@ public: virtual void DepositCharge(WarpXParIter& pti, RealVector& wp, + const int * const ion_lev, amrex::MultiFab* rho, int icomp, const long offset, diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 87c2d5e90..20f8c3e00 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -562,6 +562,7 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, void WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector& wp, + const int * const ion_lev, MultiFab* rho, int icomp, const long offset, const long np_to_depose, int thread_num, int lev, int depos_lev) @@ -623,14 +624,14 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector& wp, BL_PROFILE_VAR_START(blp_ppc_chd); if (WarpX::nox == 1){ - doChargeDepositionShapeN<1>(xp, yp, zp, wp.dataPtr()+offset, rho_arr, - np_to_depose, dx, xyzmin, lo, q); + doChargeDepositionShapeN<1>(xp, yp, zp, wp.dataPtr()+offset, ion_lev, + rho_arr, np_to_depose, dx, xyzmin, lo, q); } else if (WarpX::nox == 2){ - doChargeDepositionShapeN<2>(xp, yp, zp, wp.dataPtr()+offset, rho_arr, - np_to_depose, dx, xyzmin, lo, q); + doChargeDepositionShapeN<2>(xp, yp, zp, wp.dataPtr()+offset, ion_lev, + rho_arr, np_to_depose, dx, xyzmin, lo, q); } else if (WarpX::nox == 3){ - doChargeDepositionShapeN<3>(xp, yp, zp, wp.dataPtr()+offset, rho_arr, - np_to_depose, dx, xyzmin, lo, q); + doChargeDepositionShapeN<3>(xp, yp, zp, wp.dataPtr()+offset, ion_lev, + rho_arr, np_to_depose, dx, xyzmin, lo, q); } BL_PROFILE_VAR_STOP(blp_ppc_chd); @@ -740,7 +741,15 @@ WarpXParticleContainer::GetChargeDensity (int lev, bool local) pti.GetPosition(m_xp[thread_num], m_yp[thread_num], m_zp[thread_num]); - DepositCharge(pti, wp, rho.get(), 0, 0, np, thread_num, lev, lev); + int* AMREX_RESTRICT ion_lev; + if (do_field_ionization){ + ion_lev = pti.GetiAttribs(particle_icomps["ionization_level"]).dataPtr(); + } else { + ion_lev = nullptr; + } + + DepositCharge(pti, wp, ion_lev, rho.get(), 0, 0, np, + thread_num, lev, lev); } #ifdef _OPENMP } -- cgit v1.2.3 From d871e6f89e33aa5e9d5b308d19f84e86c5383d9a Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Wed, 21 Aug 2019 09:23:45 -0700 Subject: add comments for charge deposition --- Source/Particles/PhysicalParticleContainer.cpp | 3 ++- Source/Particles/WarpXParticleContainer.cpp | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 3db413104..a82c97945 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1179,7 +1179,7 @@ PhysicalParticleContainer::Evolve (int lev, BL_PROFILE_VAR_STOP(blp_copy); if (rho) { - + // Deposit charge before particle push, in component 0 of MultiFab rho. int* AMREX_RESTRICT ion_lev; if (do_field_ionization){ ion_lev = pti.GetiAttribs(particle_icomps["ionization_level"]).dataPtr(); @@ -1336,6 +1336,7 @@ PhysicalParticleContainer::Evolve (int lev, } if (rho) { + // Deposit charge after particle push, in component 1 of MultiFab rho. int* AMREX_RESTRICT ion_lev; if (do_field_ionization){ ion_lev = pti.GetiAttribs(particle_icomps["ionization_level"]).dataPtr(); diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 20f8c3e00..af56c58fb 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -416,6 +416,10 @@ WarpXParticleContainer::DepositCurrentFortran(WarpXParIter& pti, * \param pti : Particle iterator * \param wp : Array of particle weights * \param uxp uyp uzp : Array of particle + * \param ion_lev : Pointer to array of particle ionization level. This is + required to have the charge of each macroparticle + since q is a scalar. For non-ionizable species, + ion_lev is a null pointer. * \param jx jy jz : Full array of current density * \param offset : Index of first particle for which current is deposited * \param np_to_depose: Number of particles for which current is deposited. @@ -560,6 +564,24 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, #endif } +/* \brief Charge Deposition for thread thread_num + * \param pti : Particle iterator + * \param wp : Array of particle weights + * \param ion_lev : Pointer to array of particle ionization level. This is + required to have the charge of each macroparticle + since q is a scalar. For non-ionizable species, + ion_lev is a null pointer. + * \param rho : Full array of charge density + * \param icomp : Component of rho into which charge is deposited. + 0: old value (before particle push). + 1: new value (after particle push). + * \param offset : Index of first particle for which charge is deposited + * \param np_to_depose: Number of particles for which charge is deposited. + Particles [offset,offset+np_tp_depose] deposit charge + * \param thread_num : Thread number (if tiling) + * \param lev : Level of box that contains particles + * \param depos_lev : Level on which particles deposit (if buffers are used) + */ void WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector& wp, const int * const ion_lev, -- cgit v1.2.3 From fe26d39a01c785b946ef42103555154c13674f9e Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Wed, 21 Aug 2019 09:39:24 -0700 Subject: add back element to plot flags --- Source/Particles/PhysicalParticleContainer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index a82c97945..147f8fac6 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -2007,7 +2007,7 @@ void PhysicalParticleContainer::InitIonizationModule () pp.get("physical_element", physical_element); // Add runtime integer component for ionization level AddIntComp("ionization_level"); - // plot_flags.resize(PIdx::nattribs + 1, 1); + plot_flags.resize(PIdx::nattribs + 1, 1); // Get atomic number and ionization energies from file int ion_element_id = ion_map_ids[physical_element]; ion_atomic_number = ion_atomic_numbers[ion_element_id]; -- cgit v1.2.3 From b675832da2c1f96eb62c5fe1c2a1e71733296af2 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Wed, 21 Aug 2019 10:17:28 -0700 Subject: fix bug in particle output --- Source/Diagnostics/ParticleIO.cpp | 6 +++--- Source/Particles/PhysicalParticleContainer.cpp | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index be4d87746..743df4ce6 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -76,14 +76,15 @@ MultiParticleContainer::Checkpoint (const std::string& dir) const void MultiParticleContainer::WritePlotFile (const std::string& dir) const { - Vector int_names; - Vector int_flags; for (unsigned i = 0, n = species_names.size(); i < n; ++i) { auto& pc = allcontainers[i]; if (pc->plot_species) { Vector real_names; + Vector int_names; + Vector int_flags; + real_names.push_back("weight"); real_names.push_back("momentum_x"); @@ -121,7 +122,6 @@ MultiParticleContainer::WritePlotFile (const std::string& dir) const // when ionization is on. int_flags.resize(1, 1); } - // Convert momentum to SI pc->ConvertUnits(ConvertDirection::WarpX_to_SI); // real_names contains a list of all particle attributes. diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 147f8fac6..18a2c9646 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -2007,7 +2007,6 @@ void PhysicalParticleContainer::InitIonizationModule () pp.get("physical_element", physical_element); // Add runtime integer component for ionization level AddIntComp("ionization_level"); - plot_flags.resize(PIdx::nattribs + 1, 1); // Get atomic number and ionization energies from file int ion_element_id = ion_map_ids[physical_element]; ion_atomic_number = ion_atomic_numbers[ion_element_id]; -- cgit v1.2.3 From eb4080b05342edd63c09091f393638c9053b7b19 Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Wed, 21 Aug 2019 11:31:10 -0700 Subject: reimplement the boosted frame diagnostics without the runtime components --- Source/Diagnostics/ParticleIO.cpp | 11 ---- Source/Evolve/WarpXEvolveEM.cpp | 16 +++--- Source/Particles/MultiParticleContainer.cpp | 20 ------- Source/Particles/PhysicalParticleContainer.cpp | 65 ++++++---------------- .../Particles/RigidInjectedParticleContainer.cpp | 43 +++++--------- Source/Particles/WarpXParticleContainer.H | 2 + Source/Particles/WarpXParticleContainer.cpp | 27 --------- 7 files changed, 41 insertions(+), 143 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index f159e5302..4ac5cb6a1 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -102,17 +102,6 @@ MultiParticleContainer::WritePlotFile (const std::string& dir) const real_names.push_back("theta"); #endif - if (WarpX::do_boosted_frame_diagnostic && pc->DoBoostedFrameDiags()) - { - real_names.push_back("xold"); - real_names.push_back("yold"); - real_names.push_back("zold"); - - real_names.push_back("uxold"); - real_names.push_back("uyold"); - real_names.push_back("uzold"); - } - // Convert momentum to SI pc->ConvertUnits(ConvertDirection::WarpX_to_SI); // real_names contains a list of all particle attributes. diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index 16b5905d1..ff55724f8 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -140,6 +140,14 @@ WarpX::EvolveEM (int numsteps) bool do_insitu = ((step+1) >= insitu_start) && (insitu_int > 0) && ((step+1) % insitu_int == 0); + if (do_boosted_frame_diagnostic) { + std::unique_ptr cell_centered_data = nullptr; + if (WarpX::do_boosted_frame_fields) { + cell_centered_data = GetCellCenteredData(); + } + myBFD->writeLabFrameData(cell_centered_data.get(), *mypc, geom[0], cur_time, dt[0]); + } + bool move_j = is_synchronized || to_make_plot || do_insitu; // If is_synchronized we need to shift j too so that next step we can evolve E by dt/2. // We might need to move j because we are going to make a plotfile. @@ -173,14 +181,6 @@ WarpX::EvolveEM (int numsteps) t_new[i] = cur_time; } - if (do_boosted_frame_diagnostic) { - std::unique_ptr cell_centered_data = nullptr; - if (WarpX::do_boosted_frame_fields) { - cell_centered_data = GetCellCenteredData(); - } - myBFD->writeLabFrameData(cell_centered_data.get(), *mypc, geom[0], cur_time, dt[0]); - } - // slice gen // if (to_make_plot || do_insitu || to_make_slice_plot) { diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 982e04e39..4980bc62a 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -42,26 +42,6 @@ MultiParticleContainer::MultiParticleContainer (AmrCore* amr_core) nspecies_boosted_frame_diags += 1; } } - - if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) - { - for (int i = 0; i < nspecies_boosted_frame_diags; ++i) - { - int is = map_species_boosted_frame_diags[i]; - allcontainers[is]->AddRealComp("xold"); - allcontainers[is]->AddRealComp("yold"); - allcontainers[is]->AddRealComp("zold"); - allcontainers[is]->AddRealComp("uxold"); - allcontainers[is]->AddRealComp("uyold"); - allcontainers[is]->AddRealComp("uzold"); - } - pc_tmp->AddRealComp("xold"); - pc_tmp->AddRealComp("yold"); - pc_tmp->AddRealComp("zold"); - pc_tmp->AddRealComp("uxold"); - pc_tmp->AddRealComp("uyold"); - pc_tmp->AddRealComp("uzold"); - } } void diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index d10390204..14807d0d9 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -196,18 +196,6 @@ PhysicalParticleContainer::CheckAndAddParticle(Real x, Real y, Real z, attribs[PIdx::uz] = u[2]; attribs[PIdx::w ] = weight; - if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) - { - // need to create old values - auto& particle_tile = DefineAndReturnParticleTile(0, 0, 0); - particle_tile.push_back_real(particle_comps["xold"], x); - particle_tile.push_back_real(particle_comps["yold"], y); - particle_tile.push_back_real(particle_comps["zold"], z); - - particle_tile.push_back_real(particle_comps["uxold"], u[0]); - particle_tile.push_back_real(particle_comps["uyold"], u[1]); - particle_tile.push_back_real(particle_comps["uzold"], u[2]); - } // add particle AddOneParticle(0, 0, 0, x, y, z, attribs); } @@ -430,15 +418,6 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) for (int ia = 0; ia < PIdx::nattribs; ++ia) { pa[ia] = soa.GetRealData(ia).data() + old_size; } - GpuArray pb; - if (do_boosted) { - pb[0] = soa.GetRealData(particle_comps[ "xold"]).data() + old_size; - pb[1] = soa.GetRealData(particle_comps[ "yold"]).data() + old_size; - pb[2] = soa.GetRealData(particle_comps[ "zold"]).data() + old_size; - pb[3] = soa.GetRealData(particle_comps["uxold"]).data() + old_size; - pb[4] = soa.GetRealData(particle_comps["uyold"]).data() + old_size; - pb[5] = soa.GetRealData(particle_comps["uzold"]).data() + old_size; - } const GpuArray overlap_corner {AMREX_D_DECL(overlap_realbox.lo(0), @@ -589,15 +568,6 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) pa[PIdx::uy][ip] = u.y; pa[PIdx::uz][ip] = u.z; - if (do_boosted) { - pb[0][ip] = x; - pb[1][ip] = y; - pb[2][ip] = z; - pb[3][ip] = u.x; - pb[4][ip] = u.y; - pb[5][ip] = u.z; - } - #if (AMREX_SPACEDIM == 3) p.pos(0) = x; p.pos(1) = y; @@ -1633,21 +1603,22 @@ PhysicalParticleContainer::PushP (int lev, Real dt, void PhysicalParticleContainer::copy_attribs(WarpXParIter& pti,const Real* xp, const Real* yp, const Real* zp) { - auto& attribs = pti.GetAttribs(); - Real* AMREX_RESTRICT uxp = attribs[PIdx::ux].dataPtr(); Real* AMREX_RESTRICT uyp = attribs[PIdx::uy].dataPtr(); Real* AMREX_RESTRICT uzp = attribs[PIdx::uz].dataPtr(); - Real* AMREX_RESTRICT xpold = pti.GetAttribs(particle_comps["xold"]).dataPtr(); - Real* AMREX_RESTRICT ypold = pti.GetAttribs(particle_comps["yold"]).dataPtr(); - Real* AMREX_RESTRICT zpold = pti.GetAttribs(particle_comps["zold"]).dataPtr(); - Real* AMREX_RESTRICT uxpold = pti.GetAttribs(particle_comps["uxold"]).dataPtr(); - Real* AMREX_RESTRICT uypold = pti.GetAttribs(particle_comps["uyold"]).dataPtr(); - Real* AMREX_RESTRICT uzpold = pti.GetAttribs(particle_comps["uzold"]).dataPtr(); - - const long np = pti.numParticles(); + const auto np = pti.numParticles(); + const auto lev = pti.GetLevel(); + const auto index = pti.GetPairIndex(); + tmp_particle_data.resize(finestLevel()+1); + for (int i = 0; i < 6; ++i) tmp_particle_data[lev][index][i].resize(np); + Real* AMREX_RESTRICT xpold = tmp_particle_data[lev][index][0].dataPtr(); + Real* AMREX_RESTRICT ypold = tmp_particle_data[lev][index][1].dataPtr(); + Real* AMREX_RESTRICT zpold = tmp_particle_data[lev][index][2].dataPtr(); + Real* AMREX_RESTRICT uxpold = tmp_particle_data[lev][index][3].dataPtr(); + Real* AMREX_RESTRICT uypold = tmp_particle_data[lev][index][4].dataPtr(); + Real* AMREX_RESTRICT uzpold = tmp_particle_data[lev][index][5].dataPtr(); ParallelFor( np, [=] AMREX_GPU_DEVICE (long i) { @@ -1733,15 +1704,15 @@ void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real auto& uyp_new = attribs[PIdx::uy ]; auto& uzp_new = attribs[PIdx::uz ]; - auto& xp_old = pti.GetAttribs(particle_comps["xold"]); - auto& yp_old = pti.GetAttribs(particle_comps["yold"]); - auto& zp_old = pti.GetAttribs(particle_comps["zold"]); - auto& uxp_old = pti.GetAttribs(particle_comps["uxold"]); - auto& uyp_old = pti.GetAttribs(particle_comps["uyold"]); - auto& uzp_old = pti.GetAttribs(particle_comps["uzold"]); + auto& xp_old = tmp_particle_data[lev][index][0]; + auto& yp_old = tmp_particle_data[lev][index][1]; + auto& zp_old = tmp_particle_data[lev][index][2]; + auto& uxp_old = tmp_particle_data[lev][index][3]; + auto& uyp_old = tmp_particle_data[lev][index][4]; + auto& uzp_old = tmp_particle_data[lev][index][5]; const long np = pti.numParticles(); - + Real uzfrm = -WarpX::gamma_boost*WarpX::beta_boost*PhysConst::c; Real inv_c2 = 1.0/PhysConst::c/PhysConst::c; diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index 36cb9d224..0c94d1e33 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -239,15 +239,13 @@ RigidInjectedParticleContainer::PushPX(WarpXParIter& pti, Real* const AMREX_RESTRICT Bzp = attribs[PIdx::Bz].dataPtr(); if (!done_injecting_lev) { - if (!(WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags)) { - // If the old values are not already saved, create copies here. - xp_save = xp; - yp_save = yp; - zp_save = zp; - uxp_save = uxp; - uyp_save = uyp; - uzp_save = uzp; - } + // If the old values are not already saved, create copies here. + xp_save = xp; + yp_save = yp; + zp_save = zp; + uxp_save = uxp; + uyp_save = uyp; + uzp_save = uzp; // Scale the fields of particles about to cross the injection plane. // This only approximates what should be happening. The particles @@ -275,27 +273,12 @@ RigidInjectedParticleContainer::PushPX(WarpXParIter& pti, if (!done_injecting_lev) { - Real* AMREX_RESTRICT x_save; - Real* AMREX_RESTRICT y_save; - Real* AMREX_RESTRICT z_save; - Real* AMREX_RESTRICT ux_save; - Real* AMREX_RESTRICT uy_save; - Real* AMREX_RESTRICT uz_save; - if (!(WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags)) { - x_save = xp_save.dataPtr(); - y_save = yp_save.dataPtr(); - z_save = zp_save.dataPtr(); - ux_save = uxp_save.dataPtr(); - uy_save = uyp_save.dataPtr(); - uz_save = uzp_save.dataPtr(); - } else { - x_save = pti.GetAttribs(particle_comps["xold"]).dataPtr(); - y_save = pti.GetAttribs(particle_comps["yold"]).dataPtr(); - z_save = pti.GetAttribs(particle_comps["zold"]).dataPtr(); - ux_save = pti.GetAttribs(particle_comps["uxold"]).dataPtr(); - uy_save = pti.GetAttribs(particle_comps["uyold"]).dataPtr(); - uz_save = pti.GetAttribs(particle_comps["uzold"]).dataPtr(); - } + Real* AMREX_RESTRICT x_save = xp_save.dataPtr(); + Real* AMREX_RESTRICT y_save = yp_save.dataPtr(); + Real* AMREX_RESTRICT z_save = zp_save.dataPtr(); + Real* AMREX_RESTRICT ux_save = uxp_save.dataPtr(); + Real* AMREX_RESTRICT uy_save = uyp_save.dataPtr(); + Real* AMREX_RESTRICT uz_save = uzp_save.dataPtr(); // Undo the push for particles not injected yet. // The zp are advanced a fixed amount. diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index ac5b47ada..cbdf0cdbc 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -308,6 +308,8 @@ protected: // list of names of attributes to dump. amrex::Vector plot_vars; + amrex::Vector, std::array, 6> > > tmp_particle_data; + private: virtual void particlePostLocate(ParticleType& p, const amrex::ParticleLocData& pld, const int lev) override; diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index befa5cfed..db0f683c7 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -85,17 +85,6 @@ WarpXParticleContainer::WarpXParticleContainer (AmrCore* amr_core, int ispecies) particle_comps["theta"] = PIdx::theta; #endif - if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) - { - particle_comps["xold"] = PIdx::nattribs; - particle_comps["yold"] = PIdx::nattribs+1; - particle_comps["zold"] = PIdx::nattribs+2; - particle_comps["uxold"] = PIdx::nattribs+3; - particle_comps["uyold"] = PIdx::nattribs+4; - particle_comps["uzold"] = PIdx::nattribs+5; - - } - // Initialize temporary local arrays for charge/current deposition int num_threads = 1; #ifdef _OPENMP @@ -238,14 +227,6 @@ WarpXParticleContainer::AddNParticles (int lev, p.pos(1) = z[i]; #endif - if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) - { - auto& ptile = DefineAndReturnParticleTile(0, 0, 0); - ptile.push_back_real(particle_comps["xold"], x[i]); - ptile.push_back_real(particle_comps["yold"], y[i]); - ptile.push_back_real(particle_comps["zold"], z[i]); - } - particle_tile.push_back(p); } @@ -256,14 +237,6 @@ WarpXParticleContainer::AddNParticles (int lev, particle_tile.push_back_real(PIdx::uy, vy + ibegin, vy + iend); particle_tile.push_back_real(PIdx::uz, vz + ibegin, vz + iend); - if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) - { - auto& ptile = DefineAndReturnParticleTile(0, 0, 0); - ptile.push_back_real(particle_comps["uxold"], vx + ibegin, vx + iend); - ptile.push_back_real(particle_comps["uyold"], vy + ibegin, vy + iend); - ptile.push_back_real(particle_comps["uzold"], vz + ibegin, vz + iend); - } - for (int comp = PIdx::uz+1; comp < PIdx::nattribs; ++comp) { #ifdef WARPX_DIM_RZ -- cgit v1.2.3 From d2a176f231788f1a04472f09bbe7134a09d14ac5 Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Wed, 21 Aug 2019 11:43:28 -0700 Subject: some typedefs and an enum for clarity --- Source/Particles/PhysicalParticleContainer.cpp | 34 ++++++++++---------------- Source/Particles/WarpXParticleContainer.H | 18 +++++++++++--- 2 files changed, 28 insertions(+), 24 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 14807d0d9..c6d3cd9e7 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -277,9 +277,6 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) const int grid_id = mfi.index(); const int tile_id = mfi.LocalTileIndex(); GetParticles(lev)[std::make_pair(grid_id, tile_id)]; - if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) { - DefineAndReturnParticleTile(lev, grid_id, tile_id); - } } #endif @@ -403,11 +400,6 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) const int cpuid = ParallelDescriptor::MyProc(); auto& particle_tile = GetParticles(lev)[std::make_pair(grid_id,tile_id)]; - bool do_boosted = false; - if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) { - do_boosted = true; - DefineAndReturnParticleTile(lev, grid_id, tile_id); - } auto old_size = particle_tile.GetArrayOfStructs().size(); auto new_size = old_size + max_new_particles; particle_tile.resize(new_size); @@ -1612,13 +1604,13 @@ void PhysicalParticleContainer::copy_attribs(WarpXParIter& pti,const Real* xp, const auto lev = pti.GetLevel(); const auto index = pti.GetPairIndex(); tmp_particle_data.resize(finestLevel()+1); - for (int i = 0; i < 6; ++i) tmp_particle_data[lev][index][i].resize(np); - Real* AMREX_RESTRICT xpold = tmp_particle_data[lev][index][0].dataPtr(); - Real* AMREX_RESTRICT ypold = tmp_particle_data[lev][index][1].dataPtr(); - Real* AMREX_RESTRICT zpold = tmp_particle_data[lev][index][2].dataPtr(); - Real* AMREX_RESTRICT uxpold = tmp_particle_data[lev][index][3].dataPtr(); - Real* AMREX_RESTRICT uypold = tmp_particle_data[lev][index][4].dataPtr(); - Real* AMREX_RESTRICT uzpold = tmp_particle_data[lev][index][5].dataPtr(); + for (int i = 0; i < TmpIdx::nattribs; ++i) tmp_particle_data[lev][index][i].resize(np); + Real* AMREX_RESTRICT xpold = tmp_particle_data[lev][index][TmpIdx::xold ].dataPtr(); + Real* AMREX_RESTRICT ypold = tmp_particle_data[lev][index][TmpIdx::yold ].dataPtr(); + Real* AMREX_RESTRICT zpold = tmp_particle_data[lev][index][TmpIdx::zold ].dataPtr(); + Real* AMREX_RESTRICT uxpold = tmp_particle_data[lev][index][TmpIdx::uxold].dataPtr(); + Real* AMREX_RESTRICT uypold = tmp_particle_data[lev][index][TmpIdx::uyold].dataPtr(); + Real* AMREX_RESTRICT uzpold = tmp_particle_data[lev][index][TmpIdx::uzold].dataPtr(); ParallelFor( np, [=] AMREX_GPU_DEVICE (long i) { @@ -1704,12 +1696,12 @@ void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real auto& uyp_new = attribs[PIdx::uy ]; auto& uzp_new = attribs[PIdx::uz ]; - auto& xp_old = tmp_particle_data[lev][index][0]; - auto& yp_old = tmp_particle_data[lev][index][1]; - auto& zp_old = tmp_particle_data[lev][index][2]; - auto& uxp_old = tmp_particle_data[lev][index][3]; - auto& uyp_old = tmp_particle_data[lev][index][4]; - auto& uzp_old = tmp_particle_data[lev][index][5]; + auto& xp_old = tmp_particle_data[lev][index][TmpIdx::xold]; + auto& yp_old = tmp_particle_data[lev][index][TmpIdx::yold]; + auto& zp_old = tmp_particle_data[lev][index][TmpIdx::zold]; + auto& uxp_old = tmp_particle_data[lev][index][TmpIdx::uxold]; + auto& uyp_old = tmp_particle_data[lev][index][TmpIdx::uyold]; + auto& uzp_old = tmp_particle_data[lev][index][TmpIdx::uzold]; const long np = pti.numParticles(); diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index cbdf0cdbc..a21506ec3 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -29,6 +29,15 @@ struct DiagIdx }; }; +struct TmpIdx +{ + enum { + xold = 0, + yold, zold, uxold, uyold, uzold, + nattribs + }; +}; + namespace ParticleStringNames { const std::map to_index = { @@ -297,7 +306,10 @@ protected: amrex::Vector local_jy; amrex::Vector local_jz; - amrex::Vector > m_xp, m_yp, m_zp, m_giv; + using DataContainer = amrex::Gpu::ManagedDeviceVector; + using PairIndex = std::pair; + + amrex::Vector m_xp, m_yp, m_zp, m_giv; // Whether to dump particle quantities. // If true, particle position is always dumped. @@ -307,8 +319,8 @@ protected: amrex::Vector plot_flags; // list of names of attributes to dump. amrex::Vector plot_vars; - - amrex::Vector, std::array, 6> > > tmp_particle_data; + + amrex::Vector > > tmp_particle_data; private: virtual void particlePostLocate(ParticleType& p, const amrex::ParticleLocData& pld, -- cgit v1.2.3 From d5fc8e40d0373bd4ba1e306f4ce5dc0a895ba6e7 Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Wed, 21 Aug 2019 12:30:35 -0700 Subject: touch all map entries for tmp_particle_data --- Source/Particles/PhysicalParticleContainer.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index c6d3cd9e7..95909feba 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -274,9 +274,10 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) #ifdef _OPENMP // First touch all tiles in the map in serial for (MFIter mfi = MakeMFIter(lev); mfi.isValid(); ++mfi) { - const int grid_id = mfi.index(); - const int tile_id = mfi.LocalTileIndex(); - GetParticles(lev)[std::make_pair(grid_id, tile_id)]; + auto index = std::make_pair(mfi.index(), mfi.LocalTileIndex()); + GetParticles(lev)[index]; + tmp_particle_data.resize(finestLevel()+1); + tmp_particle_data[lev][index]; } #endif @@ -1611,7 +1612,7 @@ void PhysicalParticleContainer::copy_attribs(WarpXParIter& pti,const Real* xp, Real* AMREX_RESTRICT uxpold = tmp_particle_data[lev][index][TmpIdx::uxold].dataPtr(); Real* AMREX_RESTRICT uypold = tmp_particle_data[lev][index][TmpIdx::uyold].dataPtr(); Real* AMREX_RESTRICT uzpold = tmp_particle_data[lev][index][TmpIdx::uzold].dataPtr(); - + ParallelFor( np, [=] AMREX_GPU_DEVICE (long i) { xpold[i]=xp[i]; -- cgit v1.2.3 From e71f1cc255f169652ac0d6c0138035ea732b8e65 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Wed, 21 Aug 2019 12:54:08 -0700 Subject: error if ionization on GPU --- Source/Particles/PhysicalParticleContainer.cpp | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 18a2c9646..1d1d6c1b7 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -39,6 +39,12 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp pp.query("do_boosted_frame_diags", do_boosted_frame_diags); pp.query("do_field_ionization", do_field_ionization); +#ifdef AMREX_USE_GPU + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + do_field_ionization == 0, + "Field ionization does not work on GPU so far, because the current " + "version of Redistribute in AMReX does not work with runtime parameters"); +#endif pp.query("plot_species", plot_species); int do_user_plot_vars; -- cgit v1.2.3 From 0318924f6065a92725b43460aa3b13d7f6cf696a Mon Sep 17 00:00:00 2001 From: Dave Grote Date: Wed, 21 Aug 2019 13:21:30 -0700 Subject: Various bug fixes for RZ multimode --- .../langmuir_PICMI_rz_multimode_analyze.py | 34 ++++++++++++---------- Source/Diagnostics/FieldIO.cpp | 4 +-- Source/FieldSolver/WarpXPushFieldsEM.cpp | 6 ++-- Source/FieldSolver/WarpX_FDTD.H | 22 +++++++------- Source/Initialization/InjectorPosition.H | 2 +- Source/Initialization/PlasmaInjector.cpp | 2 +- Source/Particles/Deposition/CurrentDeposition.H | 5 ++-- Source/Particles/PhysicalParticleContainer.cpp | 20 ++++++++++--- Source/Particles/WarpXParticleContainer.H | 2 +- Source/WarpX.cpp | 7 ++--- 10 files changed, 57 insertions(+), 47 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Examples/Tests/Langmuir/langmuir_PICMI_rz_multimode_analyze.py b/Examples/Tests/Langmuir/langmuir_PICMI_rz_multimode_analyze.py index 9d9fa40f6..f72ab6f00 100644 --- a/Examples/Tests/Langmuir/langmuir_PICMI_rz_multimode_analyze.py +++ b/Examples/Tests/Langmuir/langmuir_PICMI_rz_multimode_analyze.py @@ -8,6 +8,8 @@ matplotlib.use('Agg') import matplotlib.pyplot as plt from pywarpx import picmi +constants = picmi.constants + nr = 64 nz = 200 @@ -18,9 +20,9 @@ zmax = +40.e-6 # Parameters describing particle distribution density = 2.e24 -epsilon0 = 0.001 -epsilon1 = 0.001 -epsilon2 = 0.001 +epsilon0 = 0.001*constants.c +epsilon1 = 0.001*constants.c +epsilon2 = 0.001*constants.c w0 = 5.e-6 n_osc_z = 3 @@ -28,8 +30,8 @@ n_osc_z = 3 k0 = 2.*np.pi*n_osc_z/(zmax - zmin) # Plasma frequency -wp = np.sqrt((density*picmi.q_e**2)/(picmi.m_e*picmi.ep0)) -kp = wp/picmi.c +wp = np.sqrt((density*constants.q_e**2)/(constants.m_e*constants.ep0)) +kp = wp/constants.c uniform_plasma = picmi.UniformDistribution(density = density, upper_bound = [+18e-6, +18e-6, None], @@ -80,8 +82,8 @@ sim = picmi.Simulation(solver = solver, warpx_field_gathering_algo = 'standard', warpx_particle_pusher_algo = 'boris') -sim.add_species(electrons, layout=picmi.GriddedLayout(n_macroparticle_per_cell=[2,8,2], grid=grid)) -sim.add_species(ions, layout=picmi.GriddedLayout(n_macroparticle_per_cell=[2,8,2], grid=grid)) +sim.add_species(electrons, layout=picmi.GriddedLayout(n_macroparticle_per_cell=[2,16,2], grid=grid)) +sim.add_species(ions, layout=picmi.GriddedLayout(n_macroparticle_per_cell=[2,16,2], grid=grid)) # write_inputs will create an inputs file that can be used to run # with the compiled version. @@ -101,15 +103,15 @@ def calcEr( z, r, k0, w0, wp, t, epsilons) : of the same length as z and r, in the half-plane theta=0 """ Er_array = ( - epsilons[0] * picmi.m_e*picmi.c**2/picmi.q_e * 2*r/w0**2 * + epsilons[0] * constants.m_e*constants.c/constants.q_e * 2*r/w0**2 * np.exp( -r**2/w0**2 ) * np.sin( k0*z ) * np.sin( wp*t ) - - epsilons[1] * picmi.m_e*picmi.c**2/picmi.q_e * 2/w0 * + - epsilons[1] * constants.m_e*constants.c/constants.q_e * 2/w0 * np.exp( -r**2/w0**2 ) * np.sin( k0*z ) * np.sin( wp*t ) - + epsilons[1] * picmi.m_e*picmi.c**2/picmi.q_e * 4*r**2/w0**3 * + + epsilons[1] * constants.m_e*constants.c/constants.q_e * 4*r**2/w0**3 * np.exp( -r**2/w0**2 ) * np.sin( k0*z ) * np.sin( wp*t ) - - epsilons[2] * picmi.m_e*picmi.c**2/picmi.q_e * 8*r/w0**2 * + - epsilons[2] * constants.m_e*constants.c/constants.q_e * 8*r/w0**2 * np.exp( -r**2/w0**2 ) * np.sin( k0*z ) * np.sin( wp*t ) - + epsilons[2] * picmi.m_e*picmi.c**2/picmi.q_e * 8*r**3/w0**4 * + + epsilons[2] * constants.m_e*constants.c/constants.q_e * 8*r**3/w0**4 * np.exp( -r**2/w0**2 ) * np.sin( k0*z ) * np.sin( wp*t )) return( Er_array ) @@ -119,11 +121,11 @@ def calcEz( z, r, k0, w0, wp, t, epsilons) : of the same length as z and r, in the half-plane theta=0 """ Ez_array = ( - - epsilons[0] * picmi.m_e*picmi.c**2/picmi.q_e * k0 * + - epsilons[0] * constants.m_e*constants.c/constants.q_e * k0 * np.exp( -r**2/w0**2 ) * np.cos( k0*z ) * np.sin( wp*t ) - - epsilons[1] * picmi.m_e*picmi.c**2/picmi.q_e * k0 * 2*r/w0 * + - epsilons[1] * constants.m_e*constants.c/constants.q_e * k0 * 2*r/w0 * np.exp( -r**2/w0**2 ) * np.cos( k0*z ) * np.sin( wp*t ) - - epsilons[2] * picmi.m_e*picmi.c**2/picmi.q_e * k0 * 4*r**2/w0**2 * + - epsilons[2] * constants.m_e*constants.c/constants.q_e * k0 * 4*r**2/w0**2 * np.exp( -r**2/w0**2 ) * np.cos( k0*z ) * np.sin( wp*t )) return( Ez_array ) @@ -153,7 +155,7 @@ Ez_th = calcEz(zz[:,:-1] + dz/2., rr[:,:-1], k0, w0, wp, t0, [epsilon0, epsilon1 max_error_Er = abs(Er_sim - Er_th).max()/abs(Er_th).max() max_error_Ez = abs(Ez_sim - Ez_th).max()/abs(Ez_th).max() print("Max error Er %e"%max_error_Er) -print("Max error Ez %e"%max_error_Er) +print("Max error Ez %e"%max_error_Ez) # Plot the last field from the loop (Ez at iteration 40) plt.subplot2grid( (1,2), (0,0) ) diff --git a/Source/Diagnostics/FieldIO.cpp b/Source/Diagnostics/FieldIO.cpp index a1fd61b25..93e0cce80 100644 --- a/Source/Diagnostics/FieldIO.cpp +++ b/Source/Diagnostics/FieldIO.cpp @@ -278,7 +278,7 @@ AverageAndPackVectorField( MultiFab& mf_avg, // and copies over directly the y (or theta) component (which is // already cell centered). if (vector_field[0]->nComp() > 1) { -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ // When there are more than one components, the total // fields needs to be constructed in temporary MultiFabs // Note that mf_total is declared in the same way as @@ -324,7 +324,7 @@ AverageAndPackVectorField( MultiFab& mf_avg, // See comment above, though here, the y (or theta) component // has node centering. if (vector_field[0]->nComp() > 1) { -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ // When there are more than one components, the total // fields needs to be constructed in temporary MultiFabs // Note that mf_total is declared in the same way as diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 0455762d6..39003b82d 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -683,7 +683,6 @@ WarpX::ApplyInverseVolumeScalingToCurrentDensity (MultiFab* Jx, MultiFab* Jy, Mu // Apply the inverse volume scaling // Jt is forced to zero on axis. - const amrex::Real r = std::abs(rmin + (i - irmin)*dr); if (r == 0.) { Jt_arr(i,j,0,2*imode) = 0.; Jt_arr(i,j,0,2*imode+1) = 0.; @@ -719,12 +718,11 @@ WarpX::ApplyInverseVolumeScalingToCurrentDensity (MultiFab* Jx, MultiFab* Jy, Mu // to the cells above the axis. // Jz is located on the boundary if (rmin == 0. && 0 < i && i <= ngJ) { - Jz_arr(i,j,0,2*imode) += Jz_arr(-i,j,0,2*imode); - Jz_arr(i,j,0,2*imode+1) += Jz_arr(-i,j,0,2*imode+1); + Jz_arr(i,j,0,2*imode) += ifact*Jz_arr(-i,j,0,2*imode); + Jz_arr(i,j,0,2*imode+1) += ifact*Jz_arr(-i,j,0,2*imode+1); } // Apply the inverse volume scaling - const amrex::Real r = std::abs(rmin + (i - irmin)*dr); if (r == 0.) { // Verboncoeur JCP 164, 421-427 (2001) : corrected volume on axis Jz_arr(i,j,0,2*imode) /= (MathConst::pi*dr/3.); diff --git a/Source/FieldSolver/WarpX_FDTD.H b/Source/FieldSolver/WarpX_FDTD.H index 4a5139d50..650e222e1 100644 --- a/Source/FieldSolver/WarpX_FDTD.H +++ b/Source/FieldSolver/WarpX_FDTD.H @@ -28,10 +28,10 @@ void warpx_push_bx_yee(int i, int j, int k, Array4 const& Bx, // (due to the 1/r terms). The following expressions regularize // these divergences by assuming, on axis : // Ez/r = 0/r + dEz/dr - Bx(i,j,2*imode) = Bx(i,j,0,2*imode) - imode*dtsdx*Ez(i+1,j,0,2*imode+1) & - + dtsdz*(Ey(i,j+1,0,2*imode) - Ey(i,j,0,2*imode)); - Bx(i,j,2*imode+1) = Bx(i,j,0,2*imode+1) + imode*dtsdx*Ez(i+1,j,0,2*imode) & - + dtsdz*(Ey(i,j+1,0,2*imode+1) - Ey(i,j,0,2*imode+1)); + Bx(i,j,0,2*imode) = Bx(i,j,0,2*imode) - imode*dtsdx*Ez(i+1,j,0,2*imode+1) & + + dtsdz*(Ey(i,j+1,0,2*imode) - Ey(i,j,0,2*imode)); + Bx(i,j,0,2*imode+1) = Bx(i,j,0,2*imode+1) + imode*dtsdx*Ez(i+1,j,0,2*imode) & + + dtsdz*(Ey(i,j+1,0,2*imode+1) - Ey(i,j,0,2*imode+1)); } else { Bx(i,j,0,2*imode) = 0.; Bx(i,j,0,2*imode+1) = 0.; @@ -39,10 +39,10 @@ void warpx_push_bx_yee(int i, int j, int k, Array4 const& Bx, } else { // Br(i,j,m) = Br(i,j,m) + I*m*dt*Ez(i,j,m)/r + dtsdz*(Et(i,j+1,m) - Et(i,j,m)) const Real r = rmin*dxinv + i; - Bx(i,j,2*imode) = Bx(i,j,0,2*imode) - imode*dtsdx*Ez(i,j,0,2*imode+1)/r & - + dtsdz*(Ey(i,j+1,0,2*imode) - Ey(i,j,0,2*imode)); - Bx(i,j,2*imode+1) = Bx(i,j,0,2*imode+1) + imode*dtsdx*Ez(i,j,0,2*imode)/r & - + dtsdz*(Ey(i,j+1,0,2*imode+1) - Ey(i,j,0,2*imode+1)); + Bx(i,j,0,2*imode) = Bx(i,j,0,2*imode) - imode*dtsdx*Ez(i,j,0,2*imode+1)/r & + + dtsdz*(Ey(i,j+1,0,2*imode) - Ey(i,j,0,2*imode)); + Bx(i,j,0,2*imode+1) = Bx(i,j,0,2*imode+1) + imode*dtsdx*Ez(i,j,0,2*imode)/r & + + dtsdz*(Ey(i,j+1,0,2*imode+1) - Ey(i,j,0,2*imode+1)); } } #endif @@ -90,8 +90,8 @@ void warpx_push_bz_yee(int i, int j, int k, Array4 const& Bz, Bz(i,j,0,0) += - dtsdx*(ru*Ey(i+1,j,0,0) - rd*Ey(i,j,0,0)); for (int imode=1 ; imode < nmodes ; imode++) { // Bz(i,j,m) = Bz(i,j,m) - dtsdr*(ru*Et(i+1,j,m) - rd*Et(i,j,m)) - I*m*dt*Er(i,j,m)/r - Bz(i,j,0,2*imode) += - dtsdx*(ru*Ey(i+1,j,0,2*imode) - rd*Ey(i,j,0,2*imode)) + imode*dtsdx*Ex(i,j,2*imode+1)/r; - Bz(i,j,0,2*imode+1) += - dtsdx*(ru*Ey(i+1,j,0,2*imode+1) - rd*Ey(i,j,0,2*imode+1)) - imode*dtsdx*Ex(i,j,2*imode)/r; + Bz(i,j,0,2*imode) += - dtsdx*(ru*Ey(i+1,j,0,2*imode) - rd*Ey(i,j,0,2*imode)) + imode*dtsdx*Ex(i,j,0,2*imode+1)/r; + Bz(i,j,0,2*imode+1) += - dtsdx*(ru*Ey(i+1,j,0,2*imode+1) - rd*Ey(i,j,0,2*imode+1)) - imode*dtsdx*Ex(i,j,0,2*imode)/r; } #endif } @@ -157,7 +157,7 @@ void warpx_push_ey_yee(int i, int j, int k, Array4 const& Ey, // And using the rule applying for the guards cells // Er(r=-dr/2,m=1) = Er(r=dr/2,m=1). Thus: Et(i,j,m) = -i*Er(i,j,m) Ey(i,j,0,2*imode) = Ex(i,j,0,2*imode+1); - Ey(i,j,0,2*imode+1) = -Ex(i,0,j,2*imode); + Ey(i,j,0,2*imode+1) = -Ex(i,j,0,2*imode); } else { // Etheta should remain 0 on axis, for modes different than m=1 Ey(i,j,0,2*imode) = 0.; diff --git a/Source/Initialization/InjectorPosition.H b/Source/Initialization/InjectorPosition.H index 19bb092dd..0e43f889e 100644 --- a/Source/Initialization/InjectorPosition.H +++ b/Source/Initialization/InjectorPosition.H @@ -33,7 +33,7 @@ struct InjectorPositionRegular { int nx = ref_fac*ppc.x; int ny = ref_fac*ppc.y; -#if (AMREX_SPACEDIM == 3) +#if (defined WARPX_DIM_3D) || (defined WARPX_DIM_RZ) int nz = ref_fac*ppc.z; #else int nz = 1; diff --git a/Source/Initialization/PlasmaInjector.cpp b/Source/Initialization/PlasmaInjector.cpp index 5a5c190c6..ce1258245 100644 --- a/Source/Initialization/PlasmaInjector.cpp +++ b/Source/Initialization/PlasmaInjector.cpp @@ -147,7 +147,7 @@ PlasmaInjector::PlasmaInjector (int ispecies, const std::string& name) // For 2D, only two are expected. The third is overwritten with 1. num_particles_per_cell_each_dim.assign(3, 1); pp.getarr("num_particles_per_cell_each_dim", num_particles_per_cell_each_dim); -#if ( AMREX_SPACEDIM == 2 ) && !defined(WARPX_RZ) +#if WARPX_DIM_2D num_particles_per_cell_each_dim[2] = 1; #endif // Construct InjectorPosition from InjectorPositionRegular. diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 8be67c4cd..bab0f7e2d 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -159,7 +159,7 @@ void doDepositionShapeN(const amrex::Real * const xp, } /* \brief Esirkepov Current Deposition for thread thread_num - * /param xp, yp, zp : Pointer to arrays of particle positions. + * \param xp, yp, zp : Pointer to arrays of particle positions. * \param wp : Pointer to array of particle weights. * \param uxp uyp uzp : Pointer to arrays of particle momentum. * \param Jx_arr : Array4 of current density, either full array or tile. @@ -170,7 +170,8 @@ void doDepositionShapeN(const amrex::Real * const xp, * \param dx : 3D cell size * \param xyzmin : Physical lower bounds of domain. * \param lo : Index lower bounds of domain. - * /param q : species charge. + * \param q : species charge. + * \param n_rz_azimuthal_modes: Number of azimuthal modes when using RZ geometry */ template void doEsirkepovDepositionShapeN (const amrex::Real * const xp, diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index ec360f2c4..143ea73dc 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -148,12 +148,12 @@ PhysicalParticleContainer::AddGaussianBeam(Real x_m, Real y_m, Real z_m, npart /= 4; } for (long i = 0; i < npart; ++i) { -#if ( AMREX_SPACEDIM == 3 | WARPX_DIM_RZ) +#if (defined WARPX_DIM_3D) || (WARPX_DIM_RZ) Real weight = q_tot/npart/charge; Real x = distx(mt); Real y = disty(mt); Real z = distz(mt); -#elif ( AMREX_SPACEDIM == 2 ) +#elif (defined WARPX_DIM_2D) Real weight = q_tot/npart/charge/y_rms; Real x = distx(mt); Real y = 0.; @@ -480,7 +480,12 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) #else Real x = overlap_corner[0] + (iv[0]+r.x)*dx[0]; Real y = 0.0; +#ifdef WARPX_DIM_2D Real z = overlap_corner[1] + (iv[1]+r.y)*dx[1]; +#elif defined WARPX_DIM_RZ + // Note that for RZ, r.y will be theta + Real z = overlap_corner[1] + (iv[1]+r.z)*dx[1]; +#endif #endif #if (AMREX_SPACEDIM == 3) @@ -501,9 +506,16 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) Real yb = y; #ifdef WARPX_DIM_RZ - // Replace the x and y, choosing the angle randomly. + // Replace the x and y, setting an angle theta. // These x and y are used to get the momentum and density - Real theta = 2.*MathConst::pi*amrex::Random(); + Real theta; + if (WarpX::n_rz_azimuthal_modes == 1) { + // With only 1 mode, the angle doesn't matter so + // choose it randomly. + theta = 2.*MathConst::pi*amrex::Random(); + } else { + theta = 2.*MathConst::pi*r.y; + } x = xb*std::cos(theta); y = xb*std::sin(theta); #endif diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index ee263c27b..2bd09af96 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -42,7 +42,7 @@ namespace ParticleStringNames {"Bx", PIdx::Bx }, {"By", PIdx::By }, {"Bz", PIdx::Bz } -#ifdef WARPX_RZ +#ifdef WARPX_DIM_RZ ,{"theta", PIdx::theta} #endif }; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 2d6df625f..dd24013f1 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -503,7 +503,7 @@ WarpX::ReadParameters () l_lower_order_in_v = false; } - // Only needs to be set with WARPX_RZ, otherwise defaults to 1. + // Only needs to be set with WARPX_DIM_RZ, otherwise defaults to 1. pp.query("n_rz_azimuthal_modes", n_rz_azimuthal_modes); } @@ -520,9 +520,6 @@ WarpX::ReadParameters () { ParmParse pp("algo"); - // If not in RZ mode, read use_picsar_deposition - // In RZ mode, use_picsar_deposition is on, as the C++ version - // of the deposition does not support RZ pp.query("use_picsar_deposition", use_picsar_deposition); current_deposition_algo = GetAlgorithmInteger(pp, "current_deposition"); charge_deposition_algo = GetAlgorithmInteger(pp, "charge_deposition"); @@ -776,7 +773,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm const IntVect& ngE, const IntVect& ngJ, const IntVect& ngRho, int ngF) { -#if defined WARPX_RZ +#if defined WARPX_DIM_RZ if (n_rz_azimuthal_modes > 1) { // There is a real and imaginary component for each mode ncomps = n_rz_azimuthal_modes*2; -- cgit v1.2.3 From 11d362d949107d9c51a6a9771b05d62bc8e1cd8e Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Mon, 26 Aug 2019 10:26:06 -0700 Subject: fix race condition --- Source/Particles/PhysicalParticleContainer.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 79a6f88f1..df8dcf836 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -949,6 +949,19 @@ PhysicalParticleContainer::Evolve (int lev, bool has_buffer = cEx || cjx; + if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) + { + for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) + { + const auto np = pti.numParticles(); + const auto lev = pti.GetLevel(); + const auto index = pti.GetPairIndex(); + tmp_particle_data.resize(finestLevel()+1); + for (int i = 0; i < TmpIdx::nattribs; ++i) + tmp_particle_data[lev][index][i].resize(np); + } + } + #ifdef _OPENMP #pragma omp parallel #endif @@ -1680,8 +1693,6 @@ void PhysicalParticleContainer::copy_attribs(WarpXParIter& pti,const Real* xp, const auto np = pti.numParticles(); const auto lev = pti.GetLevel(); const auto index = pti.GetPairIndex(); - tmp_particle_data.resize(finestLevel()+1); - for (int i = 0; i < TmpIdx::nattribs; ++i) tmp_particle_data[lev][index][i].resize(np); Real* AMREX_RESTRICT xpold = tmp_particle_data[lev][index][TmpIdx::xold ].dataPtr(); Real* AMREX_RESTRICT ypold = tmp_particle_data[lev][index][TmpIdx::yold ].dataPtr(); Real* AMREX_RESTRICT zpold = tmp_particle_data[lev][index][TmpIdx::zold ].dataPtr(); -- cgit v1.2.3 From 49dc1a600e30443c6d946689bfa80d3cd1091b1f Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Mon, 26 Aug 2019 11:06:56 -0700 Subject: add comment --- Source/Particles/PhysicalParticleContainer.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index df8dcf836..9169e6c1f 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -295,6 +295,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) auto index = std::make_pair(mfi.index(), mfi.LocalTileIndex()); GetParticles(lev)[index]; tmp_particle_data.resize(finestLevel()+1); + // Create map entry if not there tmp_particle_data[lev][index]; if ( (NumRuntimeRealComps()>0) || (NumRuntimeIntComps()>0) ) { DefineAndReturnParticleTile(lev, mfi.index(), mfi.LocalTileIndex()); -- cgit v1.2.3 From 7a5d2666a6a266f94e4acaa5710ffb0e453d6600 Mon Sep 17 00:00:00 2001 From: grote Date: Tue, 27 Aug 2019 13:48:43 -0700 Subject: Bug fix for RZ multimode on GPU --- Source/Particles/PhysicalParticleContainer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 15f09d409..9a3b6a565 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -339,6 +339,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) Real density_max = plasma_injector->density_max; #ifdef WARPX_DIM_RZ + const long nmodes = WarpX::n_rz_azimuthal_modes; bool radially_weighted = plasma_injector->radially_weighted; #endif @@ -533,7 +534,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) // Replace the x and y, setting an angle theta. // These x and y are used to get the momentum and density Real theta; - if (WarpX::n_rz_azimuthal_modes == 1) { + if (nmodes == 1) { // With only 1 mode, the angle doesn't matter so // choose it randomly. theta = 2.*MathConst::pi*amrex::Random(); -- cgit v1.2.3 From d40adc4e8caf3994bb24790960097b0fb779cd06 Mon Sep 17 00:00:00 2001 From: Dave Grote Date: Wed, 28 Aug 2019 13:35:40 -0700 Subject: Further changes of macro WARPX_DIM_2D to WARPX_DIM_XZ for clarity --- Source/Initialization/PlasmaInjector.cpp | 2 +- Source/Particles/PhysicalParticleContainer.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Initialization/PlasmaInjector.cpp b/Source/Initialization/PlasmaInjector.cpp index ce1258245..a65899902 100644 --- a/Source/Initialization/PlasmaInjector.cpp +++ b/Source/Initialization/PlasmaInjector.cpp @@ -147,7 +147,7 @@ PlasmaInjector::PlasmaInjector (int ispecies, const std::string& name) // For 2D, only two are expected. The third is overwritten with 1. num_particles_per_cell_each_dim.assign(3, 1); pp.getarr("num_particles_per_cell_each_dim", num_particles_per_cell_each_dim); -#if WARPX_DIM_2D +#if WARPX_DIM_XZ num_particles_per_cell_each_dim[2] = 1; #endif // Construct InjectorPosition from InjectorPositionRegular. diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 1513297a1..73724c7da 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -165,7 +165,7 @@ PhysicalParticleContainer::AddGaussianBeam(Real x_m, Real y_m, Real z_m, Real x = distx(mt); Real y = disty(mt); Real z = distz(mt); -#elif (defined WARPX_DIM_2D) +#elif (defined WARPX_DIM_XZ) Real weight = q_tot/npart/charge/y_rms; Real x = distx(mt); Real y = 0.; @@ -490,7 +490,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) #else Real x = overlap_corner[0] + (iv[0]+r.x)*dx[0]; Real y = 0.0; -#ifdef WARPX_DIM_2D +#if defined WARPX_DIM_XZ Real z = overlap_corner[1] + (iv[1]+r.y)*dx[1]; #elif defined WARPX_DIM_RZ // Note that for RZ, r.y will be theta -- cgit v1.2.3 From 30f1ec11f1f2f1f4bd57145fa3bb0ab7d11b59bd Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 9 Sep 2019 15:11:07 -0700 Subject: Particles: gather_from_main_grid Support gather for individual particle species from the main grid. Very much based on the similar option for deposition :) --- Docs/source/running_cpp/parameters.rst | 5 +++++ Source/Particles/MultiParticleContainer.H | 19 +++++++++++++++---- Source/Particles/MultiParticleContainer.cpp | 11 +++++++++++ Source/Particles/PhysicalParticleContainer.cpp | 4 ++++ Source/Particles/WarpXParticleContainer.H | 4 ++++ 5 files changed, 39 insertions(+), 4 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Docs/source/running_cpp/parameters.rst b/Docs/source/running_cpp/parameters.rst index 0e2866f58..1f5274ad6 100644 --- a/Docs/source/running_cpp/parameters.rst +++ b/Docs/source/running_cpp/parameters.rst @@ -91,6 +91,11 @@ Setting up the field mesh in the list will deposit their charge/current directly on the main grid (i.e. the coarsest level), even if they are inside a refinement patch. +* ``particles.gather_from_main_grid`` (`list of strings`) + When using mesh refinement: the particle species whose name are included + in the list will gather their fields from the main grid + (i.e. the coarsest level), even if they are inside a refinement patch. + * ``warpx.n_rz_azimuthal_modes`` (`integer`; 1 by default) When using the RZ version, this is the number of azimuthal modes. diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 1aff7edfb..5a6e9fdeb 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -2,16 +2,17 @@ #ifndef WARPX_ParticleContainer_H_ #define WARPX_ParticleContainer_H_ -#include -#include -#include - #include #include #include #include #include +#include +#include +#include +#include + // // MultiParticleContainer holds multiple (nspecies or npsecies+1 when // using laser) WarpXParticleContainer. WarpXParticleContainer is @@ -167,6 +168,12 @@ public: return r; } + int nSpeciesGatherFromMainGrid() const { + bool const fromMainGrid = true; + auto const & v = m_gather_from_main_grid; + return std::count( v.begin(), v.end(), fromMainGrid ); + } + void GetLabFrameData(const std::string& snapshot_name, const int i_lab, const int direction, const amrex::Real z_old, const amrex::Real z_new, @@ -194,8 +201,12 @@ protected: std::vector lasers_names; + //! instead of depositing (current, charge) on the finest patch level, deposit to the coarsest grid std::vector deposit_on_main_grid; + //! instead of gathering fields from the finest patch level, gather from the coarsest + std::vector m_gather_from_main_grid; + std::vector species_types; private: diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 7803bdae1..aa434a474 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -21,6 +21,7 @@ MultiParticleContainer::MultiParticleContainer (AmrCore* amr_core) allcontainers[i].reset(new RigidInjectedParticleContainer(amr_core, i, species_names[i])); } allcontainers[i]->deposit_on_main_grid = deposit_on_main_grid[i]; + allcontainers[i]->m_gather_from_main_grid = m_gather_from_main_grid[i]; } for (int i = nspecies; i < nspecies+nlasers; ++i) { @@ -69,6 +70,16 @@ MultiParticleContainer::ReadParameters () deposit_on_main_grid[i] = 1; } + m_gather_from_main_grid.resize(nspecies, 0); + std::vector tmp_gather; + pp.queryarr("gather_from_main_grid", tmp_gather); + for (auto const& name : tmp_gather) { + auto it = std::find(species_names.begin(), species_names.end(), name); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(it != species_names.end(), "ERROR: species in particles.gather_from_main_grid must be part of particles.species_names"); + int i = std::distance(species_names.begin(), it); + m_gather_from_main_grid.at(i) = true; + } + species_types.resize(nspecies, PCTypes::Physical); std::vector rigid_injected_species; diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 2dc25e6fa..c99c5278c 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1142,9 +1142,13 @@ PhysicalParticleContainer::Evolve (int lev, } } + // only deposit / gather to coarsest grid if (deposit_on_main_grid && lev > 0) { nfine_current = 0; } + if (m_gather_from_main_grid && lev > 0) { + nfine_gather = 0; + } if (nfine_current != np || nfine_gather != np) { diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index c39eec9dc..2d198de50 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -292,8 +292,12 @@ protected: amrex::Real charge; amrex::Real mass; + //! instead of depositing (current, charge) on the finest patch level, deposit to the coarsest grid bool deposit_on_main_grid = false; + //! instead of gathering fields from the finest patch level, gather from the coarsest + bool m_gather_from_main_grid = false; + static int do_not_push; // Whether to allow particles outside of the simulation domain to be -- cgit v1.2.3 From 589343c6d151fc539a865a4987b97bfd31c30ca2 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 9 Sep 2019 15:16:24 -0700 Subject: Clean Up Deposit on Main Just some small renamings, doc strings and clean-ups for deposit on main grid. --- Docs/source/running_cpp/parameters.rst | 4 ++-- Source/Particles/MultiParticleContainer.H | 12 +++++------- Source/Particles/MultiParticleContainer.cpp | 10 +++++----- Source/Particles/PhysicalParticleContainer.cpp | 6 +++--- Source/Particles/WarpXParticleContainer.H | 2 +- Source/WarpX.H | 4 ++-- 6 files changed, 18 insertions(+), 20 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Docs/source/running_cpp/parameters.rst b/Docs/source/running_cpp/parameters.rst index 1f5274ad6..11b1dfd53 100644 --- a/Docs/source/running_cpp/parameters.rst +++ b/Docs/source/running_cpp/parameters.rst @@ -81,12 +81,12 @@ Setting up the field mesh * ``warpx.n_current_deposition_buffer`` (`integer`) When using mesh refinement: the particles that are located inside - a refinement patch, but within ``n_field_gather_buffer`` cells of + a refinement patch, but within ``n_current_deposition_buffer`` cells of the edge of this patch, will deposit their charge and current to the lower refinement level, instead of depositing to the refinement patch itself. See the section :doc:`../../theory/amr` for more details. -* ``particles.deposit_on_main_grid`` (list of strings) +* ``particles.deposit_on_main_grid`` (`list of strings`) When using mesh refinement: the particle species whose name are included in the list will deposit their charge/current directly on the main grid (i.e. the coarsest level), even if they are inside a refinement patch. diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 5a6e9fdeb..305baf4ff 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -18,7 +18,7 @@ // using laser) WarpXParticleContainer. WarpXParticleContainer is // derived from amrex::ParticleContainer, and it is // polymorphic. PhysicalParticleContainer and LaserParticleContainer are -// concrete class dervied from WarpXParticlContainer. +// concrete class derived from WarpXParticlContainer. // class MultiParticleContainer @@ -161,11 +161,9 @@ public: int doBoostedFrameDiags() const {return do_boosted_frame_diags;} int nSpeciesDepositOnMainGrid () const { - int r = 0; - for (int i : deposit_on_main_grid) { - if (i) ++r; - } - return r; + bool const onMainGrid = true; + auto const & v = m_deposit_on_main_grid; + return std::count( v.begin(), v.end(), onMainGrid ); } int nSpeciesGatherFromMainGrid() const { @@ -202,7 +200,7 @@ protected: std::vector lasers_names; //! instead of depositing (current, charge) on the finest patch level, deposit to the coarsest grid - std::vector deposit_on_main_grid; + std::vector m_deposit_on_main_grid; //! instead of gathering fields from the finest patch level, gather from the coarsest std::vector m_gather_from_main_grid; diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index aa434a474..640439337 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -20,12 +20,12 @@ MultiParticleContainer::MultiParticleContainer (AmrCore* amr_core) else if (species_types[i] == PCTypes::RigidInjected) { allcontainers[i].reset(new RigidInjectedParticleContainer(amr_core, i, species_names[i])); } - allcontainers[i]->deposit_on_main_grid = deposit_on_main_grid[i]; + allcontainers[i]->m_deposit_on_main_grid = m_deposit_on_main_grid[i]; allcontainers[i]->m_gather_from_main_grid = m_gather_from_main_grid[i]; } for (int i = nspecies; i < nspecies+nlasers; ++i) { - allcontainers[i].reset(new LaserParticleContainer(amr_core,i, lasers_names[i-nspecies])); + allcontainers[i].reset(new LaserParticleContainer(amr_core, i, lasers_names[i-nspecies])); } pc_tmp.reset(new PhysicalParticleContainer(amr_core)); @@ -60,17 +60,17 @@ MultiParticleContainer::ReadParameters () pp.getarr("species_names", species_names); BL_ASSERT(species_names.size() == nspecies); - deposit_on_main_grid.resize(nspecies, 0); + m_deposit_on_main_grid.resize(nspecies, false); std::vector tmp; pp.queryarr("deposit_on_main_grid", tmp); for (auto const& name : tmp) { auto it = std::find(species_names.begin(), species_names.end(), name); AMREX_ALWAYS_ASSERT_WITH_MESSAGE(it != species_names.end(), "ERROR: species in particles.deposit_on_main_grid must be part of particles.species_names"); int i = std::distance(species_names.begin(), it); - deposit_on_main_grid[i] = 1; + m_deposit_on_main_grid[i] = true; } - m_gather_from_main_grid.resize(nspecies, 0); + m_gather_from_main_grid.resize(nspecies, false); std::vector tmp_gather; pp.queryarr("gather_from_main_grid", tmp_gather); for (auto const& name : tmp_gather) { diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index c99c5278c..318ad4664 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1087,8 +1087,8 @@ PhysicalParticleContainer::Evolve (int lev, m_giv[thread_num].resize(np); - long nfine_current = np; - long nfine_gather = np; + long nfine_current = np; //! number of particles depositing to fine grid + long nfine_gather = np; //! number of particles gathering from fine grid if (has_buffer && !do_not_push) { BL_PROFILE_VAR_START(blp_partition); @@ -1143,7 +1143,7 @@ PhysicalParticleContainer::Evolve (int lev, } // only deposit / gather to coarsest grid - if (deposit_on_main_grid && lev > 0) { + if (m_deposit_on_main_grid && lev > 0) { nfine_current = 0; } if (m_gather_from_main_grid && lev > 0) { diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 2d198de50..f7e46b2d3 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -293,7 +293,7 @@ protected: amrex::Real mass; //! instead of depositing (current, charge) on the finest patch level, deposit to the coarsest grid - bool deposit_on_main_grid = false; + bool m_deposit_on_main_grid = false; //! instead of gathering fields from the finest patch level, gather from the coarsest bool m_gather_from_main_grid = false; diff --git a/Source/WarpX.H b/Source/WarpX.H index 4f0342c8a..4da95a765 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -125,8 +125,8 @@ public: static int sort_int; // buffers - static int n_field_gather_buffer; - static int n_current_deposition_buffer; + static int n_field_gather_buffer; //! in number of cells from the edge (identical for each dimension) + static int n_current_deposition_buffer; //! in number of cells from the edge (identical for each dimension) // do nodal static int do_nodal; -- cgit v1.2.3 From fd848331f2fd6af8c56a0027cc5f088286e347c4 Mon Sep 17 00:00:00 2001 From: Tools Date: Wed, 11 Sep 2019 10:06:14 -0700 Subject: Source & Tools: No EOL Whitespaces End-of-line (EOL) whitespaces are verbose and increase diffs and merge conflicts over time. Cleaned them up for the `Source/`, `Examples/` and `Tools/` directory with the following bash one-liner: ```bash find . -type f -not -path './.git*' \ -exec sed -i 's/[[:blank:]]*$//' {} \; ``` Committed as generic user so git does not credit the many lines to me: ```bash GIT_AUTHOR_NAME="Tools" GIT_AUTHOR_EMAIL="warpx@lbl.gov" \ git commit ``` --- .../analysis_rigid_injection_LabFrame.py | 4 +- Examples/Modules/ionization/ionization_analysis.py | 4 +- .../laser_acceleration/README.md | 1 - .../laser_acceleration/laser_acceleration_PICMI.py | 16 +- .../plasma_acceleration/inputs.2d | 4 +- .../plasma_acceleration/inputs.2d.boost | 2 +- .../plasma_acceleration/inputs.3d.boost | 4 +- Examples/Tests/Larmor/plot_particle_path.py | 24 +- Examples/Tests/PML/analysis_pml_ckc.py | 2 +- Examples/Tests/PML/analysis_pml_psatd.py | 2 +- Examples/Tests/PML/analysis_pml_yee.py | 2 +- Examples/Tests/particles_in_PML/analysis.py | 6 +- Examples/Tests/subcycling/inputs.2d | 8 +- Source/Diagnostics/BoostedFrameDiagnostic.H | 62 ++-- Source/Diagnostics/BoostedFrameDiagnostic.cpp | 204 +++++------ Source/Diagnostics/ElectrostaticIO.cpp | 2 +- Source/Diagnostics/ParticleIO.cpp | 20 +- Source/Diagnostics/SliceDiagnostic.H | 26 +- Source/Diagnostics/SliceDiagnostic.cpp | 252 ++++++------- Source/Diagnostics/WarpXIO.cpp | 44 +-- Source/Evolve/WarpXEvolveEM.cpp | 6 +- .../SpectralAlgorithms/PMLPsatdAlgorithm.H | 2 +- .../SpectralAlgorithms/PsatdAlgorithm.H | 2 +- .../FieldSolver/SpectralSolver/SpectralKSpace.cpp | 2 +- Source/FieldSolver/openbc_poisson_solver.F90 | 2 +- Source/FieldSolver/solve_E_nodal.F90 | 6 +- Source/Filter/BilinearFilter.H | 2 +- Source/Filter/BilinearFilter.cpp | 4 +- Source/Filter/Filter.H | 8 +- Source/Filter/Filter.cpp | 10 +- Source/Filter/NCIGodfreyFilter.H | 2 +- Source/Filter/NCIGodfreyFilter.cpp | 18 +- Source/Initialization/CustomDensityProb.H | 2 +- Source/Initialization/InjectorDensity.H | 6 +- Source/Initialization/InjectorDensity.cpp | 2 +- Source/Initialization/InjectorMomentum.H | 10 +- Source/Initialization/InjectorPosition.H | 8 +- Source/Initialization/PlasmaInjector.H | 4 +- Source/Laser/LaserParticleContainer.H | 10 +- Source/Laser/LaserParticleContainer.cpp | 32 +- Source/Laser/LaserProfiles.cpp | 28 +- Source/Parallelization/WarpXRegrid.cpp | 8 +- Source/Parser/GpuParser.H | 2 +- Source/Parser/GpuParser.cpp | 2 +- Source/Parser/wp_parser.lex.c | 84 ++--- Source/Parser/wp_parser.lex.h | 4 +- Source/Particles/Deposition/ChargeDeposition.H | 8 +- Source/Particles/Deposition/CurrentDeposition.H | 26 +- Source/Particles/Gather/FieldGather.H | 2 +- Source/Particles/MultiParticleContainer.H | 36 +- Source/Particles/MultiParticleContainer.cpp | 46 +-- Source/Particles/PhysicalParticleContainer.H | 10 +- Source/Particles/PhysicalParticleContainer.cpp | 122 +++---- Source/Particles/RigidInjectedParticleContainer.H | 2 +- Source/Particles/ShapeFactors.H | 2 +- Source/Particles/WarpXParticleContainer.H | 72 ++-- Source/Particles/WarpXParticleContainer.cpp | 18 +- Source/Particles/deposit_cic.F90 | 4 +- Source/Particles/interpolate_cic.F90 | 34 +- Source/Particles/push_particles_ES.F90 | 22 +- Source/Python/WarpXWrappers.cpp | 14 +- Source/Python/WarpXWrappers.h | 40 +-- Source/Utils/IonizationEnergiesTable.H | 138 +++---- Source/Utils/NCIGodfreyTables.H | 400 ++++++++++----------- Source/Utils/WarpXMovingWindow.cpp | 2 +- Source/Utils/WarpXUtil.H | 2 +- Source/Utils/WarpXUtil.cpp | 6 +- Source/Utils/utils_ES.F90 | 30 +- Source/Utils/write_atomic_data_cpp.py | 2 +- Source/main.cpp | 8 +- Tools/compute_domain.py | 12 +- Tools/performance_tests/functions_perftest.py | 28 +- Tools/performance_tests/performance_log.txt | 160 ++++----- Tools/performance_tests/run_alltests.py | 26 +- Tools/performance_tests/run_alltests_1node.py | 34 +- Tools/performance_tests/run_automated.py | 74 ++-- Tools/read_lab_particles.py | 2 +- Tools/read_raw_data.py | 12 +- Tools/video_yt.py | 10 +- Tools/yt3d_mpi.py | 6 +- 80 files changed, 1182 insertions(+), 1183 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Examples/Modules/RigidInjection/analysis_rigid_injection_LabFrame.py b/Examples/Modules/RigidInjection/analysis_rigid_injection_LabFrame.py index 86214ad72..21bfcf945 100755 --- a/Examples/Modules/RigidInjection/analysis_rigid_injection_LabFrame.py +++ b/Examples/Modules/RigidInjection/analysis_rigid_injection_LabFrame.py @@ -23,7 +23,7 @@ filename = sys.argv[1] # WarpX headers include more data when rigid injection is used, # which gives an error with the last yt release. -# To avoid this issue, the three last lines of WarpXHeader are removed if +# To avoid this issue, the three last lines of WarpXHeader are removed if # needed. def remove_rigid_lines(plotfile, nlines_if_rigid): header_name = plotfile + '/WarpXHeader' @@ -32,7 +32,7 @@ def remove_rigid_lines(plotfile, nlines_if_rigid): nlines = len(file_lines) f.close() if nlines == nlines_if_rigid: - f = open(header_name, 'w') + f = open(header_name, 'w') f.writelines(file_lines[:-3]) f.close() diff --git a/Examples/Modules/ionization/ionization_analysis.py b/Examples/Modules/ionization/ionization_analysis.py index b94541f90..f512eac6e 100755 --- a/Examples/Modules/ionization/ionization_analysis.py +++ b/Examples/Modules/ionization/ionization_analysis.py @@ -4,12 +4,12 @@ This script tests the result of the ionization module in WarpX. Input files inputs.rt and inputs.bf.rt are used to reproduce the test from -Chen, JCP, 2013, figure 2 (in the lab frame and in a boosted frame, +Chen, JCP, 2013, figure 2 (in the lab frame and in a boosted frame, respectively): a plane-wave laser pulse propagates through a uniform N2+ neutral plasma and further ionizes the Nitrogen atoms. This test checks that, after the laser went through the plasma, ~32% of Nitrogen ions are N5+, in agreement with theory from Chen's article. -""" +""" import sys import yt diff --git a/Examples/Physics_applications/laser_acceleration/README.md b/Examples/Physics_applications/laser_acceleration/README.md index c556a1d9e..914eda74e 100644 --- a/Examples/Physics_applications/laser_acceleration/README.md +++ b/Examples/Physics_applications/laser_acceleration/README.md @@ -11,4 +11,3 @@ Examples are provided using the executable or Python-driven version of WarpX. ## Using the python-driven version: - laser_acceleration_PICMI.py - \ No newline at end of file diff --git a/Examples/Physics_applications/laser_acceleration/laser_acceleration_PICMI.py b/Examples/Physics_applications/laser_acceleration/laser_acceleration_PICMI.py index 4dca7151e..24d23310e 100644 --- a/Examples/Physics_applications/laser_acceleration/laser_acceleration_PICMI.py +++ b/Examples/Physics_applications/laser_acceleration/laser_acceleration_PICMI.py @@ -15,11 +15,11 @@ laser_duration = 15e-15 # Duration of the laser (in seconds) laser_polarization = np.pi/2. # Polarization angle (in rad) laser_injection_loc = 9.e-6 # Position of injection (in meters, along z) laser_focal_distance = 100.e-6 # Focal distance from the injection (in meters) -laser_t_peak = 30.e-15 # The time at which the laser reaches its peak +laser_t_peak = 30.e-15 # The time at which the laser reaches its peak # at the antenna injection location (in seconds) # --- plasma -plasma_density = 1.e24 +plasma_density = 1.e24 plasma_min = [-20.e-6, -20.e-6, 0.0e-6] plasma_max = [ 20.e-6, 20.e-6, 1.e-3] @@ -55,12 +55,12 @@ number_per_cell_each_dim = [2, 2, 1] # --- laser -laser = picmi.GaussianLaser(wavelength = laser_wavelength, - waist = laser_waist, - duration = laser_duration, - focal_position = [0., 0., laser_focal_distance + laser_injection_loc], - centroid_position = [0., 0., laser_injection_loc - picmi.c*laser_t_peak], - polarization_angle = laser_polarization, +laser = picmi.GaussianLaser(wavelength = laser_wavelength, + waist = laser_waist, + duration = laser_duration, + focal_position = [0., 0., laser_focal_distance + laser_injection_loc], + centroid_position = [0., 0., laser_injection_loc - picmi.c*laser_t_peak], + polarization_angle = laser_polarization, propagation_direction = [0,0,1], E0 = laser_a0*2.*np.pi*picmi.m_e*picmi.c**2/(picmi.q_e*laser_wavelength)) # Maximum amplitude of the laser field (in V/m) diff --git a/Examples/Physics_applications/plasma_acceleration/inputs.2d b/Examples/Physics_applications/plasma_acceleration/inputs.2d index 58f517308..fb67dd448 100644 --- a/Examples/Physics_applications/plasma_acceleration/inputs.2d +++ b/Examples/Physics_applications/plasma_acceleration/inputs.2d @@ -10,8 +10,8 @@ amr.plot_file = "plotfiles/plt" amr.plot_int = 100 geometry.coord_sys = 0 # 0: Cartesian geometry.is_periodic = 0 0 # Is periodic? -geometry.prob_lo = -125.e-6 -149.e-6 -geometry.prob_hi = 125.e-6 1.e-6 +geometry.prob_lo = -125.e-6 -149.e-6 +geometry.prob_hi = 125.e-6 1.e-6 warpx.fine_tag_lo = -12.e-6 -110.e-6 warpx.fine_tag_hi = 12.e-6 -100.e-6 diff --git a/Examples/Physics_applications/plasma_acceleration/inputs.2d.boost b/Examples/Physics_applications/plasma_acceleration/inputs.2d.boost index ba9166dee..d711815eb 100644 --- a/Examples/Physics_applications/plasma_acceleration/inputs.2d.boost +++ b/Examples/Physics_applications/plasma_acceleration/inputs.2d.boost @@ -10,7 +10,7 @@ amr.plot_file = "plotfiles/plt" amr.plot_int = 500 geometry.coord_sys = 0 # 0: Cartesian geometry.is_periodic = 0 0 # Is periodic? -geometry.prob_lo = -125.e-6 -149.e-6 +geometry.prob_lo = -125.e-6 -149.e-6 geometry.prob_hi = 125.e-6 1.e-6 ################################# diff --git a/Examples/Physics_applications/plasma_acceleration/inputs.3d.boost b/Examples/Physics_applications/plasma_acceleration/inputs.3d.boost index 1af4fac86..42446a9ac 100644 --- a/Examples/Physics_applications/plasma_acceleration/inputs.3d.boost +++ b/Examples/Physics_applications/plasma_acceleration/inputs.3d.boost @@ -10,8 +10,8 @@ amr.plot_file = "plotfiles/plt" amr.plot_int = 50 geometry.coord_sys = 0 # 0: Cartesian geometry.is_periodic = 1 1 0 # Is periodic? -geometry.prob_lo = -0.00015 -0.00015 -0.00012 -geometry.prob_hi = 0.00015 0.00015 1.e-06 +geometry.prob_lo = -0.00015 -0.00015 -0.00012 +geometry.prob_hi = 0.00015 0.00015 1.e-06 ################################# ############ NUMERICS ########### diff --git a/Examples/Tests/Larmor/plot_particle_path.py b/Examples/Tests/Larmor/plot_particle_path.py index 0a9e965e7..ef52b4f4b 100644 --- a/Examples/Tests/Larmor/plot_particle_path.py +++ b/Examples/Tests/Larmor/plot_particle_path.py @@ -3,8 +3,8 @@ import numpy as np class AMReXParticleHeader(object): ''' - This class is designed to parse and store the information - contained in an AMReX particle header file. + This class is designed to parse and store the information + contained in an AMReX particle header file. Usage: @@ -66,11 +66,11 @@ class AMReXParticleHeader(object): entry = [int(val) for val in f.readline().strip().split()] self.grids[level_num].append(tuple(entry)) - + def read_particle_data(fn, ptype="particle0"): ''' - This function returns the particle data stored in a particular + This function returns the particle data stored in a particular plot file and particle type. It returns two numpy arrays, the first containing the particle integer data, and the second the particle real data. For example, if a dataset has 3000 particles, @@ -79,22 +79,22 @@ def read_particle_data(fn, ptype="particle0"): with the shape (3000, 5). Usage: - + idata, rdata = read_particle_data("plt00000", "particle0") ''' base_fn = fn + "/" + ptype header = AMReXParticleHeader(base_fn + "/Header") - - idtype = "(%d,)i4" % header.num_int + + idtype = "(%d,)i4" % header.num_int if header.real_type == np.float64: fdtype = "(%d,)f8" % header.num_real elif header.real_type == np.float32: fdtype = "(%d,)f4" % header.num_real - + idata = np.empty((header.num_particles, header.num_int )) rdata = np.empty((header.num_particles, header.num_real)) - + ip = 0 for lvl, level_grids in enumerate(header.grids): for (which, count, where) in level_grids: @@ -107,7 +107,7 @@ def read_particle_data(fn, ptype="particle0"): floats = np.fromfile(f, dtype = fdtype, count=count) idata[ip:ip+count] = ints - rdata[ip:ip+count] = floats + rdata[ip:ip+count] = floats ip += count return idata, rdata @@ -121,10 +121,10 @@ if __name__ == "__main__": y0 = [] x1 = [] y1 = [] - + fn_list = glob.glob("plt?????") fn_list.sort() - + for fn in fn_list: idata, rdata = read_particle_data(fn, ptype="particle0") x0.append(rdata[0][0]) diff --git a/Examples/Tests/PML/analysis_pml_ckc.py b/Examples/Tests/PML/analysis_pml_ckc.py index e095864c0..d6bef942f 100755 --- a/Examples/Tests/PML/analysis_pml_ckc.py +++ b/Examples/Tests/PML/analysis_pml_ckc.py @@ -34,4 +34,4 @@ print("Reflectivity", Reflectivity) print("Reflectivity_theory", Reflectivity_theory) assert( Reflectivity < 105./100 * Reflectivity_theory ) - + diff --git a/Examples/Tests/PML/analysis_pml_psatd.py b/Examples/Tests/PML/analysis_pml_psatd.py index ff3bf8413..ba9120c8d 100755 --- a/Examples/Tests/PML/analysis_pml_psatd.py +++ b/Examples/Tests/PML/analysis_pml_psatd.py @@ -31,4 +31,4 @@ Reflectivity = energy_end/energy_start Reflectivity_theory = 1.3806831258153887e-06 assert( abs(Reflectivity-Reflectivity_theory) < 5./100 * Reflectivity_theory ) - + diff --git a/Examples/Tests/PML/analysis_pml_yee.py b/Examples/Tests/PML/analysis_pml_yee.py index 6234cd5d2..0def05450 100755 --- a/Examples/Tests/PML/analysis_pml_yee.py +++ b/Examples/Tests/PML/analysis_pml_yee.py @@ -31,4 +31,4 @@ Reflectivity = energy_end/energy_start Reflectivity_theory = 5.683000058954201e-07 assert( abs(Reflectivity-Reflectivity_theory) < 5./100 * Reflectivity_theory ) - + diff --git a/Examples/Tests/particles_in_PML/analysis.py b/Examples/Tests/particles_in_PML/analysis.py index 9d0c95d35..ab5792082 100755 --- a/Examples/Tests/particles_in_PML/analysis.py +++ b/Examples/Tests/particles_in_PML/analysis.py @@ -3,9 +3,9 @@ This script tests the absorption of particles in the PML. The input file inputs_2d/inputs is used: it features a positive and a -negative particle, going in opposite direction and eventually +negative particle, going in opposite direction and eventually leaving the box. This script tests that the field in the box -is close to 0 once the particles have left. With regular +is close to 0 once the particles have left. With regular PML, this test fails, since the particles leave a spurious charge, with associated fields, behind them. """ @@ -34,4 +34,4 @@ elif ds.dimensionality == 3: assert max_Efield < 10 else: raise ValueError("Unknown dimensionality") - + diff --git a/Examples/Tests/subcycling/inputs.2d b/Examples/Tests/subcycling/inputs.2d index 2ddd665e5..917d0fc14 100644 --- a/Examples/Tests/subcycling/inputs.2d +++ b/Examples/Tests/subcycling/inputs.2d @@ -10,7 +10,7 @@ amr.max_level = 1 amr.plot_file = "plotfiles/plt" amr.plot_int = 200 -warpx.fine_tag_lo = -2.e-6 -15.e-6 +warpx.fine_tag_lo = -2.e-6 -15.e-6 warpx.fine_tag_hi = 2.e-6 -7.e-6 @@ -28,9 +28,9 @@ warpx.use_filter = 1 warpx.do_pml = 1 warpx.do_subcycling = 1 warpx.refine_plasma = 0 -warpx.plot_raw_fields = 1 -warpx.plot_raw_fields_guards = 1 -warpx.plot_finepatch = 1 +warpx.plot_raw_fields = 1 +warpx.plot_raw_fields_guards = 1 +warpx.plot_finepatch = 1 warpx.plot_crsepatch = 1 warpx.n_current_deposition_buffer = 0 warpx.n_field_gather_buffer = 0 diff --git a/Source/Diagnostics/BoostedFrameDiagnostic.H b/Source/Diagnostics/BoostedFrameDiagnostic.H index 3a09259b0..6c8eb15db 100644 --- a/Source/Diagnostics/BoostedFrameDiagnostic.H +++ b/Source/Diagnostics/BoostedFrameDiagnostic.H @@ -14,24 +14,24 @@ #include "WarpXConst.H" /// -/// BoostedFrameDiagnostic is for handling IO when running in a boosted +/// BoostedFrameDiagnostic is for handling IO when running in a boosted /// frame of reference. Because of the relativity of simultaneity, events that -/// are synchronized in the simulation frame are not synchronized in the -/// lab frame. Thus, at a given t_boost, we must write slices of data to +/// are synchronized in the simulation frame are not synchronized in the +/// lab frame. Thus, at a given t_boost, we must write slices of data to /// multiple output files, each one corresponding to a given time in the lab frame. /// class BoostedFrameDiagnostic { /// /// LabSnapShot stores metadata corresponding to a single time - /// snapshot in the lab frame. The snapshot is written to disk + /// snapshot in the lab frame. The snapshot is written to disk /// in the directory "file_name". zmin_lab, zmax_lab, and t_lab - /// are all constant for a given snapshot. current_z_lab and - /// current_z_boost for each snapshot are updated as the + /// are all constant for a given snapshot. current_z_lab and + /// current_z_boost for each snapshot are updated as the /// simulation time in the boosted frame advances. /// struct LabSnapShot { - + std::string file_name; amrex::Real t_lab; amrex::RealBox prob_domain_lab_; @@ -42,7 +42,7 @@ class BoostedFrameDiagnostic { int ncomp_to_dump_; std::vector mesh_field_names_; - + int file_num; int initial_i; const BoostedFrameDiagnostic& my_bfd; @@ -54,12 +54,12 @@ class BoostedFrameDiagnostic { const std::vector mesh_field_names, int file_num_in, const BoostedFrameDiagnostic& bfd); - + /// /// This snapshot is at time t_lab, and the simulation is at time t_boost. /// The Lorentz transformation picks out one slice corresponding to both - /// of those times, at position current_z_boost and current_z_lab in the - /// boosted and lab frames, respectively. + /// of those times, at position current_z_boost and current_z_lab in the + /// boosted and lab frames, respectively. /// void updateCurrentZPositions(amrex::Real t_boost, amrex::Real inv_gamma, amrex::Real inv_beta); @@ -87,17 +87,17 @@ class BoostedFrameDiagnostic { amrex::Vector > data_buffer_; // particles_buffer_ is currently blind to refinement level. // particles_buffer_[i][j] is a WarpXParticleContainer::DiagnosticParticleData where - // - i is the back-transformed snapshot number + // - i is the back-transformed snapshot number // - j is the species number amrex::Vector > particles_buffer_; int num_buffer_ = 256; int max_box_size_ = 256; - // buff_counter_[i] is the number of z slices in data_buffer_[i] + // buff_counter_[i] is the number of z slices in data_buffer_[i] // for snapshot number i. amrex::Vector buff_counter_; amrex::Vector snapshots_; - + void writeParticleData(const WarpXParticleContainer::DiagnosticParticleData& pdata, const std::string& name, const int i_lab); @@ -106,44 +106,44 @@ class BoostedFrameDiagnostic { const std::string& name, const std::string& species_name); #endif public: - - BoostedFrameDiagnostic(amrex::Real zmin_lab, amrex::Real zmax_lab, + + BoostedFrameDiagnostic(amrex::Real zmin_lab, amrex::Real zmax_lab, amrex::Real v_window_lab, amrex::Real dt_snapshots_lab, int N_snapshots, amrex::Real gamma_boost, amrex::Real t_boost, amrex::Real dt_boost, int boost_direction, const amrex::Geometry& geom); - + void Flush(const amrex::Geometry& geom); - + void writeLabFrameData(const amrex::MultiFab* cell_centered_data, const MultiParticleContainer& mypc, const amrex::Geometry& geom, const amrex::Real t_boost, const amrex::Real dt); - + void writeMetaData(); private: // Map field names and component number in cell_centered_data std::map possible_fields_to_dump = { - {"Ex" , 0}, - {"Ey" , 1}, - {"Ez" , 2}, - {"Bx" , 3}, - {"By" , 4}, - {"Bz" , 5}, - {"jx" , 6}, - {"jy" , 7}, - {"jz" , 8}, + {"Ex" , 0}, + {"Ey" , 1}, + {"Ez" , 2}, + {"Bx" , 3}, + {"By" , 4}, + {"Bz" , 5}, + {"jx" , 6}, + {"jy" , 7}, + {"jz" , 8}, {"rho", 9} }; - // maps field index in data_buffer_[i] -> cell_centered_data for + // maps field index in data_buffer_[i] -> cell_centered_data for // snapshots i. By default, all fields in cell_centered_data are dumped. // Needs to be amrex::Vector because used in a ParallelFor kernel. amrex::Gpu::ManagedDeviceVector map_actual_fields_to_dump; // Name of fields to dump. By default, all fields in cell_centered_data. // Needed for file headers only. - std::vector mesh_field_names = {"Ex", "Ey", "Ez", - "Bx", "By", "Bz", + std::vector mesh_field_names = {"Ex", "Ey", "Ez", + "Bx", "By", "Bz", "jx", "jy", "jz", "rho"}; int ncomp_to_dump = 10; diff --git a/Source/Diagnostics/BoostedFrameDiagnostic.cpp b/Source/Diagnostics/BoostedFrameDiagnostic.cpp index ad4f50806..f1d437509 100644 --- a/Source/Diagnostics/BoostedFrameDiagnostic.cpp +++ b/Source/Diagnostics/BoostedFrameDiagnostic.cpp @@ -17,7 +17,7 @@ using namespace amrex; namespace { const std::vector particle_field_names = {"w", "x", "y", "z", "ux", "uy", "uz"}; - + /* Creates the HDF5 file in truncate mode and closes it. Should be run only by the root process. @@ -32,7 +32,7 @@ namespace } /* - Writes a single string attribute to the given group. + Writes a single string attribute to the given group. Should only be called by the root process. */ void write_string_attribute(hid_t& group, const std::string& key, const std::string& val) @@ -52,7 +52,7 @@ namespace } /* - Writes a single double attribute to the given group. + Writes a single double attribute to the given group. Should only be called by the root process. */ void write_double_attribute(hid_t& group, const std::string& key, const double val) @@ -109,7 +109,7 @@ namespace */ void output_create_field(const std::string& file_path, const std::string& field_path, const unsigned nx, const unsigned ny, const unsigned nz) - { + { BL_PROFILE("output_create_field"); // Open the output. @@ -121,7 +121,7 @@ namespace hsize_t dims[3] = {nx, nz}; #endif hid_t grid_space = H5Screate_simple(AMREX_SPACEDIM, dims, NULL); - + // Create the dataset. hid_t dataset = H5Dcreate(file, field_path.c_str(), H5T_IEEE_F64LE, grid_space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); @@ -141,7 +141,7 @@ namespace /* Creates a group associated with a single particle species. Should be run by all processes collectively. - */ + */ void output_create_species_group(const std::string& file_path, const std::string& species_name) { MPI_Comm comm = MPI_COMM_WORLD; @@ -152,10 +152,10 @@ namespace // Create the file access prop list. hid_t pa_plist = H5Pcreate(H5P_FILE_ACCESS); H5Pset_fapl_mpio(pa_plist, comm, info); - + // Open the output. hid_t file = H5Fopen(file_path.c_str(), H5F_ACC_RDWR, pa_plist); - + hid_t group = H5Gcreate(file, species_name.c_str(), H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); H5Gclose(group); @@ -169,7 +169,7 @@ namespace */ long output_resize_particle_field(const std::string& file_path, const std::string& field_path, const long num_to_add) - { + { BL_PROFILE("output_resize_particle_field"); // Open the output. @@ -187,7 +187,7 @@ namespace hsize_t new_size[1]; new_size[0] = dims[0] + num_to_add; status = H5Dset_extent (dataset, new_size); - + if (status < 0) { amrex::Abort("Error: set extent filed on dataset " @@ -208,7 +208,7 @@ namespace */ void output_write_particle_field(const std::string& file_path, const std::string& field_path, const Real* data_ptr, const long count, const long index) - { + { BL_PROFILE("output_write_particle_field"); MPI_Comm comm = MPI_COMM_WORLD; @@ -219,15 +219,15 @@ namespace // Create the file access prop list. hid_t pa_plist = H5Pcreate(H5P_FILE_ACCESS); H5Pset_fapl_mpio(pa_plist, comm, info); - + // Open the output. hid_t file = H5Fopen(file_path.c_str(), H5F_ACC_RDWR, pa_plist); - + int RANK = 1; hsize_t offset[1]; hsize_t dims[1]; herr_t status; - + hid_t dataset = H5Dopen (file, field_path.c_str(), H5P_DEFAULT); // Make sure the dataset is there. @@ -236,9 +236,9 @@ namespace amrex::Abort("Error on rank " + std::to_string(mpi_rank) + ". Count not find dataset " + field_path + "\n"); } - + hid_t filespace = H5Dget_space (dataset); - + offset[0] = index; dims[0] = count; @@ -247,23 +247,23 @@ namespace H5Pset_dxpl_mpio(collective_plist, H5FD_MPIO_INDEPENDENT); if (count > 0) { - + /* Define memory space */ hid_t memspace = H5Screate_simple (RANK, dims, NULL); - + status = H5Sselect_hyperslab (filespace, H5S_SELECT_SET, offset, NULL, dims, NULL); - + if (status < 0) { amrex::Abort("Error on rank " + std::to_string(ParallelDescriptor::MyProc()) + " could not select hyperslab.\n"); } - + /* Write the data to the extended portion of dataset */ status = H5Dwrite(dataset, H5T_NATIVE_DOUBLE, memspace, filespace, collective_plist, data_ptr); - + if (status < 0) { amrex::Abort("Error on rank " + std::to_string(ParallelDescriptor::MyProc()) + @@ -272,9 +272,9 @@ namespace status = H5Sclose (memspace); } - + ParallelDescriptor::Barrier(); - + // Close resources. H5Pclose(collective_plist); H5Sclose(filespace); @@ -282,13 +282,13 @@ namespace H5Fclose(file); H5Pclose(pa_plist); } - + /* Creates an extendible dataset, suitable for storing particle data. Should be run on all ranks collectively. */ void output_create_particle_field(const std::string& file_path, const std::string& field_path) - { + { BL_PROFILE("output_create_particle_field"); MPI_Comm comm = MPI_COMM_WORLD; @@ -299,7 +299,7 @@ namespace // Create the file access prop list. hid_t pa_plist = H5Pcreate(H5P_FILE_ACCESS); H5Pset_fapl_mpio(pa_plist, comm, info); - + // Open the output. hid_t file = H5Fopen(file_path.c_str(), H5F_ACC_RDWR, pa_plist); @@ -307,7 +307,7 @@ namespace hsize_t dims[1] = {0}; hsize_t maxdims[1] = {H5S_UNLIMITED}; hsize_t chunk_dims[2] = {4}; - + hid_t dataspace = H5Screate_simple (RANK, dims, maxdims); // Enable chunking @@ -316,7 +316,7 @@ namespace hid_t dataset = H5Dcreate2 (file, field_path.c_str(), H5T_NATIVE_DOUBLE, dataspace, H5P_DEFAULT, prop, H5P_DEFAULT); - + if (dataset < 0) { amrex::Abort("Error: could not create dataset. H5 returned " @@ -329,7 +329,7 @@ namespace H5Sclose(dataspace); H5Fclose(file); } - + /* Write the only component in the multifab to the dataset given by field_name. Uses hdf5-parallel. @@ -361,7 +361,7 @@ namespace amrex::Abort("Error on rank " + std::to_string(mpi_rank) + ". Count not find dataset " + field_path + "\n"); } - + // Grab the dataspace of the field dataset from file. hid_t file_dataspace = H5Dget_space(dataset); @@ -388,9 +388,9 @@ namespace const Box& box = mfi.validbox(); const int *lo_vec = box.loVect(); const int *hi_vec = box.hiVect(); - + transposed_data.resize(box.numPts(), 0.0); - + // Set slab offset and shape. for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { @@ -399,7 +399,7 @@ namespace slab_offsets[idim] = lo_vec[idim]; slab_dims[idim] = hi_vec[idim] - lo_vec[idim] + 1; } - + int cnt = 0; AMREX_D_TERM( for (int i = lo_vec[0]; i <= hi_vec[0]; ++i), @@ -433,7 +433,7 @@ namespace } ParallelDescriptor::Barrier(); - + // Close HDF5 resources. H5Pclose(collective_plist); H5Sclose(file_dataspace); @@ -447,7 +447,7 @@ namespace namespace { void - CopySlice(MultiFab& tmp, MultiFab& buf, int k_lab, + CopySlice(MultiFab& tmp, MultiFab& buf, int k_lab, const Gpu::ManagedDeviceVector& map_actual_fields_to_dump) { const int ncomp_to_dump = map_actual_fields_to_dump.size(); @@ -458,11 +458,11 @@ namespace for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) { Array4< Real> tmp_arr = tmp[mfi].array(); Array4< Real> buf_arr = buf[mfi].array(); - // For 3D runs, tmp is a 2D (x,y) multifab, that contains only + // For 3D runs, tmp is a 2D (x,y) multifab, that contains only // slice to write to file. const Box& bx = mfi.tilebox(); - const auto field_map_ptr = map_actual_fields_to_dump.dataPtr(); + const auto field_map_ptr = map_actual_fields_to_dump.dataPtr(); ParallelFor(bx, ncomp_to_dump, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) { @@ -488,13 +488,13 @@ LorentzTransformZ(MultiFab& data, Real gamma_boost, Real beta_boost, int ncomp) for (MFIter mfi(data, TilingIfNotGPU()); mfi.isValid(); ++mfi) { const Box& tile_box = mfi.tilebox(); Array4< Real > arr = data[mfi].array(); - // arr(x,y,z,comp) where 0->9 comps are + // arr(x,y,z,comp) where 0->9 comps are // Ex Ey Ez Bx By Bz jx jy jz rho Real clight = PhysConst::c; ParallelFor(tile_box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - // Transform the transverse E and B fields. Note that ez and bz are not + // Transform the transverse E and B fields. Note that ez and bz are not // changed by the tranform. Real e_lab, b_lab, j_lab, r_lab; e_lab = gamma_boost * (arr(i, j, k, 0) + beta_boost*clight*arr(i, j, k, 4)); @@ -523,8 +523,8 @@ LorentzTransformZ(MultiFab& data, Real gamma_boost, Real beta_boost, int ncomp) BoostedFrameDiagnostic:: BoostedFrameDiagnostic(Real zmin_lab, Real zmax_lab, Real v_window_lab, - Real dt_snapshots_lab, int N_snapshots, - Real gamma_boost, Real t_boost, Real dt_boost, + Real dt_snapshots_lab, int N_snapshots, + Real gamma_boost, Real t_boost, Real dt_boost, int boost_direction, const Geometry& geom) : gamma_boost_(gamma_boost), dt_snapshots_lab_(dt_snapshots_lab), @@ -537,11 +537,11 @@ BoostedFrameDiagnostic(Real zmin_lab, Real zmax_lab, Real v_window_lab, AMREX_ALWAYS_ASSERT(WarpX::do_boosted_frame_fields or WarpX::do_boosted_frame_particles); - + inv_gamma_boost_ = 1.0 / gamma_boost_; beta_boost_ = std::sqrt(1.0 - inv_gamma_boost_*inv_gamma_boost_); inv_beta_boost_ = 1.0 / beta_boost_; - + dz_lab_ = PhysConst::c * dt_boost_ * inv_beta_boost_ * inv_gamma_boost_; inv_dz_lab_ = 1.0 / dz_lab_; int Nz_lab = static_cast((zmax_lab - zmin_lab) * inv_dz_lab_); @@ -563,9 +563,9 @@ BoostedFrameDiagnostic(Real zmin_lab, Real zmax_lab, Real v_window_lab, std::vector user_fields_to_dump; ParmParse pp("warpx"); bool do_user_fields; - do_user_fields = pp.queryarr("boosted_frame_diag_fields", + do_user_fields = pp.queryarr("boosted_frame_diag_fields", user_fields_to_dump); - // If user specifies fields to dump, overwrite ncomp_to_dump, + // If user specifies fields to dump, overwrite ncomp_to_dump, // map_actual_fields_to_dump and mesh_field_names. for (int i = 0; i < 10; ++i) map_actual_fields_to_dump.push_back(i); if (do_user_fields){ @@ -588,7 +588,7 @@ BoostedFrameDiagnostic(Real zmin_lab, Real zmax_lab, Real v_window_lab, prob_domain_lab.setLo(AMREX_SPACEDIM-1, zmin_lab + v_window_lab * t_lab); prob_domain_lab.setHi(AMREX_SPACEDIM-1, zmax_lab + v_window_lab * t_lab); // Construct LabSnapShot - LabSnapShot snapshot(t_lab, t_boost, prob_domain_lab, + LabSnapShot snapshot(t_lab, t_boost, prob_domain_lab, prob_ncells_lab, ncomp_to_dump, mesh_field_names, i, *this); snapshots_.push_back(snapshot); @@ -602,54 +602,54 @@ BoostedFrameDiagnostic(Real zmin_lab, Real zmax_lab, Real v_window_lab, void BoostedFrameDiagnostic::Flush(const Geometry& geom) { BL_PROFILE("BoostedFrameDiagnostic::Flush"); - + VisMF::Header::Version current_version = VisMF::GetHeaderVersion(); VisMF::SetHeaderVersion(amrex::VisMF::Header::NoFabHeader_v1); auto & mypc = WarpX::GetInstance().GetPartContainer(); const std::vector species_names = mypc.GetSpeciesNames(); - + // Loop over BFD snapshots for (int i = 0; i < N_snapshots_; ++i) { Real zmin_lab = snapshots_[i].prob_domain_lab_.lo(AMREX_SPACEDIM-1); int i_lab = (snapshots_[i].current_z_lab - zmin_lab) / dz_lab_; - + if (buff_counter_[i] != 0) { if (WarpX::do_boosted_frame_fields) { const BoxArray& ba = data_buffer_[i]->boxArray(); const int hi = ba[0].bigEnd(boost_direction_); const int lo = hi - buff_counter_[i] + 1; - + Box buff_box = geom.Domain(); buff_box.setSmall(boost_direction_, lo); buff_box.setBig(boost_direction_, hi); - + BoxArray buff_ba(buff_box); buff_ba.maxSize(max_box_size_); DistributionMapping buff_dm(buff_ba); - + const int ncomp = data_buffer_[i]->nComp(); - + MultiFab tmp(buff_ba, buff_dm, ncomp, 0); - + tmp.copy(*data_buffer_[i], 0, 0, ncomp); #ifdef WARPX_USE_HDF5 for (int comp = 0; comp < ncomp; ++comp) output_write_field(snapshots_[i].file_name, mesh_field_names[comp], tmp, comp); -#else +#else std::stringstream ss; ss << snapshots_[i].file_name << "/Level_0/" << Concatenate("buffer", i_lab, 5); VisMF::Write(tmp, ss.str()); #endif } - + if (WarpX::do_boosted_frame_particles) { // Loop over species to be dumped to BFD for (int j = 0; j < mypc.nSpeciesBoostedFrameDiags(); ++j) { // Get species name - std::string species_name = + std::string species_name = species_names[mypc.mapSpeciesBoostedFrameDiags(j)]; #ifdef WARPX_USE_HDF5 // Dump species data @@ -691,7 +691,7 @@ writeLabFrameData(const MultiFab* cell_centered_data, // Loop over snapshots for (int i = 0; i < N_snapshots_; ++i) { - + // Get updated z position of snapshot const Real old_z_boost = snapshots_[i].current_z_boost; snapshots_[i].updateCurrentZPositions(t_boost, @@ -700,15 +700,15 @@ writeLabFrameData(const MultiFab* cell_centered_data, Real zmin_lab = snapshots_[i].prob_domain_lab_.lo(AMREX_SPACEDIM-1); Real zmax_lab = snapshots_[i].prob_domain_lab_.hi(AMREX_SPACEDIM-1); - + // If snapshot out of the domain, nothing to do if ( (snapshots_[i].current_z_boost < zlo_boost) or (snapshots_[i].current_z_boost > zhi_boost) or (snapshots_[i].current_z_lab < zmin_lab) or (snapshots_[i].current_z_lab > zmax_lab) ) continue; - // Get z index of data_buffer_ (i.e. in the lab frame) where - // simulation domain (t', [zmin',zmax']), back-transformed to lab + // Get z index of data_buffer_ (i.e. in the lab frame) where + // simulation domain (t', [zmin',zmax']), back-transformed to lab // frame, intersects with snapshot. int i_lab = (snapshots_[i].current_z_lab - zmin_lab) / dz_lab_; @@ -725,7 +725,7 @@ writeLabFrameData(const MultiFab* cell_centered_data, data_buffer_[i].reset( new MultiFab(buff_ba, buff_dm, ncomp_to_dump, 0) ); } // ... reset particle buffer particles_buffer_[i] - if (WarpX::do_boosted_frame_particles) + if (WarpX::do_boosted_frame_particles) particles_buffer_[i].resize(mypc.nSpeciesBoostedFrameDiags()); } @@ -738,7 +738,7 @@ writeLabFrameData(const MultiFab* cell_centered_data, snapshots_[i].current_z_boost, *cell_centered_data, geom, start_comp, ncomp, interpolate); - + // transform it to the lab frame LorentzTransformZ(*slice, gamma_boost_, beta_boost_, ncomp); // Create a 2D box for the slice in the boosted frame @@ -753,7 +753,7 @@ writeLabFrameData(const MultiFab* cell_centered_data, // Create MultiFab tmp on slice_ba with data from slice MultiFab tmp(slice_ba, data_buffer_[i]->DistributionMap(), ncomp, 0); tmp.copy(*slice, 0, 0, ncomp); - + // Copy data from MultiFab tmp to MultiDab data_buffer[i] CopySlice(tmp, *data_buffer_[i], i_lab, map_actual_fields_to_dump); } @@ -766,7 +766,7 @@ writeLabFrameData(const MultiFab* cell_centered_data, ++buff_counter_[i]; - + // If buffer full, write to disk. if (buff_counter_[i] == num_buffer_) { @@ -781,10 +781,10 @@ writeLabFrameData(const MultiFab* cell_centered_data, VisMF::Write(*data_buffer_[i], mesh_ss.str()); #endif } - + if (WarpX::do_boosted_frame_particles) { // Loop over species to be dumped to BFD - for (int j = 0; j < mypc.nSpeciesBoostedFrameDiags(); ++j) { + for (int j = 0; j < mypc.nSpeciesBoostedFrameDiags(); ++j) { // Get species name const std::string species_name = species_names[mypc.mapSpeciesBoostedFrameDiags(j)]; #ifdef WARPX_USE_HDF5 @@ -800,14 +800,14 @@ writeLabFrameData(const MultiFab* cell_centered_data, // Write data to disk (custom) writeParticleData(particles_buffer_[i][j], part_ss.str(), i_lab); #endif - } + } particles_buffer_[i].clear(); } buff_counter_[i] = 0; } } - - VisMF::SetHeaderVersion(current_version); + + VisMF::SetHeaderVersion(current_version); } #ifdef WARPX_USE_HDF5 @@ -817,12 +817,12 @@ writeParticleDataHDF5(const WarpXParticleContainer::DiagnosticParticleData& pdat const std::string& name, const std::string& species_name) { auto np = pdata.GetRealData(DiagIdx::w).size(); - + Vector particle_counts(ParallelDescriptor::NProcs(), 0); Vector particle_offsets(ParallelDescriptor::NProcs(), 0); - + ParallelAllGather::AllGather(np, particle_counts.data(), ParallelContext::CommunicatorAll()); - + long total_np = 0; for (int i = 0; i < ParallelDescriptor::NProcs(); ++i) { particle_offsets[i] = total_np; @@ -830,7 +830,7 @@ writeParticleDataHDF5(const WarpXParticleContainer::DiagnosticParticleData& pdat } if (total_np == 0) return; - + long old_np = 0; if (ParallelDescriptor::IOProcessor()) { @@ -844,7 +844,7 @@ writeParticleDataHDF5(const WarpXParticleContainer::DiagnosticParticleData& pdat // Note, this has the effect of an MPI Barrier between the above resize operation // and the below write. ParallelDescriptor::ReduceLongMax(old_np); - + // Write data here for (int k = 0; k < static_cast(particle_field_names.size()); ++k) { @@ -853,7 +853,7 @@ writeParticleDataHDF5(const WarpXParticleContainer::DiagnosticParticleData& pdat pdata.GetRealData(k).data(), particle_counts[ParallelDescriptor::MyProc()], particle_offsets[ParallelDescriptor::MyProc()] + old_np); - } + } } #endif @@ -863,7 +863,7 @@ writeParticleData(const WarpXParticleContainer::DiagnosticParticleData& pdata, const std::string& name, const int i_lab) { BL_PROFILE("BoostedFrameDiagnostic::writeParticleData"); - + std::string field_name; std::ofstream ofs; @@ -880,27 +880,27 @@ writeParticleData(const WarpXParticleContainer::DiagnosticParticleData& pdata, field_name = name + Concatenate("x_", i_lab, 5) + "_" + std::to_string(MyProc); ofs.open(field_name.c_str(), std::ios::out|std::ios::binary); writeRealData(pdata.GetRealData(DiagIdx::x).data(), np, ofs); - ofs.close(); + ofs.close(); field_name = name + Concatenate("y_", i_lab, 5) + "_" + std::to_string(MyProc); ofs.open(field_name.c_str(), std::ios::out|std::ios::binary); writeRealData(pdata.GetRealData(DiagIdx::y).data(), np, ofs); - ofs.close(); + ofs.close(); field_name = name + Concatenate("z_", i_lab, 5) + "_" + std::to_string(MyProc); ofs.open(field_name.c_str(), std::ios::out|std::ios::binary); writeRealData(pdata.GetRealData(DiagIdx::z).data(), np, ofs); - ofs.close(); - + ofs.close(); + field_name = name + Concatenate("ux_", i_lab, 5) + "_" + std::to_string(MyProc); ofs.open(field_name.c_str(), std::ios::out|std::ios::binary); writeRealData(pdata.GetRealData(DiagIdx::ux).data(), np, ofs); - ofs.close(); + ofs.close(); field_name = name + Concatenate("uy_", i_lab, 5) + "_" + std::to_string(MyProc); ofs.open(field_name.c_str(), std::ios::out|std::ios::binary); writeRealData(pdata.GetRealData(DiagIdx::uy).data(), np, ofs); - ofs.close(); + ofs.close(); field_name = name + Concatenate("uz_", i_lab, 5) + "_" + std::to_string(MyProc); ofs.open(field_name.c_str(), std::ios::out|std::ios::binary); @@ -910,12 +910,12 @@ writeParticleData(const WarpXParticleContainer::DiagnosticParticleData& pdata, void BoostedFrameDiagnostic:: -writeMetaData () +writeMetaData () { BL_PROFILE("BoostedFrameDiagnostic::writeMetaData"); if (ParallelDescriptor::IOProcessor()) { - + if (!UtilCreateDirectory(WarpX::lab_data_directory, 0755)) CreateDirectoryFailed(WarpX::lab_data_directory); @@ -928,22 +928,22 @@ writeMetaData () std::ofstream::binary); if(!HeaderFile.good()) FileOpenFailed(HeaderFileName); - + HeaderFile.precision(17); - + HeaderFile << N_snapshots_ << "\n"; - HeaderFile << dt_snapshots_lab_ << "\n"; + HeaderFile << dt_snapshots_lab_ << "\n"; HeaderFile << gamma_boost_ << "\n"; HeaderFile << beta_boost_ << "\n"; } } BoostedFrameDiagnostic::LabSnapShot:: -LabSnapShot(Real t_lab_in, Real t_boost, RealBox prob_domain_lab, - IntVect prob_ncells_lab, +LabSnapShot(Real t_lab_in, Real t_boost, RealBox prob_domain_lab, + IntVect prob_ncells_lab, int ncomp_to_dump, std::vector mesh_field_names, - int file_num_in, + int file_num_in, const BoostedFrameDiagnostic& bfd) : t_lab(t_lab_in), prob_domain_lab_(prob_domain_lab), @@ -967,7 +967,7 @@ LabSnapShot(Real t_lab_in, Real t_boost, RealBox prob_domain_lab, } ParallelDescriptor::Barrier(); - + if (ParallelDescriptor::IOProcessor()) { if (WarpX::do_boosted_frame_fields) @@ -986,7 +986,7 @@ LabSnapShot(Real t_lab_in, Real t_boost, RealBox prob_domain_lab, } ParallelDescriptor::Barrier(); - + if (WarpX::do_boosted_frame_particles){ auto & mypc = WarpX::GetInstance().GetPartContainer(); const std::vector species_names = mypc.GetSpeciesNames(); @@ -994,7 +994,7 @@ LabSnapShot(Real t_lab_in, Real t_boost, RealBox prob_domain_lab, for (int j = 0; j < mypc.nSpeciesBoostedFrameDiags(); ++j) { // Loop over species to be dumped to BFD - std::string species_name = + std::string species_name = species_names[mypc.mapSpeciesBoostedFrameDiags(j)]; output_create_species_group(file_name, species_name); for (int k = 0; k < static_cast(particle_field_names.size()); ++k) @@ -1003,10 +1003,10 @@ LabSnapShot(Real t_lab_in, Real t_boost, RealBox prob_domain_lab, output_create_particle_field(file_name, field_path); } } - } -#else + } +#else if (ParallelDescriptor::IOProcessor()) { - + if (!UtilCreateDirectory(file_name, 0755)) CreateDirectoryFailed(file_name); @@ -1018,13 +1018,13 @@ LabSnapShot(Real t_lab_in, Real t_boost, RealBox prob_domain_lab, } auto & mypc = WarpX::GetInstance().GetPartContainer(); - const std::vector species_names = mypc.GetSpeciesNames(); - + const std::vector species_names = mypc.GetSpeciesNames(); + const std::string particles_prefix = "particle"; // Loop over species to be dumped to BFD for(int i = 0; i < mypc.nSpeciesBoostedFrameDiags(); ++i) { // Get species name - std::string species_name = + std::string species_name = species_names[mypc.mapSpeciesBoostedFrameDiags(i)]; const std::string fullpath = file_name + "/" + species_name; if (!UtilCreateDirectory(fullpath, 0755)) @@ -1058,9 +1058,9 @@ writeSnapShotHeader() { std::ofstream::binary); if(!HeaderFile.good()) FileOpenFailed(HeaderFileName); - + HeaderFile.precision(17); - + HeaderFile << t_lab << "\n"; // Write domain number of cells HeaderFile << prob_ncells_lab_[0] << ' ' diff --git a/Source/Diagnostics/ElectrostaticIO.cpp b/Source/Diagnostics/ElectrostaticIO.cpp index a8dffac2b..75be2a24c 100644 --- a/Source/Diagnostics/ElectrostaticIO.cpp +++ b/Source/Diagnostics/ElectrostaticIO.cpp @@ -114,7 +114,7 @@ WritePlotFileES (const amrex::Vector >& rho, particle_varnames.push_back("Bz"); Vector int_names; - + mypc->Checkpoint(plotfilename, particle_varnames, int_names); WriteJobInfo(plotfilename); diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index 8bfa45a59..741d3f50d 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -22,8 +22,8 @@ RigidInjectedParticleContainer::ReadHeader (std::istream& is) int zinject_plane_tmp; is >> zinject_plane_tmp; zinject_plane_levels.push_back(zinject_plane_tmp); - WarpX::GotoNextLine(is); - } + WarpX::GotoNextLine(is); + } for (int i = 0; i < nlevs; ++i) { @@ -31,7 +31,7 @@ RigidInjectedParticleContainer::ReadHeader (std::istream& is) is >> done_injecting_tmp; done_injecting.push_back(done_injecting_tmp); WarpX::GotoNextLine(is); - } + } } void @@ -78,7 +78,7 @@ MultiParticleContainer::WritePlotFile (const std::string& dir) const { for (unsigned i = 0, n = species_names.size(); i < n; ++i) { - auto& pc = allcontainers[i]; + auto& pc = allcontainers[i]; if (pc->plot_species) { Vector real_names; @@ -90,19 +90,19 @@ MultiParticleContainer::WritePlotFile (const std::string& dir) const real_names.push_back("momentum_x"); real_names.push_back("momentum_y"); real_names.push_back("momentum_z"); - + real_names.push_back("Ex"); real_names.push_back("Ey"); real_names.push_back("Ez"); - + real_names.push_back("Bx"); real_names.push_back("By"); real_names.push_back("Bz"); - + #ifdef WARPX_DIM_RZ real_names.push_back("theta"); #endif - + if(pc->do_field_ionization){ int_names.push_back("ionization_level"); // int_flags specifies, for each integer attribs, whether it is @@ -134,7 +134,7 @@ MultiParticleContainer::Restart (const std::string& dir) } void -MultiParticleContainer::ReadHeader (std::istream& is) +MultiParticleContainer::ReadHeader (std::istream& is) { for (auto& pc : allcontainers) { pc->ReadHeader(is); @@ -149,7 +149,7 @@ MultiParticleContainer::WriteHeader (std::ostream& os) const } } -// Particle momentum is defined as gamma*velocity, which is neither +// Particle momentum is defined as gamma*velocity, which is neither // SI mass*gamma*velocity nor normalized gamma*velocity/c. // This converts momentum to SI units (or vice-versa) to write SI data // to file. diff --git a/Source/Diagnostics/SliceDiagnostic.H b/Source/Diagnostics/SliceDiagnostic.H index 31eea83be..1b9ca3967 100644 --- a/Source/Diagnostics/SliceDiagnostic.H +++ b/Source/Diagnostics/SliceDiagnostic.H @@ -16,26 +16,26 @@ -std::unique_ptr CreateSlice( const amrex::MultiFab& mf, - const amrex::Vector &dom_geom, - amrex::RealBox &slice_realbox, +std::unique_ptr CreateSlice( const amrex::MultiFab& mf, + const amrex::Vector &dom_geom, + amrex::RealBox &slice_realbox, amrex::IntVect &slice_cr_ratio ); -void CheckSliceInput( const amrex::RealBox real_box, - amrex::RealBox &slice_cc_nd_box, amrex::RealBox &slice_realbox, +void CheckSliceInput( const amrex::RealBox real_box, + amrex::RealBox &slice_cc_nd_box, amrex::RealBox &slice_realbox, amrex::IntVect &slice_cr_ratio, amrex::Vector dom_geom, - amrex::IntVect const SliceType, amrex::IntVect &slice_lo, + amrex::IntVect const SliceType, amrex::IntVect &slice_lo, amrex::IntVect &slice_hi, amrex::IntVect &interp_lo); -void InterpolateSliceValues( amrex::MultiFab& smf, - amrex::IntVect interp_lo, amrex::RealBox slice_realbox, - amrex::Vector geom, int ncomp, int nghost, - amrex::IntVect slice_lo, amrex::IntVect slice_hi, +void InterpolateSliceValues( amrex::MultiFab& smf, + amrex::IntVect interp_lo, amrex::RealBox slice_realbox, + amrex::Vector geom, int ncomp, int nghost, + amrex::IntVect slice_lo, amrex::IntVect slice_hi, amrex::IntVect SliceType, const amrex::RealBox real_box); -void InterpolateLo( const amrex::Box& bx, amrex::FArrayBox &fabox, - amrex::IntVect slice_lo, amrex::Vector geom, - int idir, amrex::IntVect IndType, amrex::RealBox slice_realbox, +void InterpolateLo( const amrex::Box& bx, amrex::FArrayBox &fabox, + amrex::IntVect slice_lo, amrex::Vector geom, + int idir, amrex::IntVect IndType, amrex::RealBox slice_realbox, int srccomp, int ncomp, int nghost, const amrex::RealBox real_box); #endif diff --git a/Source/Diagnostics/SliceDiagnostic.cpp b/Source/Diagnostics/SliceDiagnostic.cpp index 0ec528e32..79f03985b 100644 --- a/Source/Diagnostics/SliceDiagnostic.cpp +++ b/Source/Diagnostics/SliceDiagnostic.cpp @@ -10,23 +10,23 @@ using namespace amrex; /* \brief * The functions creates the slice for diagnostics based on the user-input. - * The slice can be 1D, 2D, or 3D and it inherts the index type of the underlying data. - * The implementation assumes that the slice is aligned with the coordinate axes. - * The input parameters are modified if the user-input does not comply with requirements of coarsenability or if the slice extent is not contained within the simulation domain. - * First a slice multifab (smf) with cell size equal to that of the simulation grid is created such that it extends from slice.dim_lo to slice.dim_hi and shares the same index space as the source multifab (mf) + * The slice can be 1D, 2D, or 3D and it inherts the index type of the underlying data. + * The implementation assumes that the slice is aligned with the coordinate axes. + * The input parameters are modified if the user-input does not comply with requirements of coarsenability or if the slice extent is not contained within the simulation domain. + * First a slice multifab (smf) with cell size equal to that of the simulation grid is created such that it extends from slice.dim_lo to slice.dim_hi and shares the same index space as the source multifab (mf) * The values are copied from src mf to dst smf using amrex::ParallelCopy - * If interpolation is required, then on the smf, using data points stored in the ghost cells, the data in interpolated. - * If coarsening is required, then a coarse slice multifab is generated (cs_mf) and the - * values of the refined slice (smf) is averaged down to obtain the coarse slice. - * \param mf is the source multifab containing the field data - * \param dom_geom is the geometry of the domain and used in the function to obtain the - * CellSize of the underlying grid. - * \param slice_realbox defines the extent of the slice - * \param slice_cr_ratio provides the coarsening ratio for diagnostics + * If interpolation is required, then on the smf, using data points stored in the ghost cells, the data in interpolated. + * If coarsening is required, then a coarse slice multifab is generated (cs_mf) and the + * values of the refined slice (smf) is averaged down to obtain the coarse slice. + * \param mf is the source multifab containing the field data + * \param dom_geom is the geometry of the domain and used in the function to obtain the + * CellSize of the underlying grid. + * \param slice_realbox defines the extent of the slice + * \param slice_cr_ratio provides the coarsening ratio for diagnostics */ -std::unique_ptr -CreateSlice( const MultiFab& mf, const Vector &dom_geom, +std::unique_ptr +CreateSlice( const MultiFab& mf, const Vector &dom_geom, RealBox &slice_realbox, IntVect &slice_cr_ratio ) { std::unique_ptr smf; @@ -36,10 +36,10 @@ CreateSlice( const MultiFab& mf, const Vector &dom_geom, int nghost = 1; int nlevels = dom_geom.size(); int ncomp = (mf).nComp(); - - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( nlevels==1, + + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( nlevels==1, "Slice diagnostics does not work with mesh refinement yet (TO DO)."); - + const auto conversionType = (mf).ixType(); IntVect SliceType(AMREX_D_DECL(0,0,0)); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim ) @@ -50,7 +50,7 @@ CreateSlice( const MultiFab& mf, const Vector &dom_geom, const RealBox& real_box = dom_geom[0].ProbDomain(); RealBox slice_cc_nd_box; int slice_grid_size = 32; - + bool interpolate = false; bool coarsen = false; @@ -60,45 +60,45 @@ CreateSlice( const MultiFab& mf, const Vector &dom_geom, IntVect interp_lo(AMREX_D_DECL(0,0,0)); CheckSliceInput(real_box, slice_cc_nd_box, slice_realbox, slice_cr_ratio, - dom_geom, SliceType, slice_lo, + dom_geom, SliceType, slice_lo, slice_hi, interp_lo); int configuration_dim = 0; // Determine if interpolation is required and number of cells in slice // - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + // Flag for interpolation if required // if ( interp_lo[idim] == 1) { - interpolate = 1; + interpolate = 1; } // For the case when a dimension is reduced // - if ( ( slice_hi[idim] - slice_lo[idim]) == 1) { + if ( ( slice_hi[idim] - slice_lo[idim]) == 1) { slice_ncells[idim] = 1; } - else { - slice_ncells[idim] = ( slice_hi[idim] - slice_lo[idim] + 1 ) + else { + slice_ncells[idim] = ( slice_hi[idim] - slice_lo[idim] + 1 ) / slice_cr_ratio[idim]; int refined_ncells = slice_hi[idim] - slice_lo[idim] + 1 ; - if ( slice_cr_ratio[idim] > 1) { + if ( slice_cr_ratio[idim] > 1) { coarsen = true; // modify slice_grid_size if >= refines_cells // if ( slice_grid_size >= refined_ncells ) { slice_grid_size = refined_ncells - 1; } - + } configuration_dim += 1; } } if (configuration_dim==1) { amrex::Warning("The slice configuration is 1D and cannot be visualized using yt."); - } + } // Slice generation with index type inheritance // Box slice(slice_lo, slice_hi); - + Vector sba(1); sba[0].define(slice); sba[0].maxSize(slice_grid_size); @@ -106,7 +106,7 @@ CreateSlice( const MultiFab& mf, const Vector &dom_geom, // Distribution mapping for slice can be different from that of domain // Vector sdmap(1); sdmap[0] = DistributionMapping{sba[0]}; - + smf.reset(new MultiFab(amrex::convert(sba[0],SliceType), sdmap[0], ncomp, nghost)); @@ -115,12 +115,12 @@ CreateSlice( const MultiFab& mf, const Vector &dom_geom, smf->ParallelCopy(mf, 0, 0, ncomp,nghost,nghost); // inteprolate if required on refined slice // - if (interpolate == 1 ) { - InterpolateSliceValues( *smf, interp_lo, slice_cc_nd_box, dom_geom, + if (interpolate == 1 ) { + InterpolateSliceValues( *smf, interp_lo, slice_cc_nd_box, dom_geom, ncomp, nghost, slice_lo, slice_hi, SliceType, real_box); } - + if (coarsen == false) { return smf; } @@ -131,14 +131,14 @@ CreateSlice( const MultiFab& mf, const Vector &dom_geom, AMREX_ALWAYS_ASSERT(crse_ba[0].size() == sba[0].size()); - cs_mf.reset( new MultiFab(amrex::convert(crse_ba[0],SliceType), + cs_mf.reset( new MultiFab(amrex::convert(crse_ba[0],SliceType), sdmap[0], ncomp,nghost)); MultiFab& mfSrc = *smf; MultiFab& mfDst = *cs_mf; MFIter mfi_dst(mfDst); - for (MFIter mfi(mfSrc); mfi.isValid(); ++mfi) { + for (MFIter mfi(mfSrc); mfi.isValid(); ++mfi) { Array4 const& Src_fabox = mfSrc.const_array(mfi); @@ -150,7 +150,7 @@ CreateSlice( const MultiFab& mf, const Vector &dom_geom, IntVect cctype(AMREX_D_DECL(0,0,0)); if( SliceType==cctype ) { - amrex::amrex_avgdown(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, + amrex::amrex_avgdown(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio); } IntVect ndtype(AMREX_D_DECL(1,1,1)); @@ -159,11 +159,11 @@ CreateSlice( const MultiFab& mf, const Vector &dom_geom, scomp, ncomp, slice_cr_ratio); } if( SliceType == WarpX::Ex_nodal_flag ) { - amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, + amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 0); } if( SliceType == WarpX::Ey_nodal_flag) { - amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, + amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 1); } if( SliceType == WarpX::Ez_nodal_flag ) { @@ -171,19 +171,19 @@ CreateSlice( const MultiFab& mf, const Vector &dom_geom, scomp, ncomp, slice_cr_ratio, 2); } if( SliceType == WarpX::Bx_nodal_flag) { - amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, + amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 0); } if( SliceType == WarpX::By_nodal_flag ) { - amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, + amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 1); } if( SliceType == WarpX::Bz_nodal_flag ) { - amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, + amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 2); } - if ( mfi_dst.isValid() ) { + if ( mfi_dst.isValid() ) { ++mfi_dst; } @@ -197,66 +197,66 @@ CreateSlice( const MultiFab& mf, const Vector &dom_geom, /* \brief - * This function modifies the slice input parameters under certain conditions. - * The coarsening ratio, slice_cr_ratio is modified if the input is not an exponent of 2. - * for example, if the coarsening ratio is 3, 5 or 6, which is not an exponent of 2, - * then the value of coarsening ratio is modified to the nearest exponent of 2. - * The default value for coarsening ratio is 1. - * slice_realbox.lo and slice_realbox.hi are set equal to the simulation domain lo and hi - * if for the user-input for the slice lo and hi coordinates are outside the domain. - * If the slice_realbox.lo and slice_realbox.hi coordinates do not align with the data + * This function modifies the slice input parameters under certain conditions. + * The coarsening ratio, slice_cr_ratio is modified if the input is not an exponent of 2. + * for example, if the coarsening ratio is 3, 5 or 6, which is not an exponent of 2, + * then the value of coarsening ratio is modified to the nearest exponent of 2. + * The default value for coarsening ratio is 1. + * slice_realbox.lo and slice_realbox.hi are set equal to the simulation domain lo and hi + * if for the user-input for the slice lo and hi coordinates are outside the domain. + * If the slice_realbox.lo and slice_realbox.hi coordinates do not align with the data * points and the number of cells in that dimension is greater than 1, and if the extent of * the slice in that dimension is not coarsenable, then the value lo and hi coordinates are * shifted to the nearest coarsenable point to include some extra data points in the slice. * If slice_realbox.lo==slice_realbox.hi, then that dimension has only one cell and no - * modifications are made to the value. If the lo and hi do not align with a data point, - * then it is flagged for interpolation. - * \param real_box a Real box defined for the underlying domain. - * \param slice_realbox a Real box for defining the slice dimension. - * \param slice_cc_nd_box a Real box for defining the modified lo and hi of the slice + * modifications are made to the value. If the lo and hi do not align with a data point, + * then it is flagged for interpolation. + * \param real_box a Real box defined for the underlying domain. + * \param slice_realbox a Real box for defining the slice dimension. + * \param slice_cc_nd_box a Real box for defining the modified lo and hi of the slice * such that the coordinates align with the underlying data points. - * If the dimension is reduced to have only one cell, the slice_realbox is not modified and * instead the values are interpolated to the coordinate from the nearest data points. - * \param slice_cr_ratio contains values of the coarsening ratio which may be modified - * if the input values do not satisfy coarsenability conditions. + * If the dimension is reduced to have only one cell, the slice_realbox is not modified and * instead the values are interpolated to the coordinate from the nearest data points. + * \param slice_cr_ratio contains values of the coarsening ratio which may be modified + * if the input values do not satisfy coarsenability conditions. * \param slice_lo and slice_hi are the index values of the slice - * \param interp_lo are set to 0 or 1 if they are flagged for interpolation. - * The slice shares the same index space as that of the simulation domain. + * \param interp_lo are set to 0 or 1 if they are flagged for interpolation. + * The slice shares the same index space as that of the simulation domain. */ -void -CheckSliceInput( const RealBox real_box, RealBox &slice_cc_nd_box, - RealBox &slice_realbox, IntVect &slice_cr_ratio, - Vector dom_geom, IntVect const SliceType, +void +CheckSliceInput( const RealBox real_box, RealBox &slice_cc_nd_box, + RealBox &slice_realbox, IntVect &slice_cr_ratio, + Vector dom_geom, IntVect const SliceType, IntVect &slice_lo, IntVect &slice_hi, IntVect &interp_lo) { - + IntVect slice_lo2(AMREX_D_DECL(0,0,0)); - for ( int idim = 0; idim < AMREX_SPACEDIM; ++idim) - { + for ( int idim = 0; idim < AMREX_SPACEDIM; ++idim) + { // Modify coarsening ratio if the input value is not an exponent of 2 for AMR // - if ( slice_cr_ratio[idim] > 0 ) { + if ( slice_cr_ratio[idim] > 0 ) { int log_cr_ratio = floor ( log2( double(slice_cr_ratio[idim]))); slice_cr_ratio[idim] = exp2( double(log_cr_ratio) ); } - + //// Default coarsening ratio is 1 // // Modify lo if input is out of bounds // - if ( slice_realbox.lo(idim) < real_box.lo(idim) ) { + if ( slice_realbox.lo(idim) < real_box.lo(idim) ) { slice_realbox.setLo( idim, real_box.lo(idim)); - amrex::Print() << " slice lo is out of bounds. " << - " Modified it in dimension " << idim << + amrex::Print() << " slice lo is out of bounds. " << + " Modified it in dimension " << idim << " to be aligned with the domain box\n"; - } - + } + // Modify hi if input in out od bounds // - if ( slice_realbox.hi(idim) > real_box.hi(idim) ) { + if ( slice_realbox.hi(idim) > real_box.hi(idim) ) { slice_realbox.setHi( idim, real_box.hi(idim)); - amrex::Print() << " slice hi is out of bounds." << - " Modified it in dimension " << idim << + amrex::Print() << " slice hi is out of bounds." << + " Modified it in dimension " << idim << " to be aligned with the domain box\n"; } - + // Factor to ensure index values computation depending on index type // double fac = ( 1.0 - SliceType[idim] )*dom_geom[0].CellSize(idim)*0.5; // if dimension is reduced to one cell length // @@ -264,58 +264,58 @@ CheckSliceInput( const RealBox real_box, RealBox &slice_cc_nd_box, { slice_cc_nd_box.setLo( idim, slice_realbox.lo(idim) ); slice_cc_nd_box.setHi( idim, slice_realbox.hi(idim) ); - + if ( slice_cr_ratio[idim] > 1) slice_cr_ratio[idim] = 1; - + // check for interpolation -- compute index lo with floor and ceil - if ( slice_cc_nd_box.lo(idim) - real_box.lo(idim) >= fac ) { - slice_lo[idim] = floor( ( (slice_cc_nd_box.lo(idim) - - (real_box.lo(idim) + fac ) ) + if ( slice_cc_nd_box.lo(idim) - real_box.lo(idim) >= fac ) { + slice_lo[idim] = floor( ( (slice_cc_nd_box.lo(idim) + - (real_box.lo(idim) + fac ) ) / dom_geom[0].CellSize(idim)) + fac * 1E-10); - slice_lo2[idim] = ceil( ( (slice_cc_nd_box.lo(idim) - - (real_box.lo(idim) + fac) ) - / dom_geom[0].CellSize(idim)) - fac * 1E-10 ); - } - else { - slice_lo[idim] = round( (slice_cc_nd_box.lo(idim) - - (real_box.lo(idim) ) ) + slice_lo2[idim] = ceil( ( (slice_cc_nd_box.lo(idim) + - (real_box.lo(idim) + fac) ) + / dom_geom[0].CellSize(idim)) - fac * 1E-10 ); + } + else { + slice_lo[idim] = round( (slice_cc_nd_box.lo(idim) + - (real_box.lo(idim) ) ) / dom_geom[0].CellSize(idim)); - slice_lo2[idim] = ceil((slice_cc_nd_box.lo(idim) - - (real_box.lo(idim) ) ) + slice_lo2[idim] = ceil((slice_cc_nd_box.lo(idim) + - (real_box.lo(idim) ) ) / dom_geom[0].CellSize(idim) ); } - + // flag for interpolation -- if reduced dimension location // // does not align with data point // - if ( slice_lo[idim] == slice_lo2[idim]) { + if ( slice_lo[idim] == slice_lo2[idim]) { if ( slice_cc_nd_box.lo(idim) - real_box.lo(idim) < fac ) { interp_lo[idim] = 1; } } - else { + else { interp_lo[idim] = 1; } - + // ncells = 1 if dimension is reduced // slice_hi[idim] = slice_lo[idim] + 1; } else { // moving realbox.lo and reabox.hi to nearest coarsenable grid point // - int index_lo = floor(((slice_realbox.lo(idim) + 1E-10 + int index_lo = floor(((slice_realbox.lo(idim) + 1E-10 - (real_box.lo(idim))) / dom_geom[0].CellSize(idim))); int index_hi = ceil(((slice_realbox.hi(idim) - 1E-10 - (real_box.lo(idim))) / dom_geom[0].CellSize(idim))); bool modify_cr = true; - - while ( modify_cr == true) { + + while ( modify_cr == true) { int lo_new = index_lo; - int hi_new = index_hi; + int hi_new = index_hi; int mod_lo = index_lo % slice_cr_ratio[idim]; int mod_hi = index_hi % slice_cr_ratio[idim]; modify_cr = false; - + // To ensure that the index.lo is coarsenable // if ( mod_lo > 0) { lo_new = index_lo - mod_lo; @@ -324,36 +324,36 @@ CheckSliceInput( const RealBox real_box, RealBox &slice_cc_nd_box, if ( mod_hi > 0) { hi_new = index_hi + (slice_cr_ratio[idim] - mod_hi); } - - // If modified index.hi is > baselinebox.hi, move the point // + + // If modified index.hi is > baselinebox.hi, move the point // // to the previous coarsenable point // - if ( (hi_new * dom_geom[0].CellSize(idim)) + if ( (hi_new * dom_geom[0].CellSize(idim)) > real_box.hi(idim) - real_box.lo(idim) + dom_geom[0].CellSize(idim)*0.01 ) { hi_new = index_hi - mod_hi; } - - if ( (hi_new - lo_new) == 0 ){ + + if ( (hi_new - lo_new) == 0 ){ amrex::Print() << " Diagnostic Warning :: "; amrex::Print() << " Coarsening ratio "; amrex::Print() << slice_cr_ratio[idim] << " in dim "<< idim; amrex::Print() << "is leading to zero cells for slice."; amrex::Print() << " Thus reducing cr_ratio by half.\n"; - + slice_cr_ratio[idim] = slice_cr_ratio[idim]/2; modify_cr = true; } - - if ( modify_cr == false ) { + + if ( modify_cr == false ) { index_lo = lo_new; index_hi = hi_new; } slice_lo[idim] = index_lo; - slice_hi[idim] = index_hi - 1; // since default is cell-centered + slice_hi[idim] = index_hi - 1; // since default is cell-centered } - slice_realbox.setLo( idim, index_lo * dom_geom[0].CellSize(idim) + slice_realbox.setLo( idim, index_lo * dom_geom[0].CellSize(idim) + real_box.lo(idim) ); - slice_realbox.setHi( idim, index_hi * dom_geom[0].CellSize(idim) + slice_realbox.setHi( idim, index_hi * dom_geom[0].CellSize(idim) + real_box.lo(idim) ); slice_cc_nd_box.setLo( idim, slice_realbox.lo(idim) + fac ); slice_cc_nd_box.setHi( idim, slice_realbox.hi(idim) - fac ); @@ -363,14 +363,14 @@ CheckSliceInput( const RealBox real_box, RealBox &slice_cc_nd_box, /* \brief - * This function is called if the coordinates of the slice do not align with data points - * \param interp_lo is an IntVect which is flagged as 1, if interpolation - is required in the dimension. + * This function is called if the coordinates of the slice do not align with data points + * \param interp_lo is an IntVect which is flagged as 1, if interpolation + is required in the dimension. */ -void +void InterpolateSliceValues(MultiFab& smf, IntVect interp_lo, RealBox slice_realbox, - Vector geom, int ncomp, int nghost, - IntVect slice_lo, IntVect slice_hi, IntVect SliceType, + Vector geom, int ncomp, int nghost, + IntVect slice_lo, IntVect slice_hi, IntVect SliceType, const RealBox real_box) { for (MFIter mfi(smf); mfi.isValid(); ++mfi) @@ -381,9 +381,9 @@ InterpolateSliceValues(MultiFab& smf, IntVect interp_lo, RealBox slice_realbox, const auto hi = amrex::ubound(bx); FArrayBox& fabox = smf[mfi]; - for ( int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if ( interp_lo[idim] == 1 ) { - InterpolateLo( bx, fabox, slice_lo, geom, idim, SliceType, + for ( int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + if ( interp_lo[idim] == 1 ) { + InterpolateLo( bx, fabox, slice_lo, geom, idim, SliceType, slice_realbox, 0, ncomp, nghost, real_box); } } @@ -391,10 +391,10 @@ InterpolateSliceValues(MultiFab& smf, IntVect interp_lo, RealBox slice_realbox, } -void -InterpolateLo(const Box& bx, FArrayBox &fabox, IntVect slice_lo, - Vector geom, int idir, IntVect IndType, - RealBox slice_realbox, int srccomp, int ncomp, +void +InterpolateLo(const Box& bx, FArrayBox &fabox, IntVect slice_lo, + Vector geom, int idir, IntVect IndType, + RealBox slice_realbox, int srccomp, int ncomp, int nghost, const RealBox real_box ) { auto fabarr = fabox.array(); @@ -428,7 +428,7 @@ InterpolateLo(const Box& bx, FArrayBox &fabox, IntVect slice_lo, break; } case 1: - { + { if ( imin >= lo.y && imin <= lo.y) { for (int n = srccomp; n < srccomp+ncomp; ++n) { for (int k = lo.z; k <= hi.z; ++k) { diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index f79079b0e..6aa1f7a5b 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -750,7 +750,7 @@ WarpX::WriteJobInfo (const std::string& dir) const /* \brief * The raw slice data is written out in the plotfile format and can be visualized using yt. - * The slice data is written to diags/slice_plotfiles/pltXXXXX at the plotting interval. + * The slice data is written to diags/slice_plotfiles/pltXXXXX at the plotting interval. */ void WarpX::WriteSlicePlotFile () const @@ -763,36 +763,36 @@ WarpX::WriteSlicePlotFile () const VisMF::Header::Version current_version = VisMF::GetHeaderVersion(); VisMF::SetHeaderVersion(slice_plotfile_headerversion); rfs.emplace_back("raw_fields"); - + const int nlevels = finestLevel() + 1; - + // creating a temporary cell-centered dummy multifab // // to get around the issue of yt complaining about no field data // Vector< std::unique_ptr > dummy_mf(nlevels); const DistributionMapping &dm2 = Efield_slice[0][0]->DistributionMap(); Vector varnames; IntVect cc(AMREX_D_DECL(0,0,0)); - for (int lev = 0; lev < nlevels; ++lev) + for (int lev = 0; lev < nlevels; ++lev) { - dummy_mf[lev].reset(new MultiFab( - amrex::convert(Efield_slice[lev][0]->boxArray(),cc), + dummy_mf[lev].reset(new MultiFab( + amrex::convert(Efield_slice[lev][0]->boxArray(),cc), dm2, 1, 0 )); dummy_mf[lev]->setVal(0.0); - } + } amrex::WriteMultiLevelPlotfile(slice_plotfilename, nlevels, - GetVecOfConstPtrs(dummy_mf), + GetVecOfConstPtrs(dummy_mf), varnames, Geom(), t_new[0], istep, refRatio(), - "HyperCLaw-V1.1", + "HyperCLaw-V1.1", "Level_", "Cell", rfs); - for (int lev = 0; lev < nlevels; ++lev) + for (int lev = 0; lev < nlevels; ++lev) { const std::unique_ptr empty_ptr; const std::string raw_spltname = slice_plotfilename + "/raw_fields"; amrex::Print() << " raw spltname " << raw_spltname << "\n"; const DistributionMapping &dm = Efield_slice[lev][0]->DistributionMap(); - + WriteRawField( *Efield_slice[lev][0], dm, raw_spltname, level_prefix, "Ex_slice", lev, 0); WriteRawField( *Efield_slice[lev][1], dm, raw_spltname, level_prefix, "Ey_slice", lev, 0); WriteRawField( *Efield_slice[lev][2], dm, raw_spltname, level_prefix, "Ez_slice", lev, 0); @@ -807,8 +807,8 @@ WarpX::WriteSlicePlotFile () const MultiFab rho_new(*rho_slice[lev], amrex::make_alias, 1, 1); WriteRawField( rho_new, dm, raw_spltname, level_prefix, "rho_slice", lev, 0); } - } - + } + WriteJobInfo(slice_plotfilename); WriteWarpXHeader(slice_plotfilename); @@ -817,7 +817,7 @@ WarpX::WriteSlicePlotFile () const } -void +void WarpX::InitializeSliceMultiFabs () { @@ -828,12 +828,12 @@ WarpX::InitializeSliceMultiFabs () current_slice.resize(nlevels); Efield_slice.resize(nlevels); Bfield_slice.resize(nlevels); - + } // To generate slice that inherits index type of underlying data // -void +void WarpX::SliceGenerationForDiagnostics () { @@ -841,20 +841,20 @@ WarpX::SliceGenerationForDiagnostics () dom_geom = Geom(); if (F_fp[0] ) { - F_slice[0] = CreateSlice( *F_fp[0].get(), dom_geom, slice_realbox, + F_slice[0] = CreateSlice( *F_fp[0].get(), dom_geom, slice_realbox, slice_cr_ratio ); } if (rho_fp[0]) { - rho_slice[0] = CreateSlice( *rho_fp[0].get(), dom_geom, slice_realbox, + rho_slice[0] = CreateSlice( *rho_fp[0].get(), dom_geom, slice_realbox, slice_cr_ratio ); } for (int idim = 0; idim < 3; ++idim) { - Efield_slice[0][idim] = CreateSlice( *Efield_fp[0][idim].get(), + Efield_slice[0][idim] = CreateSlice( *Efield_fp[0][idim].get(), dom_geom, slice_realbox, slice_cr_ratio ); - Bfield_slice[0][idim] = CreateSlice( *Bfield_fp[0][idim].get(), + Bfield_slice[0][idim] = CreateSlice( *Bfield_fp[0][idim].get(), dom_geom, slice_realbox, slice_cr_ratio ); - current_slice[0][idim] = CreateSlice( *current_fp[0][idim].get(), + current_slice[0][idim] = CreateSlice( *current_fp[0][idim].get(), dom_geom, slice_realbox, slice_cr_ratio ); } @@ -862,7 +862,7 @@ WarpX::SliceGenerationForDiagnostics () } -void +void WarpX::ClearSliceMultiFabs () { diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index 8f800665d..a4ed803ee 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -148,7 +148,7 @@ WarpX::EvolveEM (int numsteps) } myBFD->writeLabFrameData(cell_centered_data.get(), *mypc, geom[0], cur_time, dt[0]); } - + bool move_j = is_synchronized || to_make_plot || do_insitu; // If is_synchronized we need to shift j too so that next step we can evolve E by dt/2. // We might need to move j because we are going to make a plotfile. @@ -277,7 +277,7 @@ WarpX::EvolveEM (int numsteps) void WarpX::OneStep_nosub (Real cur_time) { - // Loop over species. For each ionizable species, create particles in + // Loop over species. For each ionizable species, create particles in // product species. mypc->doFieldIonization(); // Push particle from x^{n} to x^{n+1} @@ -349,7 +349,7 @@ WarpX::OneStep_sub1 (Real curtime) { // TODO: we could save some charge depositions - // Loop over species. For each ionizable species, create particles in + // Loop over species. For each ionizable species, create particles in // product species. mypc->doFieldIonization(); diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PMLPsatdAlgorithm.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PMLPsatdAlgorithm.H index a2511b6b7..50eb5c9b1 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PMLPsatdAlgorithm.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PMLPsatdAlgorithm.H @@ -17,7 +17,7 @@ class PMLPsatdAlgorithm : public SpectralBaseAlgorithm void InitializeSpectralCoefficients( const SpectralKSpace& spectral_kspace, - const amrex::DistributionMapping& dm, + const amrex::DistributionMapping& dm, const amrex::Real dt); // Redefine functions from base class diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithm.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithm.H index 825d04dc2..bc7f90fac 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithm.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithm.H @@ -20,7 +20,7 @@ class PsatdAlgorithm : public SpectralBaseAlgorithm virtual int getRequiredNumberOfFields() const override final { return SpectralFieldIndex::n_fields; } - + void InitializeSpectralCoefficients(const SpectralKSpace& spectral_kspace, const amrex::DistributionMapping& dm, const amrex::Real dt); diff --git a/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp b/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp index 6fe5e3939..aee131324 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp @@ -149,7 +149,7 @@ SpectralKSpace::getSpectralShiftFactor( const DistributionMapping& dm, #else shift[i] = std::exp( I*sign*k[i]*0.5*dx[i_dim] ); #endif - + } } return shift_factor; diff --git a/Source/FieldSolver/openbc_poisson_solver.F90 b/Source/FieldSolver/openbc_poisson_solver.F90 index c18b1db24..8e962a2ef 100644 --- a/Source/FieldSolver/openbc_poisson_solver.F90 +++ b/Source/FieldSolver/openbc_poisson_solver.F90 @@ -4,7 +4,7 @@ module warpx_openbc_module implicit none integer, parameter :: idecomp = 0 ! 0=xyz, 1=xy, 2=yz, 3=xz, 4=x, 5=y, 6=z. - integer, parameter :: igfflag = 1 ! =0 for ordinary 1/r Green function; + integer, parameter :: igfflag = 1 ! =0 for ordinary 1/r Green function; ! =1 for integrated Green function integer, save :: gb_lo(3), gb_hi(3) diff --git a/Source/FieldSolver/solve_E_nodal.F90 b/Source/FieldSolver/solve_E_nodal.F90 index e5ac50b2a..4ed9bb845 100644 --- a/Source/FieldSolver/solve_E_nodal.F90 +++ b/Source/FieldSolver/solve_E_nodal.F90 @@ -8,7 +8,7 @@ module warpx_ES_solve_E_nodal contains ! This routine computes the node-centered electric field given a node-centered phi. -! The gradient is computed using 2nd-order centered differences. It assumes the +! The gradient is computed using 2nd-order centered differences. It assumes the ! Boundary conditions have already been set and that you have two rows of ghost cells ! for phi and one row of ghost cells for Ex, Ey, and Ez. ! Note that this routine includes the minus sign in E = - grad phi. @@ -35,7 +35,7 @@ contains do k = lo(3)-1, hi(3)+1 do j = lo(2)-1, hi(2)+1 do i = lo(1)-1, hi(1)+1 - + Ex(i,j,k) = fac(1) * (phi(i-1,j,k) - phi(i+1,j,k)) Ey(i,j,k) = fac(2) * (phi(i,j-1,k) - phi(i,j+1,k)) Ez(i,j,k) = fac(3) * (phi(i,j,k-1) - phi(i,j,k+1)) @@ -61,7 +61,7 @@ contains do j = lo(2)-1, hi(2)+1 do i = lo(1)-1, hi(1)+1 - + Ex(i,j) = fac(1) * (phi(i-1,j) - phi(i+1,j)) Ey(i,j) = fac(2) * (phi(i,j-1) - phi(i,j+1)) diff --git a/Source/Filter/BilinearFilter.H b/Source/Filter/BilinearFilter.H index 4fcfec14b..150ca8e08 100644 --- a/Source/Filter/BilinearFilter.H +++ b/Source/Filter/BilinearFilter.H @@ -8,7 +8,7 @@ class BilinearFilter : public Filter public: BilinearFilter() = default; - + void ComputeStencils(); amrex::IntVect npass_each_dir; diff --git a/Source/Filter/BilinearFilter.cpp b/Source/Filter/BilinearFilter.cpp index 68ebde687..23abaf8bc 100644 --- a/Source/Filter/BilinearFilter.cpp +++ b/Source/Filter/BilinearFilter.cpp @@ -23,8 +23,8 @@ namespace { new_s[0] = 0.5 * old_s[0]; if (1 v; - // Read parameters for the predefined plasma profile, + // Read parameters for the predefined plasma profile, // and store them in managed memory pp.getarr("predefined_profile_params", v); p = static_cast diff --git a/Source/Initialization/InjectorMomentum.H b/Source/Initialization/InjectorMomentum.H index 7dced1336..ba0d26fc4 100644 --- a/Source/Initialization/InjectorMomentum.H +++ b/Source/Initialization/InjectorMomentum.H @@ -23,7 +23,7 @@ private: amrex::Real m_ux, m_uy, m_uz; }; -// struct whose getMomentum returns momentum for 1 particle, from random +// struct whose getMomentum returns momentum for 1 particle, from random // gaussian distribution. struct InjectorMomentumGaussian { @@ -85,9 +85,9 @@ struct InjectorMomentumParser GpuParser m_ux_parser, m_uy_parser, m_uz_parser; }; -// Base struct for momentum injector. -// InjectorMomentum contains a union (called Object) that holds any one -// instance of: +// Base struct for momentum injector. +// InjectorMomentum contains a union (called Object) that holds any one +// instance of: // - InjectorMomentumConstant : to generate constant density; // - InjectorMomentumGaussian : to generate gaussian distribution; // - InjectorMomentumRadialExpansion: to generate radial expansion; @@ -190,7 +190,7 @@ private: Type type; // An instance of union Object constructs and stores any one of - // the objects declared (constant or custom or gaussian or + // the objects declared (constant or custom or gaussian or // radial_expansion or parser). union Object { Object (InjectorMomentumConstant*, diff --git a/Source/Initialization/InjectorPosition.H b/Source/Initialization/InjectorPosition.H index 0e43f889e..f8f16746c 100644 --- a/Source/Initialization/InjectorPosition.H +++ b/Source/Initialization/InjectorPosition.H @@ -25,7 +25,7 @@ struct InjectorPositionRegular // i_part: particle number within the cell, required to evenly space // particles within the cell. - // ref_fac: the number of particles evenly-spaced within a cell + // ref_fac: the number of particles evenly-spaced within a cell // is a_ppc*(ref_fac**AMREX_SPACEDIM). AMREX_GPU_HOST_DEVICE amrex::XDim3 @@ -47,9 +47,9 @@ private: amrex::Dim3 ppc; }; -// Base struct for position injector. -// InjectorPosition contains a union (called Object) that holds any one -// instance of: +// Base struct for position injector. +// InjectorPosition contains a union (called Object) that holds any one +// instance of: // - InjectorPositionRandom : to generate random distribution; // - InjectorPositionRegular: to generate regular distribution. // The choice is made at runtime, depending in the constructor called. diff --git a/Source/Initialization/PlasmaInjector.H b/Source/Initialization/PlasmaInjector.H index 875fdd4ad..a944165d6 100644 --- a/Source/Initialization/PlasmaInjector.H +++ b/Source/Initialization/PlasmaInjector.H @@ -96,9 +96,9 @@ protected: std::unique_ptr inj_pos; std::unique_ptr inj_rho; std::unique_ptr inj_mom; - + void parseDensity (amrex::ParmParse& pp); - void parseMomentum (amrex::ParmParse& pp); + void parseMomentum (amrex::ParmParse& pp); }; #endif diff --git a/Source/Laser/LaserParticleContainer.H b/Source/Laser/LaserParticleContainer.H index 2fd42f168..782282a26 100644 --- a/Source/Laser/LaserParticleContainer.H +++ b/Source/Laser/LaserParticleContainer.H @@ -18,12 +18,12 @@ public: virtual void InitData () final; -#ifdef WARPX_DO_ELECTROSTATIC +#ifdef WARPX_DO_ELECTROSTATIC virtual void EvolveES (const amrex::Vector, 3> >& E, amrex::Vector >& rho, amrex::Real t, amrex::Real dt) { BL_ASSERT(false); } #endif // WARPX_DO_ELECTROSTATIC - + virtual void Evolve (int lev, const amrex::MultiFab&, const amrex::MultiFab&, const amrex::MultiFab&, const amrex::MultiFab&, const amrex::MultiFab&, const amrex::MultiFab&, @@ -64,7 +64,7 @@ public: const amrex::Real dt, const int thread_num); protected: - + std::string laser_name; private: @@ -107,11 +107,11 @@ private: // Theoretical position of the antenna. Used if do_continuous_injection=1. // Track the position of the antenna until it enters the simulation domain. amrex::Vector updated_position; - + void ComputeSpacing (int lev, amrex::Real& Sx, amrex::Real& Sy) const; void ComputeWeightMobility (amrex::Real Sx, amrex::Real Sy); void InitData (int lev); - // Inject the laser antenna during the simulation, if it started + // Inject the laser antenna during the simulation, if it started // outside of the simulation domain and enters it. void ContinuousInjection(const amrex::RealBox& injection_box) override; // Update position of the antenna diff --git a/Source/Laser/LaserParticleContainer.cpp b/Source/Laser/LaserParticleContainer.cpp index 9699ee06b..d0ff51313 100644 --- a/Source/Laser/LaserParticleContainer.cpp +++ b/Source/Laser/LaserParticleContainer.cpp @@ -27,7 +27,7 @@ LaserParticleContainer::LaserParticleContainer (AmrCore* amr_core, int ispecies, charge = 1.0; mass = std::numeric_limits::max(); do_boosted_frame_diags = 0; - + ParmParse pp(laser_name); // Parse the type of laser profile and set the corresponding flag `profile` @@ -166,7 +166,7 @@ LaserParticleContainer::LaserParticleContainer (AmrCore* amr_core, int ispecies, // If laser antenna initially outside of the box, store its theoretical // position in z_antenna_th updated_position = position; - + // Sanity checks int dir = WarpX::moving_window_dir; std::vector windir(3, 0.0); @@ -175,16 +175,16 @@ LaserParticleContainer::LaserParticleContainer (AmrCore* amr_core, int ispecies, #else windir[dir] = 1.0; #endif - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( - (nvec[0]-windir[0]) + (nvec[1]-windir[1]) + (nvec[2]-windir[2]) + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + (nvec[0]-windir[0]) + (nvec[1]-windir[1]) + (nvec[2]-windir[2]) < 1.e-12, "do_continous_injection for laser particle only works" + " if moving window direction and laser propagation direction are the same"); if ( WarpX::gamma_boost>1 ){ AMREX_ALWAYS_ASSERT_WITH_MESSAGE( - (WarpX::boost_direction[0]-0)*(WarpX::boost_direction[0]-0) + - (WarpX::boost_direction[1]-0)*(WarpX::boost_direction[1]-0) + + (WarpX::boost_direction[0]-0)*(WarpX::boost_direction[0]-0) + + (WarpX::boost_direction[1]-0)*(WarpX::boost_direction[1]-0) + (WarpX::boost_direction[2]-1)*(WarpX::boost_direction[2]-1) < 1.e-12, - "do_continous_injection for laser particle only works if " + + "do_continous_injection for laser particle only works if " + "warpx.boost_direction = z. TODO: all directions."); } } @@ -192,13 +192,13 @@ LaserParticleContainer::LaserParticleContainer (AmrCore* amr_core, int ispecies, /* \brief Check if laser particles enter the box, and inject if necessary. * \param injection_box: a RealBox where particles should be injected. - */ + */ void LaserParticleContainer::ContinuousInjection (const RealBox& injection_box) { // Input parameter injection_box contains small box where injection // should occur. - // So far, LaserParticleContainer::laser_injection_box contains the + // So far, LaserParticleContainer::laser_injection_box contains the // outdated full problem domain at t=0. // Convert updated_position to Real* to use RealBox::contains(). @@ -258,7 +258,7 @@ LaserParticleContainer::InitData (int lev) ComputeSpacing(lev, S_X, S_Y); ComputeWeightMobility(S_X, S_Y); - // LaserParticleContainer::position contains the initial position of the + // LaserParticleContainer::position contains the initial position of the // laser antenna. In the boosted frame, the antenna is moving. // Update its position with updated_position. if (do_continuous_injection){ @@ -459,7 +459,7 @@ LaserParticleContainer::Evolve (int lev, if (rho) { int* AMREX_RESTRICT ion_lev = nullptr; - DepositCharge(pti, wp, ion_lev, rho, 0, 0, + DepositCharge(pti, wp, ion_lev, rho, 0, 0, np_current, thread_num, lev, lev); if (crho) { DepositCharge(pti, wp, ion_lev, crho, 0, np_current, @@ -606,7 +606,7 @@ LaserParticleContainer::PushP (int lev, Real dt, * \param np: number of laser particles * \param thread_num: thread number * \param pplane_Xp, pplane_Yp: pointers to arrays of particle positions - * in laser plane coordinate. + * in laser plane coordinate. */ void LaserParticleContainer::calculate_laser_plane_coordinates ( @@ -633,16 +633,16 @@ LaserParticleContainer::calculate_laser_plane_coordinates ( np, [=] AMREX_GPU_DEVICE (int i) { #if (AMREX_SPACEDIM == 3) - pplane_Xp[i] = + pplane_Xp[i] = tmp_u_X_0 * (xp[i] - tmp_position_0) + tmp_u_X_1 * (yp[i] - tmp_position_1) + tmp_u_X_2 * (zp[i] - tmp_position_2); - pplane_Yp[i] = + pplane_Yp[i] = tmp_u_Y_0 * (xp[i] - tmp_position_0) + tmp_u_Y_1 * (yp[i] - tmp_position_1) + tmp_u_Y_2 * (zp[i] - tmp_position_2); #elif (AMREX_SPACEDIM == 2) - pplane_Xp[i] = + pplane_Xp[i] = tmp_u_X_0 * (xp[i] - tmp_position_0) + tmp_u_X_2 * (zp[i] - tmp_position_2); pplane_Yp[i] = 0.; @@ -677,7 +677,7 @@ LaserParticleContainer::update_laser_particle( Real tmp_nvec_0 = nvec[0]; Real tmp_nvec_1 = nvec[1]; Real tmp_nvec_2 = nvec[2]; - + // Copy member variables to tmp copies for GPU runs. Real tmp_mobility = mobility; Real gamma_boost = WarpX::gamma_boost; diff --git a/Source/Laser/LaserProfiles.cpp b/Source/Laser/LaserProfiles.cpp index 704107c2d..69804b17c 100644 --- a/Source/Laser/LaserProfiles.cpp +++ b/Source/Laser/LaserProfiles.cpp @@ -9,14 +9,14 @@ using namespace amrex; * Both Xp and Yp are given in laser plane coordinate. * For each particle with position Xp and Yp, this routine computes the * amplitude of the laser electric field, stored in array amplitude. - * + * * \param np: number of laser particles * \param Xp: pointer to first component of positions of laser particles * \param Yp: pointer to second component of positions of laser particles * \param t: Current physical time * \param amplitude: pointer to array of field amplitude. */ -void +void LaserParticleContainer::gaussian_laser_profile ( const int np, Real const * AMREX_RESTRICT const Xp, Real const * AMREX_RESTRICT const Yp, Real t, Real * AMREX_RESTRICT const amplitude) @@ -34,9 +34,9 @@ LaserParticleContainer::gaussian_laser_profile ( // Time stretching due to STCs and phi2 complex envelope // (1 if zeta=0, beta=0, phi2=0) - const Complex stretch_factor = 1. + 4. * + const Complex stretch_factor = 1. + 4. * (zeta+beta*profile_focal_distance) * (zeta+beta*profile_focal_distance) - * (inv_tau2*inv_complex_waist_2) + + * (inv_tau2*inv_complex_waist_2) + 2.*I*(phi2 - beta*beta*k0*profile_focal_distance) * inv_tau2; // Amplitude and monochromatic oscillations @@ -59,10 +59,10 @@ LaserParticleContainer::gaussian_laser_profile ( Real tmp_profile_focal_distance = profile_focal_distance; // Loop through the macroparticle to calculate the proper amplitude amrex::ParallelFor( - np, + np, [=] AMREX_GPU_DEVICE (int i) { const Complex stc_exponent = 1./stretch_factor * inv_tau2 * - MathFunc::pow((t - tmp_profile_t_peak - + MathFunc::pow((t - tmp_profile_t_peak - tmp_beta*k0*(Xp[i]*std::cos(tmp_theta_stc) + Yp[i]*std::sin(tmp_theta_stc)) - 2.*I*(Xp[i]*std::cos(tmp_theta_stc) + Yp[i]*std::sin(tmp_theta_stc)) *( tmp_zeta - tmp_beta*tmp_profile_focal_distance ) * inv_complex_waist_2),2); @@ -81,14 +81,14 @@ LaserParticleContainer::gaussian_laser_profile ( * Both Xp and Yp are given in laser plane coordinate. * For each particle with position Xp and Yp, this routine computes the * amplitude of the laser electric field, stored in array amplitude. - * + * * \param np: number of laser particles * \param Xp: pointer to first component of positions of laser particles * \param Yp: pointer to second component of positions of laser particles * \param t: Current physical time * \param amplitude: pointer to array of field amplitude. */ -void +void LaserParticleContainer::harris_laser_profile ( const int np, Real const * AMREX_RESTRICT const Xp, Real const * AMREX_RESTRICT const Yp, Real t, Real * AMREX_RESTRICT const amplitude) @@ -96,14 +96,14 @@ LaserParticleContainer::harris_laser_profile ( // This function uses the Harris function as the temporal profile of the pulse const Real omega0 = 2.*MathConst::pi*PhysConst::c/wavelength; const Real zR = MathConst::pi * profile_waist*profile_waist / wavelength; - const Real wz = profile_waist * + const Real wz = profile_waist * std::sqrt(1. + profile_focal_distance*profile_focal_distance/zR*zR); const Real inv_wz_2 = 1./(wz*wz); Real inv_Rz; - if (profile_focal_distance == 0.){ + if (profile_focal_distance == 0.){ inv_Rz = 0.; } else { - inv_Rz = -profile_focal_distance / + inv_Rz = -profile_focal_distance / ( profile_focal_distance*profile_focal_distance + zR*zR ); } const Real arg_env = 2.*MathConst::pi*t/profile_duration; @@ -111,8 +111,8 @@ LaserParticleContainer::harris_laser_profile ( // time envelope is given by the Harris function Real time_envelope = 0.; if (t < profile_duration) - time_envelope = 1./32. * (10. - 15.*std::cos(arg_env) + - 6.*std::cos(2.*arg_env) - + time_envelope = 1./32. * (10. - 15.*std::cos(arg_env) + + 6.*std::cos(2.*arg_env) - std::cos(3.*arg_env)); // Copy member variables to tmp copies for GPU runs. @@ -121,7 +121,7 @@ LaserParticleContainer::harris_laser_profile ( amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (int i) { - const Real space_envelope = + const Real space_envelope = std::exp(- ( Xp[i]*Xp[i] + Yp[i]*Yp[i] ) * inv_wz_2); const Real arg_osc = omega0*t - omega0/PhysConst::c* (Xp[i]*Xp[i] + Yp[i]*Yp[i]) * inv_Rz / 2.; diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index 9b3baafe1..c737591c7 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -33,11 +33,11 @@ WarpX::RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMa { if (ParallelDescriptor::NProcs() == 1) return; -#ifdef WARPX_DO_ELECTROSTATIC +#ifdef WARPX_DO_ELECTROSTATIC AMREX_ALWAYS_ASSERT(masks[lev] == nullptr); AMREX_ALWAYS_ASSERT(gather_masks[lev] == nullptr); #endif // WARPX_DO_ELECTROSTATIC - + // Fine patch const auto& period = Geom(lev).periodicity(); @@ -203,7 +203,7 @@ WarpX::RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMa auto pmf = std::unique_ptr(new iMultiFab(current_buffer_masks[lev]->boxArray(), dm, current_buffer_masks[lev]->nComp(), ng)); // pmf->ParallelCopy(*current_buffer_masks[lev], 0, 0, current_buffer_masks[lev]->nComp(), ng, ng); - current_buffer_masks[lev] = std::move(pmf); + current_buffer_masks[lev] = std::move(pmf); } if (gather_buffer_masks[lev]) { @@ -211,7 +211,7 @@ WarpX::RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMa auto pmf = std::unique_ptr(new iMultiFab(gather_buffer_masks[lev]->boxArray(), dm, gather_buffer_masks[lev]->nComp(), ng)); // pmf->ParallelCopy(*gather_buffer_masks[lev], 0, 0, gather_buffer_masks[lev]->nComp(), ng, ng); - gather_buffer_masks[lev] = std::move(pmf); + gather_buffer_masks[lev] = std::move(pmf); } } diff --git a/Source/Parser/GpuParser.H b/Source/Parser/GpuParser.H index 1533ee6b9..e49671e06 100644 --- a/Source/Parser/GpuParser.H +++ b/Source/Parser/GpuParser.H @@ -5,7 +5,7 @@ #include // When compiled for CPU, wrap WarpXParser and enable threading. -// When compiled for GPU, store one copy of the parser in +// When compiled for GPU, store one copy of the parser in // CUDA managed memory for __device__ code, and one copy of the parser // in CUDA managed memory for __host__ code. This way, the parser can be // efficiently called from both host and device. diff --git a/Source/Parser/GpuParser.cpp b/Source/Parser/GpuParser.cpp index db1c2287d..5078b498b 100644 --- a/Source/Parser/GpuParser.cpp +++ b/Source/Parser/GpuParser.cpp @@ -28,7 +28,7 @@ GpuParser::GpuParser (WarpXParser const& wp) wp_parser_regvar(&m_cpu_parser, "x", &(m_var.x)); wp_parser_regvar(&m_cpu_parser, "y", &(m_var.y)); wp_parser_regvar(&m_cpu_parser, "z", &(m_var.z)); - + #else // not defined AMREX_USE_GPU #ifdef _OPENMP diff --git a/Source/Parser/wp_parser.lex.c b/Source/Parser/wp_parser.lex.c index 0a9c58ee9..09a1b2db8 100644 --- a/Source/Parser/wp_parser.lex.c +++ b/Source/Parser/wp_parser.lex.c @@ -34,7 +34,7 @@ #if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 says to define __STDC_LIMIT_MACROS before including stdint.h, - * if you want the limit (max/min) macros for int types. + * if you want the limit (max/min) macros for int types. */ #ifndef __STDC_LIMIT_MACROS #define __STDC_LIMIT_MACROS 1 @@ -51,7 +51,7 @@ typedef uint32_t flex_uint32_t; typedef signed char flex_int8_t; typedef short int flex_int16_t; typedef int flex_int32_t; -typedef unsigned char flex_uint8_t; +typedef unsigned char flex_uint8_t; typedef unsigned short int flex_uint16_t; typedef unsigned int flex_uint32_t; @@ -162,10 +162,10 @@ extern FILE *yyin, *yyout; #define EOB_ACT_CONTINUE_SCAN 0 #define EOB_ACT_END_OF_FILE 1 #define EOB_ACT_LAST_MATCH 2 - + #define YY_LESS_LINENO(n) #define YY_LINENO_REWIND_TO(ptr) - + /* Return all but the first "n" matched characters back to the input stream. */ #define yyless(n) \ do \ @@ -590,9 +590,9 @@ extern int yywrap ( void ); #endif #ifndef YY_NO_UNPUT - + static void yyunput ( int c, char *buf_ptr ); - + #endif #ifndef yytext_ptr @@ -719,7 +719,7 @@ YY_DECL yy_state_type yy_current_state; char *yy_cp, *yy_bp; int yy_act; - + if ( !(yy_init) ) { (yy_init) = 1; @@ -1300,7 +1300,7 @@ static int yy_get_next_buffer (void) { yy_state_type yy_current_state; char *yy_cp; - + yy_current_state = (yy_start); for ( yy_cp = (yytext_ptr) + YY_MORE_ADJ; yy_cp < (yy_c_buf_p); ++yy_cp ) @@ -1356,7 +1356,7 @@ static int yy_get_next_buffer (void) static void yyunput (int c, char * yy_bp ) { char *yy_cp; - + yy_cp = (yy_c_buf_p); /* undo effects of setting up yytext */ @@ -1401,7 +1401,7 @@ static int yy_get_next_buffer (void) { int c; - + *(yy_c_buf_p) = (yy_hold_char); if ( *(yy_c_buf_p) == YY_END_OF_BUFFER_CHAR ) @@ -1468,12 +1468,12 @@ static int yy_get_next_buffer (void) /** Immediately switch to a different input stream. * @param input_file A readable stream. - * + * * @note This function does not reset the start condition to @c INITIAL . */ void yyrestart (FILE * input_file ) { - + if ( ! YY_CURRENT_BUFFER ){ yyensure_buffer_stack (); YY_CURRENT_BUFFER_LVALUE = @@ -1486,11 +1486,11 @@ static int yy_get_next_buffer (void) /** Switch to a different input buffer. * @param new_buffer The new input buffer. - * + * */ void yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ) { - + /* TODO. We should be able to replace this entire function body * with * yypop_buffer_state(); @@ -1530,13 +1530,13 @@ static void yy_load_buffer_state (void) /** Allocate and initialize an input buffer state. * @param file A readable stream. * @param size The character buffer size in bytes. When in doubt, use @c YY_BUF_SIZE. - * + * * @return the allocated buffer state. */ YY_BUFFER_STATE yy_create_buffer (FILE * file, int size ) { YY_BUFFER_STATE b; - + b = (YY_BUFFER_STATE) yyalloc( sizeof( struct yy_buffer_state ) ); if ( ! b ) YY_FATAL_ERROR( "out of dynamic memory in yy_create_buffer()" ); @@ -1559,11 +1559,11 @@ static void yy_load_buffer_state (void) /** Destroy the buffer. * @param b a buffer created with yy_create_buffer() - * + * */ void yy_delete_buffer (YY_BUFFER_STATE b ) { - + if ( ! b ) return; @@ -1584,7 +1584,7 @@ static void yy_load_buffer_state (void) { int oerrno = errno; - + yy_flush_buffer( b ); b->yy_input_file = file; @@ -1600,13 +1600,13 @@ static void yy_load_buffer_state (void) } b->yy_is_interactive = file ? (isatty( fileno(file) ) > 0) : 0; - + errno = oerrno; } /** Discard all buffered characters. On the next scan, YY_INPUT will be called. * @param b the buffer state to be flushed, usually @c YY_CURRENT_BUFFER. - * + * */ void yy_flush_buffer (YY_BUFFER_STATE b ) { @@ -1635,7 +1635,7 @@ static void yy_load_buffer_state (void) * the current state. This function will allocate the stack * if necessary. * @param new_buffer The new state. - * + * */ void yypush_buffer_state (YY_BUFFER_STATE new_buffer ) { @@ -1665,7 +1665,7 @@ void yypush_buffer_state (YY_BUFFER_STATE new_buffer ) /** Removes and deletes the top of the stack, if present. * The next element becomes the new top. - * + * */ void yypop_buffer_state (void) { @@ -1689,7 +1689,7 @@ void yypop_buffer_state (void) static void yyensure_buffer_stack (void) { yy_size_t num_to_alloc; - + if (!(yy_buffer_stack)) { /* First allocation is just for 2 elements, since we don't know if this @@ -1732,13 +1732,13 @@ static void yyensure_buffer_stack (void) /** Setup the input buffer state to scan directly from a user-specified character buffer. * @param base the character buffer * @param size the size in bytes of the character buffer - * + * * @return the newly allocated buffer state object. */ YY_BUFFER_STATE yy_scan_buffer (char * base, yy_size_t size ) { YY_BUFFER_STATE b; - + if ( size < 2 || base[size-2] != YY_END_OF_BUFFER_CHAR || base[size-1] != YY_END_OF_BUFFER_CHAR ) @@ -1767,14 +1767,14 @@ YY_BUFFER_STATE yy_scan_buffer (char * base, yy_size_t size ) /** Setup the input buffer state to scan a string. The next call to yylex() will * scan from a @e copy of @a str. * @param yystr a NUL-terminated string to scan - * + * * @return the newly allocated buffer state object. * @note If you want to scan bytes that may contain NUL values, then use * yy_scan_bytes() instead. */ YY_BUFFER_STATE yy_scan_string (const char * yystr ) { - + return yy_scan_bytes( yystr, (int) strlen(yystr) ); } @@ -1782,7 +1782,7 @@ YY_BUFFER_STATE yy_scan_string (const char * yystr ) * scan from a @e copy of @a bytes. * @param yybytes the byte buffer to scan * @param _yybytes_len the number of bytes in the buffer pointed to by @a bytes. - * + * * @return the newly allocated buffer state object. */ YY_BUFFER_STATE yy_scan_bytes (const char * yybytes, int _yybytes_len ) @@ -1791,7 +1791,7 @@ YY_BUFFER_STATE yy_scan_bytes (const char * yybytes, int _yybytes_len ) char *buf; yy_size_t n; int i; - + /* Get memory for full buffer, including space for trailing EOB's. */ n = (yy_size_t) (_yybytes_len + 2); buf = (char *) yyalloc( n ); @@ -1845,16 +1845,16 @@ static void yynoreturn yy_fatal_error (const char* msg ) /* Accessor methods (get/set functions) to struct members. */ /** Get the current line number. - * + * */ int yyget_lineno (void) { - + return yylineno; } /** Get the input stream. - * + * */ FILE *yyget_in (void) { @@ -1862,7 +1862,7 @@ FILE *yyget_in (void) } /** Get the output stream. - * + * */ FILE *yyget_out (void) { @@ -1870,7 +1870,7 @@ FILE *yyget_out (void) } /** Get the length of the current token. - * + * */ int yyget_leng (void) { @@ -1878,7 +1878,7 @@ int yyget_leng (void) } /** Get the current token. - * + * */ char *yyget_text (void) @@ -1888,18 +1888,18 @@ char *yyget_text (void) /** Set the current line number. * @param _line_number line number - * + * */ void yyset_lineno (int _line_number ) { - + yylineno = _line_number; } /** Set the input stream. This does not discard the current * input buffer. * @param _in_str A readable stream. - * + * * @see yy_switch_to_buffer */ void yyset_in (FILE * _in_str ) @@ -1953,7 +1953,7 @@ static int yy_init_globals (void) /* yylex_destroy is for both reentrant and non-reentrant scanners. */ int yylex_destroy (void) { - + /* Pop the buffer stack, destroying each element. */ while(YY_CURRENT_BUFFER){ yy_delete_buffer( YY_CURRENT_BUFFER ); @@ -1979,7 +1979,7 @@ int yylex_destroy (void) #ifndef yytext_ptr static void yy_flex_strncpy (char* s1, const char * s2, int n ) { - + int i; for ( i = 0; i < n; ++i ) s1[i] = s2[i]; @@ -2004,7 +2004,7 @@ void *yyalloc (yy_size_t size ) void *yyrealloc (void * ptr, yy_size_t size ) { - + /* The cast to (char *) in the following accommodates both * implementations that use char* generic pointers, and those * that use void* generic pointers. It works with the latter diff --git a/Source/Parser/wp_parser.lex.h b/Source/Parser/wp_parser.lex.h index 4fd613978..557add970 100644 --- a/Source/Parser/wp_parser.lex.h +++ b/Source/Parser/wp_parser.lex.h @@ -38,7 +38,7 @@ #if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 says to define __STDC_LIMIT_MACROS before including stdint.h, - * if you want the limit (max/min) macros for int types. + * if you want the limit (max/min) macros for int types. */ #ifndef __STDC_LIMIT_MACROS #define __STDC_LIMIT_MACROS 1 @@ -55,7 +55,7 @@ typedef uint32_t flex_uint32_t; typedef signed char flex_int8_t; typedef short int flex_int16_t; typedef int flex_int32_t; -typedef unsigned char flex_uint8_t; +typedef unsigned char flex_uint8_t; typedef unsigned short int flex_uint16_t; typedef unsigned int flex_uint32_t; diff --git a/Source/Particles/Deposition/ChargeDeposition.H b/Source/Particles/Deposition/ChargeDeposition.H index f02eb1033..b9210e67c 100755 --- a/Source/Particles/Deposition/ChargeDeposition.H +++ b/Source/Particles/Deposition/ChargeDeposition.H @@ -18,8 +18,8 @@ * /param q : species charge. */ template -void doChargeDepositionShapeN(const amrex::Real * const xp, - const amrex::Real * const yp, +void doChargeDepositionShapeN(const amrex::Real * const xp, + const amrex::Real * const yp, const amrex::Real * const zp, const amrex::Real * const wp, const int * const ion_lev, @@ -69,7 +69,7 @@ void doChargeDepositionShapeN(const amrex::Real * const xp, amrex::Real sx[depos_order + 1]; // i: leftmost grid point (node-centered) that the particle touches const int i = compute_shape_factor(sx, x); - + #if (defined WARPX_DIM_3D) // y direction const amrex::Real y = (yp[ip] - ymin)*dyi; @@ -86,7 +86,7 @@ void doChargeDepositionShapeN(const amrex::Real * const xp, for (int iz=0; iz<=depos_order; iz++){ for (int ix=0; ix<=depos_order; ix++){ amrex::Gpu::Atomic::Add( - &rho_arr(lo.x+i+ix, lo.y+k+iz, 0), + &rho_arr(lo.x+i+ix, lo.y+k+iz, 0), sx[ix]*sz[iz]*wq); } } diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index c7dfde75a..7a96dab9a 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -24,8 +24,8 @@ * /param q : species charge. */ template -void doDepositionShapeN(const amrex::Real * const xp, - const amrex::Real * const yp, +void doDepositionShapeN(const amrex::Real * const xp, + const amrex::Real * const yp, const amrex::Real * const zp, const amrex::Real * const wp, const amrex::Real * const uxp, @@ -39,7 +39,7 @@ void doDepositionShapeN(const amrex::Real * const xp, const std::array& dx, const std::array xyzmin, const amrex::Dim3 lo, - const amrex::Real stagger_shift, + const amrex::Real stagger_shift, const amrex::Real q) { // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer @@ -77,7 +77,7 @@ void doDepositionShapeN(const amrex::Real * const xp, const amrex::Real vx = uxp[ip]*gaminv; const amrex::Real vy = uyp[ip]*gaminv; const amrex::Real vz = uzp[ip]*gaminv; - // wqx, wqy wqz are particle current in each direction + // wqx, wqy wqz are particle current in each direction #if (defined WARPX_DIM_RZ) // In RZ, wqx is actually wqr, and wqy is wqtheta // Convert to cylinderical at the mid point @@ -117,7 +117,7 @@ void doDepositionShapeN(const amrex::Real * const xp, amrex::Real sx0[depos_order + 1]; // j0: leftmost grid point (cell-centered) that the particle touches const int j0 = compute_shape_factor(sx0, xmid-stagger_shift); - + #if (defined WARPX_DIM_3D) // y direction const amrex::Real ymid= (yp[ip]-ymin)*dyi-dts2dy*vy; @@ -138,13 +138,13 @@ void doDepositionShapeN(const amrex::Real * const xp, for (int iz=0; iz<=depos_order; iz++){ for (int ix=0; ix<=depos_order; ix++){ amrex::Gpu::Atomic::Add( - &jx_arr(lo.x+j0+ix, lo.y+l +iz, 0), + &jx_arr(lo.x+j0+ix, lo.y+l +iz, 0), sx0[ix]*sz [iz]*wqx); amrex::Gpu::Atomic::Add( - &jy_arr(lo.x+j +ix, lo.y+l +iz, 0), + &jy_arr(lo.x+j +ix, lo.y+l +iz, 0), sx [ix]*sz [iz]*wqy); amrex::Gpu::Atomic::Add( - &jz_arr(lo.x+j +ix, lo.y+l0+iz, 0), + &jz_arr(lo.x+j +ix, lo.y+l0+iz, 0), sx [ix]*sz0[iz]*wqz); } } @@ -156,7 +156,7 @@ void doDepositionShapeN(const amrex::Real * const xp, &jx_arr(lo.x+j0+ix, lo.y+k +iy, lo.z+l +iz), sx0[ix]*sy [iy]*sz [iz]*wqx); amrex::Gpu::Atomic::Add( - &jy_arr(lo.x+j +ix, lo.y+k0+iy, lo.z+l +iz), + &jy_arr(lo.x+j +ix, lo.y+k0+iy, lo.z+l +iz), sx [ix]*sy0[iy]*sz [iz]*wqy); amrex::Gpu::Atomic::Add( &jz_arr(lo.x+j +ix, lo.y+k +iy, lo.z+l0+iz), @@ -240,7 +240,7 @@ void doEsirkepovDepositionShapeN (const amrex::Real * const xp, const amrex::Real clightsq = 1.0/PhysConst::c/PhysConst::c; // Loop over particles and deposit into Jx_arr, Jy_arr and Jz_arr - amrex::ParallelFor( + amrex::ParallelFor( np_to_depose, [=] AMREX_GPU_DEVICE (long ip) { @@ -302,7 +302,7 @@ void doEsirkepovDepositionShapeN (const amrex::Real * const xp, const amrex::Real x_new = (xp[ip] - xmin)*dxi; const amrex::Real x_old = x_new - dtsdx0*uxp[ip]*gaminv; #endif -#if (defined WARPX_DIM_3D) +#if (defined WARPX_DIM_3D) const amrex::Real y_new = (yp[ip] - ymin)*dyi; const amrex::Real y_old = y_new - dtsdy0*uyp[ip]*gaminv; #endif @@ -336,7 +336,7 @@ void doEsirkepovDepositionShapeN (const amrex::Real * const xp, #if (defined WARPX_DIM_3D) const int j_new = compute_shape_factor(sy_new+1, y_new); const int j_old = compute_shifted_shape_factor(sy_old, y_old, j_new); -#endif +#endif const int k_new = compute_shape_factor(sz_new+1, z_new); const int k_old = compute_shifted_shape_factor(sz_old, z_old, k_new); @@ -418,7 +418,7 @@ void doEsirkepovDepositionShapeN (const amrex::Real * const xp, for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { // The factor 2 comes from the normalization of the modes // The minus sign comes from the different convention with respect to Davidson et al. - const Complex djt_cmplx = -2.*I*(i_new-1 + i + xmin*dxi)*wq*invdtdx/(double)imode* + const Complex djt_cmplx = -2.*I*(i_new-1 + i + xmin*dxi)*wq*invdtdx/(double)imode* (sx_new[i]*sz_new[k]*(xy_new - xy_mid) + sx_old[i]*sz_old[k]*(xy_mid - xy_old)); amrex::Gpu::Atomic::Add( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode-1), djt_cmplx.real()); amrex::Gpu::Atomic::Add( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode), djt_cmplx.imag()); diff --git a/Source/Particles/Gather/FieldGather.H b/Source/Particles/Gather/FieldGather.H index d8d1d78ef..6727b0aa9 100644 --- a/Source/Particles/Gather/FieldGather.H +++ b/Source/Particles/Gather/FieldGather.H @@ -97,7 +97,7 @@ void doGatherShapeN(const amrex::Real * const xp, Bxp[ip] = 0; Byp[ip] = 0; Bzp[ip] = 0; - // Each field is gathered in a separate block of + // Each field is gathered in a separate block of // AMREX_SPACEDIM nested loops because the deposition // order can differ for each component of each field // when lower_in_v is set to 1 diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 305baf4ff..ac261b177 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -51,24 +51,24 @@ public: const amrex::Vector > > >& masks); /// - /// This evolves all the particles by one PIC time step, including charge deposition, the + /// This evolves all the particles by one PIC time step, including charge deposition, the /// field solve, and pushing the particles, for all the species in the MultiParticleContainer. /// This is the electrostatic version. /// - void EvolveES (const amrex::Vector, 3> >& E, - amrex::Vector >& rho, + void EvolveES (const amrex::Vector, 3> >& E, + amrex::Vector >& rho, amrex::Real t, amrex::Real dt); /// - /// This pushes the particle positions by one half time step for all the species in the + /// This pushes the particle positions by one half time step for all the species in the /// MultiParticleContainer. It is used to desynchronize the particles after initializaton /// or when restarting from a checkpoint. This is the electrostatic version. - /// + /// void PushXES (amrex::Real dt); /// /// This deposits the particle charge onto rho, accumulating the value for all the species - /// in the MultiParticleContainer. rho is assumed to contain node-centered multifabs. + /// in the MultiParticleContainer. rho is assumed to contain node-centered multifabs. /// This version is hard-coded for CIC deposition. /// void DepositCharge(amrex::Vector >& rho, bool local = false); @@ -79,7 +79,7 @@ public: /// amrex::Real sumParticleCharge(bool local = false); #endif // WARPX_DO_ELECTROSTATIC - + /// /// Performs the field gather operation using the input fields E and B, for all the species /// in the MultiParticleContainer. This is the electromagnetic version of the field gather. @@ -87,7 +87,7 @@ public: void FieldGather (int lev, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, const amrex::MultiFab& Bx, - const amrex::MultiFab& By, const amrex::MultiFab& Bz); + const amrex::MultiFab& By, const amrex::MultiFab& Bz); /// /// This evolves all the particles by one PIC time step, including current deposition, the @@ -97,8 +97,8 @@ public: void Evolve (int lev, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, - amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, - amrex::MultiFab* cjx, amrex::MultiFab* cjy, amrex::MultiFab* cjz, + amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, + amrex::MultiFab* cjx, amrex::MultiFab* cjy, amrex::MultiFab* cjz, amrex::MultiFab* rho, amrex::MultiFab* crho, const amrex::MultiFab* cEx, const amrex::MultiFab* cEy, const amrex::MultiFab* cEz, const amrex::MultiFab* cBx, const amrex::MultiFab* cBy, const amrex::MultiFab* cBz, @@ -108,7 +108,7 @@ public: /// This pushes the particle positions by one half time step for all the species in the /// MultiParticleContainer. It is used to desynchronize the particles after initializaton /// or when restarting from a checkpoint. This is the electromagnetic version. - /// + /// void PushX (amrex::Real dt); /// @@ -116,7 +116,7 @@ public: /// MultiParticleContainer. It is used to desynchronize the particles after initializaton /// or when restarting from a checkpoint. It is also used to synchronize particles at the /// the end of the run. This is the electromagnetic version. - /// + /// void PushP (int lev, amrex::Real dt, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz); @@ -124,7 +124,7 @@ public: /// /// This deposits the particle charge onto a node-centered MultiFab and returns a unique ptr /// to it. The charge density is accumulated over all the particles in the MultiParticleContainer - /// + /// std::unique_ptr GetChargeDensity(int lev, bool local = false); void doFieldIonization (); @@ -132,7 +132,7 @@ public: void Checkpoint (const std::string& dir) const; void WritePlotFile (const std::string& dir) const; - + void Restart (const std::string& dir); void PostRestart (); @@ -178,7 +178,7 @@ public: const amrex::Real t_boost, const amrex::Real t_lab, const amrex::Real dt, amrex::Vector& parts) const; - // Inject particles during the simulation (for particles entering the + // Inject particles during the simulation (for particles entering the // simulation domain after some iterations, due to flowing plasma and/or // moving window). void ContinuousInjection(const amrex::RealBox& injection_box) const; @@ -189,7 +189,7 @@ public: std::vector GetSpeciesNames() const { return species_names; } PhysicalParticleContainer& GetPCtmp () { return *pc_tmp; } - + protected: // Particle container types @@ -218,10 +218,10 @@ private: void mapSpeciesProduct (); int getSpeciesID (std::string product_str); - + // Number of species dumped in BoostedFrameDiagnostics int nspecies_boosted_frame_diags = 0; - // map_species_boosted_frame_diags[i] is the species ID in + // map_species_boosted_frame_diags[i] is the species ID in // MultiParticleContainer for 0 map_species_boosted_frame_diags; int do_boosted_frame_diags = 0; diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 640439337..612583e2f 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -23,7 +23,7 @@ MultiParticleContainer::MultiParticleContainer (AmrCore* amr_core) allcontainers[i]->m_deposit_on_main_grid = m_deposit_on_main_grid[i]; allcontainers[i]->m_gather_from_main_grid = m_gather_from_main_grid[i]; } - + for (int i = nspecies; i < nspecies+nlasers; ++i) { allcontainers[i].reset(new LaserParticleContainer(amr_core, i, lasers_names[i-nspecies])); } @@ -359,7 +359,7 @@ MultiParticleContainer { BL_PROFILE("MultiParticleContainer::GetLabFrameData"); - + // Loop over particle species for (int i = 0; i < nspecies_boosted_frame_diags; ++i){ int isp = map_species_boosted_frame_diags[i]; @@ -368,41 +368,41 @@ MultiParticleContainer pc->GetParticleSlice(direction, z_old, z_new, t_boost, t_lab, dt, diagnostic_particles); // Here, diagnostic_particles[lev][index] is a WarpXParticleContainer::DiagnosticParticleData // where "lev" is the AMR level and "index" is a [grid index][tile index] pair. - + // Loop over AMR levels for (int lev = 0; lev <= pc->finestLevel(); ++lev){ - // Loop over [grid index][tile index] pairs - // and Fills parts[species number i] with particle data from all grids and - // tiles in diagnostic_particles. parts contains particles from all + // Loop over [grid index][tile index] pairs + // and Fills parts[species number i] with particle data from all grids and + // tiles in diagnostic_particles. parts contains particles from all // AMR levels indistinctly. for (auto it = diagnostic_particles[lev].begin(); it != diagnostic_particles[lev].end(); ++it){ // it->first is the [grid index][tile index] key - // it->second is the corresponding + // it->second is the corresponding // WarpXParticleContainer::DiagnosticParticleData value parts[i].GetRealData(DiagIdx::w).insert( parts[i].GetRealData(DiagIdx::w ).end(), it->second.GetRealData(DiagIdx::w ).begin(), it->second.GetRealData(DiagIdx::w ).end()); - + parts[i].GetRealData(DiagIdx::x).insert( parts[i].GetRealData(DiagIdx::x ).end(), it->second.GetRealData(DiagIdx::x ).begin(), it->second.GetRealData(DiagIdx::x ).end()); - + parts[i].GetRealData(DiagIdx::y).insert( parts[i].GetRealData(DiagIdx::y ).end(), it->second.GetRealData(DiagIdx::y ).begin(), it->second.GetRealData(DiagIdx::y ).end()); - + parts[i].GetRealData(DiagIdx::z).insert( parts[i].GetRealData(DiagIdx::z ).end(), it->second.GetRealData(DiagIdx::z ).begin(), it->second.GetRealData(DiagIdx::z ).end()); - + parts[i].GetRealData(DiagIdx::ux).insert( parts[i].GetRealData(DiagIdx::ux).end(), it->second.GetRealData(DiagIdx::ux).begin(), it->second.GetRealData(DiagIdx::ux).end()); - + parts[i].GetRealData(DiagIdx::uy).insert( parts[i].GetRealData(DiagIdx::uy).end(), it->second.GetRealData(DiagIdx::uy).begin(), it->second.GetRealData(DiagIdx::uy).end()); - + parts[i].GetRealData(DiagIdx::uz).insert( parts[i].GetRealData(DiagIdx::uz).end(), it->second.GetRealData(DiagIdx::uz).begin(), it->second.GetRealData(DiagIdx::uz).end()); @@ -413,7 +413,7 @@ MultiParticleContainer /* \brief Continuous injection for particles initially outside of the domain. * \param injection_box: Domain where new particles should be injected. - * Loop over all WarpXParticleContainer in MultiParticleContainer and + * Loop over all WarpXParticleContainer in MultiParticleContainer and * calls virtual function ContinuousInjection. */ void @@ -429,7 +429,7 @@ MultiParticleContainer::ContinuousInjection(const RealBox& injection_box) const /* \brief Update position of continuous injection parameters. * \param dt: simulation time step (level 0) - * All classes inherited from WarpXParticleContainer do not have + * All classes inherited from WarpXParticleContainer do not have * a position to update (PhysicalParticleContainer does not do anything). */ void @@ -457,7 +457,7 @@ MultiParticleContainer::doContinuousInjection () const } /* \brief Get ID of product species of each species. - * The users specifies the name of the product species, + * The users specifies the name of the product species, * this routine get its ID. */ void @@ -465,8 +465,8 @@ MultiParticleContainer::mapSpeciesProduct () { for (int i=0; iionization_product_name and store its ID into + // If species pc has ionization on, find species with name + // pc->ionization_product_name and store its ID into // pc->ionization_product. if (pc->do_field_ionization){ int i_product = getSpeciesID(pc->ionization_product_name); @@ -577,7 +577,7 @@ namespace } // --- product runtime attribs GpuArray runtime_attribs_product; - bool do_boosted_product = WarpX::do_boosted_frame_diagnostic + bool do_boosted_product = WarpX::do_boosted_frame_diagnostic && pc_product->DoBoostedFrameDiags(); if (do_boosted_product) { std::map comps_product = pc_product->getParticleComps(); @@ -624,7 +624,7 @@ namespace attribs_product[ia][ip] = attribs_source[ia][is]; } // Update xold etc. if boosted frame diagnostics required - // for product species. Fill runtime attribs with a copy of + // for product species. Fill runtime attribs with a copy of // current properties (xold = x etc.). if (do_boosted_product) { runtime_attribs_product[0][ip] = p_source.pos(0); @@ -647,7 +647,7 @@ MultiParticleContainer::doFieldIonization () // Loop over all species. // Ionized particles in pc_source create particles in pc_product for (auto& pc_source : allcontainers){ - + // Skip if not ionizable if (!pc_source->do_field_ionization){ continue; } @@ -661,7 +661,7 @@ MultiParticleContainer::doFieldIonization () // they do not exist (or if they were defined by default, i.e., // without runtime component). #ifdef _OPENMP - // Touch all tiles of source species in serial if runtime attribs + // Touch all tiles of source species in serial if runtime attribs for (MFIter mfi = pc_source->MakeMFIter(lev); mfi.isValid(); ++mfi) { const int grid_id = mfi.index(); const int tile_id = mfi.LocalTileIndex(); @@ -683,7 +683,7 @@ MultiParticleContainer::doFieldIonization () MFItInfo info; if (pc_source->do_tiling && Gpu::notInLaunchRegion()) { AMREX_ALWAYS_ASSERT_WITH_MESSAGE( - pc_product->do_tiling, + pc_product->do_tiling, "For ionization, either all or none of the " "particle species must use tiling."); info.EnableTiling(pc_source->tile_size); diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index c7494fbdf..7946b4650 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -24,7 +24,7 @@ public: void InitIonizationModule (); -#ifdef WARPX_DO_ELECTROSTATIC +#ifdef WARPX_DO_ELECTROSTATIC virtual void FieldGatherES(const amrex::Vector, 3> >& E, const amrex::Vector > > >& masks) override; @@ -32,7 +32,7 @@ public: amrex::Vector >& rho, amrex::Real t, amrex::Real dt) override; #endif // WARPX_DO_ELECTROSTATIC - + virtual void FieldGather (int lev, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, @@ -99,14 +99,14 @@ public: const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz) override; - + void copy_attribs(WarpXParIter& pti,const amrex::Real* xp, const amrex::Real* yp, const amrex::Real* zp); virtual void PostRestart () final {} void SplitParticles(int lev); - + virtual void buildIonizationMask (const amrex::MFIter& mfi, const int lev, amrex::Gpu::ManagedDeviceVector& ionization_mask) override; @@ -125,7 +125,7 @@ public: std::array u, amrex::Real weight); virtual void GetParticleSlice(const int direction, const amrex::Real z_old, - const amrex::Real z_new, const amrex::Real t_boost, + const amrex::Real z_new, const amrex::Real t_boost, const amrex::Real t_lab, const amrex::Real dt, DiagnosticParticles& diagnostic_particles) final; diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 318ad4664..99c6973f9 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -34,7 +34,7 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp pp.query("do_splitting", do_splitting); pp.query("split_type", split_type); pp.query("do_continuous_injection", do_continuous_injection); - // Whether to plot back-transformed (lab-frame) diagnostics + // Whether to plot back-transformed (lab-frame) diagnostics // for this species. pp.query("do_boosted_frame_diags", do_boosted_frame_diags); @@ -51,7 +51,7 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp do_user_plot_vars = pp.queryarr("plot_vars", plot_vars); if (not do_user_plot_vars){ // By default, all particle variables are dumped to plotfiles, - // including {x,y,z,ux,uy,uz}old variables when running in a + // including {x,y,z,ux,uy,uz}old variables when running in a // boosted frame if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags){ plot_flags.resize(PIdx::nattribs + 6, 1); @@ -68,9 +68,9 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp // If not none, set plot_flags values to 1 for elements in plot_vars. if (plot_vars[0] != "none"){ for (const auto& var : plot_vars){ - // Return error if var not in PIdx. - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( - ParticleStringNames::to_index.count(var), + // Return error if var not in PIdx. + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + ParticleStringNames::to_index.count(var), "plot_vars argument not in ParticleStringNames"); plot_flags[ParticleStringNames::to_index.at(var)] = 1; } @@ -142,7 +142,7 @@ void PhysicalParticleContainer::MapParticletoBoostedFrame(Real& x, Real& y, Real void PhysicalParticleContainer::AddGaussianBeam(Real x_m, Real y_m, Real z_m, Real x_rms, Real y_rms, Real z_rms, - Real q_tot, long npart, + Real q_tot, long npart, int do_symmetrize) { const Geometry& geom = m_gdb->Geom(0); @@ -154,7 +154,7 @@ PhysicalParticleContainer::AddGaussianBeam(Real x_m, Real y_m, Real z_m, std::normal_distribution distz(z_m, z_rms); if (ParallelDescriptor::IOProcessor()) { - // If do_symmetrize, create 4x fewer particles, and + // If do_symmetrize, create 4x fewer particles, and // Replicate each particle 4 times (x,y) (-x,y) (x,-y) (-x,-y) if (do_symmetrize){ npart /= 4; @@ -387,11 +387,11 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) const int tile_id = mfi.LocalTileIndex(); // Max number of new particles, if particles are created in the whole - // overlap_box. All of them are created, and invalid ones are then + // overlap_box. All of them are created, and invalid ones are then // discaded int max_new_particles = overlap_box.numPts() * num_ppc; - // If refine injection, build pointer dp_cellid that holds pointer to + // If refine injection, build pointer dp_cellid that holds pointer to // array of refined cell IDs. Vector cellid_v; if (refine_injection and lev == 0) @@ -446,7 +446,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) if (do_field_ionization) { pi = soa.GetIntData(particle_icomps["ionization_level"]).data() + old_size; } - + const GpuArray overlap_corner {AMREX_D_DECL(overlap_realbox.lo(0), overlap_realbox.lo(1), @@ -458,9 +458,9 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) bool loc_do_field_ionization = do_field_ionization; int loc_ionization_initial_level = ionization_initial_level; - // Loop over all new particles and inject them (creates too many + // Loop over all new particles and inject them (creates too many // particles, in particular does not consider xmin, xmax etc.). - // The invalid ones are given negative ID and are deleted during the + // The invalid ones are given negative ID and are deleted during the // next redistribute. amrex::For(max_new_particles, [=] AMREX_GPU_DEVICE (int ip) noexcept { @@ -627,7 +627,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) p.pos(1) = z; #endif }, shared_mem_bytes); - + if (cost) { wt = (amrex::second() - wt) / tile_box.d_numPts(); Array4 const& costarr = cost->array(mfi); @@ -865,7 +865,7 @@ PhysicalParticleContainer::FieldGather (int lev, MultiFab* cost = WarpX::getCosts(lev); #ifdef _OPENMP -#pragma omp parallel +#pragma omp parallel #endif { #ifdef _OPENMP @@ -915,7 +915,7 @@ PhysicalParticleContainer::FieldGather (int lev, // int e_is_nodal = Ex.is_nodal() and Ey.is_nodal() and Ez.is_nodal(); FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, - &exfab, &eyfab, &ezfab, &bxfab, &byfab, &bzfab, + &exfab, &eyfab, &ezfab, &bxfab, &byfab, &bzfab, Ex.nGrow(), e_is_nodal, 0, np, thread_num, lev, lev); @@ -949,11 +949,11 @@ PhysicalParticleContainer::Evolve (int lev, BL_PROFILE_VAR_NS("PPC::FieldGather", blp_fg); BL_PROFILE_VAR_NS("PPC::ParticlePush", blp_ppc_pp); BL_PROFILE_VAR_NS("PPC::Evolve::partition", blp_partition); - + const std::array& dx = WarpX::CellSize(lev); const std::array& cdx = WarpX::CellSize(std::max(lev-1,0)); - // Get instances of NCI Godfrey filters + // Get instances of NCI Godfrey filters const auto& nci_godfrey_filter_exeybz = WarpX::GetInstance().nci_godfrey_filter_exeybz; const auto& nci_godfrey_filter_bxbyez = WarpX::GetInstance().nci_godfrey_filter_bxbyez; @@ -978,9 +978,9 @@ PhysicalParticleContainer::Evolve (int lev, tmp_particle_data[lev][index][i].resize(np); } } - + #ifdef _OPENMP -#pragma omp parallel +#pragma omp parallel #endif { #ifdef _OPENMP @@ -1183,7 +1183,7 @@ PhysicalParticleContainer::Evolve (int lev, } const long np_current = (cjx) ? nfine_current : np; - + // // copy data from particle container to temp arrays // @@ -1199,14 +1199,14 @@ PhysicalParticleContainer::Evolve (int lev, } else { ion_lev = nullptr; } - DepositCharge(pti, wp, ion_lev, rho, 0, 0, + DepositCharge(pti, wp, ion_lev, rho, 0, 0, np_current, thread_num, lev, lev); if (has_buffer){ DepositCharge(pti, wp, ion_lev, crho, 0, np_current, np-np_current, thread_num, lev, lev-1); } } - + if (! do_not_push) { const long np_gather = (cEx) ? nfine_gather : np; @@ -1218,7 +1218,7 @@ PhysicalParticleContainer::Evolve (int lev, // BL_PROFILE_VAR_START(blp_fg); FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, - exfab, eyfab, ezfab, bxfab, byfab, bzfab, + exfab, eyfab, ezfab, bxfab, byfab, bzfab, Ex.nGrow(), e_is_nodal, 0, np_gather, thread_num, lev, lev); @@ -1234,7 +1234,7 @@ PhysicalParticleContainer::Evolve (int lev, FArrayBox const* cbxfab = &(*cBx)[pti]; FArrayBox const* cbyfab = &(*cBy)[pti]; FArrayBox const* cbzfab = &(*cBz)[pti]; - + if (WarpX::use_fdtd_nci_corr) { #if (AMREX_SPACEDIM == 2) @@ -1272,13 +1272,13 @@ PhysicalParticleContainer::Evolve (int lev, eyeli = filtered_Ey.elixir(); nci_godfrey_filter_exeybz[lev-1]->ApplyStencil(filtered_Ey, (*cEy)[pti], filtered_Ey.box()); ceyfab = &filtered_Ey; - + // Filter Bx filtered_Bx.resize(amrex::convert(tbox,WarpX::Bx_nodal_flag)); bxeli = filtered_Bx.elixir(); nci_godfrey_filter_bxbyez[lev-1]->ApplyStencil(filtered_Bx, (*cBx)[pti], filtered_Bx.box()); cbxfab = &filtered_Bx; - + // Filter Bz filtered_Bz.resize(amrex::convert(tbox,WarpX::Bz_nodal_flag)); bzeli = filtered_Bz.elixir(); @@ -1286,14 +1286,14 @@ PhysicalParticleContainer::Evolve (int lev, cbzfab = &filtered_Bz; #endif } - + // Field gather for particles in gather buffers e_is_nodal = cEx->is_nodal() and cEy->is_nodal() and cEz->is_nodal(); - FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, cexfab, ceyfab, cezfab, cbxfab, cbyfab, cbzfab, - cEx->nGrow(), e_is_nodal, - nfine_gather, np-nfine_gather, + cEx->nGrow(), e_is_nodal, + nfine_gather, np-nfine_gather, thread_num, lev, lev-1); } @@ -1303,7 +1303,7 @@ PhysicalParticleContainer::Evolve (int lev, // Particle Push // BL_PROFILE_VAR_START(blp_ppc_pp); - PushPX(pti, m_xp[thread_num], m_yp[thread_num], m_zp[thread_num], + PushPX(pti, m_xp[thread_num], m_yp[thread_num], m_zp[thread_num], m_giv[thread_num], dt); BL_PROFILE_VAR_STOP(blp_ppc_pp); @@ -1317,7 +1317,7 @@ PhysicalParticleContainer::Evolve (int lev, } else { ion_lev = nullptr; } - + // Deposit inside domains DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, &jx, &jy, &jz, 0, np_current, thread_num, @@ -1337,7 +1337,7 @@ PhysicalParticleContainer::Evolve (int lev, pti.SetPosition(m_xp[thread_num], m_yp[thread_num], m_zp[thread_num]); BL_PROFILE_VAR_STOP(blp_copy); } - + if (rho) { // Deposit charge after particle push, in component 1 of MultiFab rho. int* AMREX_RESTRICT ion_lev; @@ -1406,7 +1406,7 @@ PhysicalParticleContainer::SplitParticles(int lev) for(int i=0; icharge; const Real m = this-> mass; if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Boris){ - amrex::ParallelFor( + amrex::ParallelFor( pti.numParticles(), [=] AMREX_GPU_DEVICE (long i) { Real qp = q; @@ -1613,7 +1613,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, int thread_num = omp_get_thread_num(); #else int thread_num = 0; -#endif +#endif for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { const Box& box = pti.validbox(); @@ -1653,7 +1653,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, int e_is_nodal = Ex.is_nodal() and Ey.is_nodal() and Ez.is_nodal(); FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, - &exfab, &eyfab, &ezfab, &bxfab, &byfab, &bzfab, + &exfab, &eyfab, &ezfab, &bxfab, &byfab, &bzfab, Ex.nGrow(), e_is_nodal, 0, np, thread_num, lev, lev); @@ -1701,7 +1701,7 @@ void PhysicalParticleContainer::copy_attribs(WarpXParIter& pti,const Real* xp, Real* AMREX_RESTRICT uxp = attribs[PIdx::ux].dataPtr(); Real* AMREX_RESTRICT uyp = attribs[PIdx::uy].dataPtr(); Real* AMREX_RESTRICT uzp = attribs[PIdx::uz].dataPtr(); - + const auto np = pti.numParticles(); const auto lev = pti.GetLevel(); const auto index = pti.GetPairIndex(); @@ -1717,7 +1717,7 @@ void PhysicalParticleContainer::copy_attribs(WarpXParIter& pti,const Real* xp, xpold[i]=xp[i]; ypold[i]=yp[i]; zpold[i]=zp[i]; - + uxpold[i]=uxp[i]; uypold[i]=uyp[i]; uzpold[i]=uzp[i]; @@ -1757,7 +1757,7 @@ void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real slice_box.setHi(direction, z_max); diagnostic_particles.resize(finestLevel()+1); - + for (int lev = 0; lev < nlevs; ++lev) { const Real* dx = Geom(lev).CellSize(); @@ -1804,7 +1804,7 @@ void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real auto& uzp_old = tmp_particle_data[lev][index][TmpIdx::uzold]; const long np = pti.numParticles(); - + Real uzfrm = -WarpX::gamma_boost*WarpX::beta_boost*PhysConst::c; Real inv_c2 = 1.0/PhysConst::c/PhysConst::c; @@ -1839,7 +1839,7 @@ void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real Real uzp = uz_old_p *weight_old + uz_new_p *weight_new; diagnostic_particles[lev][index].GetRealData(DiagIdx::w).push_back(wp[i]); - + diagnostic_particles[lev][index].GetRealData(DiagIdx::x).push_back(xp); diagnostic_particles[lev][index].GetRealData(DiagIdx::y).push_back(yp); diagnostic_particles[lev][index].GetRealData(DiagIdx::z).push_back(zp); @@ -1864,7 +1864,7 @@ PhysicalParticleContainer::ContinuousInjection(const RealBox& injection_box) AddPlasma(lev, injection_box); } -/* \brief Gather fields from FArrayBox exfab, eyfab, ezfab, bxfab, byfab, +/* \brief Gather fields from FArrayBox exfab, eyfab, ezfab, bxfab, byfab, * bzfab into arrays of fields on particles Exp, Eyp, Ezp, Bxp, Byp, Bzp. * \param Exp-Bzp: fields on particles. * \param exfab-bzfab: FAB of electric and magnetic fields for particles in pti @@ -1874,7 +1874,7 @@ PhysicalParticleContainer::ContinuousInjection(const RealBox& injection_box) * \param np_to_gather: number of particles onto which fields are gathered * \param thread_num: if using OpenMP, thread number * \param lev: level on which particles are located - * \param gather_lev: level from which particles gather fields (lev-1) for + * \param gather_lev: level from which particles gather fields (lev-1) for particles in buffers. */ void @@ -1901,14 +1901,14 @@ PhysicalParticleContainer::FieldGather (WarpXParIter& pti, AMREX_ALWAYS_ASSERT_WITH_MESSAGE((gather_lev==(lev-1)) || (gather_lev==(lev )), "Gather buffers only work for lev-1"); - + // If no particles, do not do anything if (np_to_gather == 0) return; // Get cell size on gather_lev const std::array& dx = WarpX::CellSize(std::max(gather_lev,0)); // Set staggering shift depending on e_is_nodal const Real stagger_shift = e_is_nodal ? 0.0 : 0.5; - + // Get box from which field is gathered. // If not gathering from the finest level, the box is coarsened. Box box; @@ -1918,26 +1918,26 @@ PhysicalParticleContainer::FieldGather (WarpXParIter& pti, const IntVect& ref_ratio = WarpX::RefRatio(gather_lev); box = amrex::coarsen(pti.tilebox(),ref_ratio); } - + // Add guard cells to the box. box.grow(ngE); - + const Array4& ex_arr = exfab->array(); const Array4& ey_arr = eyfab->array(); const Array4& ez_arr = ezfab->array(); const Array4& bx_arr = bxfab->array(); const Array4& by_arr = byfab->array(); const Array4& bz_arr = bzfab->array(); - + const Real * const AMREX_RESTRICT xp = m_xp[thread_num].dataPtr() + offset; const Real * const AMREX_RESTRICT zp = m_zp[thread_num].dataPtr() + offset; const Real * const AMREX_RESTRICT yp = m_yp[thread_num].dataPtr() + offset; - + // Lower corner of tile box physical domain const std::array& xyzmin = WarpX::LowerCorner(box, gather_lev); - + const Dim3 lo = lbound(box); - + // Depending on l_lower_in_v and WarpX::nox, call // different versions of template function doGatherShapeN if (WarpX::l_lower_order_in_v){ @@ -2020,10 +2020,10 @@ void PhysicalParticleContainer::InitIonizationModule () } // Compute ADK prefactors (See Chen, JCP 236 (2013), equation (2)) // For now, we assume l=0 and m=0. - // The approximate expressions are used, + // The approximate expressions are used, // without Gamma function Real wa = std::pow(PhysConst::alpha,3) * PhysConst::c / PhysConst::r_e; - Real Ea = PhysConst::m_e * PhysConst::c*PhysConst::c /PhysConst::q_e * + Real Ea = PhysConst::m_e * PhysConst::c*PhysConst::c /PhysConst::q_e * std::pow(PhysConst::alpha,4)/PhysConst::r_e; Real UH = table_ionization_energies[0]; Real l_eff = std::sqrt(UH/ionization_energies[0]) - 1.; @@ -2038,18 +2038,18 @@ void PhysicalParticleContainer::InitIonizationModule () Real C2 = std::pow(2,2*n_eff)/(n_eff*tgamma(n_eff+l_eff+1)*tgamma(n_eff-l_eff)); adk_power[i] = -(2*n_eff - 1); Real Uion = ionization_energies[i]; - adk_prefactor[i] = dt * wa * C2 * ( Uion/(2*UH) ) + adk_prefactor[i] = dt * wa * C2 * ( Uion/(2*UH) ) * std::pow(2*std::pow((Uion/UH),3./2)*Ea,2*n_eff - 1); adk_exp_prefactor[i] = -2./3 * std::pow( Uion/UH,3./2) * Ea; } } /* \brief create mask of ionized particles (1 if ionized, 0 otherwise) - * + * * \param mfi: tile or grid * \param lev: MR level * \param ionization_mask: Array with as many elements as particles in mfi. - * This function initialized the array, and set each element to 1 or 0 + * This function initialized the array, and set each element to 1 or 0 * depending on whether the particle is ionized or not. */ void @@ -2095,7 +2095,7 @@ PhysicalParticleContainer::buildIonizationMask (const amrex::MFIter& mfi, const // Loop over all particles in grid/tile. If ionized, set mask to 1 // and increment ionization level. - ParallelFor( + ParallelFor( np, [=] AMREX_GPU_DEVICE (long i) { // Get index of ionization_level diff --git a/Source/Particles/RigidInjectedParticleContainer.H b/Source/Particles/RigidInjectedParticleContainer.H index b920ece0a..a635e9aa0 100644 --- a/Source/Particles/RigidInjectedParticleContainer.H +++ b/Source/Particles/RigidInjectedParticleContainer.H @@ -60,7 +60,7 @@ public: virtual void ReadHeader (std::istream& is) override; virtual void WriteHeader (std::ostream& os) const override; - + private: // User input quantities diff --git a/Source/Particles/ShapeFactors.H b/Source/Particles/ShapeFactors.H index 9d185714a..be79a4871 100644 --- a/Source/Particles/ShapeFactors.H +++ b/Source/Particles/ShapeFactors.H @@ -1,7 +1,7 @@ #ifndef SHAPEFACTORS_H_ #define SHAPEFACTORS_H_ -// Compute shape factor and return index of leftmost cell where +// Compute shape factor and return index of leftmost cell where // particle writes. // Specialized templates are defined below for orders 0 to 3. template diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index f7e46b2d3..06540871b 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -73,19 +73,19 @@ public: const amrex::Cuda::ManagedDeviceVector& y, const amrex::Cuda::ManagedDeviceVector& z); #endif - const std::array& GetAttribs () const { - return GetStructOfArrays().GetRealData(); + const std::array& GetAttribs () const { + return GetStructOfArrays().GetRealData(); } - - std::array& GetAttribs () { - return GetStructOfArrays().GetRealData(); + + std::array& GetAttribs () { + return GetStructOfArrays().GetRealData(); } - const RealVector& GetAttribs (int comp) const { + const RealVector& GetAttribs (int comp) const { return GetStructOfArrays().GetRealData(comp); } - - RealVector& GetAttribs (int comp) { + + RealVector& GetAttribs (int comp) { return GetStructOfArrays().GetRealData(comp); } @@ -102,15 +102,15 @@ class WarpXParticleContainer public: friend MultiParticleContainer; - // amrex::StructOfArrays with DiagIdx::nattribs amrex::Real components + // amrex::StructOfArrays with DiagIdx::nattribs amrex::Real components // and 0 int components for the particle data. using DiagnosticParticleData = amrex::StructOfArrays; // DiagnosticParticles is a vector, with one element per MR level. - // DiagnosticParticles[lev] is typically a key-value pair where the key is - // a pair [grid_index, tile_index], and the value is the corresponding + // DiagnosticParticles[lev] is typically a key-value pair where the key is + // a pair [grid_index, tile_index], and the value is the corresponding // DiagnosticParticleData (see above) on this tile. using DiagnosticParticles = amrex::Vector, DiagnosticParticleData> >; - + WarpXParticleContainer (amrex::AmrCore* amr_core, int ispecies); virtual ~WarpXParticleContainer() {} @@ -124,12 +124,12 @@ public: const amrex::MultiFab& Ez, const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz) {} -#ifdef WARPX_DO_ELECTROSTATIC +#ifdef WARPX_DO_ELECTROSTATIC virtual void EvolveES (const amrex::Vector, 3> >& E, - amrex::Vector >& rho, + amrex::Vector >& rho, amrex::Real t, amrex::Real dt) = 0; #endif // WARPX_DO_ELECTROSTATIC - + virtual void Evolve (int lev, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, @@ -143,10 +143,10 @@ public: virtual void PostRestart () = 0; virtual void GetParticleSlice(const int direction, const amrex::Real z_old, - const amrex::Real z_new, const amrex::Real t_boost, + const amrex::Real z_new, const amrex::Real t_boost, const amrex::Real t_lab, const amrex::Real dt, DiagnosticParticles& diagnostic_particles) {} - + void AllocData (); /// @@ -154,7 +154,7 @@ public: /// It is used to desynchronize the particles after initializaton /// or when restarting from a checkpoint. /// This is the electrostatic version of the particle push. - /// + /// void PushXES (amrex::Real dt); /// @@ -162,13 +162,13 @@ public: /// It is used to desynchronize the particles after initializaton /// or when restarting from a checkpoint. /// This is the electromagnetic version of the particle push. - /// + /// void PushX ( amrex::Real dt); void PushX (int lev, amrex::Real dt); /// /// This pushes the particle momenta by dt. - /// + /// virtual void PushP (int lev, amrex::Real dt, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, @@ -208,9 +208,9 @@ public: int depos_lev, amrex::Real dt); - // If particles start outside of the domain, ContinuousInjection - // makes sure that they are initialized when they enter the domain, and - // NOT before. Virtual function, overriden by derived classes. + // If particles start outside of the domain, ContinuousInjection + // makes sure that they are initialized when they enter the domain, and + // NOT before. Virtual function, overriden by derived classes. // Current status: // PhysicalParticleContainer: implemented. // LaserParticleContainer: implemented. @@ -219,7 +219,7 @@ public: // Update optional sub-class-specific injection location. virtual void UpdateContinuousInjectionPosition(amrex::Real dt) {} - /// + /// /// This returns the total charge for all the particles in this ParticleContainer. /// This is needed when solving Poisson's equation with periodic boundary conditions. /// @@ -259,9 +259,9 @@ public: // split along axes (0) or diagonals (1) int split_type = 0; - using amrex::ParticleContainer<0, 0, PIdx::nattribs>::AddRealComp; + using amrex::ParticleContainer<0, 0, PIdx::nattribs>::AddRealComp; using amrex::ParticleContainer<0, 0, PIdx::nattribs>::AddIntComp; - + void AddRealComp (const std::string& name, bool comm=true) { particle_comps[name] = NumRealComps(); @@ -274,7 +274,7 @@ public: AddIntComp(comm); } - int DoBoostedFrameDiags () const { return do_boosted_frame_diags; } + int DoBoostedFrameDiags () const { return do_boosted_frame_diags; } virtual void buildIonizationMask (const amrex::MFIter& mfi, const int lev, amrex::Gpu::ManagedDeviceVector& ionization_mask) @@ -286,7 +286,7 @@ protected: std::map particle_comps; std::map particle_icomps; - + int species_id; amrex::Real charge; @@ -300,9 +300,9 @@ protected: static int do_not_push; - // Whether to allow particles outside of the simulation domain to be + // Whether to allow particles outside of the simulation domain to be // initialized when they enter the domain. - // This is currently required because continuous injection does not + // This is currently required because continuous injection does not // support all features allowed by direct injection. int do_continuous_injection = 0; @@ -316,7 +316,7 @@ protected: amrex::Gpu::ManagedVector adk_prefactor; amrex::Gpu::ManagedVector adk_exp_prefactor; std::string physical_element; - + int do_boosted_frame_diags = 1; amrex::Vector local_rho; @@ -326,20 +326,20 @@ protected: using DataContainer = amrex::Gpu::ManagedDeviceVector; using PairIndex = std::pair; - + amrex::Vector m_xp, m_yp, m_zp, m_giv; - // Whether to dump particle quantities. + // Whether to dump particle quantities. // If true, particle position is always dumped. int plot_species = 1; - // For all particle attribs (execept position), whether or not + // For all particle attribs (execept position), whether or not // to dump to file. amrex::Vector plot_flags; // list of names of attributes to dump. amrex::Vector plot_vars; - + amrex::Vector > > tmp_particle_data; - + private: virtual void particlePostLocate(ParticleType& p, const amrex::ParticleLocData& pld, const int lev) override; diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 4e80374d8..83de9fb0e 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -228,7 +228,7 @@ WarpXParticleContainer::AddNParticles (int lev, #endif if ( (NumRuntimeRealComps()>0) || (NumRuntimeIntComps()>0) ){ - auto& ptile = DefineAndReturnParticleTile(0, 0, 0); + auto& ptile = DefineAndReturnParticleTile(0, 0, 0); } particle_tile.push_back(p); @@ -242,7 +242,7 @@ WarpXParticleContainer::AddNParticles (int lev, particle_tile.push_back_real(PIdx::uz, vz + ibegin, vz + iend); if ( (NumRuntimeRealComps()>0) || (NumRuntimeIntComps()>0) ){ - auto& ptile = DefineAndReturnParticleTile(0, 0, 0); + auto& ptile = DefineAndReturnParticleTile(0, 0, 0); } for (int comp = PIdx::uz+1; comp < PIdx::nattribs; ++comp) @@ -289,7 +289,7 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, const long offset, const long np_to_depose, int thread_num, int lev, int depos_lev, Real dt) -{ +{ AMREX_ALWAYS_ASSERT_WITH_MESSAGE((depos_lev==(lev-1)) || (depos_lev==(lev )), "Deposition buffers only work for lev-1"); @@ -316,7 +316,7 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, const IntVect& ref_ratio = WarpX::RefRatio(depos_lev); tilebox = amrex::coarsen(pti.tilebox(),ref_ratio); } - + // Staggered tile boxes (different in each direction) Box tbx = convert(tilebox, WarpX::jx_nodal_flag); Box tby = convert(tilebox, WarpX::jy_nodal_flag); @@ -360,11 +360,11 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, // Lower corner of tile box physical domain // Note that this includes guard cells since it is after tilebox.ngrow const std::array& xyzmin = WarpX::LowerCorner(tilebox, depos_lev); - // xyzmin is built on pti.tilebox(), so it does + // xyzmin is built on pti.tilebox(), so it does // not include staggering, so the stagger_shift has to be done by hand. - // Alternatively, we could define xyzminx from tbx (and the same for 3 + // Alternatively, we could define xyzminx from tbx (and the same for 3 // directions and for jx, jy, jz). This way, sx0 would not be needed. - // Better for memory? worth trying? + // Better for memory? worth trying? const Dim3 lo = lbound(tilebox); BL_PROFILE_VAR_START(blp_deposit); @@ -431,7 +431,7 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, ion_lev is a null pointer. * \param rho : Full array of charge density * \param icomp : Component of rho into which charge is deposited. - 0: old value (before particle push). + 0: old value (before particle push). 1: new value (after particle push). * \param offset : Index of first particle for which charge is deposited * \param np_to_depose: Number of particles for which charge is deposited. @@ -471,7 +471,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector& wp, const IntVect& ref_ratio = WarpX::RefRatio(depos_lev); tilebox = amrex::coarsen(pti.tilebox(),ref_ratio); } - + tilebox.grow(ngRho); const int nc = (rho->nComp() == 1 ? 1 : rho->nComp()/2); diff --git a/Source/Particles/deposit_cic.F90 b/Source/Particles/deposit_cic.F90 index e4e1ace03..1fe80016f 100644 --- a/Source/Particles/deposit_cic.F90 +++ b/Source/Particles/deposit_cic.F90 @@ -7,11 +7,11 @@ module warpx_ES_deposit_cic contains -! This routine computes the charge density due to the particles using cloud-in-cell +! This routine computes the charge density due to the particles using cloud-in-cell ! deposition. The Fab rho is assumed to be node-centered. ! ! Arguments: -! particles : a pointer to the particle array-of-structs +! particles : a pointer to the particle array-of-structs ! ns : the stride length of particle struct (the size of the struct in number of reals) ! np : the number of particles ! weights : the particle weights diff --git a/Source/Particles/interpolate_cic.F90 b/Source/Particles/interpolate_cic.F90 index bc2c4f37e..005ab35f4 100644 --- a/Source/Particles/interpolate_cic.F90 +++ b/Source/Particles/interpolate_cic.F90 @@ -12,7 +12,7 @@ contains ! node-centered. ! ! Arguments: -! particles : a pointer to the particle array-of-structs +! particles : a pointer to the particle array-of-structs ! ns : the stride length of particle struct (the size of the struct in number of reals) ! np : the number of particles ! Ex_p : the electric field in the x-direction at the particle positions (output) @@ -91,7 +91,7 @@ contains wx_hi*wy_lo*wz_hi*Ez(i+1, j, k+1) + & wx_hi*wy_hi*wz_lo*Ez(i+1, j+1, k ) + & wx_hi*wy_hi*wz_hi*Ez(i+1, j+1, k+1) - + end do end subroutine warpx_interpolate_cic_3d @@ -184,7 +184,7 @@ contains lx = (particles(1, n) - plo(1))*inv_dx(1) ly = (particles(2, n) - plo(2))*inv_dx(2) lz = (particles(3, n) - plo(3))*inv_dx(3) - + i = floor(lx) j = floor(ly) k = floor(lz) @@ -203,11 +203,11 @@ contains wx_hi = lx - i wy_hi = ly - j wz_hi = lz - k - + wx_lo = 1.0d0 - wx_hi wy_lo = 1.0d0 - wy_hi wz_lo = 1.0d0 - wz_hi - + Ex_p(n) = wx_lo*wy_lo*wz_lo*cEx(i, j, k ) + & wx_lo*wy_lo*wz_hi*cEx(i, j, k+1) + & wx_lo*wy_hi*wz_lo*cEx(i, j+1, k ) + & @@ -225,7 +225,7 @@ contains wx_hi*wy_lo*wz_hi*cEy(i+1, j, k+1) + & wx_hi*wy_hi*wz_lo*cEy(i+1, j+1, k ) + & wx_hi*wy_hi*wz_hi*cEy(i+1, j+1, k+1) - + Ez_p(n) = wx_lo*wy_lo*wz_lo*cEz(i, j, k ) + & wx_lo*wy_lo*wz_hi*cEz(i, j, k+1) + & wx_lo*wy_hi*wz_lo*cEz(i, j+1, k ) + & @@ -234,14 +234,14 @@ contains wx_hi*wy_lo*wz_hi*cEz(i+1, j, k+1) + & wx_hi*wy_hi*wz_lo*cEz(i+1, j+1, k ) + & wx_hi*wy_hi*wz_hi*cEz(i+1, j+1, k+1) - + ! otherwise use the fine else wx_hi = lx - i wy_hi = ly - j wz_hi = lz - k - + wx_lo = 1.0d0 - wx_hi wy_lo = 1.0d0 - wy_hi wz_lo = 1.0d0 - wz_hi @@ -254,7 +254,7 @@ contains wx_hi*wy_lo*wz_hi*Ex(i+1, j, k+1) + & wx_hi*wy_hi*wz_lo*Ex(i+1, j+1, k ) + & wx_hi*wy_hi*wz_hi*Ex(i+1, j+1, k+1) - + Ey_p(n) = wx_lo*wy_lo*wz_lo*Ey(i, j, k ) + & wx_lo*wy_lo*wz_hi*Ey(i, j, k+1) + & wx_lo*wy_hi*wz_lo*Ey(i, j+1, k ) + & @@ -263,7 +263,7 @@ contains wx_hi*wy_lo*wz_hi*Ey(i+1, j, k+1) + & wx_hi*wy_hi*wz_lo*Ey(i+1, j+1, k ) + & wx_hi*wy_hi*wz_hi*Ey(i+1, j+1, k+1) - + Ez_p(n) = wx_lo*wy_lo*wz_lo*Ez(i, j, k ) + & wx_lo*wy_lo*wz_hi*Ez(i, j, k+1) + & wx_lo*wy_hi*wz_lo*Ez(i, j+1, k ) + & @@ -272,7 +272,7 @@ contains wx_hi*wy_lo*wz_hi*Ez(i+1, j, k+1) + & wx_hi*wy_hi*wz_lo*Ez(i+1, j+1, k ) + & wx_hi*wy_hi*wz_hi*Ez(i+1, j+1, k+1) - + end if end do @@ -314,7 +314,7 @@ contains lx = (particles(1, n) - plo(1))*inv_dx(1) ly = (particles(2, n) - plo(2))*inv_dx(2) - + i = floor(lx) j = floor(ly) @@ -329,10 +329,10 @@ contains wx_hi = lx - i wy_hi = ly - j - + wx_lo = 1.0d0 - wx_hi wy_lo = 1.0d0 - wy_hi - + Ex_p(n) = wx_lo*wy_lo*cEx(i, j ) + & wx_lo*wy_hi*cEx(i, j+1) + & wx_hi*wy_lo*cEx(i+1, j ) + & @@ -342,13 +342,13 @@ contains wx_lo*wy_hi*cEy(i, j+1) + & wx_hi*wy_lo*cEy(i+1, j ) + & wx_hi*wy_hi*cEy(i+1, j+1) - + ! otherwise use the fine else wx_hi = lx - i wy_hi = ly - j - + wx_lo = 1.0d0 - wx_hi wy_lo = 1.0d0 - wy_hi @@ -356,7 +356,7 @@ contains wx_lo*wy_hi*Ex(i, j+1) + & wx_hi*wy_lo*Ex(i+1, j ) + & wx_hi*wy_hi*Ex(i+1, j+1) - + Ey_p(n) = wx_lo*wy_lo*Ey(i, j ) + & wx_lo*wy_hi*Ey(i, j+1) + & wx_hi*wy_lo*Ey(i+1, j ) + & diff --git a/Source/Particles/push_particles_ES.F90 b/Source/Particles/push_particles_ES.F90 index 9c7bd9e92..53dd3c181 100644 --- a/Source/Particles/push_particles_ES.F90 +++ b/Source/Particles/push_particles_ES.F90 @@ -8,13 +8,13 @@ module warpx_ES_push_particles contains ! -! This routine updates the particle positions and velocities using the +! This routine updates the particle positions and velocities using the ! leapfrog time integration algorithm, given the electric fields at the ! particle positions. It also enforces specular reflection off the domain ! walls. ! ! Arguments: -! particles : a pointer to the particle array-of-structs +! particles : a pointer to the particle array-of-structs ! ns : the stride length of particle struct (the size of the struct in number of reals) ! np : the number of particles ! vx_p : the particle x-velocities @@ -30,7 +30,7 @@ contains ! prob_hi : the right-hand corner of the problem domain ! subroutine warpx_push_leapfrog_3d(particles, ns, np, & - vx_p, vy_p, vz_p, & + vx_p, vy_p, vz_p, & Ex_p, Ey_p, Ez_p, & charge, mass, dt, & prob_lo, prob_hi) & @@ -43,7 +43,7 @@ contains real(amrex_real), intent(in) :: mass real(amrex_real), intent(in) :: dt real(amrex_real), intent(in) :: prob_lo(3), prob_hi(3) - + integer n real(amrex_real) fac @@ -69,7 +69,7 @@ contains vx_p(n) = -vx_p(n) end do -! ... y... +! ... y... do while (particles(2, n) .lt. prob_lo(2) .or. particles(2, n) .gt. prob_hi(2)) if (particles(2, n) .lt. prob_lo(2)) then particles(2, n) = 2.d0*prob_lo(2) - particles(2, n) @@ -107,7 +107,7 @@ contains real(amrex_real), intent(in) :: mass real(amrex_real), intent(in) :: dt real(amrex_real), intent(in) :: prob_lo(2), prob_hi(2) - + integer n real(amrex_real) fac @@ -131,7 +131,7 @@ contains vx_p(n) = -vx_p(n) end do -! ... y... +! ... y... do while (particles(2, n) .lt. prob_lo(2) .or. particles(2, n) .gt. prob_hi(2)) if (particles(2, n) .lt. prob_lo(2)) then particles(2, n) = 2.d0*prob_lo(2) - particles(2, n) @@ -152,7 +152,7 @@ contains ! from the velocities after particle initialization. ! ! Arguments: -! particles : a pointer to the particle array-of-structs +! particles : a pointer to the particle array-of-structs ! ns : the stride length of particle struct (the size of the struct in number of reals) ! np : the number of particles ! xx_p : the electric field in the x-direction at the particle positions @@ -190,7 +190,7 @@ contains vx_p(n) = -vx_p(n) end do -! ... y... +! ... y... do while (particles(2, n) .lt. prob_lo(2) .or. particles(2, n) .gt. prob_hi(2)) if (particles(2, n) .lt. prob_lo(2)) then particles(2, n) = 2.d0*prob_lo(2) - particles(2, n) @@ -209,7 +209,7 @@ contains end if vz_p(n) = -vz_p(n) end do - + end do end subroutine warpx_push_leapfrog_positions_3d @@ -241,7 +241,7 @@ contains vx_p(n) = -vx_p(n) end do -! ... y... +! ... y... do while (particles(2, n) .lt. prob_lo(2) .or. particles(2, n) .gt. prob_hi(2)) if (particles(2, n) .lt. prob_lo(2)) then particles(2, n) = 2.d0*prob_lo(2) - particles(2, n) diff --git a/Source/Python/WarpXWrappers.cpp b/Source/Python/WarpXWrappers.cpp index 2fb7200bb..b59f80495 100644 --- a/Source/Python/WarpXWrappers.cpp +++ b/Source/Python/WarpXWrappers.cpp @@ -8,7 +8,7 @@ #include #include -namespace +namespace { double** getMultiFabPointers(const amrex::MultiFab& mf, int *num_boxes, int *ncomps, int *ngrow, int **shapes) { @@ -19,7 +19,7 @@ namespace if (mf.nComp() > 1) shapesize += 1; *shapes = (int*) malloc(shapesize * (*num_boxes) * sizeof(int)); double** data = (double**) malloc((*num_boxes) * sizeof(double*)); - + #ifdef _OPENMP #pragma omp parallel #endif @@ -27,7 +27,7 @@ namespace int i = mfi.LocalIndex(); data[i] = (double*) mf[mfi].dataPtr(); for (int j = 0; j < AMREX_SPACEDIM; ++j) { - (*shapes)[shapesize*i+j] = mf[mfi].box().length(j); + (*shapes)[shapesize*i+j] = mf[mfi].box().length(j); } if (mf.nComp() > 1) (*shapes)[shapesize*i+AMREX_SPACEDIM] = mf.nComp(); } @@ -69,12 +69,12 @@ extern "C" return WarpX::l_lower_order_in_v; } - int warpx_nComps() + int warpx_nComps() { - return PIdx::nattribs; + return PIdx::nattribs; } - int warpx_SpaceDim() + int warpx_SpaceDim() { return AMREX_SPACEDIM; } @@ -87,7 +87,7 @@ extern "C" #ifdef BL_USE_MPI void amrex_init_with_inited_mpi (int argc, char* argv[], MPI_Comm mpicomm) { - amrex::Initialize(argc,argv,true,mpicomm); + amrex::Initialize(argc,argv,true,mpicomm); } #endif diff --git a/Source/Python/WarpXWrappers.h b/Source/Python/WarpXWrappers.h index 0a476b5fc..233a0b930 100644 --- a/Source/Python/WarpXWrappers.h +++ b/Source/Python/WarpXWrappers.h @@ -26,9 +26,9 @@ extern "C" { #endif void amrex_finalize (int finalize_mpi); - + void warpx_init (); - + void warpx_finalize (); typedef void(*WARPX_CALLBACK_PY_FUNC_0)(); @@ -47,41 +47,41 @@ extern "C" { void warpx_set_callback_py_appliedfields (WARPX_CALLBACK_PY_FUNC_0); void warpx_evolve (int numsteps); // -1 means the inputs parameter will be used. - + void warpx_addNParticles(int speciesnumber, int lenx, double* x, double* y, double* z, double* vx, double* vy, double* vz, int nattr, double* attr, int uniqueparticles); void warpx_ConvertLabParamsToBoost(); - + double warpx_getProbLo(int dir); - + double warpx_getProbHi(int dir); - + long warpx_getNumParticles(int speciesnumber); - - double** warpx_getEfield(int lev, int direction, + + double** warpx_getEfield(int lev, int direction, int *return_size, int* ncomps, int* ngrow, int **shapes); - - int* warpx_getEfieldLoVects(int lev, int direction, + + int* warpx_getEfieldLoVects(int lev, int direction, int *return_size, int* ngrow); - - double** warpx_getBfield(int lev, int direction, + + double** warpx_getBfield(int lev, int direction, int *return_size, int* ncomps, int* ngrow, int **shapes); - - int* warpx_getBfieldLoVects(int lev, int direction, + + int* warpx_getBfieldLoVects(int lev, int direction, int *return_size, int* ngrow); - - double** warpx_getCurrentDensity(int lev, int direction, + + double** warpx_getCurrentDensity(int lev, int direction, int *return_size, int* ncomps, int* ngrow, int **shapes); - - int* warpx_getCurrentDensityLoVects(int lev, int direction, + + int* warpx_getCurrentDensityLoVects(int lev, int direction, int *return_size, int* ngrow); - + double** warpx_getParticleStructs(int speciesnumber, int lev, int* num_tiles, int** particles_per_tile); - + double** warpx_getParticleArrays(int speciesnumber, int comp, int lev, int* num_tiles, int** particles_per_tile); diff --git a/Source/Utils/IonizationEnergiesTable.H b/Source/Utils/IonizationEnergiesTable.H index a7ab8210f..4c2898b91 100644 --- a/Source/Utils/IonizationEnergiesTable.H +++ b/Source/Utils/IonizationEnergiesTable.H @@ -32,105 +32,105 @@ std::map ion_map_ids = { const int nelements = 22; const int ion_atomic_numbers[nelements] = { - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 16, 17, 18, 36, 37, + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, 36, 37, 54, 86}; const int ion_energy_offsets[nelements] = { - 0, 1, 3, 6, 10, 15, 21, 28, 36, 45, - 55, 66, 78, 91, 105, 120, 136, 153, 171, 207, + 0, 1, 3, 6, 10, 15, 21, 28, 36, 45, + 55, 66, 78, 91, 105, 120, 136, 153, 171, 207, 244, 298}; const int energies_tab_length = 384; const amrex::Real table_ionization_energies[energies_tab_length]{ // H - 13.59843449, + 13.59843449, // He - 24.58738880, 54.4177650, + 24.58738880, 54.4177650, // Li - 5.39171495, 75.6400964, 122.4543581, + 5.39171495, 75.6400964, 122.4543581, // Be - 9.322699, 18.21115, 153.896203, 217.7185843, + 9.322699, 18.21115, 153.896203, 217.7185843, // B - 8.298019, 25.15483, 37.93058, 259.3715, 340.226020, + 8.298019, 25.15483, 37.93058, 259.3715, 340.226020, // C - 11.2602880, 24.383154, 47.88778, 64.49352, 392.090515, 489.993194, + 11.2602880, 24.383154, 47.88778, 64.49352, 392.090515, 489.993194, // N - 14.53413, 29.60125, 47.4453, 77.4735, 97.8901, 552.06732, 667.046116, + 14.53413, 29.60125, 47.4453, 77.4735, 97.8901, 552.06732, 667.046116, // O - 13.618055, 35.12112, 54.93554, 77.41350, 113.8990, 138.1189, 739.32682, - 871.40988, + 13.618055, 35.12112, 54.93554, 77.41350, 113.8990, 138.1189, 739.32682, + 871.40988, // F - 17.42282, 34.97081, 62.70798, 87.175, 114.249, 157.16311, 185.1868, - 953.89804, 1103.11747, + 17.42282, 34.97081, 62.70798, 87.175, 114.249, 157.16311, 185.1868, + 953.89804, 1103.11747, // Ne - 21.564540, 40.96297, 63.4233, 97.1900, 126.247, 157.934, 207.271, - 239.0970, 1195.80783, 1362.19915, + 21.564540, 40.96297, 63.4233, 97.1900, 126.247, 157.934, 207.271, + 239.0970, 1195.80783, 1362.19915, // Na - 5.1390769, 47.28636, 71.6200, 98.936, 138.404, 172.23, 208.504, - 264.192, 299.856, 1465.13449, 1648.70218, + 5.1390769, 47.28636, 71.6200, 98.936, 138.404, 172.23, 208.504, + 264.192, 299.856, 1465.13449, 1648.70218, // Mg - 7.646236, 15.035271, 80.1436, 109.2654, 141.33, 186.76, 225.02, - 265.924, 327.99, 367.489, 1761.80487, 1962.66365, + 7.646236, 15.035271, 80.1436, 109.2654, 141.33, 186.76, 225.02, + 265.924, 327.99, 367.489, 1761.80487, 1962.66365, // Al - 5.985769, 18.82855, 28.447642, 119.9924, 153.8252, 190.49, 241.76, - 284.64, 330.21, 398.65, 442.005, 2085.97700, 2304.14005, + 5.985769, 18.82855, 28.447642, 119.9924, 153.8252, 190.49, 241.76, + 284.64, 330.21, 398.65, 442.005, 2085.97700, 2304.14005, // Si - 8.15168, 16.34585, 33.49300, 45.14179, 166.767, 205.279, 246.57, - 303.59, 351.28, 401.38, 476.273, 523.415, 2437.65813, 2673.17753, + 8.15168, 16.34585, 33.49300, 45.14179, 166.767, 205.279, 246.57, + 303.59, 351.28, 401.38, 476.273, 523.415, 2437.65813, 2673.17753, // P - 10.486686, 19.76949, 30.20264, 51.44387, 65.02511, 220.430, 263.57, - 309.60, 372.31, 424.40, 479.44, 560.62, 611.741, 2816.90876, - 3069.8415, + 10.486686, 19.76949, 30.20264, 51.44387, 65.02511, 220.430, 263.57, + 309.60, 372.31, 424.40, 479.44, 560.62, 611.741, 2816.90876, + 3069.8415, // S - 10.36001, 23.33788, 34.86, 47.222, 72.5945, 88.0529, 280.954, - 328.794, 379.84, 447.7, 504.55, 564.41, 651.96, 706.994, - 3223.7807, 3494.1879, + 10.36001, 23.33788, 34.86, 47.222, 72.5945, 88.0529, 280.954, + 328.794, 379.84, 447.7, 504.55, 564.41, 651.96, 706.994, + 3223.7807, 3494.1879, // Cl - 12.967632, 23.81364, 39.80, 53.24, 67.68, 96.94, 114.2013, - 348.306, 400.851, 456.7, 530.0, 591.58, 656.30, 750.23, - 809.198, 3658.3437, 3946.2909, + 12.967632, 23.81364, 39.80, 53.24, 67.68, 96.94, 114.2013, + 348.306, 400.851, 456.7, 530.0, 591.58, 656.30, 750.23, + 809.198, 3658.3437, 3946.2909, // Ar - 15.7596117, 27.62967, 40.735, 59.58, 74.84, 91.290, 124.41, - 143.4567, 422.60, 479.76, 540.4, 619.0, 685.5, 755.13, - 855.5, 918.375, 4120.6656, 4426.2228, + 15.7596117, 27.62967, 40.735, 59.58, 74.84, 91.290, 124.41, + 143.4567, 422.60, 479.76, 540.4, 619.0, 685.5, 755.13, + 855.5, 918.375, 4120.6656, 4426.2228, // Kr - 13.9996053, 24.35984, 35.838, 50.85, 64.69, 78.49, 109.13, - 125.802, 233.0, 268, 308, 350, 391, 446, - 492, 540, 591, 640, 785, 831.6, 882.8, - 945, 999.0, 1042, 1155.0, 1205.23, 2928.9, 3072, - 3228, 3380, 3584, 3752.0, 3971, 4109.083, 17296.420, - 17936.209, + 13.9996053, 24.35984, 35.838, 50.85, 64.69, 78.49, 109.13, + 125.802, 233.0, 268, 308, 350, 391, 446, + 492, 540, 591, 640, 785, 831.6, 882.8, + 945, 999.0, 1042, 1155.0, 1205.23, 2928.9, 3072, + 3228, 3380, 3584, 3752.0, 3971, 4109.083, 17296.420, + 17936.209, // Rb - 4.1771280, 27.28954, 39.247, 52.20, 68.44, 82.9, 98.67, - 132.79, 150.628, 277.12, 313.1, 356.0, 400, 443, - 502, 550, 601, 654, 706.0, 857, 905.3, - 958.9, 1024, 1080, 1125, 1242.5, 1294.57, 3133.3, - 3281, 3443, 3600, 3815, 3988, 4214, 4356.865, - 18305.884, 18965.516, + 4.1771280, 27.28954, 39.247, 52.20, 68.44, 82.9, 98.67, + 132.79, 150.628, 277.12, 313.1, 356.0, 400, 443, + 502, 550, 601, 654, 706.0, 857, 905.3, + 958.9, 1024, 1080, 1125, 1242.5, 1294.57, 3133.3, + 3281, 3443, 3600, 3815, 3988, 4214, 4356.865, + 18305.884, 18965.516, // Xe - 12.1298436, 20.975, 31.05, 42.20, 54.1, 66.703, 91.6, - 105.9778, 179.84, 202.0, 229.02, 255.0, 281, 314, - 343, 374, 404, 434, 549, 582, 616, - 650, 700, 736, 818, 857.0, 1493, 1571, - 1653, 1742, 1826, 1919, 2023, 2113, 2209, - 2300, 2556, 2637, 2726, 2811, 2975, 3068, - 3243, 3333.8, 7660, 7889, 8144, 8382, 8971, - 9243, 9581, 9810.37, 40271.724, 41299.71, + 12.1298436, 20.975, 31.05, 42.20, 54.1, 66.703, 91.6, + 105.9778, 179.84, 202.0, 229.02, 255.0, 281, 314, + 343, 374, 404, 434, 549, 582, 616, + 650, 700, 736, 818, 857.0, 1493, 1571, + 1653, 1742, 1826, 1919, 2023, 2113, 2209, + 2300, 2556, 2637, 2726, 2811, 2975, 3068, + 3243, 3333.8, 7660, 7889, 8144, 8382, 8971, + 9243, 9581, 9810.37, 40271.724, 41299.71, // Rn - 10.74850, 21.4, 29.4, 36.9, 52.9, 64.0, 88.0, - 102.0, 154.0, 173.9, 195.0, 218.0, 240, 264, - 293, 317, 342, 367, 488, 520, 550, - 580, 640, 680, 760, 800, 850, 920, - 980, 1050, 1110, 1180, 1250, 1310, 1390, - 1460, 1520, 1590, 1660, 1720, 2033, 2094, - 2158, 2227, 2293, 2357, 2467, 2535, 2606, - 2674, 2944, 3010, 3082, 3149, 3433, 3510, - 3699, 3777, 6169, 6318, 6476, 6646, 6807, - 6964, 7283, 7450, 7630, 7800, 8260, 8410, - 8570, 8710, 9610, 9780, 10120, 10290, 21770, - 22160, 22600, 22990, 26310, 26830, 27490, 27903.1, + 10.74850, 21.4, 29.4, 36.9, 52.9, 64.0, 88.0, + 102.0, 154.0, 173.9, 195.0, 218.0, 240, 264, + 293, 317, 342, 367, 488, 520, 550, + 580, 640, 680, 760, 800, 850, 920, + 980, 1050, 1110, 1180, 1250, 1310, 1390, + 1460, 1520, 1590, 1660, 1720, 2033, 2094, + 2158, 2227, 2293, 2357, 2467, 2535, 2606, + 2674, 2944, 3010, 3082, 3149, 3433, 3510, + 3699, 3777, 6169, 6318, 6476, 6646, 6807, + 6964, 7283, 7450, 7630, 7800, 8260, 8410, + 8570, 8710, 9610, 9780, 10120, 10290, 21770, + 22160, 22600, 22990, 26310, 26830, 27490, 27903.1, 110842.0, 112843.7 }; #endif // #ifndef WARPX_IONIZATION_TABLE_H_ diff --git a/Source/Utils/NCIGodfreyTables.H b/Source/Utils/NCIGodfreyTables.H index 8cb105aa0..a3faf95a0 100644 --- a/Source/Utils/NCIGodfreyTables.H +++ b/Source/Utils/NCIGodfreyTables.H @@ -12,212 +12,212 @@ const int tab_length = 101; // Table of coefficient for Ex, Ey abd Bz // We typically interpolate between two lines const amrex::Real table_nci_godfrey_Ex_Ey_Bz[tab_length][tab_width]{ - -2.47536,2.04288,-0.598163,0.0314711, - -2.47536,2.04288,-0.598163,0.0314711, - -2.47545,2.04309,-0.598307,0.0315029, - -2.4756,2.04342,-0.598549,0.0315558, - -2.47581,2.0439,-0.598886,0.0316298, - -2.47608,2.0445,-0.59932,0.031725, - -2.47641,2.04525,-0.59985,0.0318412, - -2.4768,2.04612,-0.600477,0.0319785, - -2.47725,2.04714,-0.6012,0.0321367, - -2.47776,2.04829,-0.602019,0.0323158, - -2.47833,2.04957,-0.602934,0.0325158, - -2.47896,2.05099,-0.603944,0.0327364, - -2.47965,2.05254,-0.605051,0.0329777, - -2.4804,2.05423,-0.606253,0.0332396, - -2.48121,2.05606,-0.60755,0.0335218, - -2.48208,2.05802,-0.608942,0.0338243, - -2.48301,2.06012,-0.610429,0.0341469, - -2.48401,2.06235,-0.61201,0.0344895, - -2.48506,2.06471,-0.613685,0.0348519, - -2.48618,2.06721,-0.615453,0.0352339, - -2.48735,2.06984,-0.617314,0.0356353, - -2.48859,2.07261,-0.619268,0.0360559, - -2.48988,2.0755,-0.621312,0.0364954, - -2.49123,2.07853,-0.623447,0.0369536, - -2.49265,2.08169,-0.625672,0.0374302, - -2.49412,2.08498,-0.627986,0.0379248, - -2.49565,2.0884,-0.630386,0.0384372, - -2.49724,2.09194,-0.632873,0.0389669, - -2.49888,2.09561,-0.635443,0.0395135, - -2.50058,2.09939,-0.638096,0.0400766, - -2.50234,2.1033,-0.640829,0.0406557, - -2.50415,2.10732,-0.64364,0.0412502, - -2.50601,2.11145,-0.646526,0.0418594, - -2.50791,2.1157,-0.649485,0.0424828, - -2.50987,2.12004,-0.652512,0.0431196, - -2.51187,2.12448,-0.655604,0.0437688, - -2.51392,2.12901,-0.658756,0.0444297, - -2.516,2.13363,-0.661964,0.0451011, - -2.51812,2.13832,-0.665221,0.0457818, - -2.52027,2.14308,-0.668521,0.0464705, - -2.52244,2.14789,-0.671856,0.0471658, - -2.52464,2.15274,-0.675218,0.0478658, - -2.52684,2.15762,-0.678596,0.0485687, - -2.52906,2.16251,-0.68198,0.0492723, - -2.53126,2.16738,-0.685355,0.049974, - -2.53345,2.17222,-0.688706,0.0506708, - -2.53561,2.177,-0.692015,0.0513594, - -2.53773,2.18168,-0.69526,0.0520359, - -2.53978,2.18623,-0.698416,0.0526955, - -2.54175,2.19059,-0.701452,0.053333, - -2.5436,2.19471,-0.704331,0.0539417, - -2.54531,2.19852,-0.70701,0.0545141, - -2.54683,2.20193,-0.709433,0.0550409, - -2.5481,2.20483,-0.711533,0.0555106, - -2.54906,2.20709,-0.713224,0.0559094, - -2.54963,2.20852,-0.714397,0.0562198, - -2.54968,2.20888,-0.714907,0.0564196, - -2.54905,2.20785,-0.714562,0.0564797, - -2.54751,2.20496,-0.713094,0.0563618, - -2.54472,2.19955,-0.710118,0.0560124, - -2.54014,2.19058,-0.705048,0.0553544, - -2.53286,2.1763,-0.69693,0.0542684, - -2.52115,2.15344,-0.684027,0.05255, - -2.50098,2.11466,-0.66255,0.0497817, - -2.45797,2.03459,-0.620099,0.0446889, - -2.28371,1.72254,-0.465905,0.0283268, - -2.4885,2.04899,-0.599292,0.0390466, - -2.1433,1.36735,-0.220924,-0.00215633, - -2.4943,2.07019,-0.610552,0.035166, - -2.84529,2.77303,-1.00018,0.0724884, - -2.72242,2.51888,-0.847226,0.0509964, - -2.65633,2.3744,-0.750392,0.0326366, - -2.59601,2.23412,-0.646421,0.00868027, - -2.51477,2.0369,-0.491066,-0.0306397, - -2.35935,1.65155,-0.178971,-0.112713, - -1.84315,0.361693,0.876104,-0.393844, - -2.65422,2.39262,-0.789663,0.0516265, - -3.46529,4.42354,-2.45543,0.497097, - -3.15747,3.65311,-1.824,0.328432, - -3.04694,3.37613,-1.59668,0.267631, - -2.99205,3.23814,-1.48302,0.237103, - -2.96075,3.15894,-1.41733,0.219317, - -2.94172,3.11028,-1.37649,0.20811, - -2.92994,3.07962,-1.35025,0.200755, - -2.92283,3.06054,-1.33338,0.195859, - -2.91894,3.04938,-1.3229,0.192637, - -2.91736,3.04394,-1.31702,0.190612, - -2.91753,3.04278,-1.31456,0.189477, - -2.91905,3.04494,-1.31475,0.189026, - -2.92165,3.04973,-1.31705,0.189117, - -2.92512,3.05667,-1.32105,0.189646, - -2.92933,3.06539,-1.32646,0.190538, - -2.93416,3.07562,-1.33308,0.191735, - -2.93952,3.08715,-1.34072,0.193194, - -2.94535,3.09982,-1.34925,0.194881, - -2.95159,3.11349,-1.35858,0.196769, - -2.9582,3.12805,-1.36861,0.198838, - -2.96514,3.14342,-1.37929,0.201068, - -2.97239,3.15953,-1.39055,0.203448, - -2.97991,3.17632,-1.40234,0.205964, + -2.47536,2.04288,-0.598163,0.0314711, + -2.47536,2.04288,-0.598163,0.0314711, + -2.47545,2.04309,-0.598307,0.0315029, + -2.4756,2.04342,-0.598549,0.0315558, + -2.47581,2.0439,-0.598886,0.0316298, + -2.47608,2.0445,-0.59932,0.031725, + -2.47641,2.04525,-0.59985,0.0318412, + -2.4768,2.04612,-0.600477,0.0319785, + -2.47725,2.04714,-0.6012,0.0321367, + -2.47776,2.04829,-0.602019,0.0323158, + -2.47833,2.04957,-0.602934,0.0325158, + -2.47896,2.05099,-0.603944,0.0327364, + -2.47965,2.05254,-0.605051,0.0329777, + -2.4804,2.05423,-0.606253,0.0332396, + -2.48121,2.05606,-0.60755,0.0335218, + -2.48208,2.05802,-0.608942,0.0338243, + -2.48301,2.06012,-0.610429,0.0341469, + -2.48401,2.06235,-0.61201,0.0344895, + -2.48506,2.06471,-0.613685,0.0348519, + -2.48618,2.06721,-0.615453,0.0352339, + -2.48735,2.06984,-0.617314,0.0356353, + -2.48859,2.07261,-0.619268,0.0360559, + -2.48988,2.0755,-0.621312,0.0364954, + -2.49123,2.07853,-0.623447,0.0369536, + -2.49265,2.08169,-0.625672,0.0374302, + -2.49412,2.08498,-0.627986,0.0379248, + -2.49565,2.0884,-0.630386,0.0384372, + -2.49724,2.09194,-0.632873,0.0389669, + -2.49888,2.09561,-0.635443,0.0395135, + -2.50058,2.09939,-0.638096,0.0400766, + -2.50234,2.1033,-0.640829,0.0406557, + -2.50415,2.10732,-0.64364,0.0412502, + -2.50601,2.11145,-0.646526,0.0418594, + -2.50791,2.1157,-0.649485,0.0424828, + -2.50987,2.12004,-0.652512,0.0431196, + -2.51187,2.12448,-0.655604,0.0437688, + -2.51392,2.12901,-0.658756,0.0444297, + -2.516,2.13363,-0.661964,0.0451011, + -2.51812,2.13832,-0.665221,0.0457818, + -2.52027,2.14308,-0.668521,0.0464705, + -2.52244,2.14789,-0.671856,0.0471658, + -2.52464,2.15274,-0.675218,0.0478658, + -2.52684,2.15762,-0.678596,0.0485687, + -2.52906,2.16251,-0.68198,0.0492723, + -2.53126,2.16738,-0.685355,0.049974, + -2.53345,2.17222,-0.688706,0.0506708, + -2.53561,2.177,-0.692015,0.0513594, + -2.53773,2.18168,-0.69526,0.0520359, + -2.53978,2.18623,-0.698416,0.0526955, + -2.54175,2.19059,-0.701452,0.053333, + -2.5436,2.19471,-0.704331,0.0539417, + -2.54531,2.19852,-0.70701,0.0545141, + -2.54683,2.20193,-0.709433,0.0550409, + -2.5481,2.20483,-0.711533,0.0555106, + -2.54906,2.20709,-0.713224,0.0559094, + -2.54963,2.20852,-0.714397,0.0562198, + -2.54968,2.20888,-0.714907,0.0564196, + -2.54905,2.20785,-0.714562,0.0564797, + -2.54751,2.20496,-0.713094,0.0563618, + -2.54472,2.19955,-0.710118,0.0560124, + -2.54014,2.19058,-0.705048,0.0553544, + -2.53286,2.1763,-0.69693,0.0542684, + -2.52115,2.15344,-0.684027,0.05255, + -2.50098,2.11466,-0.66255,0.0497817, + -2.45797,2.03459,-0.620099,0.0446889, + -2.28371,1.72254,-0.465905,0.0283268, + -2.4885,2.04899,-0.599292,0.0390466, + -2.1433,1.36735,-0.220924,-0.00215633, + -2.4943,2.07019,-0.610552,0.035166, + -2.84529,2.77303,-1.00018,0.0724884, + -2.72242,2.51888,-0.847226,0.0509964, + -2.65633,2.3744,-0.750392,0.0326366, + -2.59601,2.23412,-0.646421,0.00868027, + -2.51477,2.0369,-0.491066,-0.0306397, + -2.35935,1.65155,-0.178971,-0.112713, + -1.84315,0.361693,0.876104,-0.393844, + -2.65422,2.39262,-0.789663,0.0516265, + -3.46529,4.42354,-2.45543,0.497097, + -3.15747,3.65311,-1.824,0.328432, + -3.04694,3.37613,-1.59668,0.267631, + -2.99205,3.23814,-1.48302,0.237103, + -2.96075,3.15894,-1.41733,0.219317, + -2.94172,3.11028,-1.37649,0.20811, + -2.92994,3.07962,-1.35025,0.200755, + -2.92283,3.06054,-1.33338,0.195859, + -2.91894,3.04938,-1.3229,0.192637, + -2.91736,3.04394,-1.31702,0.190612, + -2.91753,3.04278,-1.31456,0.189477, + -2.91905,3.04494,-1.31475,0.189026, + -2.92165,3.04973,-1.31705,0.189117, + -2.92512,3.05667,-1.32105,0.189646, + -2.92933,3.06539,-1.32646,0.190538, + -2.93416,3.07562,-1.33308,0.191735, + -2.93952,3.08715,-1.34072,0.193194, + -2.94535,3.09982,-1.34925,0.194881, + -2.95159,3.11349,-1.35858,0.196769, + -2.9582,3.12805,-1.36861,0.198838, + -2.96514,3.14342,-1.37929,0.201068, + -2.97239,3.15953,-1.39055,0.203448, + -2.97991,3.17632,-1.40234,0.205964, -2.98769,3.19374,-1.41463,0.208607 }; // Table of coefficient for Bx, By and Ez // We typically interpolate between two lines const amrex::Real table_nci_godfrey_Bx_By_Ez[tab_length][tab_width]{ - -2.80862,2.80104,-1.14615,0.154077, - -2.80862,2.80104,-1.14615,0.154077, - -2.80851,2.80078,-1.14595,0.154027, - -2.80832,2.80034,-1.14561,0.153945, - -2.80807,2.79973,-1.14514,0.153829, - -2.80774,2.79894,-1.14454,0.15368, - -2.80733,2.79798,-1.1438,0.153498, - -2.80685,2.79685,-1.14292,0.153284, - -2.8063,2.79554,-1.14192,0.153036, - -2.80568,2.79405,-1.14077,0.152756, - -2.80498,2.79239,-1.1395,0.152443, - -2.80421,2.79056,-1.13809,0.152098, - -2.80337,2.78856,-1.13656,0.151721, - -2.80246,2.78638,-1.13488,0.151312, - -2.80147,2.78404,-1.13308,0.150871, - -2.80041,2.78152,-1.13115,0.150397, - -2.79927,2.77882,-1.12908,0.149893, - -2.79807,2.77596,-1.12689,0.149356, - -2.79679,2.77292,-1.12456,0.148789, - -2.79543,2.76972,-1.12211,0.14819, - -2.79401,2.76634,-1.11953,0.14756, - -2.79251,2.76279,-1.11681,0.1469, - -2.79094,2.75907,-1.11397,0.146208, - -2.78929,2.75517,-1.111,0.145486, - -2.78757,2.7511,-1.10789,0.144733, - -2.78578,2.74686,-1.10466,0.14395, - -2.78391,2.74245,-1.1013,0.143137, - -2.78196,2.73786,-1.09781,0.142293, - -2.77994,2.73309,-1.09419,0.141419, - -2.77784,2.72814,-1.09043,0.140514, - -2.77566,2.72301,-1.08654,0.139578, - -2.7734,2.7177,-1.08252,0.138612, - -2.77106,2.7122,-1.07836,0.137614, - -2.76864,2.70651,-1.07406,0.136586, - -2.76613,2.70062,-1.06962,0.135525, - -2.76353,2.69453,-1.06503,0.134432, - -2.76084,2.68824,-1.0603,0.133307, - -2.75806,2.68173,-1.05541,0.132148, - -2.75518,2.675,-1.05037,0.130954, - -2.75219,2.66804,-1.04516,0.129725, - -2.7491,2.66084,-1.03978,0.12846, - -2.7459,2.65339,-1.03423,0.127156, - -2.74257,2.64566,-1.02848,0.125813, - -2.73912,2.63765,-1.02254,0.124428, - -2.73552,2.62934,-1.01638,0.122999, - -2.73178,2.62069,-1.01,0.121523, - -2.72787,2.61169,-1.00337,0.119996, - -2.72379,2.6023,-0.996479,0.118417, - -2.71951,2.59248,-0.989294,0.116778, - -2.71501,2.58218,-0.981786,0.115076, - -2.71026,2.57135,-0.97392,0.113303, - -2.70524,2.55991,-0.965651,0.111453, - -2.69989,2.54778,-0.956922,0.109514, - -2.69416,2.53484,-0.947666,0.107476, - -2.68799,2.52096,-0.937795,0.105324, - -2.68129,2.50596,-0.927197,0.103039, - -2.67394,2.48959,-0.915724,0.100597, - -2.66578,2.47153,-0.903179,0.097968, - -2.65657,2.4513,-0.889283,0.0951084, - -2.64598,2.42824,-0.873638,0.0919592, - -2.63347,2.40127,-0.855632,0.0884325, - -2.61813,2.36864,-0.834261,0.0843898, - -2.59821,2.32701,-0.807691,0.0795876, - -2.56971,2.26887,-0.77188,0.0735132, - -2.51823,2.16823,-0.713448,0.0645399, - -2.33537,1.8294,-0.533852,0.0409941, - -2.53143,2.14818,-0.670502,0.053982, - -2.17737,1.43641,-0.259095,0.00101255, - -2.51929,2.12931,-0.654743,0.0452381, - -2.86122,2.82221,-1.05039,0.0894636, - -2.72908,2.54506,-0.87834,0.0626188, - -2.6536,2.37954,-0.7665,0.0409117, - -2.58374,2.21923,-0.649738,0.0146791, - -2.49284,2.00346,-0.48457,-0.0255348, - -2.32762,1.60337,-0.1698,-0.105287, - -1.80149,0.316787,0.855414,-0.369652, - -2.60242,2.28418,-0.721378,0.040091, - -3.40335,4.25157,-2.29817,0.449834, - -3.0852,3.47341,-1.67791,0.28982, - -2.9642,3.17856,-1.44399,0.229852, - -2.89872,3.01966,-1.31861,0.197945, - -2.85668,2.91811,-1.23894,0.17783, - -2.82679,2.84621,-1.18287,0.163785, - -2.80401,2.79167,-1.14058,0.153278, - -2.78577,2.74819,-1.10706,0.145015, - -2.77061,2.7122,-1.07946,0.138267, - -2.75764,2.68152,-1.05606,0.132589, - -2.74627,2.65475,-1.03575,0.127695, - -2.73612,2.63093,-1.01777,0.123395, - -2.72692,2.6094,-1.00159,0.119553, - -2.71846,2.58968,-0.986841,0.116074, - -2.71061,2.57142,-0.973239,0.112887, - -2.70323,2.55434,-0.960573,0.109937, - -2.69626,2.53824,-0.948678,0.107185, - -2.68962,2.52294,-0.937429,0.104598, - -2.68327,2.50833,-0.926722,0.102151, - -2.67714,2.4943,-0.916477,0.0998223, - -2.67122,2.48076,-0.906627,0.0975966, - -2.66546,2.46764,-0.897118,0.0954599, - -2.65985,2.45489,-0.887903,0.0934011, + -2.80862,2.80104,-1.14615,0.154077, + -2.80862,2.80104,-1.14615,0.154077, + -2.80851,2.80078,-1.14595,0.154027, + -2.80832,2.80034,-1.14561,0.153945, + -2.80807,2.79973,-1.14514,0.153829, + -2.80774,2.79894,-1.14454,0.15368, + -2.80733,2.79798,-1.1438,0.153498, + -2.80685,2.79685,-1.14292,0.153284, + -2.8063,2.79554,-1.14192,0.153036, + -2.80568,2.79405,-1.14077,0.152756, + -2.80498,2.79239,-1.1395,0.152443, + -2.80421,2.79056,-1.13809,0.152098, + -2.80337,2.78856,-1.13656,0.151721, + -2.80246,2.78638,-1.13488,0.151312, + -2.80147,2.78404,-1.13308,0.150871, + -2.80041,2.78152,-1.13115,0.150397, + -2.79927,2.77882,-1.12908,0.149893, + -2.79807,2.77596,-1.12689,0.149356, + -2.79679,2.77292,-1.12456,0.148789, + -2.79543,2.76972,-1.12211,0.14819, + -2.79401,2.76634,-1.11953,0.14756, + -2.79251,2.76279,-1.11681,0.1469, + -2.79094,2.75907,-1.11397,0.146208, + -2.78929,2.75517,-1.111,0.145486, + -2.78757,2.7511,-1.10789,0.144733, + -2.78578,2.74686,-1.10466,0.14395, + -2.78391,2.74245,-1.1013,0.143137, + -2.78196,2.73786,-1.09781,0.142293, + -2.77994,2.73309,-1.09419,0.141419, + -2.77784,2.72814,-1.09043,0.140514, + -2.77566,2.72301,-1.08654,0.139578, + -2.7734,2.7177,-1.08252,0.138612, + -2.77106,2.7122,-1.07836,0.137614, + -2.76864,2.70651,-1.07406,0.136586, + -2.76613,2.70062,-1.06962,0.135525, + -2.76353,2.69453,-1.06503,0.134432, + -2.76084,2.68824,-1.0603,0.133307, + -2.75806,2.68173,-1.05541,0.132148, + -2.75518,2.675,-1.05037,0.130954, + -2.75219,2.66804,-1.04516,0.129725, + -2.7491,2.66084,-1.03978,0.12846, + -2.7459,2.65339,-1.03423,0.127156, + -2.74257,2.64566,-1.02848,0.125813, + -2.73912,2.63765,-1.02254,0.124428, + -2.73552,2.62934,-1.01638,0.122999, + -2.73178,2.62069,-1.01,0.121523, + -2.72787,2.61169,-1.00337,0.119996, + -2.72379,2.6023,-0.996479,0.118417, + -2.71951,2.59248,-0.989294,0.116778, + -2.71501,2.58218,-0.981786,0.115076, + -2.71026,2.57135,-0.97392,0.113303, + -2.70524,2.55991,-0.965651,0.111453, + -2.69989,2.54778,-0.956922,0.109514, + -2.69416,2.53484,-0.947666,0.107476, + -2.68799,2.52096,-0.937795,0.105324, + -2.68129,2.50596,-0.927197,0.103039, + -2.67394,2.48959,-0.915724,0.100597, + -2.66578,2.47153,-0.903179,0.097968, + -2.65657,2.4513,-0.889283,0.0951084, + -2.64598,2.42824,-0.873638,0.0919592, + -2.63347,2.40127,-0.855632,0.0884325, + -2.61813,2.36864,-0.834261,0.0843898, + -2.59821,2.32701,-0.807691,0.0795876, + -2.56971,2.26887,-0.77188,0.0735132, + -2.51823,2.16823,-0.713448,0.0645399, + -2.33537,1.8294,-0.533852,0.0409941, + -2.53143,2.14818,-0.670502,0.053982, + -2.17737,1.43641,-0.259095,0.00101255, + -2.51929,2.12931,-0.654743,0.0452381, + -2.86122,2.82221,-1.05039,0.0894636, + -2.72908,2.54506,-0.87834,0.0626188, + -2.6536,2.37954,-0.7665,0.0409117, + -2.58374,2.21923,-0.649738,0.0146791, + -2.49284,2.00346,-0.48457,-0.0255348, + -2.32762,1.60337,-0.1698,-0.105287, + -1.80149,0.316787,0.855414,-0.369652, + -2.60242,2.28418,-0.721378,0.040091, + -3.40335,4.25157,-2.29817,0.449834, + -3.0852,3.47341,-1.67791,0.28982, + -2.9642,3.17856,-1.44399,0.229852, + -2.89872,3.01966,-1.31861,0.197945, + -2.85668,2.91811,-1.23894,0.17783, + -2.82679,2.84621,-1.18287,0.163785, + -2.80401,2.79167,-1.14058,0.153278, + -2.78577,2.74819,-1.10706,0.145015, + -2.77061,2.7122,-1.07946,0.138267, + -2.75764,2.68152,-1.05606,0.132589, + -2.74627,2.65475,-1.03575,0.127695, + -2.73612,2.63093,-1.01777,0.123395, + -2.72692,2.6094,-1.00159,0.119553, + -2.71846,2.58968,-0.986841,0.116074, + -2.71061,2.57142,-0.973239,0.112887, + -2.70323,2.55434,-0.960573,0.109937, + -2.69626,2.53824,-0.948678,0.107185, + -2.68962,2.52294,-0.937429,0.104598, + -2.68327,2.50833,-0.926722,0.102151, + -2.67714,2.4943,-0.916477,0.0998223, + -2.67122,2.48076,-0.906627,0.0975966, + -2.66546,2.46764,-0.897118,0.0954599, + -2.65985,2.45489,-0.887903,0.0934011, -2.65437,2.44244,-0.878945,0.0914107 }; diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index a5ab7b18c..f84701a02 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -76,7 +76,7 @@ WarpX::MoveWindow (bool move_j) new_slice_lo[i] = current_slice_lo[i]; new_slice_hi[i] = current_slice_hi[i]; } - int num_shift_base_slice = static_cast ((moving_window_x - + int num_shift_base_slice = static_cast ((moving_window_x - current_slice_lo[dir]) / cdx[dir]); new_slice_lo[dir] = current_slice_lo[dir] + num_shift_base_slice*cdx[dir]; new_slice_hi[dir] = current_slice_hi[dir] + num_shift_base_slice*cdx[dir]; diff --git a/Source/Utils/WarpXUtil.H b/Source/Utils/WarpXUtil.H index 39fded8e8..fd6e72dc6 100644 --- a/Source/Utils/WarpXUtil.H +++ b/Source/Utils/WarpXUtil.H @@ -7,5 +7,5 @@ void ReadBoostedFrameParameters(amrex::Real& gamma_boost, amrex::Real& beta_boos void ConvertLabParamsToBoost(); -void NullifyMF(amrex::MultiFab& mf, int lev, amrex::Real zmin, +void NullifyMF(amrex::MultiFab& mf, int lev, amrex::Real zmin, amrex::Real zmax); diff --git a/Source/Utils/WarpXUtil.cpp b/Source/Utils/WarpXUtil.cpp index 19e898208..4b11eb69d 100644 --- a/Source/Utils/WarpXUtil.cpp +++ b/Source/Utils/WarpXUtil.cpp @@ -68,7 +68,7 @@ void ConvertLabParamsToBoost() BL_ASSERT(slice_lo.size() == AMREX_SPACEDIM); pp_slice.queryarr("dom_hi",slice_hi,0,AMREX_SPACEDIM); BL_ASSERT(slice_hi.size() == AMREX_SPACEDIM); - + pp_amr.query("max_level", max_level); if (max_level > 0){ @@ -113,7 +113,7 @@ void ConvertLabParamsToBoost() } -/* \brief Function that sets the value of MultiFab MF to zero for z between +/* \brief Function that sets the value of MultiFab MF to zero for z between * zmin and zmax. */ void NullifyMF(amrex::MultiFab& mf, int lev, amrex::Real zmin, amrex::Real zmax){ @@ -143,7 +143,7 @@ void NullifyMF(amrex::MultiFab& mf, int lev, amrex::Real zmin, amrex::Real zmax) const Real z_gridpoint = zmin_box+(k-lo_ind)*dz; #else const Real z_gridpoint = zmin_box+(j-lo_ind)*dz; -#endif +#endif if ( (z_gridpoint >= zmin) && (z_gridpoint < zmax) ) { arr(i,j,k) = 0.; }; diff --git a/Source/Utils/utils_ES.F90 b/Source/Utils/utils_ES.F90 index 887a5ef15..ce143bb94 100644 --- a/Source/Utils/utils_ES.F90 +++ b/Source/Utils/utils_ES.F90 @@ -16,9 +16,9 @@ contains integer, intent(in) :: lrat(3) real(amrex_real), intent(inout) :: crse(clo(1):chi(1),clo(2):chi(2),clo(3):chi(3)) real(amrex_real), intent(in) :: fine(flo(1):fhi(1),flo(2):fhi(2),flo(3):fhi(3)) - + integer :: i, j, k, ii, jj, kk - + do k = lo(3), hi(3) kk = k * lrat(3) do j = lo(2), hi(2) @@ -27,7 +27,7 @@ contains ii = i * lrat(1) crse(i,j,k) = fine(ii,jj,kk) + & ! These six fine nodes are shared by two coarse nodes... - 0.5d0 * (fine(ii-1,jj,kk) + fine(ii+1,jj,kk) + & + 0.5d0 * (fine(ii-1,jj,kk) + fine(ii+1,jj,kk) + & fine(ii,jj-1,kk) + fine(ii,jj+1,kk) + & fine(ii,jj,kk-1) + fine(ii,jj,kk+1)) + & ! ... these twelve are shared by four... @@ -59,16 +59,16 @@ contains integer, intent(in) :: lrat(2) real(amrex_real), intent(inout) :: crse(clo(1):chi(1),clo(2):chi(2)) real(amrex_real), intent(in) :: fine(flo(1):fhi(1),flo(2):fhi(2)) - + integer :: i, j, ii, jj - + do j = lo(2), hi(2) jj = j * lrat(2) do i = lo(1), hi(1) ii = i * lrat(1) crse(i,j) = fine(ii,jj) + & ! These four fine nodes are shared by two coarse nodes... - 0.5d0 * (fine(ii-1,jj) + fine(ii+1,jj) + & + 0.5d0 * (fine(ii-1,jj) + fine(ii+1,jj) + & fine(ii,jj-1) + fine(ii,jj+1)) + & ! ... and these four are shared by four... 0.25d0 * (fine(ii-1,jj-1) + fine(ii-1,jj+1) + & @@ -87,12 +87,12 @@ contains double precision, intent(inout) :: input_data(lo(1):hi(1),lo(2):hi(2),lo(3):hi(3)) double precision, intent(inout) :: bndry_data(lo(1)-1:hi(1)+1,lo(2)-1:hi(2)+1,lo(3)-1:hi(3)+1) integer(c_int), intent(in ) :: mask (lo(1):hi(1),lo(2):hi(2),lo(3):hi(3)) - + integer :: i, j, k - + do k = lo(3), hi(3) do j = lo(2), hi(2) - do i = lo(1), hi(1) + do i = lo(1), hi(1) if (mask(i,j,k) .eq. 1) then bndry_data(i,j,k) = input_data(i,j,k) input_data(i,j,k) = 0.d0 @@ -110,11 +110,11 @@ contains double precision, intent(inout) :: input_data(lo(1):hi(1),lo(2):hi(2)) double precision, intent(inout) :: bndry_data(lo(1)-1:hi(1)+1,lo(2)-1:hi(2)+1) integer(c_int), intent(in ) :: mask (lo(1):hi(1),lo(2):hi(2)) - + integer :: i, j - + do j = lo(2), hi(2) - do i = lo(1), hi(1) + do i = lo(1), hi(1) if (mask(i,j) .eq. 1) then bndry_data(i,j) = input_data(i,j) input_data(i,j) = 0.d0 @@ -145,7 +145,7 @@ contains end do end do end do - + if (total .gt. 0) then mask(i,j,k) = 1 else @@ -169,14 +169,14 @@ contains do j = lo(2), hi(2) do i = lo(1), hi(1) - + total = 0 do ii = i-ncells, i+ncells do jj = j-ncells, j+ncells total = total + tmp_mask(ii, jj) end do end do - + if (total .gt. 0) then mask(i,j) = 1 else diff --git a/Source/Utils/write_atomic_data_cpp.py b/Source/Utils/write_atomic_data_cpp.py index b085d50eb..1c85fa32c 100644 --- a/Source/Utils/write_atomic_data_cpp.py +++ b/Source/Utils/write_atomic_data_cpp.py @@ -60,7 +60,7 @@ for element in ion_names: '\n\s+(\d+)\s+\|\s+%s\s+\w+\s+\|\s+\+*(\d+)\s+\|\s+\(*\[*(\d+\.*\d*)' \ %element list_of_tuples = re.findall( regex_command, text_data ) - for count, energy in enumerate([x[2] for x in list_of_tuples]): + for count, energy in enumerate([x[2] for x in list_of_tuples]): if count%7==0: cpp_string += '\n ' cpp_string += energy + ', ' cpp_string = cpp_string[:-2] diff --git a/Source/main.cpp b/Source/main.cpp index d31268b12..6aca49cd3 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -25,18 +25,18 @@ int main(int argc, char* argv[]) ConvertLabParamsToBoost(); BL_PROFILE_VAR("main()", pmain); - + const Real strt_total = amrex::second(); { WarpX warpx; - + warpx.InitData(); warpx.Evolve(); - + Real end_total = amrex::second() - strt_total; - + ParallelDescriptor::ReduceRealMax(end_total ,ParallelDescriptor::IOProcessorNumber()); if (warpx.Verbose()) { amrex::Print() << "Total Time : " << end_total << '\n'; diff --git a/Tools/compute_domain.py b/Tools/compute_domain.py index 822d776e8..a078ff870 100644 --- a/Tools/compute_domain.py +++ b/Tools/compute_domain.py @@ -4,7 +4,7 @@ import scipy.constants as scc import time, copy ''' -This Python script helps a user to parallelize a WarpX simulation. +This Python script helps a user to parallelize a WarpX simulation. The user specifies the minimal size of the physical domain and the resolution in each dimension, and the scripts computes: @@ -17,7 +17,7 @@ When running in a boosted frame, the script also has the option to automatically compute the number of cells in z to satisfy dx>dz in the boosted frame. -Note that the script has no notion of blocking_factor. It is assumed that +Note that the script has no notion of blocking_factor. It is assumed that blocking_factor = max_grid_size, and that all boxes have the same size. ''' @@ -32,8 +32,8 @@ box_hi0 = np.array([ 25.e-6, 25.e-6, 60.e-6]) dx = 1.e-6 dz = dx cell_size = np.array([dx, dx, dz]) -# Use this for simulations in a boosted frame if you -# want to enforce dz < dx / dx_over_dz_boosted_frame +# Use this for simulations in a boosted frame if you +# want to enforce dz < dx / dx_over_dz_boosted_frame compute_dz_boosted_frame = True gamma_boost = 30. dx_over_dz_boosted_frame = 1.1 # >1. is usually more stable @@ -54,7 +54,7 @@ def adjust_bounds(box_lo0, box_hi0, box_ncell0, mgs): box_hi = box_ncell * cell_size * box_hi0 / (box_hi0 - box_lo0) return box_lo, box_hi, box_ncell -# Calculate parallelization for the simulation, given numerical parameters +# Calculate parallelization for the simulation, given numerical parameters # (number of cells, max_grid_size, number of threads per node etc.) def nb_nodes_mpi(box_ncell,mgs,threadspernode,ompnumthreads,ngridpernode, ndim): nmpipernode = threadspernode/ompnumthreads @@ -63,7 +63,7 @@ def nb_nodes_mpi(box_ncell,mgs,threadspernode,ompnumthreads,ngridpernode, ndim): if ndim == 2: ngrids = box_ngrids[0] * box_ngrids[1] elif ndim == 3: - ngrids = np.prod(box_ngrids) + ngrids = np.prod(box_ngrids) n_mpi = intceil( ngrids/ngridpermpi ) n_node = intceil( n_mpi/nmpipernode ) return n_node, n_mpi diff --git a/Tools/performance_tests/functions_perftest.py b/Tools/performance_tests/functions_perftest.py index 1a0767500..5e026bf12 100644 --- a/Tools/performance_tests/functions_perftest.py +++ b/Tools/performance_tests/functions_perftest.py @@ -47,7 +47,7 @@ def run_batch_nnode(test_list, res_dir, bin_name, config_command, architecture=' batch_string += '#SBATCH -q regular\n' batch_string += '#SBATCH -e error.txt\n' batch_string += '#SBATCH --account=m2852\n' - + for count, current_test in enumerate(test_list): shutil.copy(cwd + current_test.input_file, res_dir) srun_string = '' @@ -177,14 +177,14 @@ def get_nsteps(run_name): return nsteps def extract_dataframe(filename, n_steps): - # Get init time and total time through Inclusive time + # Get init time and total time through Inclusive time partition_limit_start = 'NCalls Incl. Min Incl. Avg Incl. Max Max %' with open(filename) as file_handler: output_text = file_handler.read() - # get total simulation time + # get total simulation time line_match_totaltime = re.search('TinyProfiler total time across processes.*', output_text) total_time = float(line_match_totaltime.group(0).split()[8]) - # get time performing steps as Inclusive WarpX::Evolve() time + # get time performing steps as Inclusive WarpX::Evolve() time search_area = output_text.partition(partition_limit_start)[2] line_match_looptime = re.search('\nWarpX::Evolve().*', search_area) time_wo_initialization = float(line_match_looptime.group(0).split()[3]) @@ -194,14 +194,14 @@ def extract_dataframe(filename, n_steps): time_WritePlotFile = float(line_match_WritePlotFile.group(0).split()[3]) else: time_WritePlotFile = 0. - # Get timers for all routines - # Where to start and stop in the output_file + # Get timers for all routines + # Where to start and stop in the output_file partition_limit_start = 'NCalls Excl. Min Excl. Avg Excl. Max Max %' partition_limit_end = 'NCalls Incl. Min Incl. Avg Incl. Max Max %' - # Put file content in a string + # Put file content in a string with open(filename) as file_handler: output_text = file_handler.read() - # Keep only profiling data + # Keep only profiling data search_area = output_text.partition(partition_limit_start)[2]\ .partition(partition_limit_end)[0] list_string = search_area.split('\n')[2:-4] @@ -218,26 +218,26 @@ def extract_dataframe(filename, n_steps): # df['string_output'] = partition_limit_start + '\n' + search_area return df -# Run a performance test in an interactive allocation +# Run a performance test in an interactive allocation # def run_interactive(run_name, res_dir, n_node=1, n_mpi=1, n_omp=1): -# # Clean res_dir # +# # Clean res_dir # # if os.path.exists(res_dir): # shutil.rmtree(res_dir) # os.makedirs(res_dir) -# # Copy files to res_dir # +# # Copy files to res_dir # # shutil.copyfile(bin_dir + bin_name, res_dir + bin_name) # shutil.copyfile(cwd + run_name, res_dir + 'inputs') # os.chdir(res_dir) # if args.architecture == 'cpu': -# cflag_value = max(1, int(32/n_mpi) * 2) # Follow NERSC directives # +# cflag_value = max(1, int(32/n_mpi) * 2) # Follow NERSC directives # # exec_command = 'export OMP_NUM_THREADS=' + str(n_omp) + ';' +\ # 'srun --cpu_bind=cores ' + \ # ' -n ' + str(n_node*n_mpi) + \ # ' -c ' + str(cflag_value) + \ # ' ./' + bin_name + ' inputs > perf_output.txt' # elif args.architecture == 'knl': -# # number of logical cores per MPI process # -# cflag_value = max(1,int(68/n_mpi) * 4) # Follow NERSC directives # +# # number of logical cores per MPI process # +# cflag_value = max(1,int(68/n_mpi) * 4) # Follow NERSC directives # # exec_command = 'export OMP_NUM_THREADS=' + str(n_omp) + ';' +\ # 'srun --cpu_bind=cores ' + \ # ' -n ' + str(n_node*n_mpi) + \ diff --git a/Tools/performance_tests/performance_log.txt b/Tools/performance_tests/performance_log.txt index d2fa22fe4..7f2d24534 100644 --- a/Tools/performance_tests/performance_log.txt +++ b/Tools/performance_tests/performance_log.txt @@ -1,81 +1,81 @@ ## year month day run_name compiler architecture n_node n_mpi n_omp time_initialization time_one_iteration Redistribute FillBoundary ParallelCopy CurrentDeposition FieldGather ParthiclePush Copy EvolveEM Checkpoint WriteParticles Write_FabArray WriteMultiLevelPlotfile(unit: second) RedistributeMPI -2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.14 0.3986 0.1713 0.01719 0.01615 0.06987 0.03636 0.01901 0.01999 0.003602 0 0 0 0 0.007262 -2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.39 0.4009 0.1712 0.01676 0.01583 0.07061 0.03684 0.01926 0.02011 0.003687 0 0 0 0 0.007841 -2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 2.91 0.4024 0.1716 0.01826 0.01918 0.0703 0.0363 0.01912 0.01989 0.003017 0 0 0 0 0.007256 -2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.21 0.3997 0.1717 0.01706 0.0162 0.07026 0.03655 0.01928 0.01999 0.003687 0 0 0 0 0.006799 -2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 0.89 0.4779 0.04441 0.1143 0.09117 0.1072 0.01254 0.003702 0.004217 0.01247 0 0 0 0 0.003441 -2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.58 0.4626 0.04424 0.1048 0.0851 0.1073 0.01259 0.003767 0.004282 0.01311 0 0 0 0 0.002798 -2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.63 0.4616 0.04441 0.1033 0.08398 0.1079 0.01312 0.003802 0.004224 0.01278 0 0 0 0 0.003188 -2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.72 0.461 0.04419 0.1038 0.08424 0.1074 0.01257 0.003799 0.0043 0.01318 0 0 0 0 0.002816 -2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.32 0.3986 0.1712 0.01804 0.01697 0.06999 0.03615 0.01842 0.01896 0.003445 0 0 0 0 0.00738 -2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.17 0.3974 0.1711 0.01722 0.01587 0.07016 0.03642 0.01844 0.01902 0.003431 0 0 0 0 0.007332 -2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 2.88 0.3946 0.1709 0.01686 0.01562 0.06972 0.03595 0.01848 0.01916 0.003269 0 0 0 0 0.006887 -2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 2.95 0.4094 0.1708 0.01761 0.01632 0.07001 0.03651 0.01863 0.01906 0.003314 0 0 0 0 0.01898 -2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.3 0.4787 0.04447 0.1139 0.09124 0.108 0.01287 0.003811 0.004205 0.01249 0 0 0 0 0.003045 -2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 3.16 0.4578 0.04412 0.1015 0.08339 0.1078 0.01301 0.003919 0.004182 0.0125 0 0 0 0 0.002701 -2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 2.78 0.4679 0.04418 0.1035 0.08456 0.1079 0.01303 0.003902 0.004214 0.0127 0 0 0 0 0.009118 -2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.12 0.4613 0.04425 0.1043 0.08517 0.1073 0.01242 0.003797 0.004221 0.01239 0 0 0 0 0.003665 -2018 01 31 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.48 0.1237 0.03056 0.01622 0.01468 0.02039 0.005016 0.003737 0.002632 0.00326 0 0 0 0 0.006871 -2018 01 31 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.79 0.1287 0.0308 0.01706 0.01715 0.02042 0.005452 0.003636 0.002797 0.003143 0 0 0 0 0.007324 -2018 01 31 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.9 0.1296 0.03084 0.01711 0.01731 0.02053 0.005379 0.003641 0.002843 0.003137 0 0 0 0 0.008151 -2018 01 31 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.9 0.1323 0.03081 0.01703 0.01736 0.02065 0.005339 0.003638 0.002751 0.004008 0 0 0 0 0.01015 -2018 01 31 automated_test_4_labdiags_2ppc intel knl 1 16 8 0.85 0.2896 0.03832 0.06449 0.07493 0.003507 0.002987 0.0001515 0.0001762 0.007921 0.0371 0.001537 0 0.0004387 0.03832 -2018 01 31 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.12 0.2895 0.03845 0.06423 0.07481 0.003489 0.002994 0.000152 0.0001779 0.00834 0.0357 0.001545 0 0.0005249 0.03845 -2018 01 31 automated_test_4_labdiags_2ppc intel knl 1 16 8 0.76 0.3243 0.03804 0.0646 0.07462 0.003483 0.002991 0.0001508 0.0001769 0.008051 0.05983 0.001565 0 0.005392 0.03804 -2018 01 31 automated_test_4_labdiags_2ppc intel knl 1 16 8 0.74 0.3143 0.03941 0.06478 0.07547 0.003486 0.003007 0.0001518 0.0001808 0.007845 0.05079 0.001543 0 0.0007033 0.03941 -2018 01 31 automated_test_5_loadimbalance intel knl 1 16 8 9.2 0.3845 0.08558 0.1042 0.1332 0 0 0 0 0.01226 0 0 0 0 0.08558 -2018 01 31 automated_test_5_loadimbalance intel knl 1 16 8 9.19 0.3864 0.085 0.1051 0.134 0 0 0 0 0.01202 0 0 0 0 0.085 -2018 01 31 automated_test_5_loadimbalance intel knl 1 16 8 8.98 0.3912 0.08665 0.1061 0.1356 0 0 0 0 0.01193 0 0 0 0 0.08665 -2018 01 31 automated_test_5_loadimbalance intel knl 1 16 8 9.03 0.3826 0.08484 0.1031 0.1329 0 0 0 0 0.01205 0 0 0 0 0.08484 -2018 01 31 automated_test_6_output_2ppc intel knl 1 16 8 3.6 1.086 0.0898 0.1311 0.09441 0.1345 0.027 0.008783 0.009792 0.02151 0.08454 0.04962 0 0.0008218 0.005303 -2018 01 31 automated_test_6_output_2ppc intel knl 1 16 8 4.7 1.136 0.09059 0.1437 0.09535 0.1358 0.02915 0.009238 0.01002 0.02315 0.09088 0.05006 0 0.01081 0.005381 -2018 01 31 automated_test_6_output_2ppc intel knl 1 16 8 4.0 1.132 0.09145 0.1377 0.09592 0.1365 0.02817 0.009353 0.0103 0.02447 0.066 0.05309 0 0.02047 0.009196 -2018 01 31 automated_test_6_output_2ppc intel knl 1 16 8 3.8 1.135 0.09088 0.1308 0.09623 0.135 0.02762 0.008839 0.009758 0.02561 0.1144 0.04874 0 0.0008693 0.008112 -2018 02 13 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.87 0.4053 0.1754 0.01914 0.01871 0.0691 0.03648 0.01879 0.0193 0.003268 0 0 0 0 0.007445 -2018 02 13 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 4.38 0.405 0.1741 0.01901 0.01839 0.07034 0.03718 0.01894 0.0195 0.003845 0 0 0 0 0.007187 -2018 02 13 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.79 0.3999 0.1739 0.01859 0.01631 0.06918 0.0367 0.01906 0.01952 0.003278 0 0 0 0 0.006658 -2018 02 13 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.93 0.4044 0.1746 0.01854 0.01695 0.06975 0.03721 0.0191 0.01941 0.003979 0 0 0 0 0.007381 -2018 02 13 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.83 0.4773 0.04582 0.1089 0.08772 0.1072 0.01304 0.003335 0.004231 0.01385 0 0 0 0 0.002991 -2018 02 13 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.51 0.4654 0.04556 0.1027 0.08351 0.1068 0.01292 0.003114 0.004249 0.01356 0 0 0 0 0.002748 -2018 02 13 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.62 0.4755 0.0457 0.1082 0.08761 0.1069 0.0131 0.003205 0.00431 0.01388 0 0 0 0 0.002738 -2018 02 13 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.44 0.4798 0.04561 0.1133 0.08962 0.1064 0.01246 0.003076 0.004241 0.01318 0 0 0 0 0.003164 -2018 02 13 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.92 0.1282 0.03185 0.01747 0.01557 0.01956 0.005103 0.003455 0.00274 0.00346 0 0 0 0 0.007196 -2018 02 13 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.97 0.1301 0.03157 0.01788 0.01732 0.01957 0.00508 0.003335 0.002803 0.003454 0 0 0 0 0.007446 -2018 02 13 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.91 0.1289 0.03137 0.01765 0.0155 0.02026 0.005636 0.003513 0.002716 0.003381 0 0 0 0 0.007087 -2018 02 13 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.64 0.1308 0.03142 0.0181 0.01777 0.01953 0.005204 0.003371 0.002782 0.003057 0 0 0 0 0.007769 -2018 02 13 automated_test_4_labdiags_2ppc intel knl 1 16 8 3.19 0.3005 0.0383 0.06396 0.07274 0.003502 0.003005 0.0001628 0.0001839 0.008869 0.04427 0.001522 0 0.0005522 0.0383 -2018 02 13 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.41 0.2945 0.0389 0.06251 0.0723 0.003508 0.003009 0.000164 0.0001825 0.009131 0.04042 0.001538 0 0.0005936 0.0389 -2018 02 13 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.32 0.3066 0.0387 0.06558 0.07547 0.003463 0.003017 0.0001631 0.0001837 0.008431 0.04955 0.001555 0 0.000454 0.0387 -2018 02 13 automated_test_4_labdiags_2ppc intel knl 1 16 8 0.71 0.3391 0.03987 0.06534 0.07626 0.003475 0.003004 0.0001643 0.0001821 0.008152 0.06677 0.001534 0 0.01029 0.03987 -2018 02 13 automated_test_5_loadimbalance intel knl 1 16 8 9.68 0.3956 0.08701 0.1051 0.1352 0 0 0 0 0.01387 0 0 0 0 0.08701 -2018 02 13 automated_test_5_loadimbalance intel knl 1 16 8 10.65 0.3987 0.0866 0.1051 0.1332 0 0 0 0 0.0191 0 0 0 0 0.0866 -2018 02 13 automated_test_5_loadimbalance intel knl 1 16 8 10.11 0.4013 0.08782 0.1087 0.1359 0 0 0 0 0.01379 0 0 0 0 0.08782 -2018 02 13 automated_test_5_loadimbalance intel knl 1 16 8 9.94 0.39 0.08702 0.1028 0.132 0 0 0 0 0.0142 0 0 0 0 0.08702 -2018 02 13 automated_test_6_output_2ppc intel knl 1 16 8 1.292 0.2639 0.01424 0.03424 0.01742 0.01893 0.003449 0.001364 0.001712 0.009362 0.04053 0.01765 0 0.002558 0.001185 -2018 02 13 automated_test_6_output_2ppc intel knl 1 16 8 0.779 0.3155 0.01125 0.03605 0.01628 0.02431 0.009672 0.002843 0.001334 0.008876 0.05925 0.02047 0 0.001897 0.0006917 -2018 02 13 automated_test_6_output_2ppc intel knl 1 16 8 0.635 0.2568 0.01083 0.03443 0.01592 0.01963 0.003027 0.001439 0.001286 0.009288 0.03879 0.01815 0 0.001509 0.0007743 -2018 02 13 automated_test_6_output_2ppc intel knl 1 16 8 1.371 0.2648 0.01401 0.03376 0.01593 0.01936 0.003443 0.001351 0.00169 0.01161 0.03936 0.01785 0 0.002107 0.001171 -2018 02 20 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.7 0.4573 0.01159 0.02139 0.02206 0.06934 0.03845 0.0192 0.02062 0.003496 0 0 0 0 0.01159 -2018 02 20 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.45 0.4603 0.01356 0.02085 0.02488 0.06946 0.03777 0.01908 0.02031 0.003356 0 0 0 0 0.01356 -2018 02 20 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.72 0.4552 0.01245 0.02003 0.02013 0.06874 0.03766 0.01907 0.0203 0.003667 0 0 0 0 0.01245 -2018 02 20 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 2.94 0.4557 0.01381 0.01979 0.02053 0.0687 0.03694 0.01886 0.02012 0.006396 0 0 0 0 0.01381 -2018 02 20 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.33 0.4937 0.005316 0.1103 0.09802 0.1071 0.01258 0.00326 0.004435 0.01347 0 0 0 0 0.005316 -2018 02 20 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.27 0.5063 0.004948 0.1213 0.1019 0.1067 0.01183 0.003056 0.004479 0.01327 0 0 0 0 0.004948 -2018 02 20 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 2.2 0.4983 0.005787 0.1141 0.1002 0.1067 0.0121 0.00307 0.00445 0.01343 0 0 0 0 0.005787 -2018 02 20 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.39 0.5018 0.005339 0.1152 0.1007 0.1073 0.01249 0.003196 0.004484 0.01348 0 0 0 0 0.005339 -2018 02 20 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.98 0.1342 0.007843 0.01855 0.01798 0.01936 0.005198 0.003471 0.002626 0.003161 0 0 0 0 0.007843 -2018 02 20 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.63 0.1367 0.008055 0.01917 0.01818 0.01992 0.006097 0.003388 0.002639 0.003079 0 0 0 0 0.008055 -2018 02 20 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.66 0.1365 0.008017 0.0196 0.01819 0.01979 0.005769 0.00331 0.002668 0.003111 0 0 0 0 0.008017 -2018 02 20 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.89 0.1367 0.008249 0.01947 0.01818 0.01956 0.005585 0.003341 0.002697 0.003217 0 0 0 0 0.008249 -2018 02 20 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.14 0.3087 0.04174 0.0637 0.0734 0.00345 0.002967 0.0001664 0.0001849 0.008714 0.05156 0.001539 0 0.0004984 0.04174 -2018 02 20 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.21 0.3407 0.07513 0.07261 0.0713 0.003428 0.002994 0.0001638 0.0001848 0.009408 0.003442 0.00173 0 0.0005256 0.07513 -2018 02 20 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.73 0.347 0.04077 0.06476 0.07148 0.00345 0.002998 0.0001637 0.0001829 0.009379 0.03947 0.001574 0 0.04989 0.04077 -2018 02 20 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.52 0.3469 0.04088 0.06365 0.07183 0.003493 0.002957 0.0001659 0.0001827 0.009064 0.04694 0.001959 0 0.04099 0.04088 -2018 02 20 automated_test_5_loadimbalance intel knl 1 16 8 9.92 0.4206 0.08811 0.1186 0.1402 0 0 0 0 0.01443 0 0 0 0 0.08811 -2018 02 20 automated_test_5_loadimbalance intel knl 1 16 8 9.12 0.3884 0.08626 0.1027 0.1305 0 0 0 0 0.01368 0 0 0 0 0.08626 -2018 02 20 automated_test_5_loadimbalance intel knl 1 16 8 9.91 0.4097 0.08598 0.1119 0.1381 0 0 0 0 0.01414 0 0 0 0 0.08598 -2018 02 20 automated_test_5_loadimbalance intel knl 1 16 8 9.63 0.4257 0.0876 0.1213 0.1441 0 0 0 0 0.01422 0 0 0 0 0.0876 -2018 02 20 automated_test_6_output_2ppc intel knl 1 16 8 1.23 0.274 0.003227 0.03782 0.01724 0.01945 0.003219 0.001468 0.0014 0.01094 0.03943 0.0175 0 0.00509 0.001122 -2018 02 20 automated_test_6_output_2ppc intel knl 1 16 8 2.076 0.3023 0.002995 0.035 0.01619 0.02462 0.01126 0.006984 0.001548 0.01009 0.04604 0.01734 0 0.08398 0.001151 -2018 02 20 automated_test_6_output_2ppc intel knl 1 16 8 1.378 0.273 0.004545 0.03721 0.01754 0.02039 0.003415 0.00145 0.001561 0.01058 0.04009 0.01763 0 0.002519 0.001187 -2018 02 20 automated_test_6_output_2ppc intel knl 1 16 8 1.61 0.2911 0.004065 0.03726 0.01782 0.02439 0.01289 0.003463 0.001689 0.008778 0.03975 0.01723 0 0.00247 0.00129 +2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.14 0.3986 0.1713 0.01719 0.01615 0.06987 0.03636 0.01901 0.01999 0.003602 0 0 0 0 0.007262 +2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.39 0.4009 0.1712 0.01676 0.01583 0.07061 0.03684 0.01926 0.02011 0.003687 0 0 0 0 0.007841 +2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 2.91 0.4024 0.1716 0.01826 0.01918 0.0703 0.0363 0.01912 0.01989 0.003017 0 0 0 0 0.007256 +2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.21 0.3997 0.1717 0.01706 0.0162 0.07026 0.03655 0.01928 0.01999 0.003687 0 0 0 0 0.006799 +2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 0.89 0.4779 0.04441 0.1143 0.09117 0.1072 0.01254 0.003702 0.004217 0.01247 0 0 0 0 0.003441 +2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.58 0.4626 0.04424 0.1048 0.0851 0.1073 0.01259 0.003767 0.004282 0.01311 0 0 0 0 0.002798 +2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.63 0.4616 0.04441 0.1033 0.08398 0.1079 0.01312 0.003802 0.004224 0.01278 0 0 0 0 0.003188 +2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.72 0.461 0.04419 0.1038 0.08424 0.1074 0.01257 0.003799 0.0043 0.01318 0 0 0 0 0.002816 +2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.32 0.3986 0.1712 0.01804 0.01697 0.06999 0.03615 0.01842 0.01896 0.003445 0 0 0 0 0.00738 +2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.17 0.3974 0.1711 0.01722 0.01587 0.07016 0.03642 0.01844 0.01902 0.003431 0 0 0 0 0.007332 +2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 2.88 0.3946 0.1709 0.01686 0.01562 0.06972 0.03595 0.01848 0.01916 0.003269 0 0 0 0 0.006887 +2018 01 31 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 2.95 0.4094 0.1708 0.01761 0.01632 0.07001 0.03651 0.01863 0.01906 0.003314 0 0 0 0 0.01898 +2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.3 0.4787 0.04447 0.1139 0.09124 0.108 0.01287 0.003811 0.004205 0.01249 0 0 0 0 0.003045 +2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 3.16 0.4578 0.04412 0.1015 0.08339 0.1078 0.01301 0.003919 0.004182 0.0125 0 0 0 0 0.002701 +2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 2.78 0.4679 0.04418 0.1035 0.08456 0.1079 0.01303 0.003902 0.004214 0.0127 0 0 0 0 0.009118 +2018 01 31 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.12 0.4613 0.04425 0.1043 0.08517 0.1073 0.01242 0.003797 0.004221 0.01239 0 0 0 0 0.003665 +2018 01 31 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.48 0.1237 0.03056 0.01622 0.01468 0.02039 0.005016 0.003737 0.002632 0.00326 0 0 0 0 0.006871 +2018 01 31 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.79 0.1287 0.0308 0.01706 0.01715 0.02042 0.005452 0.003636 0.002797 0.003143 0 0 0 0 0.007324 +2018 01 31 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.9 0.1296 0.03084 0.01711 0.01731 0.02053 0.005379 0.003641 0.002843 0.003137 0 0 0 0 0.008151 +2018 01 31 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.9 0.1323 0.03081 0.01703 0.01736 0.02065 0.005339 0.003638 0.002751 0.004008 0 0 0 0 0.01015 +2018 01 31 automated_test_4_labdiags_2ppc intel knl 1 16 8 0.85 0.2896 0.03832 0.06449 0.07493 0.003507 0.002987 0.0001515 0.0001762 0.007921 0.0371 0.001537 0 0.0004387 0.03832 +2018 01 31 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.12 0.2895 0.03845 0.06423 0.07481 0.003489 0.002994 0.000152 0.0001779 0.00834 0.0357 0.001545 0 0.0005249 0.03845 +2018 01 31 automated_test_4_labdiags_2ppc intel knl 1 16 8 0.76 0.3243 0.03804 0.0646 0.07462 0.003483 0.002991 0.0001508 0.0001769 0.008051 0.05983 0.001565 0 0.005392 0.03804 +2018 01 31 automated_test_4_labdiags_2ppc intel knl 1 16 8 0.74 0.3143 0.03941 0.06478 0.07547 0.003486 0.003007 0.0001518 0.0001808 0.007845 0.05079 0.001543 0 0.0007033 0.03941 +2018 01 31 automated_test_5_loadimbalance intel knl 1 16 8 9.2 0.3845 0.08558 0.1042 0.1332 0 0 0 0 0.01226 0 0 0 0 0.08558 +2018 01 31 automated_test_5_loadimbalance intel knl 1 16 8 9.19 0.3864 0.085 0.1051 0.134 0 0 0 0 0.01202 0 0 0 0 0.085 +2018 01 31 automated_test_5_loadimbalance intel knl 1 16 8 8.98 0.3912 0.08665 0.1061 0.1356 0 0 0 0 0.01193 0 0 0 0 0.08665 +2018 01 31 automated_test_5_loadimbalance intel knl 1 16 8 9.03 0.3826 0.08484 0.1031 0.1329 0 0 0 0 0.01205 0 0 0 0 0.08484 +2018 01 31 automated_test_6_output_2ppc intel knl 1 16 8 3.6 1.086 0.0898 0.1311 0.09441 0.1345 0.027 0.008783 0.009792 0.02151 0.08454 0.04962 0 0.0008218 0.005303 +2018 01 31 automated_test_6_output_2ppc intel knl 1 16 8 4.7 1.136 0.09059 0.1437 0.09535 0.1358 0.02915 0.009238 0.01002 0.02315 0.09088 0.05006 0 0.01081 0.005381 +2018 01 31 automated_test_6_output_2ppc intel knl 1 16 8 4.0 1.132 0.09145 0.1377 0.09592 0.1365 0.02817 0.009353 0.0103 0.02447 0.066 0.05309 0 0.02047 0.009196 +2018 01 31 automated_test_6_output_2ppc intel knl 1 16 8 3.8 1.135 0.09088 0.1308 0.09623 0.135 0.02762 0.008839 0.009758 0.02561 0.1144 0.04874 0 0.0008693 0.008112 +2018 02 13 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.87 0.4053 0.1754 0.01914 0.01871 0.0691 0.03648 0.01879 0.0193 0.003268 0 0 0 0 0.007445 +2018 02 13 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 4.38 0.405 0.1741 0.01901 0.01839 0.07034 0.03718 0.01894 0.0195 0.003845 0 0 0 0 0.007187 +2018 02 13 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.79 0.3999 0.1739 0.01859 0.01631 0.06918 0.0367 0.01906 0.01952 0.003278 0 0 0 0 0.006658 +2018 02 13 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.93 0.4044 0.1746 0.01854 0.01695 0.06975 0.03721 0.0191 0.01941 0.003979 0 0 0 0 0.007381 +2018 02 13 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.83 0.4773 0.04582 0.1089 0.08772 0.1072 0.01304 0.003335 0.004231 0.01385 0 0 0 0 0.002991 +2018 02 13 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.51 0.4654 0.04556 0.1027 0.08351 0.1068 0.01292 0.003114 0.004249 0.01356 0 0 0 0 0.002748 +2018 02 13 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.62 0.4755 0.0457 0.1082 0.08761 0.1069 0.0131 0.003205 0.00431 0.01388 0 0 0 0 0.002738 +2018 02 13 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.44 0.4798 0.04561 0.1133 0.08962 0.1064 0.01246 0.003076 0.004241 0.01318 0 0 0 0 0.003164 +2018 02 13 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.92 0.1282 0.03185 0.01747 0.01557 0.01956 0.005103 0.003455 0.00274 0.00346 0 0 0 0 0.007196 +2018 02 13 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.97 0.1301 0.03157 0.01788 0.01732 0.01957 0.00508 0.003335 0.002803 0.003454 0 0 0 0 0.007446 +2018 02 13 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.91 0.1289 0.03137 0.01765 0.0155 0.02026 0.005636 0.003513 0.002716 0.003381 0 0 0 0 0.007087 +2018 02 13 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.64 0.1308 0.03142 0.0181 0.01777 0.01953 0.005204 0.003371 0.002782 0.003057 0 0 0 0 0.007769 +2018 02 13 automated_test_4_labdiags_2ppc intel knl 1 16 8 3.19 0.3005 0.0383 0.06396 0.07274 0.003502 0.003005 0.0001628 0.0001839 0.008869 0.04427 0.001522 0 0.0005522 0.0383 +2018 02 13 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.41 0.2945 0.0389 0.06251 0.0723 0.003508 0.003009 0.000164 0.0001825 0.009131 0.04042 0.001538 0 0.0005936 0.0389 +2018 02 13 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.32 0.3066 0.0387 0.06558 0.07547 0.003463 0.003017 0.0001631 0.0001837 0.008431 0.04955 0.001555 0 0.000454 0.0387 +2018 02 13 automated_test_4_labdiags_2ppc intel knl 1 16 8 0.71 0.3391 0.03987 0.06534 0.07626 0.003475 0.003004 0.0001643 0.0001821 0.008152 0.06677 0.001534 0 0.01029 0.03987 +2018 02 13 automated_test_5_loadimbalance intel knl 1 16 8 9.68 0.3956 0.08701 0.1051 0.1352 0 0 0 0 0.01387 0 0 0 0 0.08701 +2018 02 13 automated_test_5_loadimbalance intel knl 1 16 8 10.65 0.3987 0.0866 0.1051 0.1332 0 0 0 0 0.0191 0 0 0 0 0.0866 +2018 02 13 automated_test_5_loadimbalance intel knl 1 16 8 10.11 0.4013 0.08782 0.1087 0.1359 0 0 0 0 0.01379 0 0 0 0 0.08782 +2018 02 13 automated_test_5_loadimbalance intel knl 1 16 8 9.94 0.39 0.08702 0.1028 0.132 0 0 0 0 0.0142 0 0 0 0 0.08702 +2018 02 13 automated_test_6_output_2ppc intel knl 1 16 8 1.292 0.2639 0.01424 0.03424 0.01742 0.01893 0.003449 0.001364 0.001712 0.009362 0.04053 0.01765 0 0.002558 0.001185 +2018 02 13 automated_test_6_output_2ppc intel knl 1 16 8 0.779 0.3155 0.01125 0.03605 0.01628 0.02431 0.009672 0.002843 0.001334 0.008876 0.05925 0.02047 0 0.001897 0.0006917 +2018 02 13 automated_test_6_output_2ppc intel knl 1 16 8 0.635 0.2568 0.01083 0.03443 0.01592 0.01963 0.003027 0.001439 0.001286 0.009288 0.03879 0.01815 0 0.001509 0.0007743 +2018 02 13 automated_test_6_output_2ppc intel knl 1 16 8 1.371 0.2648 0.01401 0.03376 0.01593 0.01936 0.003443 0.001351 0.00169 0.01161 0.03936 0.01785 0 0.002107 0.001171 +2018 02 20 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.7 0.4573 0.01159 0.02139 0.02206 0.06934 0.03845 0.0192 0.02062 0.003496 0 0 0 0 0.01159 +2018 02 20 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.45 0.4603 0.01356 0.02085 0.02488 0.06946 0.03777 0.01908 0.02031 0.003356 0 0 0 0 0.01356 +2018 02 20 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 3.72 0.4552 0.01245 0.02003 0.02013 0.06874 0.03766 0.01907 0.0203 0.003667 0 0 0 0 0.01245 +2018 02 20 automated_test_1_uniform_rest_32ppc intel knl 1 16 8 2.94 0.4557 0.01381 0.01979 0.02053 0.0687 0.03694 0.01886 0.02012 0.006396 0 0 0 0 0.01381 +2018 02 20 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.33 0.4937 0.005316 0.1103 0.09802 0.1071 0.01258 0.00326 0.004435 0.01347 0 0 0 0 0.005316 +2018 02 20 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.27 0.5063 0.004948 0.1213 0.1019 0.1067 0.01183 0.003056 0.004479 0.01327 0 0 0 0 0.004948 +2018 02 20 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 2.2 0.4983 0.005787 0.1141 0.1002 0.1067 0.0121 0.00307 0.00445 0.01343 0 0 0 0 0.005787 +2018 02 20 automated_test_2_uniform_rest_1ppc intel knl 1 16 8 1.39 0.5018 0.005339 0.1152 0.1007 0.1073 0.01249 0.003196 0.004484 0.01348 0 0 0 0 0.005339 +2018 02 20 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.98 0.1342 0.007843 0.01855 0.01798 0.01936 0.005198 0.003471 0.002626 0.003161 0 0 0 0 0.007843 +2018 02 20 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.63 0.1367 0.008055 0.01917 0.01818 0.01992 0.006097 0.003388 0.002639 0.003079 0 0 0 0 0.008055 +2018 02 20 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.66 0.1365 0.008017 0.0196 0.01819 0.01979 0.005769 0.00331 0.002668 0.003111 0 0 0 0 0.008017 +2018 02 20 automated_test_3_uniform_drift_4ppc intel knl 1 16 8 0.89 0.1367 0.008249 0.01947 0.01818 0.01956 0.005585 0.003341 0.002697 0.003217 0 0 0 0 0.008249 +2018 02 20 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.14 0.3087 0.04174 0.0637 0.0734 0.00345 0.002967 0.0001664 0.0001849 0.008714 0.05156 0.001539 0 0.0004984 0.04174 +2018 02 20 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.21 0.3407 0.07513 0.07261 0.0713 0.003428 0.002994 0.0001638 0.0001848 0.009408 0.003442 0.00173 0 0.0005256 0.07513 +2018 02 20 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.73 0.347 0.04077 0.06476 0.07148 0.00345 0.002998 0.0001637 0.0001829 0.009379 0.03947 0.001574 0 0.04989 0.04077 +2018 02 20 automated_test_4_labdiags_2ppc intel knl 1 16 8 1.52 0.3469 0.04088 0.06365 0.07183 0.003493 0.002957 0.0001659 0.0001827 0.009064 0.04694 0.001959 0 0.04099 0.04088 +2018 02 20 automated_test_5_loadimbalance intel knl 1 16 8 9.92 0.4206 0.08811 0.1186 0.1402 0 0 0 0 0.01443 0 0 0 0 0.08811 +2018 02 20 automated_test_5_loadimbalance intel knl 1 16 8 9.12 0.3884 0.08626 0.1027 0.1305 0 0 0 0 0.01368 0 0 0 0 0.08626 +2018 02 20 automated_test_5_loadimbalance intel knl 1 16 8 9.91 0.4097 0.08598 0.1119 0.1381 0 0 0 0 0.01414 0 0 0 0 0.08598 +2018 02 20 automated_test_5_loadimbalance intel knl 1 16 8 9.63 0.4257 0.0876 0.1213 0.1441 0 0 0 0 0.01422 0 0 0 0 0.0876 +2018 02 20 automated_test_6_output_2ppc intel knl 1 16 8 1.23 0.274 0.003227 0.03782 0.01724 0.01945 0.003219 0.001468 0.0014 0.01094 0.03943 0.0175 0 0.00509 0.001122 +2018 02 20 automated_test_6_output_2ppc intel knl 1 16 8 2.076 0.3023 0.002995 0.035 0.01619 0.02462 0.01126 0.006984 0.001548 0.01009 0.04604 0.01734 0 0.08398 0.001151 +2018 02 20 automated_test_6_output_2ppc intel knl 1 16 8 1.378 0.273 0.004545 0.03721 0.01754 0.02039 0.003415 0.00145 0.001561 0.01058 0.04009 0.01763 0 0.002519 0.001187 +2018 02 20 automated_test_6_output_2ppc intel knl 1 16 8 1.61 0.2911 0.004065 0.03726 0.01782 0.02439 0.01289 0.003463 0.001689 0.008778 0.03975 0.01723 0 0.00247 0.00129 diff --git a/Tools/performance_tests/run_alltests.py b/Tools/performance_tests/run_alltests.py index 15fbe10d4..3d2f90871 100644 --- a/Tools/performance_tests/run_alltests.py +++ b/Tools/performance_tests/run_alltests.py @@ -8,11 +8,11 @@ import datetime # results in file performance_log.txt in warpx/performance_tests/ # ---- User's manual ---- -# Before running performance tests, make sure you have the latest version +# Before running performance tests, make sure you have the latest version # of performance_log.txt # A typical execution reads: # > python run_alltests.py --no-recompile --compiler=gnu --architecture=cpu --mode=run --log_file='my_performance_log.txt' -# These are default values, and will give the same result as +# These are default values, and will give the same result as # > python run_alltests.py # To add a new test item, extent the test_list with a line like # test_list.extend([['my_input_file', n_node, n_mpi, n_omp]]*3) @@ -30,7 +30,7 @@ import datetime # This last job runs once all others are completed # - 'read' mode: Get performance data from all test items # create performance log file if does not exist -# loop over test_file +# loop over test_file # read initialization time and step time # write data into the performance log file # push file performance_log.txt on the repo @@ -146,7 +146,7 @@ year = time.strftime('%Y') if args.mode == 'run': # Set default options for compilation and execution config_command = '' - config_command += 'module unload darshan;' + config_command += 'module unload darshan;' config_command += 'module load craype-hugepages4M;' if args.architecture == 'knl': if args.compiler == 'intel': @@ -168,7 +168,7 @@ if args.mode == 'run': config_command += 'module load craype-haswell;' # Create main result directory if does not exist if not os.path.exists(res_dir_base): - os.mkdir(res_dir_base) + os.mkdir(res_dir_base) # Recompile if requested if args.recompile == True: @@ -213,7 +213,7 @@ def process_analysis(): os.system('chmod 700 ' + batch_file) os.system('sbatch --dependency afterok:' + dependencies[0:-1] + ' ' + batch_file) return 0 - + # Loop over the tests and return run time + details # ------------------------------------------------- if args.mode == 'run': @@ -243,7 +243,7 @@ if args.mode == 'run': process_analysis() if args.mode == 'read': - # Create log_file for performance tests if does not exist + # Create log_file for performance tests if does not exist if not os.path.isfile(log_dir + log_file): log_line = '## year month day run_name compiler architecture n_node n_mpi ' +\ 'n_omp time_initialization time_one_iteration Redistribute '+\ @@ -270,7 +270,7 @@ if args.mode == 'read': res_dir += '_'.join([run_name, args.compiler,\ args.architecture, str(n_node), str(n_mpi),\ str(n_omp), str(count)]) + '/' - # Read to store in text file + # Read to store in text file # -------------------------- output_filename = 'perf_output.txt' timing_list = read_run_perf(res_dir + output_filename, n_steps) @@ -282,9 +282,9 @@ if args.mode == 'read': # Read data for all test to put in hdf5 a database # ------------------------------------------------ - # This is an hdf5 file containing ALL the simulation parameters and results. Might be too large for a repo + # This is an hdf5 file containing ALL the simulation parameters and results. Might be too large for a repo df_newline = extract_dataframe(res_dir + 'perf_output.txt', n_steps) - # Add all simulation parameters to the dataframe + # Add all simulation parameters to the dataframe df_newline['run_name'] = run_name df_newline['n_node'] = n_node df_newline['n_mpi'] = n_mpi @@ -303,7 +303,7 @@ if args.mode == 'read': updated_df = df_newline updated_df.to_hdf(perf_database_file, key='all_data', mode='w') - # Store test parameters for record if requested + # Store test parameters for record if requested if store_test == True: dir_record_base = './perf_warpx_record/' if not os.path.exists(dir_record_base): @@ -319,7 +319,7 @@ if args.mode == 'read': shutil.copy(current_run[0], dir_record) if do_rename == True: - # Rename files if requested + # Rename files if requested for count, current_run in enumerate(test_list): run_name = current_run[0] n_node = current_run[1] @@ -335,7 +335,7 @@ if args.mode == 'read': str(n_omp), str(count)]) + '/' os.rename(res_dir, res_dir_arch) - # Commit results to the Repo + # Commit results to the Repo if args.commit == True: os.system('git add ' + log_dir + log_file + ';'\ 'git commit -m "performance tests";'\ diff --git a/Tools/performance_tests/run_alltests_1node.py b/Tools/performance_tests/run_alltests_1node.py index 5a90e1000..b4992c09d 100644 --- a/Tools/performance_tests/run_alltests_1node.py +++ b/Tools/performance_tests/run_alltests_1node.py @@ -7,19 +7,19 @@ from functions_perftest import * # results in file performance_log.txt in warpx/performance_tests/ # ---- User's manual ---- -# Before running performance tests, make sure you have the latest version +# Before running performance tests, make sure you have the latest version # of performance_log.txt # ---- Running a custom set of performance tests ---- -# > python run_alltests_1node.py --no-recompile --compiler=intel -# > --architecture=knl --mode=run --input_file=uniform_plasma +# > python run_alltests_1node.py --no-recompile --compiler=intel +# > --architecture=knl --mode=run --input_file=uniform_plasma # > --n_node=1 --log_file='my_performance_log.txt' # ---- Running the pre-drefined automated tests ---- # Compile and run: # > python run_alltests_1node.py --automated --recompile # Just run: -# > python run_alltests_1node.py --automated +# > python run_alltests_1node.py --automated # To add a new test item, extent the test_list with a line like # test_list.extend([['my_input_file', n_node, n_mpi, n_omp]]*n_repeat) @@ -35,7 +35,7 @@ from functions_perftest import * # This last job runs once all others are completed # - 'read' mode: Get performance data from all test items # create performance log file if does not exist -# loop over test_file +# loop over test_file # read initialization time and step time # write data into the performance log file # push file performance_log.txt on the repo @@ -57,9 +57,9 @@ parser.add_argument( '--mode', choices=['run', 'read'], default='run', parser.add_argument( '--log_file', dest = 'log_file', default='my_performance_log.txt', help='name of log file where data will be written. ignored if option --commit is used') parser.add_argument('--n_node', dest='n_node', default=1, help='nomber of nodes for the runs') -parser.add_argument('--input_file', dest='input_file', default='input_file.pixr', +parser.add_argument('--input_file', dest='input_file', default='input_file.pixr', type=str, help='input file to run') -parser.add_argument('--automated', dest='automated', action='store_true', default=False, +parser.add_argument('--automated', dest='automated', action='store_true', default=False, help='Use to run the automated test list') args = parser.parse_args() @@ -67,7 +67,7 @@ log_file = args.log_file do_commit = args.commit run_name = args.input_file -# list of tests to run and analyse. +# list of tests to run and analyse. # Note: This is overwritten if option --automated is used # each element of test_list contains # [str input_file, int n_node, int n_mpi PER NODE, int n_omp] @@ -78,7 +78,7 @@ test_list.extend([[filename1, 1, 128, 1]]*n_repeat) test_list.extend([[filename1, 1, 64, 2]]*n_repeat) # Nothing should be changed after this line -# if flag --automated is used, test_list and do_commit are +# if flag --automated is used, test_list and do_commit are # overwritten if args.automated == True: @@ -89,7 +89,7 @@ if args.automated == True: test_list.extend([['automated_test_3_uniform_drift_4ppc', 1, 16, 8]]*n_repeat) test_list.extend([['automated_test_4_labdiags_2ppc', 1, 16, 8]]*n_repeat) test_list.extend([['automated_test_5_loadimbalance', 1, 16, 8]]*n_repeat) - test_list.extend([['automated_test_6_output_2ppc', 1, 16, 8]]*n_repeat) + test_list.extend([['automated_test_6_output_2ppc', 1, 16, 8]]*n_repeat) do_commit = False run_name = 'automated_tests' @@ -123,7 +123,7 @@ perf_database_file = cwd + 'perf_database_warpx.h5' if args.mode == 'run': # Set default options for compilation and execution config_command = '' - config_command += 'module unload darshan;' + config_command += 'module unload darshan;' config_command += 'module load craype-hugepages4M;' if args.architecture == 'knl': if args.compiler == 'intel': @@ -145,7 +145,7 @@ if args.mode == 'run': config_command += 'module load craype-haswell;' # Create main result directory if does not exist if not os.path.exists(res_dir_base): - os.mkdir(res_dir_base) + os.mkdir(res_dir_base) # Recompile if requested if args.recompile == True: @@ -156,7 +156,7 @@ if args.recompile == True: makefile_handler.write( makefile_text ) os.system(config_command + " make -f GNUmakefile_perftest realclean ; " + " rm -r tmp_build_dir *.mod; make -j 8 -f GNUmakefile_perftest") -# This function runs a batch script with dependencies to perform the analysis +# This function runs a batch script with dependencies to perform the analysis # when performance runs are done. def process_analysis(): dependencies = '' @@ -192,7 +192,7 @@ def process_analysis(): os.system('chmod 700 ' + batch_file) os.system('sbatch --dependency afterok:' + dependencies[0:-1] + ' ' + batch_file) return 0 - + # Loop over the tests and return run time + details # ------------------------------------------------- if args.mode == 'run': @@ -248,7 +248,7 @@ if args.mode == 'read': # ------------------------------------------------ # This is an hdf5 file containing ALL the simulation parameters and results. Might be too large for a repo df_newline = extract_dataframe(res_dir + output_filename, n_steps) - # Add all simulation parameters to the dataframe + # Add all simulation parameters to the dataframe df_newline['run_name'] = run_name df_newline['n_node'] = n_node df_newline['n_mpi'] = n_mpi @@ -293,7 +293,7 @@ if args.mode == 'read': os.system('git add ' + log_dir + log_file + ';'\ 'git commit -m "performance tests";'\ 'git push -u origin master') - + # Plot file import numpy as np import matplotlib @@ -348,4 +348,4 @@ if args.mode == 'read': plt.legend(loc='best') plt.legend(bbox_to_anchor=(1.1, 1.05)) plt.savefig( selector_string + '.pdf', bbox_inches='tight') - plt.savefig( selector_string + '.png', bbox_inches='tight') + plt.savefig( selector_string + '.png', bbox_inches='tight') diff --git a/Tools/performance_tests/run_automated.py b/Tools/performance_tests/run_automated.py index dca038c6c..8f79750d4 100644 --- a/Tools/performance_tests/run_automated.py +++ b/Tools/performance_tests/run_automated.py @@ -24,7 +24,7 @@ parser.add_argument('--commit', parser.add_argument('--automated', dest='automated', action='store_true', - default=False, + default=False, help='Use to run the automated test list') parser.add_argument('--n_node_list', dest='n_node_list', @@ -34,7 +34,7 @@ parser.add_argument('--start_date', dest='start_date' ) parser.add_argument('--compiler', choices=['gnu', 'intel'], - default='intel', + default='intel', help='which compiler to use') parser.add_argument('--architecture', choices=['cpu', 'knl'], @@ -42,14 +42,14 @@ parser.add_argument('--architecture', help='which architecture to cross-compile for NERSC machines') parser.add_argument('--mode', choices=['run', 'read', 'browse_output_files', 'write_csv'], - default='run', + default='run', help='whether to run perftests or read their perf output. run calls read') args = parser.parse_args() n_node_list_string = args.n_node_list.split(',') n_node_list = [int(i) for i in n_node_list_string] start_date = args.start_date -# Set behavior variables +# Set behavior variables ######################## write_csv = False browse_output_files = False @@ -74,7 +74,7 @@ if args.automated == True: # Each instance of this class contains information for a single test. class test_element(): - def __init__(self, input_file=None, n_node=None, n_mpi_per_node=None, + def __init__(self, input_file=None, n_node=None, n_mpi_per_node=None, n_omp=None, n_cell=None, n_step=None): self.input_file = input_file self.n_node = n_node @@ -99,35 +99,35 @@ test_list_unq = [] n_repeat = 2 # n_node is kept to None and passed in functions as an external argument # That way, several test_element_instance run with the same n_node on the same batch job -test_list_unq.append( test_element(input_file='automated_test_1_uniform_rest_32ppc', - n_mpi_per_node=8, - n_omp=8, - n_cell=[128, 128, 128], +test_list_unq.append( test_element(input_file='automated_test_1_uniform_rest_32ppc', + n_mpi_per_node=8, + n_omp=8, + n_cell=[128, 128, 128], n_step=10) ) -test_list_unq.append( test_element(input_file='automated_test_2_uniform_rest_1ppc', - n_mpi_per_node=8, - n_omp=8, - n_cell=[256, 256, 512], +test_list_unq.append( test_element(input_file='automated_test_2_uniform_rest_1ppc', + n_mpi_per_node=8, + n_omp=8, + n_cell=[256, 256, 512], n_step=10) ) -test_list_unq.append( test_element(input_file='automated_test_3_uniform_drift_4ppc', - n_mpi_per_node=8, - n_omp=8, - n_cell=[128, 128, 128], +test_list_unq.append( test_element(input_file='automated_test_3_uniform_drift_4ppc', + n_mpi_per_node=8, + n_omp=8, + n_cell=[128, 128, 128], n_step=10) ) -test_list_unq.append( test_element(input_file='automated_test_4_labdiags_2ppc', - n_mpi_per_node=8, - n_omp=8, - n_cell=[64, 64, 128], +test_list_unq.append( test_element(input_file='automated_test_4_labdiags_2ppc', + n_mpi_per_node=8, + n_omp=8, + n_cell=[64, 64, 128], n_step=50) ) -test_list_unq.append( test_element(input_file='automated_test_5_loadimbalance', - n_mpi_per_node=8, - n_omp=8, - n_cell=[128, 128, 128], +test_list_unq.append( test_element(input_file='automated_test_5_loadimbalance', + n_mpi_per_node=8, + n_omp=8, + n_cell=[128, 128, 128], n_step=10) ) -test_list_unq.append( test_element(input_file='automated_test_6_output_2ppc', - n_mpi_per_node=8, - n_omp=8, - n_cell=[128, 256, 256], +test_list_unq.append( test_element(input_file='automated_test_6_output_2ppc', + n_mpi_per_node=8, + n_omp=8, + n_cell=[128, 256, 256], n_step=0) ) test_list = [copy.deepcopy(item) for item in test_list_unq for _ in range(n_repeat) ] @@ -160,7 +160,7 @@ if args.mode == 'run': start_date = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S") # Set default options for compilation and execution config_command = '' - config_command += 'module unload darshan;' + config_command += 'module unload darshan;' config_command += 'module load craype-hugepages4M;' if args.architecture == 'knl': if args.compiler == 'intel': @@ -182,7 +182,7 @@ if args.mode == 'run': config_command += 'module load craype-haswell;' # Create main result directory if does not exist if not os.path.exists(res_dir_base): - os.mkdir(res_dir_base) + os.mkdir(res_dir_base) # Recompile if requested # ---------------------- @@ -206,8 +206,8 @@ if args.mode == 'run': store_git_hash(repo_path=amrex_dir , filename=cwd + 'store_git_hashes.txt', name='amrex' ) store_git_hash(repo_path=warpx_dir , filename=cwd + 'store_git_hashes.txt', name='warpx' ) -# This function runs a batch script with -# dependencies to perform the analysis +# This function runs a batch script with +# dependencies to perform the analysis # after all performance tests are done. def process_analysis(): dependencies = '' @@ -246,7 +246,7 @@ def process_analysis(): # Loop over the tests and run all simulations: # One batch job submitted per n_node. Several -# tests run within the same batch job. +# tests run within the same batch job. # -------------------------------------------- if args.mode == 'run': if os.path.exists( 'log_jobids_tmp.txt' ): @@ -310,10 +310,10 @@ for n_node in n_node_list: updated_df = df_base.append(df_newline, ignore_index=True) else: updated_df = df_newline - # Write dataframe to file perf_database_file + # Write dataframe to file perf_database_file # (overwrite if file exists) updated_df.to_hdf(perf_database_file, key='all_data', mode='w') - + # Rename directory with precise date+hour for archive purpose if rename_archive == True: loc_counter = 0 @@ -331,7 +331,7 @@ for n_node in n_node_list: # csv file and copy this file to perf_logs repo # ------------------------------------------------- if write_csv: - # Extract small data from data frame and write them to + # Extract small data from data frame and write them to # First, generate csv files df = pd.read_hdf( perf_database_file ) # One large file diff --git a/Tools/read_lab_particles.py b/Tools/read_lab_particles.py index 53f228ba1..96f5f059c 100644 --- a/Tools/read_lab_particles.py +++ b/Tools/read_lab_particles.py @@ -15,7 +15,7 @@ def get_particle_field(field): data = np.fromfile(f) all_data = np.concatenate((all_data, data)) return all_data - + x = get_particle_field('x') z = get_particle_field('z') diff --git a/Tools/read_raw_data.py b/Tools/read_raw_data.py index 363c3a007..96d266df7 100644 --- a/Tools/read_raw_data.py +++ b/Tools/read_raw_data.py @@ -72,7 +72,7 @@ def read_lab_snapshot(snapshot, global_header): direction = 1 else: direction = 2 - + buffer_fullsize = 0 buffer_allsizes = [0] for i, hdr in enumerate(hdrs): @@ -80,7 +80,7 @@ def read_lab_snapshot(snapshot, global_header): buffer_fullsize += buffer_data[field1].shape[direction] buffer_allsizes.append(buffer_data[field1].shape[direction]) buffer_allstarts = np.cumsum(buffer_allsizes) - + data = {} for i in range(header.ncomp): if space_dim == 3: @@ -95,10 +95,10 @@ def read_lab_snapshot(snapshot, global_header): else: for k,v in buffer_data.items(): data[k][..., buffer_allstarts[i]:buffer_allstarts[i+1]] = v[...] - + info = local_info - # Add some handy info + # Add some handy info x = np.linspace(local_info['xmin'], local_info['xmax'], local_info['nx']) y = np.linspace(local_info['ymin'], local_info['ymax'], local_info['ny']) z = np.linspace(local_info['zmin'], local_info['zmax'], local_info['nz']) @@ -114,7 +114,7 @@ def read_lab_snapshot(snapshot, global_header): # xmin = local_info['axes_lo'][0] # xmax = local_info['axes_hi'][0] # x = np.linspace(xmin, xmax, data['Bx'].shape[0]) -# info.update({ 'xmin' : xmin, 'xmax' : xmax, 'x' : x }) +# info.update({ 'xmin' : xmin, 'xmax' : xmax, 'x' : x }) # zmin = local_info['axes_lo'][-1] # zmax = local_info['axes_hi'][-1] # z = np.linspace(zmin, zmax, data['Bx'].shape[-1]) @@ -126,7 +126,7 @@ def read_lab_snapshot(snapshot, global_header): # info.update({ 'ymin' : ymin, 'ymax' : ymax, 'y' : y }) # return data, info -# For the moment, the back-transformed diagnostics must be read with +# For the moment, the back-transformed diagnostics must be read with # custom functions like this one. # It should be OpenPMD-compliant hdf5 files soon, making this part outdated. def get_particle_field(snapshot, species, field): diff --git a/Tools/video_yt.py b/Tools/video_yt.py index a4cf409ad..a63881cb5 100644 --- a/Tools/video_yt.py +++ b/Tools/video_yt.py @@ -11,11 +11,11 @@ def img_onestep(filename): # Load the data ds = yt.load( filename ) - ad = ds.all_data() + ad = ds.all_data() # Calculate the z position of the box. -# You can use ds.domain_right_edge[2] instead. However, if a moving window -# was used in the simulation, the rendering shows some jitter. +# You can use ds.domain_right_edge[2] instead. However, if a moving window +# was used in the simulation, the rendering shows some jitter. # This is because a cell is added in z at some iterations but not all. # These lines calculate this jitter z_shift and remove it from the camera position and focus iteration=int(filename[-5:]) @@ -39,7 +39,7 @@ def img_onestep(filename): z0 = ad['particle0','particle_position_z'].v vertices0 = np.column_stack((x0,y0,z0)) colors0 = np.tile(colors0_vect,(vertices0.shape[0], 1)) - colors0[:,3] = .01 + colors0[:,3] = .01 point0 = yt.visualization.volume_rendering.render_source.PointSource(vertices0, colors=colors0, radii=2) # particle1: read data and create a yt source object x1 = ad['particle1','particle_position_x'].v @@ -95,7 +95,7 @@ def img_onestep(filename): # Save file sc.save(filename + '_quarter.png', sigma_clip=1.) return 0 - + # Get plt folders in current folder and loop over them. file_list = glob.glob('./plt?????') for filename in file_list: diff --git a/Tools/yt3d_mpi.py b/Tools/yt3d_mpi.py index de6202458..f18788570 100644 --- a/Tools/yt3d_mpi.py +++ b/Tools/yt3d_mpi.py @@ -29,7 +29,7 @@ def jitter_shift(ds, ad, cfl, iteration): elif maxwell_solver == 'ckc': dt = cfl * min( [ ad['dx'][-1], ad['dy'][-1], ad['dz'][-1] ] ) / scc.c z_front = dt * float(iteration) * scc.c + 7.5e-6*yt.units.meter - z_shift = z_front-ds.domain_right_edge[2] + z_shift = z_front-ds.domain_right_edge[2] return z_shift def get_species_ytpoints(ad, species, color_vec): @@ -80,7 +80,7 @@ def img_onestep(filename): tf.add_gaussian(.04 *my_max, width=8*w, height=[1.0, 1.0, 0.2, 0.2]) tf.add_gaussian(.2 *my_max, width=5*w, height=[1.0, 1.0, 0.2, 0.5]) tf.add_gaussian(.6 *my_max, width=w, height=[1.0, 1.0, 0.0, 1.]) - + ###################### ### plot particles ### ###################### @@ -110,7 +110,7 @@ def img_onestep(filename): sc.save('img_' + str(my_number_list[count]).zfill(4), sigma_clip=5.) if rendering_type == 'layers': sc.save('img_' + str(my_number_list[count]).zfill(4), sigma_clip=2.) - + file_list.sort() # Total number of files nfiles = len(file_list) -- cgit v1.2.3 From d6f8ce933459c52552772ae8bba683ec2e8cb3ad Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 11 Sep 2019 16:06:07 -0700 Subject: Source: tabs2spaces Manually fix tabs to four spaces and alignments for consistent prepresentation of source code over all machines. --- Source/BoundaryConditions/PML.H | 4 +- Source/BoundaryConditions/WarpXEvolvePML.cpp | 28 +- Source/Diagnostics/BoostedFrameDiagnostic.H | 2 +- Source/Diagnostics/BoostedFrameDiagnostic.cpp | 4 +- Source/Diagnostics/ElectrostaticIO.cpp | 6 +- Source/Diagnostics/ParticleIO.cpp | 8 +- Source/Diagnostics/WarpXIO.cpp | 399 ++++++++++----------- Source/Evolve/WarpXEvolveEM.cpp | 6 +- Source/Evolve/WarpXEvolveES.cpp | 38 +- .../PicsarHybridSpectralSolver.cpp | 34 +- .../SpectralAlgorithms/PMLPsatdAlgorithm.H | 6 +- Source/FieldSolver/WarpXPushFieldsEM.cpp | 77 ++-- Source/FortranInterface/WarpX_f.H | 16 +- Source/Initialization/PlasmaInjector.cpp | 2 +- Source/Initialization/WarpXInitData.cpp | 38 +- Source/Laser/LaserParticleContainer.cpp | 258 ++++++------- Source/Parallelization/WarpXComm.cpp | 6 +- Source/Parallelization/WarpXRegrid.cpp | 2 +- Source/Particles/MultiParticleContainer.H | 14 +- Source/Particles/MultiParticleContainer.cpp | 4 +- Source/Particles/PhysicalParticleContainer.H | 8 +- Source/Particles/PhysicalParticleContainer.cpp | 22 +- Source/Particles/WarpXParticleContainer.H | 16 +- Source/Particles/WarpXParticleContainer.cpp | 46 +-- Source/Python/WarpXWrappers.cpp | 30 +- Source/WarpX.H | 6 +- Source/WarpX.cpp | 245 +++++++------ Source/main.cpp | 14 +- 28 files changed, 672 insertions(+), 667 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/BoundaryConditions/PML.H b/Source/BoundaryConditions/PML.H index 9e04322f5..2db7befd6 100644 --- a/Source/BoundaryConditions/PML.H +++ b/Source/BoundaryConditions/PML.H @@ -141,9 +141,9 @@ public: const std::array& j_cp); void ExchangeB (PatchType patch_type, - const std::array& Bp, int do_pml_in_domain); + const std::array& Bp, int do_pml_in_domain); void ExchangeE (PatchType patch_type, - const std::array& Ep, int do_pml_in_domain); + const std::array& Ep, int do_pml_in_domain); void CopyJtoPMLs (PatchType patch_type, const std::array& jp); diff --git a/Source/BoundaryConditions/WarpXEvolvePML.cpp b/Source/BoundaryConditions/WarpXEvolvePML.cpp index f5c231ddf..67b179f9c 100644 --- a/Source/BoundaryConditions/WarpXEvolvePML.cpp +++ b/Source/BoundaryConditions/WarpXEvolvePML.cpp @@ -58,24 +58,24 @@ WarpX::DampPML (int lev, PatchType patch_type) const Box& tby = mfi.tilebox(By_nodal_flag); const Box& tbz = mfi.tilebox(Bz_nodal_flag); WRPX_DAMP_PML(tex.loVect(), tex.hiVect(), - tey.loVect(), tey.hiVect(), - tez.loVect(), tez.hiVect(), - tbx.loVect(), tbx.hiVect(), - tby.loVect(), tby.hiVect(), - tbz.loVect(), tbz.hiVect(), - BL_TO_FORTRAN_3D((*pml_E[0])[mfi]), - BL_TO_FORTRAN_3D((*pml_E[1])[mfi]), - BL_TO_FORTRAN_3D((*pml_E[2])[mfi]), - BL_TO_FORTRAN_3D((*pml_B[0])[mfi]), - BL_TO_FORTRAN_3D((*pml_B[1])[mfi]), - BL_TO_FORTRAN_3D((*pml_B[2])[mfi]), - WRPX_PML_TO_FORTRAN(sigba[mfi])); + tey.loVect(), tey.hiVect(), + tez.loVect(), tez.hiVect(), + tbx.loVect(), tbx.hiVect(), + tby.loVect(), tby.hiVect(), + tbz.loVect(), tbz.hiVect(), + BL_TO_FORTRAN_3D((*pml_E[0])[mfi]), + BL_TO_FORTRAN_3D((*pml_E[1])[mfi]), + BL_TO_FORTRAN_3D((*pml_E[2])[mfi]), + BL_TO_FORTRAN_3D((*pml_B[0])[mfi]), + BL_TO_FORTRAN_3D((*pml_B[1])[mfi]), + BL_TO_FORTRAN_3D((*pml_B[2])[mfi]), + WRPX_PML_TO_FORTRAN(sigba[mfi])); if (pml_F) { const Box& tnd = mfi.nodaltilebox(); WRPX_DAMP_PML_F(tnd.loVect(), tnd.hiVect(), - BL_TO_FORTRAN_3D((*pml_F)[mfi]), - WRPX_PML_TO_FORTRAN(sigba[mfi])); + BL_TO_FORTRAN_3D((*pml_F)[mfi]), + WRPX_PML_TO_FORTRAN(sigba[mfi])); } } } diff --git a/Source/Diagnostics/BoostedFrameDiagnostic.H b/Source/Diagnostics/BoostedFrameDiagnostic.H index 6c8eb15db..4263af8d0 100644 --- a/Source/Diagnostics/BoostedFrameDiagnostic.H +++ b/Source/Diagnostics/BoostedFrameDiagnostic.H @@ -139,7 +139,7 @@ private: // maps field index in data_buffer_[i] -> cell_centered_data for // snapshots i. By default, all fields in cell_centered_data are dumped. // Needs to be amrex::Vector because used in a ParallelFor kernel. - amrex::Gpu::ManagedDeviceVector map_actual_fields_to_dump; + amrex::Gpu::ManagedDeviceVector map_actual_fields_to_dump; // Name of fields to dump. By default, all fields in cell_centered_data. // Needed for file headers only. std::vector mesh_field_names = {"Ex", "Ey", "Ez", diff --git a/Source/Diagnostics/BoostedFrameDiagnostic.cpp b/Source/Diagnostics/BoostedFrameDiagnostic.cpp index f1d437509..abd9e99fe 100644 --- a/Source/Diagnostics/BoostedFrameDiagnostic.cpp +++ b/Source/Diagnostics/BoostedFrameDiagnostic.cpp @@ -567,7 +567,9 @@ BoostedFrameDiagnostic(Real zmin_lab, Real zmax_lab, Real v_window_lab, user_fields_to_dump); // If user specifies fields to dump, overwrite ncomp_to_dump, // map_actual_fields_to_dump and mesh_field_names. - for (int i = 0; i < 10; ++i) map_actual_fields_to_dump.push_back(i); + for (int i = 0; i < 10; ++i) + map_actual_fields_to_dump.push_back(i); + if (do_user_fields){ ncomp_to_dump = user_fields_to_dump.size(); map_actual_fields_to_dump.resize(ncomp_to_dump); diff --git a/Source/Diagnostics/ElectrostaticIO.cpp b/Source/Diagnostics/ElectrostaticIO.cpp index 75be2a24c..a023da0b7 100644 --- a/Source/Diagnostics/ElectrostaticIO.cpp +++ b/Source/Diagnostics/ElectrostaticIO.cpp @@ -29,10 +29,10 @@ WritePlotFileES (const amrex::Vector >& rho, const int nlevels = finestLevel()+1; { - Vector varnames; - Vector > mf(finest_level+1); + Vector varnames; + Vector > mf(finest_level+1); - for (int lev = 0; lev <= finest_level; ++lev) { + for (int lev = 0; lev <= finest_level; ++lev) { int ncomp = 5; const int ngrow = 0; mf[lev].reset(new MultiFab(grids[lev], dmap[lev], ncomp, ngrow)); diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index 741d3f50d..5cf3f3047 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -69,7 +69,7 @@ void MultiParticleContainer::Checkpoint (const std::string& dir) const { for (unsigned i = 0, n = species_names.size(); i < n; ++i) { - allcontainers[i]->Checkpoint(dir, species_names[i]); + allcontainers[i]->Checkpoint(dir, species_names[i]); } } @@ -129,7 +129,7 @@ void MultiParticleContainer::Restart (const std::string& dir) { for (unsigned i = 0, n = species_names.size(); i < n; ++i) { - allcontainers[i]->Restart(dir, species_names[i]); + allcontainers[i]->Restart(dir, species_names[i]); } } @@ -137,7 +137,7 @@ void MultiParticleContainer::ReadHeader (std::istream& is) { for (auto& pc : allcontainers) { - pc->ReadHeader(is); + pc->ReadHeader(is); } } @@ -145,7 +145,7 @@ void MultiParticleContainer::WriteHeader (std::ostream& os) const { for (const auto& pc : allcontainers) { - pc->WriteHeader(os); + pc->WriteHeader(os); } } diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index 6aa1f7a5b..2ac3a5eb8 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -35,72 +35,71 @@ WarpX::GotoNextLine (std::istream& is) void WarpX::WriteWarpXHeader(const std::string& name) const { - if (ParallelDescriptor::IOProcessor()) + if (ParallelDescriptor::IOProcessor()) { - VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size); - std::ofstream HeaderFile; - HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size()); - std::string HeaderFileName(name + "/WarpXHeader"); + VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size); + std::ofstream HeaderFile; + HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size()); + std::string HeaderFileName(name + "/WarpXHeader"); HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out | std::ofstream::trunc | std::ofstream::binary); - if( ! HeaderFile.good()) { - amrex::FileOpenFailed(HeaderFileName); - } + if( ! HeaderFile.good()) + amrex::FileOpenFailed(HeaderFileName); - HeaderFile.precision(17); + HeaderFile.precision(17); - HeaderFile << "Checkpoint version: 1\n"; + HeaderFile << "Checkpoint version: 1\n"; - const int nlevels = finestLevel()+1; - HeaderFile << nlevels << "\n"; + const int nlevels = finestLevel()+1; + HeaderFile << nlevels << "\n"; - for (int i = 0; i < istep.size(); ++i) { - HeaderFile << istep[i] << " "; - } - HeaderFile << "\n"; + for (int i = 0; i < istep.size(); ++i) { + HeaderFile << istep[i] << " "; + } + HeaderFile << "\n"; - for (int i = 0; i < nsubsteps.size(); ++i) { - HeaderFile << nsubsteps[i] << " "; - } - HeaderFile << "\n"; + for (int i = 0; i < nsubsteps.size(); ++i) { + HeaderFile << nsubsteps[i] << " "; + } + HeaderFile << "\n"; - for (int i = 0; i < t_new.size(); ++i) { - HeaderFile << t_new[i] << " "; - } - HeaderFile << "\n"; + for (int i = 0; i < t_new.size(); ++i) { + HeaderFile << t_new[i] << " "; + } + HeaderFile << "\n"; - for (int i = 0; i < t_old.size(); ++i) { - HeaderFile << t_old[i] << " "; - } - HeaderFile << "\n"; + for (int i = 0; i < t_old.size(); ++i) { + HeaderFile << t_old[i] << " "; + } + HeaderFile << "\n"; - for (int i = 0; i < dt.size(); ++i) { - HeaderFile << dt[i] << " "; - } - HeaderFile << "\n"; + for (int i = 0; i < dt.size(); ++i) { + HeaderFile << dt[i] << " "; + } + HeaderFile << "\n"; - HeaderFile << moving_window_x << "\n"; + HeaderFile << moving_window_x << "\n"; HeaderFile << is_synchronized << "\n"; - // Geometry - for (int i = 0; i < AMREX_SPACEDIM; ++i) { + // Geometry + for (int i = 0; i < AMREX_SPACEDIM; ++i) { HeaderFile << Geom(0).ProbLo(i) << ' '; - } + } HeaderFile << '\n'; for (int i = 0; i < AMREX_SPACEDIM; ++i) { HeaderFile << Geom(0).ProbHi(i) << ' '; - } + } HeaderFile << '\n'; - // BoxArray - for (int lev = 0; lev < nlevels; ++lev) { - boxArray(lev).writeOn(HeaderFile); - HeaderFile << '\n'; - } + // BoxArray + for (int lev = 0; lev < nlevels; ++lev) { + boxArray(lev).writeOn(HeaderFile); + HeaderFile << '\n'; + } - mypc->WriteHeader(HeaderFile); + mypc->WriteHeader(HeaderFile); } } @@ -125,18 +124,18 @@ WarpX::WriteCheckPointFile() const for (int lev = 0; lev < nlevels; ++lev) { - VisMF::Write(*Efield_fp[lev][0], - amrex::MultiFabFileFullPrefix(lev, checkpointname, level_prefix, "Ex_fp")); - VisMF::Write(*Efield_fp[lev][1], - amrex::MultiFabFileFullPrefix(lev, checkpointname, level_prefix, "Ey_fp")); - VisMF::Write(*Efield_fp[lev][2], - amrex::MultiFabFileFullPrefix(lev, checkpointname, level_prefix, "Ez_fp")); - VisMF::Write(*Bfield_fp[lev][0], - amrex::MultiFabFileFullPrefix(lev, checkpointname, level_prefix, "Bx_fp")); - VisMF::Write(*Bfield_fp[lev][1], - amrex::MultiFabFileFullPrefix(lev, checkpointname, level_prefix, "By_fp")); - VisMF::Write(*Bfield_fp[lev][2], - amrex::MultiFabFileFullPrefix(lev, checkpointname, level_prefix, "Bz_fp")); + VisMF::Write(*Efield_fp[lev][0], + amrex::MultiFabFileFullPrefix(lev, checkpointname, level_prefix, "Ex_fp")); + VisMF::Write(*Efield_fp[lev][1], + amrex::MultiFabFileFullPrefix(lev, checkpointname, level_prefix, "Ey_fp")); + VisMF::Write(*Efield_fp[lev][2], + amrex::MultiFabFileFullPrefix(lev, checkpointname, level_prefix, "Ez_fp")); + VisMF::Write(*Bfield_fp[lev][0], + amrex::MultiFabFileFullPrefix(lev, checkpointname, level_prefix, "Bx_fp")); + VisMF::Write(*Bfield_fp[lev][1], + amrex::MultiFabFileFullPrefix(lev, checkpointname, level_prefix, "By_fp")); + VisMF::Write(*Bfield_fp[lev][2], + amrex::MultiFabFileFullPrefix(lev, checkpointname, level_prefix, "Bz_fp")); if (is_synchronized) { // Need to save j if synchronized because after restart we need j to evolve E by dt/2. VisMF::Write(*current_fp[lev][0], @@ -196,108 +195,108 @@ WarpX::InitFromCheckpoint () // Header { - std::string File(restart_chkfile + "/WarpXHeader"); - - VisMF::IO_Buffer io_buffer(VisMF::GetIOBufferSize()); - - Vector fileCharPtr; - ParallelDescriptor::ReadAndBcastFile(File, fileCharPtr); - std::string fileCharPtrString(fileCharPtr.dataPtr()); - std::istringstream is(fileCharPtrString, std::istringstream::in); - - std::string line, word; - - std::getline(is, line); - - int nlevs; - is >> nlevs; - GotoNextLine(is); - finest_level = nlevs-1; - - std::getline(is, line); - { - std::istringstream lis(line); - int i = 0; - while (lis >> word) { - istep[i++] = std::stoi(word); - } - } - - std::getline(is, line); - { - std::istringstream lis(line); - int i = 0; - while (lis >> word) { - nsubsteps[i++] = std::stoi(word); - } - } - - std::getline(is, line); - { - std::istringstream lis(line); - int i = 0; - while (lis >> word) { - t_new[i++] = std::stod(word); - } - } - - std::getline(is, line); - { - std::istringstream lis(line); - int i = 0; - while (lis >> word) { - t_old[i++] = std::stod(word); - } - } - - std::getline(is, line); - { - std::istringstream lis(line); - int i = 0; - while (lis >> word) { - dt[i++] = std::stod(word); - } - } - - is >> moving_window_x; - GotoNextLine(is); + std::string File(restart_chkfile + "/WarpXHeader"); + + VisMF::IO_Buffer io_buffer(VisMF::GetIOBufferSize()); + + Vector fileCharPtr; + ParallelDescriptor::ReadAndBcastFile(File, fileCharPtr); + std::string fileCharPtrString(fileCharPtr.dataPtr()); + std::istringstream is(fileCharPtrString, std::istringstream::in); + + std::string line, word; + + std::getline(is, line); + + int nlevs; + is >> nlevs; + GotoNextLine(is); + finest_level = nlevs-1; + + std::getline(is, line); + { + std::istringstream lis(line); + int i = 0; + while (lis >> word) { + istep[i++] = std::stoi(word); + } + } + + std::getline(is, line); + { + std::istringstream lis(line); + int i = 0; + while (lis >> word) { + nsubsteps[i++] = std::stoi(word); + } + } + + std::getline(is, line); + { + std::istringstream lis(line); + int i = 0; + while (lis >> word) { + t_new[i++] = std::stod(word); + } + } + + std::getline(is, line); + { + std::istringstream lis(line); + int i = 0; + while (lis >> word) { + t_old[i++] = std::stod(word); + } + } + + std::getline(is, line); + { + std::istringstream lis(line); + int i = 0; + while (lis >> word) { + dt[i++] = std::stod(word); + } + } + + is >> moving_window_x; + GotoNextLine(is); is >> is_synchronized; - GotoNextLine(is); - - Real prob_lo[AMREX_SPACEDIM]; - std::getline(is, line); - { - std::istringstream lis(line); - int i = 0; - while (lis >> word) { - prob_lo[i++] = std::stod(word); - } - } - - Real prob_hi[AMREX_SPACEDIM]; - std::getline(is, line); - { - std::istringstream lis(line); - int i = 0; - while (lis >> word) { - prob_hi[i++] = std::stod(word); - } - } + GotoNextLine(is); + + Real prob_lo[AMREX_SPACEDIM]; + std::getline(is, line); + { + std::istringstream lis(line); + int i = 0; + while (lis >> word) { + prob_lo[i++] = std::stod(word); + } + } + + Real prob_hi[AMREX_SPACEDIM]; + std::getline(is, line); + { + std::istringstream lis(line); + int i = 0; + while (lis >> word) { + prob_hi[i++] = std::stod(word); + } + } ResetProbDomain(RealBox(prob_lo,prob_hi)); - for (int lev = 0; lev < nlevs; ++lev) { - BoxArray ba; - ba.readFrom(is); - GotoNextLine(is); - DistributionMapping dm { ba, ParallelDescriptor::NProcs() }; + for (int lev = 0; lev < nlevs; ++lev) { + BoxArray ba; + ba.readFrom(is); + GotoNextLine(is); + DistributionMapping dm { ba, ParallelDescriptor::NProcs() }; SetBoxArray(lev, ba); SetDistributionMap(lev, dm); - AllocLevelData(lev, ba, dm); - } + AllocLevelData(lev, ba, dm); + } - mypc->ReadHeader(is); + mypc->ReadHeader(is); } const int nlevs = finestLevel()+1; @@ -633,43 +632,43 @@ WarpX::WriteJobInfo (const std::string& dir) const { if (ParallelDescriptor::IOProcessor()) { - // job_info file with details about the run - std::ofstream jobInfoFile; - std::string FullPathJobInfoFile = dir; + // job_info file with details about the run + std::ofstream jobInfoFile; + std::string FullPathJobInfoFile = dir; std::string PrettyLine = std::string(78, '=') + "\n"; // std::string OtherLine = std::string(78, '-') + "\n"; // std::string SkipSpace = std::string(8, ' ') + "\n"; - FullPathJobInfoFile += "/warpx_job_info"; - jobInfoFile.open(FullPathJobInfoFile.c_str(), std::ios::out); + FullPathJobInfoFile += "/warpx_job_info"; + jobInfoFile.open(FullPathJobInfoFile.c_str(), std::ios::out); - // job information - jobInfoFile << PrettyLine; - jobInfoFile << " WarpX Job Information\n"; - jobInfoFile << PrettyLine; + // job information + jobInfoFile << PrettyLine; + jobInfoFile << " WarpX Job Information\n"; + jobInfoFile << PrettyLine; - jobInfoFile << "number of MPI processes: " << ParallelDescriptor::NProcs() << "\n"; + jobInfoFile << "number of MPI processes: " << ParallelDescriptor::NProcs() << "\n"; #ifdef _OPENMP - jobInfoFile << "number of threads: " << omp_get_max_threads() << "\n"; + jobInfoFile << "number of threads: " << omp_get_max_threads() << "\n"; #endif - jobInfoFile << "\n\n"; + jobInfoFile << "\n\n"; // build information - jobInfoFile << PrettyLine; - jobInfoFile << " Build Information\n"; - jobInfoFile << PrettyLine; + jobInfoFile << PrettyLine; + jobInfoFile << " Build Information\n"; + jobInfoFile << PrettyLine; - jobInfoFile << "build date: " << buildInfoGetBuildDate() << "\n"; - jobInfoFile << "build machine: " << buildInfoGetBuildMachine() << "\n"; - jobInfoFile << "build dir: " << buildInfoGetBuildDir() << "\n"; - jobInfoFile << "AMReX dir: " << buildInfoGetAMReXDir() << "\n"; + jobInfoFile << "build date: " << buildInfoGetBuildDate() << "\n"; + jobInfoFile << "build machine: " << buildInfoGetBuildMachine() << "\n"; + jobInfoFile << "build dir: " << buildInfoGetBuildDir() << "\n"; + jobInfoFile << "AMReX dir: " << buildInfoGetAMReXDir() << "\n"; - jobInfoFile << "\n"; + jobInfoFile << "\n"; - jobInfoFile << "COMP: " << buildInfoGetComp() << "\n"; - jobInfoFile << "COMP version: " << buildInfoGetCompVersion() << "\n"; + jobInfoFile << "COMP: " << buildInfoGetComp() << "\n"; + jobInfoFile << "COMP version: " << buildInfoGetCompVersion() << "\n"; jobInfoFile << "\n"; @@ -686,64 +685,64 @@ WarpX::WriteJobInfo (const std::string& dir) const jobInfoFile << "Link flags: " << buildInfoGetLinkFlags() << "\n"; jobInfoFile << "Libraries: " << buildInfoGetLibraries() << "\n"; - jobInfoFile << "\n"; + jobInfoFile << "\n"; - const char* githash1 = buildInfoGetGitHash(1); - const char* githash2 = buildInfoGetGitHash(2); - const char* githash3 = buildInfoGetGitHash(3); - if (strlen(githash1) > 0) { - jobInfoFile << "WarpX git describe: " << githash1 << "\n"; - } - if (strlen(githash2) > 0) { - jobInfoFile << "AMReX git describe: " << githash2 << "\n"; - } - if (strlen(githash3) > 0) { - jobInfoFile << "PICSAR git describe: " << githash3 << "\n"; - } + const char* githash1 = buildInfoGetGitHash(1); + const char* githash2 = buildInfoGetGitHash(2); + const char* githash3 = buildInfoGetGitHash(3); + if (strlen(githash1) > 0) { + jobInfoFile << "WarpX git describe: " << githash1 << "\n"; + } + if (strlen(githash2) > 0) { + jobInfoFile << "AMReX git describe: " << githash2 << "\n"; + } + if (strlen(githash3) > 0) { + jobInfoFile << "PICSAR git describe: " << githash3 << "\n"; + } - jobInfoFile << "\n\n"; + jobInfoFile << "\n\n"; - // grid information + // grid information jobInfoFile << PrettyLine; jobInfoFile << " Grid Information\n"; jobInfoFile << PrettyLine; for (int i = 0; i <= finest_level; i++) - { + { jobInfoFile << " level: " << i << "\n"; jobInfoFile << " number of boxes = " << grids[i].size() << "\n"; jobInfoFile << " maximum zones = "; for (int n = 0; n < AMREX_SPACEDIM; n++) - { + { jobInfoFile << geom[i].Domain().length(n) << " "; - } + } jobInfoFile << "\n\n"; - } + } jobInfoFile << " Boundary conditions\n"; jobInfoFile << " -x: " << "interior" << "\n"; jobInfoFile << " +x: " << "interior" << "\n"; if (AMREX_SPACEDIM >= 2) { - jobInfoFile << " -y: " << "interior" << "\n"; - jobInfoFile << " +y: " << "interior" << "\n"; + jobInfoFile << " -y: " << "interior" << "\n"; + jobInfoFile << " +y: " << "interior" << "\n"; } if (AMREX_SPACEDIM == 3) { - jobInfoFile << " -z: " << "interior" << "\n"; - jobInfoFile << " +z: " << "interior" << "\n"; + jobInfoFile << " -z: " << "interior" << "\n"; + jobInfoFile << " +z: " << "interior" << "\n"; } jobInfoFile << "\n\n"; - // runtime parameters - jobInfoFile << PrettyLine; - jobInfoFile << " Inputs File Parameters\n"; - jobInfoFile << PrettyLine; + // runtime parameters + jobInfoFile << PrettyLine; + jobInfoFile << " Inputs File Parameters\n"; + jobInfoFile << PrettyLine; - ParmParse::dumpTable(jobInfoFile, true); + ParmParse::dumpTable(jobInfoFile, true); - jobInfoFile.close(); + jobInfoFile.close(); } } diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index a4ed803ee..95e086bbf 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -183,7 +183,7 @@ WarpX::EvolveEM (int numsteps) } // slice gen // - if (to_make_plot || do_insitu || to_make_slice_plot) + if (to_make_plot || do_insitu || to_make_slice_plot) { FillBoundaryE(); FillBoundaryB(); @@ -199,7 +199,7 @@ WarpX::EvolveEM (int numsteps) last_insitu_step = step+1; if (to_make_plot) - WritePlotFile(); + WritePlotFile(); if (to_make_slice_plot) { @@ -211,7 +211,7 @@ WarpX::EvolveEM (int numsteps) if (do_insitu) UpdateInSitu(); - } + } if (check_int > 0 && (step+1) % check_int == 0) { last_check_file_step = step+1; diff --git a/Source/Evolve/WarpXEvolveES.cpp b/Source/Evolve/WarpXEvolveES.cpp index e1e4a3929..effd6ec96 100644 --- a/Source/Evolve/WarpXEvolveES.cpp +++ b/Source/Evolve/WarpXEvolveES.cpp @@ -61,7 +61,7 @@ WarpX::EvolveES (int numsteps) { for (int step = istep[0]; step < numsteps_max && cur_time < stop_time; ++step) { - // Start loop on time steps + // Start loop on time steps amrex::Print() << "\nSTEP " << step+1 << " starts ...\n"; #ifdef WARPX_USE_PY if (warpx_py_beforestep) warpx_py_beforestep(); @@ -124,36 +124,36 @@ WarpX::EvolveES (int numsteps) { amrex::Print()<< "STEP " << step+1 << " ends." << " TIME = " << cur_time << " DT = " << dt[0] << "\n"; - // sync up time - for (int i = 0; i <= finest_level; ++i) { - t_new[i] = cur_time; - } + // sync up time + for (int i = 0; i <= finest_level; ++i) { + t_new[i] = cur_time; + } - if (to_make_plot) { + if (to_make_plot) { // replace with ES field Gather mypc->DepositCharge(rhoNodal); computePhi(rhoNodal, phiNodal); phiNodal[0]->FillBoundary(Geom(0).periodicity()); computeE(eFieldNodal, phiNodal); mypc->FieldGatherES(eFieldNodal, gather_masks); - last_plot_file_step = step+1; - WritePlotFileES(rhoNodal, phiNodal, eFieldNodal); - } + last_plot_file_step = step+1; + WritePlotFileES(rhoNodal, phiNodal, eFieldNodal); + } - if (check_int > 0 && (step+1) % check_int == 0) { - last_check_file_step = step+1; - WriteCheckPointFile(); - } + if (check_int > 0 && (step+1) % check_int == 0) { + last_check_file_step = step+1; + WriteCheckPointFile(); + } - if (cur_time >= stop_time - 1.e-3*dt[0]) { - max_time_reached = true; - break; - } + if (cur_time >= stop_time - 1.e-3*dt[0]) { + max_time_reached = true; + break; + } #ifdef WARPX_USE_PY if (warpx_py_afterstep) warpx_py_afterstep(); #endif - // End loop on time steps + // End loop on time steps } if (plot_int > 0 && istep[0] > last_plot_file_step && (max_time_reached || istep[0] >= max_step)) { @@ -161,7 +161,7 @@ WarpX::EvolveES (int numsteps) { } if (check_int > 0 && istep[0] > last_check_file_step && (max_time_reached || istep[0] >= max_step)) { - WriteCheckPointFile(); + WriteCheckPointFile(); } } diff --git a/Source/FieldSolver/PicsarHybridSpectralSolver/PicsarHybridSpectralSolver.cpp b/Source/FieldSolver/PicsarHybridSpectralSolver/PicsarHybridSpectralSolver.cpp index 26c93086a..6b7a34271 100644 --- a/Source/FieldSolver/PicsarHybridSpectralSolver/PicsarHybridSpectralSolver.cpp +++ b/Source/FieldSolver/PicsarHybridSpectralSolver/PicsarHybridSpectralSolver.cpp @@ -336,7 +336,7 @@ WarpX::InitFFTDataPlan (int lev) // No FFT patch on this MPI rank (may happen with FFTW) // Still need to call the MPI-FFT initialization routines { - nullfftdata.reset(new FFTData()); + nullfftdata.reset(new FFTData()); warpx_fft_dataplan_init(&nox_fft, &noy_fft, &noz_fft, nullfftdata->data, &FFTData::N, dx_fp.data(), &dt[lev], &fftw_plan_measure, @@ -392,8 +392,8 @@ WarpX::PushPSATD_hybridFFT (int lev, amrex::Real /* dt */) if (Efield_fp_fft[lev][0]->local_size() == 1) //Only one FFT patch on this MPI { - for (MFIter mfi(*Efield_fp_fft[lev][0]); mfi.isValid(); ++mfi) - { + for (MFIter mfi(*Efield_fp_fft[lev][0]); mfi.isValid(); ++mfi) + { warpx_fft_push_eb(WARPX_TO_FORTRAN_ANYD((*Efield_fp_fft[lev][0])[mfi]), WARPX_TO_FORTRAN_ANYD((*Efield_fp_fft[lev][1])[mfi]), WARPX_TO_FORTRAN_ANYD((*Efield_fp_fft[lev][2])[mfi]), @@ -405,29 +405,29 @@ WarpX::PushPSATD_hybridFFT (int lev, amrex::Real /* dt */) WARPX_TO_FORTRAN_ANYD((*current_fp_fft[lev][2])[mfi]), WARPX_TO_FORTRAN_N_ANYD((*rho_fp_fft[lev])[mfi],0), WARPX_TO_FORTRAN_N_ANYD((*rho_fp_fft[lev])[mfi],1)); - } + } } else if (Efield_fp_fft[lev][0]->local_size() == 0) // No FFT patch on this MPI rank // Still need to call the MPI-FFT routine. { - FArrayBox fab(Box(IntVect::TheZeroVector(), IntVect::TheUnitVector())); - warpx_fft_push_eb(WARPX_TO_FORTRAN_ANYD(fab), - WARPX_TO_FORTRAN_ANYD(fab), - WARPX_TO_FORTRAN_ANYD(fab), - WARPX_TO_FORTRAN_ANYD(fab), - WARPX_TO_FORTRAN_ANYD(fab), - WARPX_TO_FORTRAN_ANYD(fab), - WARPX_TO_FORTRAN_ANYD(fab), - WARPX_TO_FORTRAN_ANYD(fab), - WARPX_TO_FORTRAN_ANYD(fab), - WARPX_TO_FORTRAN_ANYD(fab), - WARPX_TO_FORTRAN_ANYD(fab)); + FArrayBox fab(Box(IntVect::TheZeroVector(), IntVect::TheUnitVector())); + warpx_fft_push_eb(WARPX_TO_FORTRAN_ANYD(fab), + WARPX_TO_FORTRAN_ANYD(fab), + WARPX_TO_FORTRAN_ANYD(fab), + WARPX_TO_FORTRAN_ANYD(fab), + WARPX_TO_FORTRAN_ANYD(fab), + WARPX_TO_FORTRAN_ANYD(fab), + WARPX_TO_FORTRAN_ANYD(fab), + WARPX_TO_FORTRAN_ANYD(fab), + WARPX_TO_FORTRAN_ANYD(fab), + WARPX_TO_FORTRAN_ANYD(fab), + WARPX_TO_FORTRAN_ANYD(fab)); } else // Multiple FFT patches on this MPI rank { - amrex::Abort("WarpX::PushPSATD: TODO"); + amrex::Abort("WarpX::PushPSATD: TODO"); } BL_PROFILE_VAR_STOP(blp_push_eb); diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PMLPsatdAlgorithm.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PMLPsatdAlgorithm.H index 50eb5c9b1..80ceb2e93 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PMLPsatdAlgorithm.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PMLPsatdAlgorithm.H @@ -16,9 +16,9 @@ class PMLPsatdAlgorithm : public SpectralBaseAlgorithm const amrex::Real dt); void InitializeSpectralCoefficients( - const SpectralKSpace& spectral_kspace, - const amrex::DistributionMapping& dm, - const amrex::Real dt); + const SpectralKSpace& spectral_kspace, + const amrex::DistributionMapping& dm, + const amrex::Real dt); // Redefine functions from base class virtual void pushSpectralFields(SpectralFieldData& f) const override final; diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index b3d8e9d76..91641e9da 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -249,17 +249,18 @@ WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real a_dt) const Box& tbz = mfi.tilebox(Bz_nodal_flag); WRPX_PUSH_PML_BVEC( - tbx.loVect(), tbx.hiVect(), - tby.loVect(), tby.hiVect(), - tbz.loVect(), tbz.hiVect(), - BL_TO_FORTRAN_3D((*pml_E[0])[mfi]), - BL_TO_FORTRAN_3D((*pml_E[1])[mfi]), - BL_TO_FORTRAN_3D((*pml_E[2])[mfi]), - BL_TO_FORTRAN_3D((*pml_B[0])[mfi]), - BL_TO_FORTRAN_3D((*pml_B[1])[mfi]), - BL_TO_FORTRAN_3D((*pml_B[2])[mfi]), - &dtsdx, &dtsdy, &dtsdz, - &WarpX::maxwell_fdtd_solver_id); + tbx.loVect(), tbx.hiVect(), + tby.loVect(), tby.hiVect(), + tbz.loVect(), tbz.hiVect(), + BL_TO_FORTRAN_3D((*pml_E[0])[mfi]), + BL_TO_FORTRAN_3D((*pml_E[1])[mfi]), + BL_TO_FORTRAN_3D((*pml_E[2])[mfi]), + BL_TO_FORTRAN_3D((*pml_B[0])[mfi]), + BL_TO_FORTRAN_3D((*pml_B[1])[mfi]), + BL_TO_FORTRAN_3D((*pml_B[2])[mfi]), + &dtsdx, &dtsdy, &dtsdz, + &WarpX::maxwell_fdtd_solver_id + ); } } } @@ -471,16 +472,17 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) auto const& pml_Ezfab = pml_E[2]->array(mfi); WRPX_PUSH_PML_EVEC( - tex.loVect(), tex.hiVect(), - tey.loVect(), tey.hiVect(), - tez.loVect(), tez.hiVect(), - BL_TO_FORTRAN_3D((*pml_E[0])[mfi]), - BL_TO_FORTRAN_3D((*pml_E[1])[mfi]), - BL_TO_FORTRAN_3D((*pml_E[2])[mfi]), - BL_TO_FORTRAN_3D((*pml_B[0])[mfi]), - BL_TO_FORTRAN_3D((*pml_B[1])[mfi]), - BL_TO_FORTRAN_3D((*pml_B[2])[mfi]), - &dtsdx_c2, &dtsdy_c2, &dtsdz_c2); + tex.loVect(), tex.hiVect(), + tey.loVect(), tey.hiVect(), + tez.loVect(), tez.hiVect(), + BL_TO_FORTRAN_3D((*pml_E[0])[mfi]), + BL_TO_FORTRAN_3D((*pml_E[1])[mfi]), + BL_TO_FORTRAN_3D((*pml_E[2])[mfi]), + BL_TO_FORTRAN_3D((*pml_B[0])[mfi]), + BL_TO_FORTRAN_3D((*pml_B[1])[mfi]), + BL_TO_FORTRAN_3D((*pml_B[2])[mfi]), + &dtsdx_c2, &dtsdy_c2, &dtsdz_c2 + ); if (pml_has_particles) { // Update the E field in the PML, using the current @@ -523,15 +525,16 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) if (pml_F) { WRPX_PUSH_PML_EVEC_F( - tex.loVect(), tex.hiVect(), - tey.loVect(), tey.hiVect(), - tez.loVect(), tez.hiVect(), - BL_TO_FORTRAN_3D((*pml_E[0])[mfi]), - BL_TO_FORTRAN_3D((*pml_E[1])[mfi]), - BL_TO_FORTRAN_3D((*pml_E[2])[mfi]), - BL_TO_FORTRAN_3D((*pml_F )[mfi]), - &dtsdx_c2, &dtsdy_c2, &dtsdz_c2, - &WarpX::maxwell_fdtd_solver_id); + tex.loVect(), tex.hiVect(), + tey.loVect(), tey.hiVect(), + tez.loVect(), tez.hiVect(), + BL_TO_FORTRAN_3D((*pml_E[0])[mfi]), + BL_TO_FORTRAN_3D((*pml_E[1])[mfi]), + BL_TO_FORTRAN_3D((*pml_E[2])[mfi]), + BL_TO_FORTRAN_3D((*pml_F )[mfi]), + &dtsdx_c2, &dtsdy_c2, &dtsdz_c2, + &WarpX::maxwell_fdtd_solver_id + ); } } } @@ -606,12 +609,14 @@ WarpX::EvolveF (int lev, PatchType patch_type, Real a_dt, DtType a_dt_type) for ( MFIter mfi(*pml_F, TilingIfNotGPU()); mfi.isValid(); ++mfi ) { const Box& bx = mfi.tilebox(); - WRPX_PUSH_PML_F(bx.loVect(), bx.hiVect(), - BL_TO_FORTRAN_ANYD((*pml_F )[mfi]), - BL_TO_FORTRAN_ANYD((*pml_E[0])[mfi]), - BL_TO_FORTRAN_ANYD((*pml_E[1])[mfi]), - BL_TO_FORTRAN_ANYD((*pml_E[2])[mfi]), - &dtsdx[0], &dtsdx[1], &dtsdx[2]); + WRPX_PUSH_PML_F( + bx.loVect(), bx.hiVect(), + BL_TO_FORTRAN_ANYD((*pml_F )[mfi]), + BL_TO_FORTRAN_ANYD((*pml_E[0])[mfi]), + BL_TO_FORTRAN_ANYD((*pml_E[1])[mfi]), + BL_TO_FORTRAN_ANYD((*pml_E[2])[mfi]), + &dtsdx[0], &dtsdx[1], &dtsdx[2] + ); } } } diff --git a/Source/FortranInterface/WarpX_f.H b/Source/FortranInterface/WarpX_f.H index 825d9effd..4f96d6d08 100644 --- a/Source/FortranInterface/WarpX_f.H +++ b/Source/FortranInterface/WarpX_f.H @@ -70,11 +70,11 @@ extern "C" #endif void warpx_compute_E (const int* lo, const int* hi, - const BL_FORT_FAB_ARG_3D(phi), - BL_FORT_FAB_ARG_3D(Ex), - BL_FORT_FAB_ARG_3D(Ey), - BL_FORT_FAB_ARG_3D(Ez), - const amrex::Real* dx); + const BL_FORT_FAB_ARG_3D(phi), + BL_FORT_FAB_ARG_3D(Ex), + BL_FORT_FAB_ARG_3D(Ey), + BL_FORT_FAB_ARG_3D(Ez), + const amrex::Real* dx); /// /// These functions are used in electrostatic mode. @@ -169,9 +169,9 @@ extern "C" BL_FORT_FAB_ARG_3D(by), BL_FORT_FAB_ARG_3D(bz), const amrex::Real* dtsdx, - const amrex::Real* dtsdy, - const amrex::Real* dtsdz, - const int* maxwell_fdtd_solver_id); + const amrex::Real* dtsdy, + const amrex::Real* dtsdz, + const int* maxwell_fdtd_solver_id); void WRPX_PUSH_PML_EVEC(const int* xlo, const int* xhi, diff --git a/Source/Initialization/PlasmaInjector.cpp b/Source/Initialization/PlasmaInjector.cpp index 4dc16cb63..af04c6b31 100644 --- a/Source/Initialization/PlasmaInjector.cpp +++ b/Source/Initialization/PlasmaInjector.cpp @@ -46,7 +46,7 @@ namespace { } else if (name == "m_p"){ return PhysConst::m_p; } else if (name == "inf"){ - return std::numeric_limits::infinity(); + return std::numeric_limits::infinity(); } else if (pp.query("mass", result)) { return result; } else { diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 843773505..9c0cf2fd3 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -25,11 +25,11 @@ WarpX::InitData () } else { - InitFromCheckpoint(); + InitFromCheckpoint(); if (is_synchronized) { ComputeDt(); } - PostRestart(); + PostRestart(); } ComputePMLFactors(); @@ -87,12 +87,12 @@ WarpX::InitDiagnostics () { const Real* current_hi = geom[0].ProbHi(); Real dt_boost = dt[0]; - // Find the positions of the lab-frame box that corresponds to the boosted-frame box at t=0 - Real zmin_lab = current_lo[moving_window_dir]/( (1.+beta_boost)*gamma_boost ); - Real zmax_lab = current_hi[moving_window_dir]/( (1.+beta_boost)*gamma_boost ); + // Find the positions of the lab-frame box that corresponds to the boosted-frame box at t=0 + Real zmin_lab = current_lo[moving_window_dir]/( (1.+beta_boost)*gamma_boost ); + Real zmax_lab = current_hi[moving_window_dir]/( (1.+beta_boost)*gamma_boost ); myBFD.reset(new BoostedFrameDiagnostic(zmin_lab, - zmax_lab, + zmax_lab, moving_window_v, dt_snapshots_lab, num_snapshots_lab, gamma_boost, t_new[0], dt_boost, @@ -251,9 +251,9 @@ WarpX::InitOpenbc () BoxList bl{IndexType::TheNodeType()}; for (int i = 0; i < nprocs; ++i) { - bl.push_back(Box(IntVect(alllohi[6*i ],alllohi[6*i+1],alllohi[6*i+2]), - IntVect(alllohi[6*i+3],alllohi[6*i+4],alllohi[6*i+5]), - IndexType::TheNodeType())); + bl.push_back(Box(IntVect(alllohi[6*i ],alllohi[6*i+1],alllohi[6*i+2]), + IntVect(alllohi[6*i+3],alllohi[6*i+4],alllohi[6*i+5]), + IndexType::TheNodeType())); } BoxArray ba{bl}; @@ -286,13 +286,13 @@ WarpX::InitOpenbc () #endif for (MFIter mfi(phi); mfi.isValid(); ++mfi) { - const Box& bx = mfi.validbox(); - warpx_compute_E(bx.loVect(), bx.hiVect(), - BL_TO_FORTRAN_3D(phi[mfi]), - BL_TO_FORTRAN_3D((*Efield[lev][0])[mfi]), - BL_TO_FORTRAN_3D((*Efield[lev][1])[mfi]), - BL_TO_FORTRAN_3D((*Efield[lev][2])[mfi]), - dx); + const Box& bx = mfi.validbox(); + warpx_compute_E(bx.loVect(), bx.hiVect(), + BL_TO_FORTRAN_3D(phi[mfi]), + BL_TO_FORTRAN_3D((*Efield[lev][0])[mfi]), + BL_TO_FORTRAN_3D((*Efield[lev][1])[mfi]), + BL_TO_FORTRAN_3D((*Efield[lev][2])[mfi]), + dx); } } #endif @@ -301,9 +301,9 @@ void WarpX::InitLevelData (int lev, Real time) { for (int i = 0; i < 3; ++i) { - current_fp[lev][i]->setVal(0.0); - Efield_fp[lev][i]->setVal(0.0); - Bfield_fp[lev][i]->setVal(0.0); + current_fp[lev][i]->setVal(0.0); + Efield_fp[lev][i]->setVal(0.0); + Bfield_fp[lev][i]->setVal(0.0); } if (lev > 0) { diff --git a/Source/Laser/LaserParticleContainer.cpp b/Source/Laser/LaserParticleContainer.cpp index d0ff51313..ec602a593 100644 --- a/Source/Laser/LaserParticleContainer.cpp +++ b/Source/Laser/LaserParticleContainer.cpp @@ -30,125 +30,125 @@ LaserParticleContainer::LaserParticleContainer (AmrCore* amr_core, int ispecies, ParmParse pp(laser_name); - // Parse the type of laser profile and set the corresponding flag `profile` - std::string laser_type_s; - pp.get("profile", laser_type_s); - std::transform(laser_type_s.begin(), laser_type_s.end(), laser_type_s.begin(), ::tolower); - if (laser_type_s == "gaussian") { - profile = laser_t::Gaussian; + // Parse the type of laser profile and set the corresponding flag `profile` + std::string laser_type_s; + pp.get("profile", laser_type_s); + std::transform(laser_type_s.begin(), laser_type_s.end(), laser_type_s.begin(), ::tolower); + if (laser_type_s == "gaussian") { + profile = laser_t::Gaussian; } else if(laser_type_s == "harris") { profile = laser_t::Harris; } else if(laser_type_s == "parse_field_function") { profile = laser_t::parse_field_function; - } else { - amrex::Abort("Unknown laser type"); - } - - // Parse the properties of the antenna - pp.getarr("position", position); - pp.getarr("direction", nvec); - pp.getarr("polarization", p_X); - pp.query("pusher_algo", pusher_algo); - pp.get("wavelength", wavelength); - pp.get("e_max", e_max); - pp.query("do_continuous_injection", do_continuous_injection); - - if ( profile == laser_t::Gaussian ) { - // Parse the properties of the Gaussian profile - pp.get("profile_waist", profile_waist); - pp.get("profile_duration", profile_duration); - pp.get("profile_t_peak", profile_t_peak); - pp.get("profile_focal_distance", profile_focal_distance); - stc_direction = p_X; - pp.queryarr("stc_direction", stc_direction); - pp.query("zeta", zeta); - pp.query("beta", beta); - pp.query("phi2", phi2); - } - - if ( profile == laser_t::Harris ) { - // Parse the properties of the Harris profile - pp.get("profile_waist", profile_waist); - pp.get("profile_duration", profile_duration); - pp.get("profile_focal_distance", profile_focal_distance); - } + } else { + amrex::Abort("Unknown laser type"); + } - if ( profile == laser_t::parse_field_function ) { - // Parse the properties of the parse_field_function profile - pp.get("field_function(X,Y,t)", field_function); - parser.define(field_function); - parser.registerVariables({"X","Y","t"}); - - ParmParse ppc("my_constants"); - std::set symbols = parser.symbols(); - symbols.erase("X"); - symbols.erase("Y"); - symbols.erase("t"); // after removing variables, we are left with constants - for (auto it = symbols.begin(); it != symbols.end(); ) { - Real v; - if (ppc.query(it->c_str(), v)) { - parser.setConstant(*it, v); - it = symbols.erase(it); - } else { - ++it; - } + // Parse the properties of the antenna + pp.getarr("position", position); + pp.getarr("direction", nvec); + pp.getarr("polarization", p_X); + pp.query("pusher_algo", pusher_algo); + pp.get("wavelength", wavelength); + pp.get("e_max", e_max); + pp.query("do_continuous_injection", do_continuous_injection); + + if ( profile == laser_t::Gaussian ) { + // Parse the properties of the Gaussian profile + pp.get("profile_waist", profile_waist); + pp.get("profile_duration", profile_duration); + pp.get("profile_t_peak", profile_t_peak); + pp.get("profile_focal_distance", profile_focal_distance); + stc_direction = p_X; + pp.queryarr("stc_direction", stc_direction); + pp.query("zeta", zeta); + pp.query("beta", beta); + pp.query("phi2", phi2); } - for (auto const& s : symbols) { // make sure there no unknown symbols - amrex::Abort("Laser Profile: Unknown symbol "+s); + + if ( profile == laser_t::Harris ) { + // Parse the properties of the Harris profile + pp.get("profile_waist", profile_waist); + pp.get("profile_duration", profile_duration); + pp.get("profile_focal_distance", profile_focal_distance); } - } - // Plane normal - Real s = 1.0/std::sqrt(nvec[0]*nvec[0] + nvec[1]*nvec[1] + nvec[2]*nvec[2]); - nvec = { nvec[0]*s, nvec[1]*s, nvec[2]*s }; - - if (WarpX::gamma_boost > 1.) { - // Check that the laser direction is equal to the boost direction - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( nvec[0]*WarpX::boost_direction[0] - + nvec[1]*WarpX::boost_direction[1] - + nvec[2]*WarpX::boost_direction[2] - 1. < 1.e-12, - "The Lorentz boost should be in the same direction as the laser propagation"); - // Get the position of the plane, along the boost direction, in the lab frame - // and convert the position of the antenna to the boosted frame - Z0_lab = nvec[0]*position[0] + nvec[1]*position[1] + nvec[2]*position[2]; - Real Z0_boost = Z0_lab/WarpX::gamma_boost; - position[0] += (Z0_boost-Z0_lab)*nvec[0]; - position[1] += (Z0_boost-Z0_lab)*nvec[1]; - position[2] += (Z0_boost-Z0_lab)*nvec[2]; - } + if ( profile == laser_t::parse_field_function ) { + // Parse the properties of the parse_field_function profile + pp.get("field_function(X,Y,t)", field_function); + parser.define(field_function); + parser.registerVariables({"X","Y","t"}); + + ParmParse ppc("my_constants"); + std::set symbols = parser.symbols(); + symbols.erase("X"); + symbols.erase("Y"); + symbols.erase("t"); // after removing variables, we are left with constants + for (auto it = symbols.begin(); it != symbols.end(); ) { + Real v; + if (ppc.query(it->c_str(), v)) { + parser.setConstant(*it, v); + it = symbols.erase(it); + } else { + ++it; + } + } + for (auto const& s : symbols) { // make sure there no unknown symbols + amrex::Abort("Laser Profile: Unknown symbol "+s); + } + } + + // Plane normal + Real s = 1.0/std::sqrt(nvec[0]*nvec[0] + nvec[1]*nvec[1] + nvec[2]*nvec[2]); + nvec = { nvec[0]*s, nvec[1]*s, nvec[2]*s }; + + if (WarpX::gamma_boost > 1.) { + // Check that the laser direction is equal to the boost direction + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( nvec[0]*WarpX::boost_direction[0] + + nvec[1]*WarpX::boost_direction[1] + + nvec[2]*WarpX::boost_direction[2] - 1. < 1.e-12, + "The Lorentz boost should be in the same direction as the laser propagation"); + // Get the position of the plane, along the boost direction, in the lab frame + // and convert the position of the antenna to the boosted frame + Z0_lab = nvec[0]*position[0] + nvec[1]*position[1] + nvec[2]*position[2]; + Real Z0_boost = Z0_lab/WarpX::gamma_boost; + position[0] += (Z0_boost-Z0_lab)*nvec[0]; + position[1] += (Z0_boost-Z0_lab)*nvec[1]; + position[2] += (Z0_boost-Z0_lab)*nvec[2]; + } - // The first polarization vector - s = 1.0/std::sqrt(p_X[0]*p_X[0] + p_X[1]*p_X[1] + p_X[2]*p_X[2]); - p_X = { p_X[0]*s, p_X[1]*s, p_X[2]*s }; + // The first polarization vector + s = 1.0/std::sqrt(p_X[0]*p_X[0] + p_X[1]*p_X[1] + p_X[2]*p_X[2]); + p_X = { p_X[0]*s, p_X[1]*s, p_X[2]*s }; - Real dp = std::inner_product(nvec.begin(), nvec.end(), p_X.begin(), 0.0); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(std::abs(dp) < 1.0e-14, - "Laser plane vector is not perpendicular to the main polarization vector"); + Real dp = std::inner_product(nvec.begin(), nvec.end(), p_X.begin(), 0.0); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(std::abs(dp) < 1.0e-14, + "Laser plane vector is not perpendicular to the main polarization vector"); - p_Y = CrossProduct(nvec, p_X); // The second polarization vector + p_Y = CrossProduct(nvec, p_X); // The second polarization vector - s = 1.0/std::sqrt(stc_direction[0]*stc_direction[0] + stc_direction[1]*stc_direction[1] + stc_direction[2]*stc_direction[2]); - stc_direction = { stc_direction[0]*s, stc_direction[1]*s, stc_direction[2]*s }; - dp = std::inner_product(nvec.begin(), nvec.end(), stc_direction.begin(), 0.0); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(std::abs(dp) < 1.0e-14, - "stc_direction is not perpendicular to the laser plane vector"); + s = 1.0/std::sqrt(stc_direction[0]*stc_direction[0] + stc_direction[1]*stc_direction[1] + stc_direction[2]*stc_direction[2]); + stc_direction = { stc_direction[0]*s, stc_direction[1]*s, stc_direction[2]*s }; + dp = std::inner_product(nvec.begin(), nvec.end(), stc_direction.begin(), 0.0); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(std::abs(dp) < 1.0e-14, + "stc_direction is not perpendicular to the laser plane vector"); - // Get angle between p_X and stc_direction - // in 2d, stcs are in the simulation plane + // Get angle between p_X and stc_direction + // in 2d, stcs are in the simulation plane #if AMREX_SPACEDIM == 3 - theta_stc = acos(stc_direction[0]*p_X[0] + + theta_stc = acos(stc_direction[0]*p_X[0] + stc_direction[1]*p_X[1] + stc_direction[2]*p_X[2]); #else - theta_stc = 0.; + theta_stc = 0.; #endif #if AMREX_SPACEDIM == 3 - u_X = p_X; - u_Y = p_Y; + u_X = p_X; + u_Y = p_Y; #else - u_X = CrossProduct({0., 1., 0.}, nvec); - u_Y = {0., 1., 0.}; + u_X = CrossProduct({0., 1., 0.}, nvec); + u_Y = {0., 1., 0.}; #endif laser_injection_box= Geom(0).ProbDomain(); @@ -327,19 +327,19 @@ LaserParticleContainer::InitData (int lev) IntVect(plane_hi[0],plane_hi[1],0)}; BoxArray plane_ba {plane_box}; { - IntVect chunk(plane_box.size()); - const int min_size = 8; - while (plane_ba.size() < nprocs && chunk[0] > min_size && chunk[1] > min_size) - { - for (int j = 1; j >= 0 ; j--) - { - chunk[j] /= 2; - - if (plane_ba.size() < nprocs) { - plane_ba.maxSize(chunk); - } - } - } + IntVect chunk(plane_box.size()); + const int min_size = 8; + while (plane_ba.size() < nprocs && chunk[0] > min_size && chunk[1] > min_size) + { + for (int j = 1; j >= 0 ; j--) + { + chunk[j] /= 2; + + if (plane_ba.size() < nprocs) { + plane_ba.maxSize(chunk); + } + } + } } #else BoxArray plane_ba { Box {IntVect(plane_lo[0],0), IntVect(plane_hi[0],0)} }; @@ -351,29 +351,29 @@ LaserParticleContainer::InitData (int lev) const Vector& procmap = plane_dm.ProcessorMap(); for (int i = 0, n = plane_ba.size(); i < n; ++i) { - if (procmap[i] == myproc) - { - const Box& bx = plane_ba[i]; - for (IntVect cell = bx.smallEnd(); cell <= bx.bigEnd(); bx.next(cell)) - { - const Vector& pos = Transform(cell[0], cell[1]); + if (procmap[i] == myproc) + { + const Box& bx = plane_ba[i]; + for (IntVect cell = bx.smallEnd(); cell <= bx.bigEnd(); bx.next(cell)) + { + const Vector& pos = Transform(cell[0], cell[1]); #if (AMREX_SPACEDIM == 3) - const Real* x = pos.dataPtr(); + const Real* x = pos.dataPtr(); #else - const Real x[2] = {pos[0], pos[2]}; + const Real x[2] = {pos[0], pos[2]}; #endif - if (laser_injection_box.contains(x)) - { - for (int k = 0; k<2; ++k) { - particle_x.push_back(pos[0]); - particle_y.push_back(pos[1]); - particle_z.push_back(pos[2]); + if (laser_injection_box.contains(x)) + { + for (int k = 0; k<2; ++k) { + particle_x.push_back(pos[0]); + particle_y.push_back(pos[1]); + particle_z.push_back(pos[2]); + } + particle_w.push_back( weight); + particle_w.push_back(-weight); } - particle_w.push_back( weight); - particle_w.push_back(-weight); } - } - } + } } const int np = particle_z.size(); RealVector particle_ux(np, 0.0); diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 93fc12799..ea939cef3 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -252,12 +252,12 @@ WarpX::FillBoundaryE (int lev, PatchType patch_type) { if (do_pml && pml[lev]->ok()) { - pml[lev]->ExchangeE(patch_type, + pml[lev]->ExchangeE(patch_type, { Efield_fp[lev][0].get(), Efield_fp[lev][1].get(), Efield_fp[lev][2].get() }, do_pml_in_domain); - pml[lev]->FillBoundaryE(patch_type); + pml[lev]->FillBoundaryE(patch_type); } const auto& period = Geom(lev).periodicity(); @@ -296,7 +296,7 @@ WarpX::FillBoundaryB (int lev, PatchType patch_type) { if (do_pml && pml[lev]->ok()) { - pml[lev]->ExchangeB(patch_type, + pml[lev]->ExchangeB(patch_type, { Bfield_fp[lev][0].get(), Bfield_fp[lev][1].get(), Bfield_fp[lev][2].get() }, diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index c737591c7..5441755f5 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -18,7 +18,7 @@ WarpX::LoadBalance () const Real nprocs = ParallelDescriptor::NProcs(); const int nmax = static_cast(std::ceil(nboxes/nprocs*load_balance_knapsack_factor)); const DistributionMapping newdm = (load_balance_with_sfc) - ? DistributionMapping::makeSFC(*costs[lev], false) + ? DistributionMapping::makeSFC(*costs[lev], false) : DistributionMapping::makeKnapSack(*costs[lev], nmax); RemakeLevel(lev, t_new[lev], boxArray(lev), newdm); } diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index ac261b177..2b18e174d 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -31,7 +31,7 @@ public: ~MultiParticleContainer() {} WarpXParticleContainer& GetParticleContainer (int ispecies) { - return *allcontainers[ispecies]; + return *allcontainers[ispecies]; } std::array meanParticleVelocity(int ispecies) { @@ -95,13 +95,13 @@ public: /// This is the electromagnetic version. /// void Evolve (int lev, - const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, - amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, - amrex::MultiFab* cjx, amrex::MultiFab* cjy, amrex::MultiFab* cjz, + const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, + const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, + amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, + amrex::MultiFab* cjx, amrex::MultiFab* cjy, amrex::MultiFab* cjz, amrex::MultiFab* rho, amrex::MultiFab* crho, - const amrex::MultiFab* cEx, const amrex::MultiFab* cEy, const amrex::MultiFab* cEz, - const amrex::MultiFab* cBx, const amrex::MultiFab* cBy, const amrex::MultiFab* cBz, + const amrex::MultiFab* cEx, const amrex::MultiFab* cEy, const amrex::MultiFab* cEz, + const amrex::MultiFab* cBx, const amrex::MultiFab* cBy, const amrex::MultiFab* cBz, amrex::Real t, amrex::Real dt); /// diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 612583e2f..d5ad7a366 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -239,8 +239,8 @@ MultiParticleContainer::Evolve (int lev, if (rho) rho->setVal(0.0); if (crho) crho->setVal(0.0); for (auto& pc : allcontainers) { - pc->Evolve(lev, Ex, Ey, Ez, Bx, By, Bz, jx, jy, jz, cjx, cjy, cjz, - rho, crho, cEx, cEy, cEz, cBx, cBy, cBz, t, dt); + pc->Evolve(lev, Ex, Ey, Ez, Bx, By, Bz, jx, jy, jz, cjx, cjy, cjz, + rho, crho, cEx, cEy, cEz, cBx, cBy, cBz, t, dt); } } diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index 7946b4650..529bffab1 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -62,13 +62,13 @@ public: int depos_lev); virtual void Evolve (int lev, - const amrex::MultiFab& Ex, + const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, + const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, - amrex::MultiFab& jx, + amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, amrex::MultiFab* cjx, @@ -86,7 +86,7 @@ public: amrex::Real dt) override; virtual void PushPX(WarpXParIter& pti, - amrex::Cuda::ManagedDeviceVector& xp, + amrex::Cuda::ManagedDeviceVector& xp, amrex::Cuda::ManagedDeviceVector& yp, amrex::Cuda::ManagedDeviceVector& zp, amrex::Cuda::ManagedDeviceVector& giv, diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 99c6973f9..0b82df7b5 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -455,8 +455,8 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) std::size_t shared_mem_bytes = plasma_injector->sharedMemoryNeeded(); int lrrfac = rrfac; - bool loc_do_field_ionization = do_field_ionization; - int loc_ionization_initial_level = ionization_initial_level; + bool loc_do_field_ionization = do_field_ionization; + int loc_ionization_initial_level = ionization_initial_level; // Loop over all new particles and inject them (creates too many // particles, in particular does not consider xmin, xmax etc.). @@ -1501,13 +1501,13 @@ PhysicalParticleContainer::SplitParticles(int lev) } } } - // Add local arrays psplit_x etc. to the temporary - // particle container pctmp_split. Split particles - // are tagged with p.id()=NoSplitParticleID so that - // they are not re-split when entering a higher level - // AddNParticles calls Redistribute, so that particles - // in pctmp_split are in the proper grids and tiles - pctmp_split.AddNParticles(lev, + // Add local arrays psplit_x etc. to the temporary + // particle container pctmp_split. Split particles + // are tagged with p.id()=NoSplitParticleID so that + // they are not re-split when entering a higher level + // AddNParticles calls Redistribute, so that particles + // in pctmp_split are in the proper grids and tiles + pctmp_split.AddNParticles(lev, np_split_to_add, psplit_x.dataPtr(), psplit_y.dataPtr(), @@ -1518,9 +1518,9 @@ PhysicalParticleContainer::SplitParticles(int lev) 1, psplit_w.dataPtr(), 1, NoSplitParticleID); - // Copy particles from tmp to current particle container + // Copy particles from tmp to current particle container addParticles(pctmp_split,1); - // Clear tmp container + // Clear tmp container pctmp_split.clearParticles(); } diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 06540871b..4411436fa 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -11,12 +11,12 @@ enum struct ConvertDirection{WarpX_to_SI, SI_to_WarpX}; struct PIdx { enum { // Particle Attributes stored in amrex::ParticleContainer's struct of array - w = 0, // weight - ux, uy, uz, Ex, Ey, Ez, Bx, By, Bz, + w = 0, // weight + ux, uy, uz, Ex, Ey, Ez, Bx, By, Bz, #ifdef WARPX_DIM_RZ theta, // RZ needs all three position components #endif - nattribs + nattribs }; }; @@ -131,9 +131,9 @@ public: #endif // WARPX_DO_ELECTROSTATIC virtual void Evolve (int lev, - const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, - const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, - amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, + const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, + const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, + amrex::MultiFab& jx, amrex::MultiFab& jy, amrex::MultiFab& jz, amrex::MultiFab* cjx, amrex::MultiFab* cjy, amrex::MultiFab* cjz, amrex::MultiFab* rho, amrex::MultiFab* crho, const amrex::MultiFab* cEx, const amrex::MultiFab* cEy, const amrex::MultiFab* cEz, @@ -231,8 +231,8 @@ public: void AddNParticles (int lev, int n, const amrex::Real* x, const amrex::Real* y, const amrex::Real* z, - const amrex::Real* vx, const amrex::Real* vy, const amrex::Real* vz, - int nattr, const amrex::Real* attr, int uniqueparticles, int id=-1); + const amrex::Real* vx, const amrex::Real* vy, const amrex::Real* vz, + int nattr, const amrex::Real* attr, int uniqueparticles, int id=-1); void AddOneParticle (int lev, int grid, int tile, amrex::Real x, amrex::Real y, amrex::Real z, diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 83de9fb0e..6de7a6011 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -108,7 +108,7 @@ WarpXParticleContainer::ReadParameters () static bool initialized = false; if (!initialized) { - ParmParse pp("particles"); + ParmParse pp("particles"); #ifdef AMREX_USE_GPU do_tiling = false; // By default, tiling is off on GPU @@ -118,7 +118,7 @@ WarpXParticleContainer::ReadParameters () pp.query("do_tiling", do_tiling); pp.query("do_not_push", do_not_push); - initialized = true; + initialized = true; } } @@ -168,28 +168,28 @@ WarpXParticleContainer::AddOneParticle (ParticleTileType& particle_tile, void WarpXParticleContainer::AddNParticles (int lev, int n, const Real* x, const Real* y, const Real* z, - const Real* vx, const Real* vy, const Real* vz, - int nattr, const Real* attr, int uniqueparticles, int id) + const Real* vx, const Real* vy, const Real* vz, + int nattr, const Real* attr, int uniqueparticles, int id) { BL_ASSERT(nattr == 1); const Real* weight = attr; int ibegin, iend; if (uniqueparticles) { - ibegin = 0; - iend = n; + ibegin = 0; + iend = n; } else { - int myproc = ParallelDescriptor::MyProc(); - int nprocs = ParallelDescriptor::NProcs(); - int navg = n/nprocs; - int nleft = n - navg * nprocs; - if (myproc < nleft) { - ibegin = myproc*(navg+1); - iend = ibegin + navg+1; - } else { - ibegin = myproc*navg + nleft; - iend = ibegin + navg; - } + int myproc = ParallelDescriptor::MyProc(); + int nprocs = ParallelDescriptor::NProcs(); + int navg = n/nprocs; + int nleft = n - navg * nprocs; + if (myproc < nleft) { + ibegin = myproc*(navg+1); + iend = ibegin + navg+1; + } else { + ibegin = myproc*navg + nleft; + iend = ibegin + navg; + } } // Add to grid 0 and tile 0 @@ -206,12 +206,12 @@ WarpXParticleContainer::AddNParticles (int lev, for (int i = ibegin; i < iend; ++i) { ParticleType p; - if (id==-1) - { - p.id() = ParticleType::NextID(); - } else { - p.id() = id; - } + if (id==-1) + { + p.id() = ParticleType::NextID(); + } else { + p.id() = id; + } p.cpu() = ParallelDescriptor::MyProc(); #if (AMREX_SPACEDIM == 3) p.pos(0) = x[i]; diff --git a/Source/Python/WarpXWrappers.cpp b/Source/Python/WarpXWrappers.cpp index b59f80495..c6d7dfdb8 100644 --- a/Source/Python/WarpXWrappers.cpp +++ b/Source/Python/WarpXWrappers.cpp @@ -55,18 +55,18 @@ extern "C" int warpx_nSpecies() { - auto & mypc = WarpX::GetInstance().GetPartContainer(); + auto & mypc = WarpX::GetInstance().GetPartContainer(); return mypc.nSpecies(); } bool warpx_use_fdtd_nci_corr() { - return WarpX::use_fdtd_nci_corr; + return WarpX::use_fdtd_nci_corr; } int warpx_l_lower_order_in_v() { - return WarpX::l_lower_order_in_v; + return WarpX::l_lower_order_in_v; } int warpx_nComps() @@ -81,32 +81,32 @@ extern "C" void amrex_init (int argc, char* argv[]) { - amrex::Initialize(argc,argv); + amrex::Initialize(argc,argv); } #ifdef BL_USE_MPI void amrex_init_with_inited_mpi (int argc, char* argv[], MPI_Comm mpicomm) { - amrex::Initialize(argc,argv,true,mpicomm); + amrex::Initialize(argc,argv,true,mpicomm); } #endif void amrex_finalize (int finalize_mpi) { - amrex::Finalize(); + amrex::Finalize(); } void warpx_init () { - WarpX& warpx = WarpX::GetInstance(); - warpx.InitData(); + WarpX& warpx = WarpX::GetInstance(); + warpx.InitData(); if (warpx_py_afterinit) warpx_py_afterinit(); if (warpx_py_particleloader) warpx_py_particleloader(); } void warpx_finalize () { - WarpX::ResetInstance(); + WarpX::ResetInstance(); } void warpx_set_callback_py_afterinit (WARPX_CALLBACK_PY_FUNC_0 callback) @@ -160,8 +160,8 @@ extern "C" void warpx_evolve (int numsteps) { - WarpX& warpx = WarpX::GetInstance(); - warpx.Evolve(numsteps); + WarpX& warpx = WarpX::GetInstance(); + warpx.Evolve(numsteps); } void warpx_addNParticles(int speciesnumber, int lenx, @@ -169,10 +169,10 @@ extern "C" double* vx, double* vy, double* vz, int nattr, double* attr, int uniqueparticles) { - auto & mypc = WarpX::GetInstance().GetPartContainer(); - auto & myspc = mypc.GetParticleContainer(speciesnumber); + auto & mypc = WarpX::GetInstance().GetPartContainer(); + auto & myspc = mypc.GetParticleContainer(speciesnumber); const int lev = 0; - myspc.AddNParticles(lev, lenx, x, y, z, vx, vy, vz, nattr, attr, uniqueparticles); + myspc.AddNParticles(lev, lenx, x, y, z, vx, vy, vz, nattr, attr, uniqueparticles); } void warpx_ConvertLabParamsToBoost() @@ -444,7 +444,7 @@ extern "C" } void mypc_Redistribute () { - auto & mypc = WarpX::GetInstance().GetPartContainer(); + auto & mypc = WarpX::GetInstance().GetPartContainer(); mypc.Redistribute(); } diff --git a/Source/WarpX.H b/Source/WarpX.H index 7131355d2..0cfd9c623 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -300,20 +300,20 @@ protected: //! DistributionMapping. Only used during initialization. Called //! by AmrCoreInitFromScratch. virtual void MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray& ba, - const amrex::DistributionMapping& dm) final; + const amrex::DistributionMapping& dm) final; //! Make a new level using provided BoxArray and //! DistributionMapping and fill with interpolated coarse level //! data. Called by AmrCore::regrid. virtual void MakeNewLevelFromCoarse (int lev, amrex::Real time, const amrex::BoxArray& ba, - const amrex::DistributionMapping& dm) final + const amrex::DistributionMapping& dm) final { amrex::Abort("MakeNewLevelFromCoarse: To be implemented"); } //! Remake an existing level using provided BoxArray and //! DistributionMapping and fill with existing fine and coarse //! data. Called by AmrCore::regrid. virtual void RemakeLevel (int lev, amrex::Real time, const amrex::BoxArray& ba, - const amrex::DistributionMapping& dm) final; + const amrex::DistributionMapping& dm) final; //! Delete level data. Called by AmrCore::regrid. virtual void ClearLevel (int lev) final; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index e57d833c8..d2ccc71ef 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -112,7 +112,7 @@ WarpX& WarpX::GetInstance () { if (!m_instance) { - m_instance = new WarpX(); + m_instance = new WarpX(); } return *m_instance; } @@ -142,7 +142,7 @@ WarpX::WarpX () #if 0 // no subcycling yet for (int lev = 1; lev <= maxLevel(); ++lev) { - nsubsteps[lev] = MaxRefRatio(lev-1); + nsubsteps[lev] = MaxRefRatio(lev-1); } #endif @@ -251,142 +251,142 @@ void WarpX::ReadParameters () { { - ParmParse pp; // Traditionally, max_step and stop_time do not have prefix. - pp.query("max_step", max_step); - pp.query("stop_time", stop_time); + ParmParse pp; // Traditionally, max_step and stop_time do not have prefix. + pp.query("max_step", max_step); + pp.query("stop_time", stop_time); } { - ParmParse pp("amr"); // Traditionally, these have prefix, amr. + ParmParse pp("amr"); // Traditionally, these have prefix, amr. - pp.query("check_file", check_file); - pp.query("check_int", check_int); + pp.query("check_file", check_file); + pp.query("check_int", check_int); - pp.query("plot_file", plot_file); - pp.query("plot_int", plot_int); + pp.query("plot_file", plot_file); + pp.query("plot_int", plot_int); - pp.query("restart", restart_chkfile); + pp.query("restart", restart_chkfile); } { - ParmParse pp("warpx"); + ParmParse pp("warpx"); - pp.query("cfl", cfl); - pp.query("verbose", verbose); - pp.query("regrid_int", regrid_int); + pp.query("cfl", cfl); + pp.query("verbose", verbose); + pp.query("regrid_int", regrid_int); pp.query("do_subcycling", do_subcycling); pp.query("override_sync_int", override_sync_int); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(do_subcycling != 1 || max_level <= 1, - "Subcycling method 1 only works for 2 levels."); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(do_subcycling != 1 || max_level <= 1, + "Subcycling method 1 only works for 2 levels."); - ReadBoostedFrameParameters(gamma_boost, beta_boost, boost_direction); + ReadBoostedFrameParameters(gamma_boost, beta_boost, boost_direction); - // pp.query returns 1 if argument zmax_plasma_to_compute_max_step is - // specified by the user, 0 otherwise. - do_compute_max_step_from_zmax = - pp.query("zmax_plasma_to_compute_max_step", - zmax_plasma_to_compute_max_step); + // pp.query returns 1 if argument zmax_plasma_to_compute_max_step is + // specified by the user, 0 otherwise. + do_compute_max_step_from_zmax = + pp.query("zmax_plasma_to_compute_max_step", + zmax_plasma_to_compute_max_step); - pp.queryarr("B_external", B_external); + pp.queryarr("B_external", B_external); - pp.query("do_moving_window", do_moving_window); - if (do_moving_window) - { - std::string s; - pp.get("moving_window_dir", s); - if (s == "x" || s == "X") { - moving_window_dir = 0; - } + pp.query("do_moving_window", do_moving_window); + if (do_moving_window) + { + std::string s; + pp.get("moving_window_dir", s); + if (s == "x" || s == "X") { + moving_window_dir = 0; + } #if (AMREX_SPACEDIM == 3) - else if (s == "y" || s == "Y") { - moving_window_dir = 1; - } + else if (s == "y" || s == "Y") { + moving_window_dir = 1; + } #endif - else if (s == "z" || s == "Z") { - moving_window_dir = AMREX_SPACEDIM-1; - } - else { - const std::string msg = "Unknown moving_window_dir: "+s; - amrex::Abort(msg.c_str()); - } - - moving_window_x = geom[0].ProbLo(moving_window_dir); - - pp.get("moving_window_v", moving_window_v); - moving_window_v *= PhysConst::c; - } - - pp.query("do_boosted_frame_diagnostic", do_boosted_frame_diagnostic); - if (do_boosted_frame_diagnostic) { - - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(gamma_boost > 1.0, - "gamma_boost must be > 1 to use the boosted frame diagnostic."); - - pp.query("lab_data_directory", lab_data_directory); - - std::string s; - pp.get("boost_direction", s); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( (s == "z" || s == "Z"), - "The boosted frame diagnostic currently only works if the boost is in the z direction."); - - pp.get("num_snapshots_lab", num_snapshots_lab); - - // Read either dz_snapshots_lab or dt_snapshots_lab - bool snapshot_interval_is_specified = 0; - Real dz_snapshots_lab = 0; - snapshot_interval_is_specified += pp.query("dt_snapshots_lab", dt_snapshots_lab); - if ( pp.query("dz_snapshots_lab", dz_snapshots_lab) ){ - dt_snapshots_lab = dz_snapshots_lab/PhysConst::c; - snapshot_interval_is_specified = 1; + else if (s == "z" || s == "Z") { + moving_window_dir = AMREX_SPACEDIM-1; + } + else { + const std::string msg = "Unknown moving_window_dir: "+s; + amrex::Abort(msg.c_str()); + } + + moving_window_x = geom[0].ProbLo(moving_window_dir); + + pp.get("moving_window_v", moving_window_v); + moving_window_v *= PhysConst::c; } - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( - snapshot_interval_is_specified, - "When using back-transformed diagnostics, user should specify either dz_snapshots_lab or dt_snapshots_lab."); - pp.get("gamma_boost", gamma_boost); + pp.query("do_boosted_frame_diagnostic", do_boosted_frame_diagnostic); + if (do_boosted_frame_diagnostic) { - pp.query("do_boosted_frame_fields", do_boosted_frame_fields); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(gamma_boost > 1.0, + "gamma_boost must be > 1 to use the boosted frame diagnostic."); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(do_moving_window, - "The moving window should be on if using the boosted frame diagnostic."); + pp.query("lab_data_directory", lab_data_directory); - pp.get("moving_window_dir", s); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( (s == "z" || s == "Z"), - "The boosted frame diagnostic currently only works if the moving window is in the z direction."); - } + std::string s; + pp.get("boost_direction", s); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( (s == "z" || s == "Z"), + "The boosted frame diagnostic currently only works if the boost is in the z direction."); - pp.query("do_electrostatic", do_electrostatic); - pp.query("n_buffer", n_buffer); - pp.query("const_dt", const_dt); - - // Read filter and fill IntVect filter_npass_each_dir with - // proper size for AMREX_SPACEDIM - pp.query("use_filter", use_filter); - Vector parse_filter_npass_each_dir(AMREX_SPACEDIM,1); - pp.queryarr("filter_npass_each_dir", parse_filter_npass_each_dir); - filter_npass_each_dir[0] = parse_filter_npass_each_dir[0]; - filter_npass_each_dir[1] = parse_filter_npass_each_dir[1]; + pp.get("num_snapshots_lab", num_snapshots_lab); + + // Read either dz_snapshots_lab or dt_snapshots_lab + bool snapshot_interval_is_specified = 0; + Real dz_snapshots_lab = 0; + snapshot_interval_is_specified += pp.query("dt_snapshots_lab", dt_snapshots_lab); + if ( pp.query("dz_snapshots_lab", dz_snapshots_lab) ){ + dt_snapshots_lab = dz_snapshots_lab/PhysConst::c; + snapshot_interval_is_specified = 1; + } + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + snapshot_interval_is_specified, + "When using back-transformed diagnostics, user should specify either dz_snapshots_lab or dt_snapshots_lab."); + + pp.get("gamma_boost", gamma_boost); + + pp.query("do_boosted_frame_fields", do_boosted_frame_fields); + + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(do_moving_window, + "The moving window should be on if using the boosted frame diagnostic."); + + pp.get("moving_window_dir", s); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( (s == "z" || s == "Z"), + "The boosted frame diagnostic currently only works if the moving window is in the z direction."); + } + + pp.query("do_electrostatic", do_electrostatic); + pp.query("n_buffer", n_buffer); + pp.query("const_dt", const_dt); + + // Read filter and fill IntVect filter_npass_each_dir with + // proper size for AMREX_SPACEDIM + pp.query("use_filter", use_filter); + Vector parse_filter_npass_each_dir(AMREX_SPACEDIM,1); + pp.queryarr("filter_npass_each_dir", parse_filter_npass_each_dir); + filter_npass_each_dir[0] = parse_filter_npass_each_dir[0]; + filter_npass_each_dir[1] = parse_filter_npass_each_dir[1]; #if (AMREX_SPACEDIM == 3) - filter_npass_each_dir[2] = parse_filter_npass_each_dir[2]; + filter_npass_each_dir[2] = parse_filter_npass_each_dir[2]; #endif - pp.query("num_mirrors", num_mirrors); - if (num_mirrors>0){ - mirror_z.resize(num_mirrors); - pp.getarr("mirror_z", mirror_z, 0, num_mirrors); - mirror_z_width.resize(num_mirrors); - pp.getarr("mirror_z_width", mirror_z_width, 0, num_mirrors); - mirror_z_npoints.resize(num_mirrors); - pp.getarr("mirror_z_npoints", mirror_z_npoints, 0, num_mirrors); - } + pp.query("num_mirrors", num_mirrors); + if (num_mirrors>0){ + mirror_z.resize(num_mirrors); + pp.getarr("mirror_z", mirror_z, 0, num_mirrors); + mirror_z_width.resize(num_mirrors); + pp.getarr("mirror_z_width", mirror_z_width, 0, num_mirrors); + mirror_z_npoints.resize(num_mirrors); + pp.getarr("mirror_z_npoints", mirror_z_npoints, 0, num_mirrors); + } - pp.query("serialize_ics", serialize_ics); - pp.query("refine_plasma", refine_plasma); + pp.query("serialize_ics", serialize_ics); + pp.query("refine_plasma", refine_plasma); pp.query("do_dive_cleaning", do_dive_cleaning); pp.query("n_field_gather_buffer", n_field_gather_buffer); pp.query("n_current_deposition_buffer", n_current_deposition_buffer); - pp.query("sort_int", sort_int); + pp.query("sort_int", sort_int); pp.query("do_pml", do_pml); pp.query("pml_ncell", pml_ncell); @@ -411,7 +411,7 @@ WarpX::ReadParameters () #endif if ( (do_pml_j_damping==1)&&(do_pml_in_domain==0) ){ - amrex::Abort("J-damping can only be done when PML are inside simulation domain (do_pml_in_domain=1)"); + amrex::Abort("J-damping can only be done when PML are inside simulation domain (do_pml_in_domain=1)"); } pp.query("dump_openpmd", dump_openpmd); @@ -517,16 +517,15 @@ WarpX::ReadParameters () // Only needs to be set with WARPX_DIM_RZ, otherwise defaults to 1. pp.query("n_rz_azimuthal_modes", n_rz_azimuthal_modes); - } { - ParmParse pp("interpolation"); - pp.query("nox", nox); - pp.query("noy", noy); - pp.query("noz", noz); + ParmParse pp("interpolation"); + pp.query("nox", nox); + pp.query("noy", noy); + pp.query("noz", noz); AMREX_ALWAYS_ASSERT_WITH_MESSAGE( nox == noy and nox == noz , - "warpx.nox, noy and noz must be equal"); + "warpx.nox, noy and noz must be equal"); AMREX_ALWAYS_ASSERT_WITH_MESSAGE( nox >= 1, "warpx.nox must >= 1"); } @@ -617,21 +616,21 @@ void WarpX::ClearLevel (int lev) { for (int i = 0; i < 3; ++i) { - Efield_aux[lev][i].reset(); - Bfield_aux[lev][i].reset(); + Efield_aux[lev][i].reset(); + Bfield_aux[lev][i].reset(); - current_fp[lev][i].reset(); - Efield_fp [lev][i].reset(); - Bfield_fp [lev][i].reset(); + current_fp[lev][i].reset(); + Efield_fp [lev][i].reset(); + Bfield_fp [lev][i].reset(); current_store[lev][i].reset(); - current_cp[lev][i].reset(); - Efield_cp [lev][i].reset(); - Bfield_cp [lev][i].reset(); + current_cp[lev][i].reset(); + Efield_cp [lev][i].reset(); + Bfield_cp [lev][i].reset(); - Efield_cax[lev][i].reset(); - Bfield_cax[lev][i].reset(); + Efield_cax[lev][i].reset(); + Bfield_cax[lev][i].reset(); current_buf[lev][i].reset(); } diff --git a/Source/main.cpp b/Source/main.cpp index 6aca49cd3..551be9c30 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -29,20 +29,20 @@ int main(int argc, char* argv[]) const Real strt_total = amrex::second(); { - WarpX warpx; + WarpX warpx; - warpx.InitData(); + warpx.InitData(); - warpx.Evolve(); + warpx.Evolve(); - Real end_total = amrex::second() - strt_total; + Real end_total = amrex::second() - strt_total; - ParallelDescriptor::ReduceRealMax(end_total ,ParallelDescriptor::IOProcessorNumber()); - if (warpx.Verbose()) { + ParallelDescriptor::ReduceRealMax(end_total, ParallelDescriptor::IOProcessorNumber()); + if (warpx.Verbose()) { amrex::Print() << "Total Time : " << end_total << '\n'; amrex::Print() << "WarpX Version: " << WarpX::Version() << '\n'; amrex::Print() << "PICSAR Version: " << WarpX::PicsarVersion() << '\n'; - } + } } BL_PROFILE_VAR_STOP(pmain); -- cgit v1.2.3 From 1e0126346ff5cc46e01e8bb006a1677b71776afb Mon Sep 17 00:00:00 2001 From: Dave Grote Date: Fri, 13 Sep 2019 10:40:50 -0700 Subject: Removed m_giv which was being set but not used. --- Source/Laser/LaserParticleContainer.cpp | 4 ---- Source/Particles/PhotonParticleContainer.H | 1 - Source/Particles/PhotonParticleContainer.cpp | 2 -- Source/Particles/PhysicalParticleContainer.H | 1 - Source/Particles/PhysicalParticleContainer.cpp | 18 +++++------------- Source/Particles/Pusher/UpdateMomentumBoris.H | 3 +-- Source/Particles/Pusher/UpdateMomentumVay.H | 3 +-- Source/Particles/RigidInjectedParticleContainer.H | 1 - Source/Particles/RigidInjectedParticleContainer.cpp | 15 +++++---------- Source/Particles/WarpXParticleContainer.H | 2 +- Source/Particles/WarpXParticleContainer.cpp | 1 - 11 files changed, 13 insertions(+), 38 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Laser/LaserParticleContainer.cpp b/Source/Laser/LaserParticleContainer.cpp index 15e82e940..2adf751e1 100644 --- a/Source/Laser/LaserParticleContainer.cpp +++ b/Source/Laser/LaserParticleContainer.cpp @@ -444,8 +444,6 @@ LaserParticleContainer::Evolve (int lev, // For now, laser particles do not take the current buffers into account const long np_current = np; - m_giv[thread_num].resize(np); - plane_Xp.resize(np); plane_Yp.resize(np); amplitude_E.resize(np); @@ -670,7 +668,6 @@ LaserParticleContainer::update_laser_particle( Real * const AMREX_RESTRICT xp = m_xp[thread_num].dataPtr(); Real * const AMREX_RESTRICT yp = m_yp[thread_num].dataPtr(); Real * const AMREX_RESTRICT zp = m_zp[thread_num].dataPtr(); - Real * const AMREX_RESTRICT giv = m_giv[thread_num].dataPtr(); Real tmp_p_X_0 = p_X[0]; Real tmp_p_X_1 = p_X[1]; Real tmp_p_X_2 = p_X[2]; @@ -700,7 +697,6 @@ LaserParticleContainer::update_laser_particle( } // Get the corresponding momenta const Real gamma = gamma_boost/std::sqrt(1. - v_over_c*v_over_c); - giv[i] = 1./gamma; puxp[i] = gamma * vx; puyp[i] = gamma * vy; puzp[i] = gamma * vz; diff --git a/Source/Particles/PhotonParticleContainer.H b/Source/Particles/PhotonParticleContainer.H index 4f9fbc047..94c8f03ef 100644 --- a/Source/Particles/PhotonParticleContainer.H +++ b/Source/Particles/PhotonParticleContainer.H @@ -43,7 +43,6 @@ public: amrex::Cuda::ManagedDeviceVector& xp, amrex::Cuda::ManagedDeviceVector& yp, amrex::Cuda::ManagedDeviceVector& zp, - amrex::Cuda::ManagedDeviceVector& giv, amrex::Real dt) override; diff --git a/Source/Particles/PhotonParticleContainer.cpp b/Source/Particles/PhotonParticleContainer.cpp index c0a159f1b..55dc839eb 100644 --- a/Source/Particles/PhotonParticleContainer.cpp +++ b/Source/Particles/PhotonParticleContainer.cpp @@ -37,7 +37,6 @@ PhotonParticleContainer::PushPX(WarpXParIter& pti, Cuda::ManagedDeviceVector& xp, Cuda::ManagedDeviceVector& yp, Cuda::ManagedDeviceVector& zp, - Cuda::ManagedDeviceVector& giv, Real dt) { @@ -47,7 +46,6 @@ PhotonParticleContainer::PushPX(WarpXParIter& pti, Real* const AMREX_RESTRICT x = xp.dataPtr(); Real* const AMREX_RESTRICT y = yp.dataPtr(); Real* const AMREX_RESTRICT z = zp.dataPtr(); - Real* const AMREX_RESTRICT gi = giv.dataPtr(); Real* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); Real* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); Real* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index 529bffab1..db25e9c85 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -89,7 +89,6 @@ public: amrex::Cuda::ManagedDeviceVector& xp, amrex::Cuda::ManagedDeviceVector& yp, amrex::Cuda::ManagedDeviceVector& zp, - amrex::Cuda::ManagedDeviceVector& giv, amrex::Real dt); virtual void PushP (int lev, amrex::Real dt, diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 0b82df7b5..4bc0ee16e 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1085,8 +1085,6 @@ PhysicalParticleContainer::Evolve (int lev, Byp.assign(np,WarpX::B_external[1]); Bzp.assign(np,WarpX::B_external[2]); - m_giv[thread_num].resize(np); - long nfine_current = np; //! number of particles depositing to fine grid long nfine_gather = np; //! number of particles gathering from fine grid if (has_buffer && !do_not_push) @@ -1303,8 +1301,7 @@ PhysicalParticleContainer::Evolve (int lev, // Particle Push // BL_PROFILE_VAR_START(blp_ppc_pp); - PushPX(pti, m_xp[thread_num], m_yp[thread_num], m_zp[thread_num], - m_giv[thread_num], dt); + PushPX(pti, m_xp[thread_num], m_yp[thread_num], m_zp[thread_num], dt); BL_PROFILE_VAR_STOP(blp_ppc_pp); // @@ -1529,7 +1526,6 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti, Cuda::ManagedDeviceVector& xp, Cuda::ManagedDeviceVector& yp, Cuda::ManagedDeviceVector& zp, - Cuda::ManagedDeviceVector& giv, Real dt) { @@ -1539,7 +1535,6 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti, Real* const AMREX_RESTRICT x = xp.dataPtr(); Real* const AMREX_RESTRICT y = yp.dataPtr(); Real* const AMREX_RESTRICT z = zp.dataPtr(); - Real* const AMREX_RESTRICT gi = giv.dataPtr(); Real* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); Real* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); Real* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); @@ -1569,7 +1564,7 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti, [=] AMREX_GPU_DEVICE (long i) { Real qp = q; if (ion_lev){ qp *= ion_lev[i]; } - UpdateMomentumBoris( ux[i], uy[i], uz[i], gi[i], + UpdateMomentumBoris( ux[i], uy[i], uz[i], Ex[i], Ey[i], Ez[i], Bx[i], By[i], Bz[i], qp, m, dt); UpdatePosition( x[i], y[i], z[i], @@ -1582,7 +1577,7 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti, [=] AMREX_GPU_DEVICE (long i) { Real qp = q; if (ion_lev){ qp *= ion_lev[i]; } - UpdateMomentumVay( ux[i], uy[i], uz[i], gi[i], + UpdateMomentumVay( ux[i], uy[i], uz[i], Ex[i], Ey[i], Ez[i], Bx[i], By[i], Bz[i], qp, m, dt); UpdatePosition( x[i], y[i], z[i], @@ -1644,8 +1639,6 @@ PhysicalParticleContainer::PushP (int lev, Real dt, Byp.assign(np,WarpX::B_external[1]); Bzp.assign(np,WarpX::B_external[2]); - m_giv[thread_num].resize(np); - // // copy data from particle container to temp arrays // @@ -1659,7 +1652,6 @@ PhysicalParticleContainer::PushP (int lev, Real dt, // This wraps the momentum advance so that inheritors can modify the call. // Extract pointers to the different particle quantities - Real* const AMREX_RESTRICT gi = m_giv[thread_num].dataPtr(); Real* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); Real* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); Real* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); @@ -1676,14 +1668,14 @@ PhysicalParticleContainer::PushP (int lev, Real dt, if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Boris){ amrex::ParallelFor( pti.numParticles(), [=] AMREX_GPU_DEVICE (long i) { - UpdateMomentumBoris( ux[i], uy[i], uz[i], gi[i], + UpdateMomentumBoris( ux[i], uy[i], uz[i], Expp[i], Eypp[i], Ezpp[i], Bxpp[i], Bypp[i], Bzpp[i], q, m, dt); } ); } else if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Vay) { amrex::ParallelFor( pti.numParticles(), [=] AMREX_GPU_DEVICE (long i) { - UpdateMomentumVay( ux[i], uy[i], uz[i], gi[i], + UpdateMomentumVay( ux[i], uy[i], uz[i], Expp[i], Eypp[i], Ezpp[i], Bxpp[i], Bypp[i], Bzpp[i], q, m, dt); } ); diff --git a/Source/Particles/Pusher/UpdateMomentumBoris.H b/Source/Particles/Pusher/UpdateMomentumBoris.H index 71e9a8ed1..a33058347 100644 --- a/Source/Particles/Pusher/UpdateMomentumBoris.H +++ b/Source/Particles/Pusher/UpdateMomentumBoris.H @@ -7,7 +7,7 @@ * given the value of its momenta `ux`, `uy`, `uz` */ AMREX_GPU_HOST_DEVICE AMREX_INLINE void UpdateMomentumBoris( - amrex::Real& ux, amrex::Real& uy, amrex::Real& uz, amrex::Real& gaminv, + amrex::Real& ux, amrex::Real& uy, amrex::Real& uz, const amrex::Real Ex, const amrex::Real Ey, const amrex::Real Ez, const amrex::Real Bx, const amrex::Real By, const amrex::Real Bz, const amrex::Real q, const amrex::Real m, const amrex::Real dt ) @@ -41,7 +41,6 @@ void UpdateMomentumBoris( ux += econst*Ex; uy += econst*Ey; uz += econst*Ez; - gaminv = 1./std::sqrt(1. + (ux*ux + uy*uy + uz*uz)*inv_c2); } #endif // WARPX_PARTICLES_PUSHER_UPDATEMOMENTUM_BORIS_H_ diff --git a/Source/Particles/Pusher/UpdateMomentumVay.H b/Source/Particles/Pusher/UpdateMomentumVay.H index 044297e22..1f0f19e63 100644 --- a/Source/Particles/Pusher/UpdateMomentumVay.H +++ b/Source/Particles/Pusher/UpdateMomentumVay.H @@ -9,7 +9,7 @@ * given the value of its momenta `ux`, `uy`, `uz` */ AMREX_GPU_HOST_DEVICE AMREX_INLINE void UpdateMomentumVay( - amrex::Real& ux, amrex::Real& uy, amrex::Real& uz, amrex::Real& gaminv, + amrex::Real& ux, amrex::Real& uy, amrex::Real& uz, const amrex::Real Ex, const amrex::Real Ey, const amrex::Real Ez, const amrex::Real Bx, const amrex::Real By, const amrex::Real Bz, const amrex::Real q, const amrex::Real m, const amrex::Real dt ) @@ -48,7 +48,6 @@ void UpdateMomentumVay( ux = s*(uxpr+tx*tu+uypr*tz-uzpr*ty); uy = s*(uypr+ty*tu+uzpr*tx-uxpr*tz); uz = s*(uzpr+tz*tu+uxpr*ty-uypr*tx); - gaminv = 1./std::sqrt(1. + (ux*ux + uy*uy + uz*uz)*invclightsq); } #endif // WARPX_PARTICLES_PUSHER_UPDATEMOMENTUM_VAY_H_ diff --git a/Source/Particles/RigidInjectedParticleContainer.H b/Source/Particles/RigidInjectedParticleContainer.H index a635e9aa0..c22979d0f 100644 --- a/Source/Particles/RigidInjectedParticleContainer.H +++ b/Source/Particles/RigidInjectedParticleContainer.H @@ -46,7 +46,6 @@ public: amrex::Cuda::ManagedDeviceVector& xp, amrex::Cuda::ManagedDeviceVector& yp, amrex::Cuda::ManagedDeviceVector& zp, - amrex::Cuda::ManagedDeviceVector& giv, amrex::Real dt) override; virtual void PushP (int lev, amrex::Real dt, diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index 4893b3294..cd5e34770 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -210,7 +210,6 @@ RigidInjectedParticleContainer::PushPX(WarpXParIter& pti, Cuda::ManagedDeviceVector& xp, Cuda::ManagedDeviceVector& yp, Cuda::ManagedDeviceVector& zp, - Cuda::ManagedDeviceVector& giv, Real dt) { @@ -227,7 +226,6 @@ RigidInjectedParticleContainer::PushPX(WarpXParIter& pti, Real* const AMREX_RESTRICT x = xp.dataPtr(); Real* const AMREX_RESTRICT y = yp.dataPtr(); Real* const AMREX_RESTRICT z = zp.dataPtr(); - Real* const AMREX_RESTRICT gi = giv.dataPtr(); Real* const AMREX_RESTRICT ux = uxp.dataPtr(); Real* const AMREX_RESTRICT uy = uyp.dataPtr(); Real* const AMREX_RESTRICT uz = uzp.dataPtr(); @@ -269,7 +267,7 @@ RigidInjectedParticleContainer::PushPX(WarpXParIter& pti, ); } - PhysicalParticleContainer::PushPX(pti, xp, yp, zp, giv, dt); + PhysicalParticleContainer::PushPX(pti, xp, yp, zp, dt); if (!done_injecting_lev) { @@ -292,14 +290,14 @@ RigidInjectedParticleContainer::PushPX(WarpXParIter& pti, ux[i] = ux_save[i]; uy[i] = uy_save[i]; uz[i] = uz_save[i]; - gi[i] = 1./std::sqrt(1. + (ux[i]*ux[i] + uy[i]*uy[i] + uz[i]*uz[i])*inv_csq); x[i] = x_save[i]; y[i] = y_save[i]; if (rigid) { z[i] = z_save[i] + dt*vz_ave_boosted; } else { - z[i] = z_save[i] + dt*uz[i]*gi[i]; + const Real gi = 1./std::sqrt(1. + (ux[i]*ux[i] + uy[i]*uy[i] + uz[i]*uz[i])*inv_csq); + z[i] = z_save[i] + dt*uz[i]*gi; } } } @@ -399,8 +397,6 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, Byp.assign(np,WarpX::B_external[1]); Bzp.assign(np,WarpX::B_external[2]); - m_giv[thread_num].resize(np); - // // copy data from particle container to temp arrays // @@ -420,7 +416,6 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, // This wraps the momentum advance so that inheritors can modify the call. // Extract pointers to the different particle quantities const Real* const AMREX_RESTRICT zp = m_zp[thread_num].dataPtr(); - Real* const AMREX_RESTRICT gi = m_giv[thread_num].dataPtr(); Real* const AMREX_RESTRICT uxpp = uxp.dataPtr(); Real* const AMREX_RESTRICT uypp = uyp.dataPtr(); Real* const AMREX_RESTRICT uzpp = uzp.dataPtr(); @@ -437,14 +432,14 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Boris){ amrex::ParallelFor( pti.numParticles(), [=] AMREX_GPU_DEVICE (long i) { - UpdateMomentumBoris( uxpp[i], uypp[i], uzpp[i], gi[i], + UpdateMomentumBoris( uxpp[i], uypp[i], uzpp[i], Expp[i], Eypp[i], Ezpp[i], Bxpp[i], Bypp[i], Bzpp[i], q, m, dt); } ); } else if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Vay) { amrex::ParallelFor( pti.numParticles(), [=] AMREX_GPU_DEVICE (long i) { - UpdateMomentumVay( uxpp[i], uypp[i], uzpp[i], gi[i], + UpdateMomentumVay( uxpp[i], uypp[i], uzpp[i], Expp[i], Eypp[i], Ezpp[i], Bxpp[i], Bypp[i], Bzpp[i], q, m, dt); } ); diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 4411436fa..e96e78c7a 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -327,7 +327,7 @@ protected: using DataContainer = amrex::Gpu::ManagedDeviceVector; using PairIndex = std::pair; - amrex::Vector m_xp, m_yp, m_zp, m_giv; + amrex::Vector m_xp, m_yp, m_zp; // Whether to dump particle quantities. // If true, particle position is always dumped. diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index ee606aadb..176c147da 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -99,7 +99,6 @@ WarpXParticleContainer::WarpXParticleContainer (AmrCore* amr_core, int ispecies) m_xp.resize(num_threads); m_yp.resize(num_threads); m_zp.resize(num_threads); - m_giv.resize(num_threads); } void -- cgit v1.2.3