diff options
| -rw-r--r-- | Docs/source/running_cpp/parameters.rst | 16 | ||||
| -rw-r--r-- | Examples/Tests/Langmuir/inputs.multi.rz.rt | 1 | ||||
| -rw-r--r-- | Examples/Tests/particles_in_PML/inputs | 1 | ||||
| -rw-r--r-- | Examples/Tests/particles_in_PML/inputs2d | 1 | ||||
| -rw-r--r-- | Source/FieldSolver/WarpXPushFieldsEM.cpp | 4 | ||||
| -rw-r--r-- | Source/FortranInterface/Make.package | 1 | ||||
| -rw-r--r-- | Source/FortranInterface/WarpX_f.H | 118 | ||||
| -rw-r--r-- | Source/FortranInterface/WarpX_picsar.F90 | 132 | ||||
| -rw-r--r-- | Source/Laser/LaserParticleContainer.cpp | 8 | ||||
| -rw-r--r-- | Source/Particles/MultiParticleContainer.H | 1 | ||||
| -rw-r--r-- | Source/Particles/PhysicalParticleContainer.cpp | 51 | ||||
| -rw-r--r-- | Source/Particles/WarpXParticleContainer.H | 15 | ||||
| -rw-r--r-- | Source/Particles/WarpXParticleContainer.cpp | 130 | ||||
| -rw-r--r-- | Source/Utils/WarpXAlgorithmSelection.H | 18 | ||||
| -rw-r--r-- | Source/Utils/WarpXAlgorithmSelection.cpp | 10 | ||||
| -rw-r--r-- | Source/WarpX.H | 1 | ||||
| -rw-r--r-- | Source/WarpX.cpp | 5 | ||||
| -rw-r--r-- | Tools/performance_tests/functions_perftest.py | 6 |
18 files changed, 38 insertions, 481 deletions
diff --git a/Docs/source/running_cpp/parameters.rst b/Docs/source/running_cpp/parameters.rst index 1c4e477d1..9beb234ae 100644 --- a/Docs/source/running_cpp/parameters.rst +++ b/Docs/source/running_cpp/parameters.rst @@ -559,10 +559,6 @@ Numerics and algorithms (see `Esirkepov, Comp. Phys. Comm. (2001) <https://www.sciencedirect.com/science/article/pii/S0010465500002289>`__) - ``direct``: simpler current deposition algorithm, described in the section :doc:`../theory/picsar_theory`. Note that this algorithm is not strictly charge-conserving. - - ``direct-vectorized`` (only available in 3D, and when running on CPU/KNL - as opposed to GPU): - mathematically equivalent to ``direct``, but uses an optimized algorithm - for vectorization on CPU/KNL (see `Vincenti, Comp. Phys. Comm. (2017) - <https://www.sciencedirect.com/science/article/pii/S0010465516302764>`__) If ``algo.current_deposition`` is not specified, the default is ``esirkepov``. @@ -571,24 +567,12 @@ Numerics and algorithms - ``standard``: standard charge deposition algorithm, described in the section :doc:`../theory/picsar_theory`. - - ``vectorized`` (only available in 3D, and when running on CPU/KNL - as opposed to GPU): - mathematically equivalent to ``standard``, but uses an optimized algorithm - for vectorization on CPU/KNL (see `Vincenti, Comp. Phys. Comm. (2017) - <https://www.sciencedirect.com/science/article/pii/S0010465516302764>`__) - - If ``algo.charge_deposition`` is not specified, ``vectorized`` is the default - whenever it is available ; ``standard`` is the default otherwise. * ``algo.field_gathering`` (`string`, optional) The algorithm for field gathering. Available options are: - ``standard``: gathers directly from the grid points (either staggered or nodal gridpoints depending on ``warpx.do_nodal``). - - ``vectorized`` (not available when running on GPU): mathematically - equivalent to ``standard``, but uses optimized vector instructions for CPU/KNL. - - If ``algo.field_gathering`` is not specified, ``vectorized`` is the default - on CPU/KNL ; ``standard`` is the default on GPU. * ``algo.particle_pusher`` (`string`, optional) The algorithm for the particle pusher. Available options are: diff --git a/Examples/Tests/Langmuir/inputs.multi.rz.rt b/Examples/Tests/Langmuir/inputs.multi.rz.rt index 873efb9fc..b602e8da5 100644 --- a/Examples/Tests/Langmuir/inputs.multi.rz.rt +++ b/Examples/Tests/Langmuir/inputs.multi.rz.rt @@ -28,7 +28,6 @@ warpx.verbose = 1 # Algorithms algo.field_gathering = standard algo.current_deposition = esirkepov -algo.use_picsar_deposition = 0 # Interpolation interpolation.nox = 1 diff --git a/Examples/Tests/particles_in_PML/inputs b/Examples/Tests/particles_in_PML/inputs index 32d72d452..a1bda17bc 100644 --- a/Examples/Tests/particles_in_PML/inputs +++ b/Examples/Tests/particles_in_PML/inputs @@ -26,7 +26,6 @@ warpx.do_pml_j_damping = 1 # Algorithms algo.current_deposition = esirkepov algo.charge_deposition = standard -algo.field_gathering = vectorized algo.particle_pusher = vay algo.maxwell_fdtd_solver = ckc warpx.cfl = 1.0 diff --git a/Examples/Tests/particles_in_PML/inputs2d b/Examples/Tests/particles_in_PML/inputs2d index 076d7a55d..9c5a64921 100644 --- a/Examples/Tests/particles_in_PML/inputs2d +++ b/Examples/Tests/particles_in_PML/inputs2d @@ -26,7 +26,6 @@ warpx.do_pml_j_damping = 1 # Algorithms algo.current_deposition = esirkepov algo.charge_deposition = standard -algo.field_gathering = vectorized algo.particle_pusher = vay algo.maxwell_fdtd_solver = ckc warpx.cfl = 1.0 diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 609a50a17..11d598b98 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -138,7 +138,7 @@ WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real a_dt) MultiFab* cost = costs[lev].get(); const IntVect& rr = (lev > 0) ? refRatio(lev-1) : IntVect::TheUnitVector(); - // xmin is only used by the picsar kernel with cylindrical geometry, + // xmin is only used by the kernel for cylindrical geometry, // in which case it is actually rmin. const Real xmin = Geom(0).ProbLo(0); @@ -325,7 +325,7 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) MultiFab* cost = costs[lev].get(); const IntVect& rr = (lev > 0) ? refRatio(lev-1) : IntVect::TheUnitVector(); - // xmin is only used by the picsar kernel with cylindrical geometry, + // xmin is only used by the kernel for cylindrical geometry, // in which case it is actually rmin. const Real xmin = Geom(0).ProbLo(0); diff --git a/Source/FortranInterface/Make.package b/Source/FortranInterface/Make.package index fe0c2ba3d..d7b17fa56 100644 --- a/Source/FortranInterface/Make.package +++ b/Source/FortranInterface/Make.package @@ -1,6 +1,5 @@ CEXE_headers += WarpX_f.H F90EXE_sources += WarpX_f.F90 -F90EXE_sources += WarpX_picsar.F90 INCLUDE_LOCATIONS += $(WARPX_HOME)/Source/FortranInterface VPATH_LOCATIONS += $(WARPX_HOME)/Source/FortranInterface diff --git a/Source/FortranInterface/WarpX_f.H b/Source/FortranInterface/WarpX_f.H index c17160e59..60a9817a2 100644 --- a/Source/FortranInterface/WarpX_f.H +++ b/Source/FortranInterface/WarpX_f.H @@ -16,8 +16,6 @@ #if (AMREX_SPACEDIM == 3) -#define WRPX_COMPUTE_DIVB warpx_compute_divb_3d -#define WRPX_COMPUTE_DIVE warpx_compute_dive_3d #define WRPX_SYNC_CURRENT warpx_sync_current_3d #define WRPX_SYNC_RHO warpx_sync_rho_3d @@ -27,7 +25,6 @@ #define WRPX_PUSH_PML_F warpx_push_pml_f_3d #define WRPX_DAMP_PML warpx_damp_pml_3d #define WRPX_DAMP_PML_F warpx_damp_pml_f_3d - #define WRPX_SUM_FINE_TO_CRSE_NODAL warpx_sum_fine_to_crse_nodal_3d #define WRPX_ZERO_OUT_BNDRY warpx_zero_out_bndry_3d #define WRPX_BUILD_MASK warpx_build_mask_3d @@ -40,7 +37,6 @@ #elif (AMREX_SPACEDIM == 2) -#define WRPX_COMPUTE_DIVB warpx_compute_divb_2d #define WRPX_SYNC_CURRENT warpx_sync_current_2d #define WRPX_SYNC_RHO warpx_sync_rho_2d @@ -61,57 +57,13 @@ #define WRPX_PUSH_LEAPFROG warpx_push_leapfrog_2d #define WRPX_PUSH_LEAPFROG_POSITIONS warpx_push_leapfrog_positions_2d -#ifdef WARPX_DIM_RZ -#define WRPX_COMPUTE_DIVE warpx_compute_dive_rz -#else -#define WRPX_COMPUTE_DIVE warpx_compute_dive_2d -#endif - #endif #ifdef __cplusplus extern "C" { #endif - - // Current deposition - void warpx_current_deposition( - amrex::Real* jx, const long* jx_ng, const int* jx_ntot, - amrex::Real* jy, const long* jy_ng, const int* jy_ntot, - amrex::Real* jz, const long* jz_ng, const int* jz_ntot, - const long* np, - const amrex::Real* xp, const amrex::Real* yp, const amrex::Real* zp, - const amrex::Real* uxp, const amrex::Real* uyp,const amrex::Real* uzp, - const amrex::Real* gip, const amrex::Real* w, const amrex::Real* q, - const amrex::Real* xmin, const amrex::Real* ymin, const amrex::Real* zmin, - const amrex::Real* dt, - const amrex::Real* dx, const amrex::Real* dy, const amrex::Real* dz, - const long* nox, const long* noy,const long* noz, - const int* l_nodal, const long* lvect, const long* current_depo_algo); - - // Particle pusher (velocity and position) - - void warpx_particle_pusher(const long* np, - amrex::Real* xp, amrex::Real* yp, amrex::Real* zp, - amrex::Real* uxp, amrex::Real* uyp, amrex::Real* uzp, amrex::Real* gaminv, - const amrex::Real* exp, const amrex::Real* eyp,const amrex::Real* ezp, - const amrex::Real* bxp, const amrex::Real* byp,const amrex::Real* bzp, - const amrex::Real* charge, const amrex::Real* mass, const amrex::Real* dt, - const long* particle_pusher_algo); - - // Particle pusher (velocity) - - void warpx_particle_pusher_momenta(const long* np, - amrex::Real* xp, amrex::Real* yp, amrex::Real* zp, - amrex::Real* uxp, amrex::Real* uyp, amrex::Real* uzp, amrex::Real* gaminv, - const amrex::Real* exp, const amrex::Real* eyp,const amrex::Real* ezp, - const amrex::Real* bxp, const amrex::Real* byp,const amrex::Real* bzp, - const amrex::Real* charge, const amrex::Real* mass, const amrex::Real* dt, - const long* particle_pusher_algo); - - // Laser pusher - void warpx_gaussian_laser( const long* np, amrex::Real* Xp, amrex::Real* Yp, amrex::Real* t, amrex::Real* wavelength, amrex::Real* e_max, amrex::Real* waist, amrex::Real* duration, amrex::Real* t_peak, amrex::Real* f, amrex::Real* amplitude, @@ -137,58 +89,6 @@ extern "C" amrex::Real* mobility, amrex::Real* dt, const amrex::Real* c, const amrex::Real* beta_boost, const amrex::Real* gamma_boost ); - // Maxwell solver - - void warpx_push_evec( - const int* xlo, const int* xhi, - const int* ylo, const int* yhi, - const int* zlo, const int* zhi, - BL_FORT_FAB_ARG_3D(ex), - BL_FORT_FAB_ARG_3D(ey), - BL_FORT_FAB_ARG_3D(ez), - const BL_FORT_FAB_ARG_3D(bx), - const BL_FORT_FAB_ARG_3D(by), - const BL_FORT_FAB_ARG_3D(bz), - const BL_FORT_FAB_ARG_3D(jx), - const BL_FORT_FAB_ARG_3D(jy), - const BL_FORT_FAB_ARG_3D(jz), - const amrex::Real* mudt, - const amrex::Real* dtsdx, - const amrex::Real* dtsdy, - const amrex::Real* dtsdz, - const amrex::Real* xmin, - const amrex::Real* dx); - - void warpx_push_bvec( - const int* xlo, const int* xhi, - const int* ylo, const int* yhi, - const int* zlo, const int* zhi, - const BL_FORT_FAB_ARG_3D(ex), - const BL_FORT_FAB_ARG_3D(ey), - const BL_FORT_FAB_ARG_3D(ez), - BL_FORT_FAB_ARG_3D(bx), - BL_FORT_FAB_ARG_3D(by), - BL_FORT_FAB_ARG_3D(bz), - const amrex::Real* dtsdx, - const amrex::Real* dtsdy, - const amrex::Real* dtsdz, - const amrex::Real* xmin, - const amrex::Real* dx, - const int* maxwell_fdtd_solver_id); - - void warpx_push_evec_f( - const int* xlo, const int* xhi, - const int* ylo, const int* yhi, - const int* zlo, const int* zhi, - BL_FORT_FAB_ARG_3D(ex), - BL_FORT_FAB_ARG_3D(ey), - BL_FORT_FAB_ARG_3D(ez), - const BL_FORT_FAB_ARG_3D(f), - const amrex::Real* dtsdx_c2, - const amrex::Real* dtsdy_c2, - const amrex::Real* dtsdz_c2, - const int* maxwell_fdtd_solver_id); - #ifdef USE_OPENBC_POISSON void warpx_openbc_potential (amrex::Real* rho, amrex::Real* phi, const amrex::Real* dx); void warpx_openbc_decompose (const int*, const int*, int*, int*); @@ -284,24 +184,6 @@ extern "C" // These functions are used to evolve E and B in the PML - void WRPX_COMPUTE_DIVB (const int* lo, const int* hi, - BL_FORT_FAB_ARG_ANYD(divb), - const BL_FORT_FAB_ARG_ANYD(Bx), - const BL_FORT_FAB_ARG_ANYD(By), - const BL_FORT_FAB_ARG_ANYD(Bz), - const amrex::Real* dx); - - void WRPX_COMPUTE_DIVE (const int* lo, const int* hi, - BL_FORT_FAB_ARG_ANYD(dive), - const BL_FORT_FAB_ARG_ANYD(ex), - const BL_FORT_FAB_ARG_ANYD(ey), - const BL_FORT_FAB_ARG_ANYD(ez), - const amrex::Real* dx -#ifdef WARPX_DIM_RZ - ,const amrex::Real* rmin -#endif - ); - void WRPX_PUSH_PML_BVEC(const int* xlo, const int* xhi, const int* ylo, const int* yhi, const int* zlo, const int* zhi, diff --git a/Source/FortranInterface/WarpX_picsar.F90 b/Source/FortranInterface/WarpX_picsar.F90 deleted file mode 100644 index 34084d753..000000000 --- a/Source/FortranInterface/WarpX_picsar.F90 +++ /dev/null @@ -1,132 +0,0 @@ -#if (AMREX_SPACEDIM == 3) - -#define WRPX_PXR_CURRENT_DEPOSITION depose_jxjyjz_generic - -#elif (AMREX_SPACEDIM == 2) - -#ifdef WARPX_DIM_RZ - -#define WRPX_PXR_CURRENT_DEPOSITION depose_jrjtjz_generic_rz - -#else - -#define WRPX_PXR_CURRENT_DEPOSITION depose_jxjyjz_generic_2d - -#endif - -#endif - -#define LVECT_CURRDEPO 8_c_long -#define LVECT_FIELDGATHE 64_c_long - -! _________________________________________________________________ -! -!> @brief -!> Module that contains subroutines to be called with AMReX -!> and that uses subroutines of Picsar -!> -!> @details -!> This avoids the use of interface with bind in the core of Picsar -!> This enables the use of integer in AMReX and Logical in Picsar -!> wihtout compatibility issue -!> -!> @author -!> Weiqun Zhang -!> Ann Almgren -!> Remi Lehe -!> Mathieu Lobet -!> -module warpx_to_pxr_module -! _________________________________________________________________ - - use iso_c_binding - use amrex_fort_module, only : amrex_real - - implicit none - - integer, parameter :: pxr_logical = 8 - -contains - - ! _________________________________________________________________ - !> - !> @brief - !> Main subroutine for the current deposition - !> - !> @details - !> This subroutines enable to controle the interpolation order - !> via the parameters nox,noy,noz and the type of algorithm via - !> the parameter current_depo_algo - ! - !> @param[inout] jx,jy,jz current arrays - !> @param[in] np number of particles - !> @param[in] xp,yp,zp particle position arrays - !> @param[in] uxp,uyp,uzp particle momentum arrays - !> @param[in] gaminv inverve of the particle Lorentz factor (array) - !> @param[in] w particle weight arrays - !> @param[in] q particle species charge - !> @param[in] xmin,ymin,zmin tile grid minimum position - !> @param[in] dx,dy,dz space discretization steps - !> @param[in] nx,ny,nz number of cells - !> @param[in] nxguard,nyguard,nzguard number of guard cells - !> @param[in] nox,noy,noz interpolation order - !> @param[in] lvect vector length - !> @param[in] charge_depo_algo algorithm choice for the charge deposition - !> - subroutine warpx_current_deposition( & - jx,jx_ng,jx_ntot,jy,jy_ng,jy_ntot,jz,jz_ng,jz_ntot, & - np,xp,yp,zp,uxp,uyp,uzp,gaminv,w,q,xmin,ymin,zmin,dt,dx,dy,dz,nox,noy,noz,& - l_nodal,lvect,current_depo_algo) & - bind(C, name="warpx_current_deposition") - - integer, intent(in) :: jx_ntot(AMREX_SPACEDIM), jy_ntot(AMREX_SPACEDIM), jz_ntot(AMREX_SPACEDIM) - integer(c_long), intent(in) :: jx_ng, jy_ng, jz_ng - integer(c_long), intent(IN) :: np - integer(c_long), intent(IN) :: nox,noy,noz - integer(c_int), intent(in) :: l_nodal - real(amrex_real), intent(IN OUT):: jx(*), jy(*), jz(*) - real(amrex_real), intent(IN) :: q - real(amrex_real), intent(IN) :: dx,dy,dz - real(amrex_real), intent(IN) :: dt - real(amrex_real), intent(IN) :: xmin,ymin,zmin - real(amrex_real), intent(IN), dimension(np) :: xp,yp,zp,w - real(amrex_real), intent(IN), dimension(np) :: uxp,uyp,uzp - real(amrex_real), intent(IN), dimension(np) :: gaminv - integer(c_long), intent(IN) :: lvect - integer(c_long), intent(IN) :: current_depo_algo - logical(pxr_logical) :: pxr_l_nodal - - ! Compute the number of valid cells and guard cells - integer(c_long) :: jx_nvalid(AMREX_SPACEDIM), jy_nvalid(AMREX_SPACEDIM), jz_nvalid(AMREX_SPACEDIM), & - jx_nguards(AMREX_SPACEDIM), jy_nguards(AMREX_SPACEDIM), jz_nguards(AMREX_SPACEDIM) - jx_nvalid = jx_ntot - 2*jx_ng - jy_nvalid = jy_ntot - 2*jy_ng - jz_nvalid = jz_ntot - 2*jz_ng - jx_nguards = jx_ng - jy_nguards = jy_ng - jz_nguards = jz_ng - pxr_l_nodal = l_nodal .eq. 1 - -! Dimension 3 -#if (AMREX_SPACEDIM==3) - CALL WRPX_PXR_CURRENT_DEPOSITION( & - jx,jx_nguards,jx_nvalid, & - jy,jy_nguards,jy_nvalid, & - jz,jz_nguards,jz_nvalid, & - np,xp,yp,zp,uxp,uyp,uzp,gaminv,w,q, & - xmin,ymin,zmin,dt,dx,dy,dz, & - nox,noy,noz,pxr_l_nodal,current_depo_algo) -! Dimension 2 -#elif (AMREX_SPACEDIM==2) - CALL WRPX_PXR_CURRENT_DEPOSITION( & - jx,jx_nguards,jx_nvalid, & - jy,jy_nguards,jy_nvalid, & - jz,jz_nguards,jz_nvalid, & - np,xp,yp,zp,uxp,uyp,uzp,gaminv,w,q, & - xmin,zmin,dt,dx,dz,nox,noz,pxr_l_nodal, & - lvect,current_depo_algo) -#endif - - end subroutine warpx_current_deposition - -end module warpx_to_pxr_module diff --git a/Source/Laser/LaserParticleContainer.cpp b/Source/Laser/LaserParticleContainer.cpp index c141ae4b0..3fdd8730b 100644 --- a/Source/Laser/LaserParticleContainer.cpp +++ b/Source/Laser/LaserParticleContainer.cpp @@ -397,8 +397,8 @@ LaserParticleContainer::Evolve (int lev, { BL_PROFILE("Laser::Evolve()"); BL_PROFILE_VAR_NS("Laser::Evolve::Copy", blp_copy); - BL_PROFILE_VAR_NS("PICSAR::LaserParticlePush", blp_pxr_pp); - BL_PROFILE_VAR_NS("PICSAR::LaserCurrentDepo", blp_pxr_cd); + BL_PROFILE_VAR_NS("Laser::ParticlePush", blp_pp); + BL_PROFILE_VAR_NS("Laser::CurrentDepo", blp_cd); BL_PROFILE_VAR_NS("Laser::Evolve::Accumulate", blp_accumulate); Real t_lab = t; @@ -466,7 +466,7 @@ LaserParticleContainer::Evolve (int lev, // // Particle Push // - BL_PROFILE_VAR_START(blp_pxr_pp); + BL_PROFILE_VAR_START(blp_pp); // Find the coordinates of the particles in the emission plane calculate_laser_plane_coordinates( &np, m_xp[thread_num].dataPtr(), @@ -506,7 +506,7 @@ LaserParticleContainer::Evolve (int lev, wp.dataPtr(), amplitude_E.dataPtr(), &p_X[0], &p_X[1], &p_X[2], &nvec[0], &nvec[1], &nvec[2], &mobility, &dt, &PhysConst::c, &WarpX::beta_boost, &WarpX::gamma_boost ); - BL_PROFILE_VAR_STOP(blp_pxr_pp); + BL_PROFILE_VAR_STOP(blp_pp); // // Current Deposition diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index aaed96423..1aff7edfb 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -123,7 +123,6 @@ public: /// /// This deposits the particle charge onto a node-centered MultiFab and returns a unique ptr /// to it. The charge density is accumulated over all the particles in the MultiParticleContainer - /// This version uses PICSAR routines to perform the deposition. /// std::unique_ptr<amrex::MultiFab> GetChargeDensity(int lev, bool local = false); diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 73acd60fa..015482e3f 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -932,7 +932,7 @@ PhysicalParticleContainer::Evolve (int lev, { BL_PROFILE("PPC::Evolve()"); BL_PROFILE_VAR_NS("PPC::Evolve::Copy", blp_copy); - BL_PROFILE_VAR_NS("PICSAR::FieldGather", blp_pxr_fg); + BL_PROFILE_VAR_NS("PPC::FieldGather", blp_fg); BL_PROFILE_VAR_NS("PPC::ParticlePush", blp_ppc_pp); BL_PROFILE_VAR_NS("PPC::Evolve::partition", blp_partition); @@ -1198,7 +1198,7 @@ PhysicalParticleContainer::Evolve (int lev, // // Field Gather of Aux Data (i.e., the full solution) // - BL_PROFILE_VAR_START(blp_pxr_fg); + BL_PROFILE_VAR_START(blp_fg); FieldGather(pti, Exp, Eyp, Ezp, Bxp, Byp, Bzp, exfab, eyfab, ezfab, bxfab, byfab, bzfab, Ex.nGrow(), e_is_nodal, 0, np_gather, thread_num, lev, lev); @@ -1278,7 +1278,7 @@ PhysicalParticleContainer::Evolve (int lev, thread_num, lev, lev-1); } - BL_PROFILE_VAR_STOP(blp_pxr_fg); + BL_PROFILE_VAR_STOP(blp_fg); // // Particle Push @@ -1291,36 +1291,25 @@ PhysicalParticleContainer::Evolve (int lev, // // Current Deposition // - if (WarpX::use_picsar_deposition) { - // Deposit inside domains - DepositCurrentFortran(pti, wp, uxp, uyp, uzp, &jx, &jy, &jz, - 0, np_current, thread_num, - lev, lev, dt); - if (has_buffer){ - // Deposit in buffers - DepositCurrentFortran(pti, wp, uxp, uyp, uzp, cjx, cjy, cjz, - np_current, np-np_current, thread_num, - lev, lev-1, dt); - } + + int* AMREX_RESTRICT ion_lev; + if (do_field_ionization){ + ion_lev = pti.GetiAttribs(particle_icomps["ionization_level"]).dataPtr(); } else { - int* AMREX_RESTRICT ion_lev; - if (do_field_ionization){ - ion_lev = pti.GetiAttribs(particle_icomps["ionization_level"]).dataPtr(); - } else { - ion_lev = nullptr; - } - - // Deposit inside domains - DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, &jx, &jy, &jz, - 0, np_current, thread_num, - lev, lev, dt); - if (has_buffer){ - // Deposit in buffers - DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, cjx, cjy, cjz, - np_current, np-np_current, thread_num, - lev, lev-1, dt); - } + ion_lev = nullptr; } + + // Deposit inside domains + DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, &jx, &jy, &jz, + 0, np_current, thread_num, + lev, lev, dt); + if (has_buffer){ + // Deposit in buffers + DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, cjx, cjy, cjz, + np_current, np-np_current, thread_num, + lev, lev-1, dt); + } + // // copy particle data back diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index c4bb5e410..7393f7301 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -205,21 +205,6 @@ public: int depos_lev, amrex::Real dt); - virtual void DepositCurrentFortran(WarpXParIter& pti, - RealVector& wp, - RealVector& uxp, - RealVector& uyp, - RealVector& uzp, - amrex::MultiFab* jx, - amrex::MultiFab* jy, - amrex::MultiFab* jz, - const long offset, - const long np_to_depose, - int thread_num, - int lev, - int depos_lev, - amrex::Real dt); - // If particles start outside of the domain, ContinuousInjection // makes sure that they are initialized when they enter the domain, and // NOT before. Virtual function, overriden by derived classes. diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index b34e71178..80d1caa0f 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -263,132 +263,6 @@ WarpXParticleContainer::AddNParticles (int lev, Redistribute(); } -/* \brief Current Deposition for thread thread_num using PICSAR - * \param pti : Particle iterator - * \param wp : Array of particle weights - * \param uxp uyp uzp : Array of particle - * \param jx jy jz : Full array of current density - * \param offset : Index of first particle for which current is deposited - * \param np_to_depose: Number of particles for which current is deposited. - Particles [offset,offset+np_tp_depose] deposit their - current - * \param thread_num : Thread number (if tiling) - * \param lev : Level of box that contains particles - * \param depos_lev : Level on which particles deposit (if buffers are used) - * \param dt : Time step for particle level - */ -void -WarpXParticleContainer::DepositCurrentFortran(WarpXParIter& pti, - RealVector& wp, RealVector& uxp, - RealVector& uyp, RealVector& uzp, - MultiFab* jx, MultiFab* jy, MultiFab* jz, - const long offset, const long np_to_depose, - int thread_num, int lev, int depos_lev, - Real dt) -{ - AMREX_ALWAYS_ASSERT_WITH_MESSAGE((depos_lev==(lev-1)) || - (depos_lev==(lev )), - "Deposition buffers only work for lev-1"); - // If no particles, do not do anything - if (np_to_depose == 0) return; - - const long ngJ = jx->nGrow(); - const std::array<Real,3>& dx = WarpX::CellSize(std::max(depos_lev,0)); - const long lvect = 8; - int j_is_nodal = jx->is_nodal() and jy->is_nodal() and jz->is_nodal(); - - BL_PROFILE_VAR_NS("PICSAR::CurrentDeposition", blp_pxr_cd); - BL_PROFILE_VAR_NS("PPC::Evolve::Accumulate", blp_accumulate); - - // Get tile box where current is deposited. - // The tile box is different when depositing in the buffers (depos_lev<lev) - // or when depositing inside the level (depos_lev=lev) - Box tilebox; - if (lev == depos_lev) { - tilebox = pti.tilebox(); - } else { - const IntVect& ref_ratio = WarpX::RefRatio(depos_lev); - tilebox = amrex::coarsen(pti.tilebox(),ref_ratio); - } - - // Staggered tile boxes (different in each direction) - Box tbx = convert(tilebox, WarpX::jx_nodal_flag); - Box tby = convert(tilebox, WarpX::jy_nodal_flag); - Box tbz = convert(tilebox, WarpX::jz_nodal_flag); - - // Lower corner of tile box physical domain - const std::array<Real,3>& xyzmin_tile = WarpX::LowerCorner(tilebox, depos_lev); - const std::array<Real, 3>& xyzmin = xyzmin_tile; - -#ifdef AMREX_USE_GPU - // No tiling on GPU: jx_ptr points to the full - // jx array (same for jy_ptr and jz_ptr). - Real* jx_ptr = (*jx)[pti].dataPtr(); - Real* jy_ptr = (*jy)[pti].dataPtr(); - Real* jz_ptr = (*jz)[pti].dataPtr(); - - auto jxntot = (*jx)[pti].length(); - auto jyntot = (*jy)[pti].length(); - auto jzntot = (*jz)[pti].length(); -#else - // Tiling is on: jx_ptr points to local_jx[thread_num] - // (same for jy_ptr and jz_ptr) - tbx.grow(ngJ); - tby.grow(ngJ); - tbz.grow(ngJ); - - local_jx[thread_num].resize(tbx); - local_jy[thread_num].resize(tby); - local_jz[thread_num].resize(tbz); - - Real* jx_ptr = local_jx[thread_num].dataPtr(); - Real* jy_ptr = local_jy[thread_num].dataPtr(); - Real* jz_ptr = local_jz[thread_num].dataPtr(); - - // local_jx[thread_num] is set to zero - local_jx[thread_num].setVal(0.0); - local_jy[thread_num].setVal(0.0); - local_jz[thread_num].setVal(0.0); - - auto jxntot = local_jx[thread_num].length(); - auto jyntot = local_jy[thread_num].length(); - auto jzntot = local_jz[thread_num].length(); -#endif - // GPU, no tiling: deposit directly in jx - // CPU, tiling: deposit into local_jx - // (same for jx and jz) - BL_PROFILE_VAR_START(blp_pxr_cd); - warpx_current_deposition( - jx_ptr, &ngJ, jxntot.getVect(), - jy_ptr, &ngJ, jyntot.getVect(), - jz_ptr, &ngJ, jzntot.getVect(), - &np_to_depose, - m_xp[thread_num].dataPtr() + offset, - m_yp[thread_num].dataPtr() + offset, - m_zp[thread_num].dataPtr() + offset, - uxp.dataPtr() + offset, - uyp.dataPtr() + offset, - uzp.dataPtr() + offset, - m_giv[thread_num].dataPtr() + offset, - wp.dataPtr() + offset, &this->charge, - &xyzmin[0], &xyzmin[1], &xyzmin[2], - &dt, &dx[0], &dx[1], &dx[2], - &WarpX::nox,&WarpX::noy,&WarpX::noz, &j_is_nodal, - &lvect,&WarpX::current_deposition_algo); - - BL_PROFILE_VAR_STOP(blp_pxr_cd); - -#ifndef AMREX_USE_GPU - BL_PROFILE_VAR_START(blp_accumulate); - // CPU, tiling: atomicAdd local_jx into jx - // (same for jx and jz) - (*jx)[pti].atomicAdd(local_jx[thread_num], tbx, tbx, 0, 0, 1); - (*jy)[pti].atomicAdd(local_jy[thread_num], tby, tby, 0, 0, 1); - (*jz)[pti].atomicAdd(local_jz[thread_num], tbz, tbz, 0, 0, 1); - BL_PROFILE_VAR_STOP(blp_accumulate); -#endif -} - /* \brief Current Deposition for thread thread_num * \param pti : Particle iterator * \param wp : Array of particle weights @@ -429,6 +303,8 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, const Real stagger_shift = j_is_nodal ? 0.0 : 0.5; BL_PROFILE_VAR_NS("PPC::Evolve::Accumulate", blp_accumulate); + BL_PROFILE_VAR_NS("PPC::CurrentDeposition", blp_deposit); + // Get tile box where current is deposited. // The tile box is different when depositing in the buffers (depos_lev<lev) @@ -491,6 +367,7 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, // Better for memory? worth trying? const Dim3 lo = lbound(tilebox); + BL_PROFILE_VAR_START(blp_deposit); if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { if (WarpX::nox == 1){ doEsirkepovDepositionShapeN<1>( @@ -529,6 +406,7 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, stagger_shift, q); } } + BL_PROFILE_VAR_STOP(blp_deposit); #ifndef AMREX_USE_GPU BL_PROFILE_VAR_START(blp_accumulate); diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index 6a32513b7..54c721abf 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -5,8 +5,6 @@ #include <string> struct MaxwellSolverAlgo { - // These numbers corresponds to the algorithm code in WarpX's - // `warpx_push_bvec` and `warpx_push_evec_f` enum { Yee = 0, CKC = 1 @@ -14,8 +12,6 @@ struct MaxwellSolverAlgo { }; struct ParticlePusherAlgo { - // These numbers correspond to the algorithm code in WarpX's - // `warpx_particle_pusher` enum { Boris = 0, Vay = 1 @@ -24,12 +20,8 @@ struct ParticlePusherAlgo { struct CurrentDepositionAlgo { enum { - // These numbers corresponds to the algorithm code in PICSAR's - // `depose_jxjyjz_generic` and `depose_jxjyjz_generic_2d` - Direct = 3, - DirectVectorized = 2, - EsirkepovNonOptimized = 1, - Esirkepov = 0 + Esirkepov = 0, + Direct = 1 }; }; @@ -41,11 +33,9 @@ struct ChargeDepositionAlgo { }; struct GatheringAlgo { - // These numbers corresponds to the algorithm code in PICSAR's - // `geteb3d_energy_conserving_generic` function + // Only the Standard algorithm is implemented enum { - Vectorized = 0, - Standard = 1 + Standard = 0 }; }; diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index 842085a36..216199103 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -4,7 +4,7 @@ #include <cstring> // Define dictionary with correspondance between user-input strings, -// and corresponding integer for use inside the code (e.g. in PICSAR). +// and corresponding integer for use inside the code const std::map<std::string, int> maxwell_solver_algo_to_int = { {"yee", MaxwellSolverAlgo::Yee }, @@ -23,9 +23,6 @@ const std::map<std::string, int> particle_pusher_algo_to_int = { const std::map<std::string, int> current_deposition_algo_to_int = { {"esirkepov", CurrentDepositionAlgo::Esirkepov }, {"direct", CurrentDepositionAlgo::Direct }, -#if (!defined AMREX_USE_GPU)&&(AMREX_SPACEDIM == 3) // Only available on CPU and 3D - {"direct-vectorized", CurrentDepositionAlgo::DirectVectorized }, -#endif {"default", CurrentDepositionAlgo::Esirkepov } }; @@ -36,12 +33,7 @@ const std::map<std::string, int> charge_deposition_algo_to_int = { const std::map<std::string, int> gathering_algo_to_int = { {"standard", GatheringAlgo::Standard }, -#ifndef AMREX_USE_GPU // Only available on CPU - {"vectorized", GatheringAlgo::Vectorized }, - {"default", GatheringAlgo::Vectorized } -#else {"default", GatheringAlgo::Standard } -#endif }; diff --git a/Source/WarpX.H b/Source/WarpX.H index 6cc10cef8..36d2a8f35 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -83,7 +83,6 @@ public: static amrex::Vector<amrex::Real> B_external; // Algorithms - static long use_picsar_deposition; static long current_deposition_algo; static long charge_deposition_algo; static long field_gathering_algo; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index e676ad5a7..517fb2332 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -38,7 +38,6 @@ Vector<int> WarpX::boost_direction = {0,0,0}; int WarpX::do_compute_max_step_from_zmax = 0; Real WarpX::zmax_plasma_to_compute_max_step = 0.; -long WarpX::use_picsar_deposition = 0; long WarpX::current_deposition_algo; long WarpX::charge_deposition_algo; long WarpX::field_gathering_algo; @@ -532,10 +531,6 @@ WarpX::ReadParameters () { ParmParse pp("algo"); - // If not in RZ mode, read use_picsar_deposition - // In RZ mode, use_picsar_deposition is on, as the C++ version - // of the deposition does not support RZ - pp.query("use_picsar_deposition", use_picsar_deposition); current_deposition_algo = GetAlgorithmInteger(pp, "current_deposition"); charge_deposition_algo = GetAlgorithmInteger(pp, "charge_deposition"); field_gathering_algo = GetAlgorithmInteger(pp, "field_gathering"); diff --git a/Tools/performance_tests/functions_perftest.py b/Tools/performance_tests/functions_perftest.py index 90e014f88..1a0767500 100644 --- a/Tools/performance_tests/functions_perftest.py +++ b/Tools/performance_tests/functions_perftest.py @@ -144,9 +144,9 @@ def read_run_perf(filename, n_steps): pattern_list = ['\nParticleContainer::Redistribute().*',\ '\nFabArray::FillBoundary().*',\ '\nFabArray::ParallelCopy().*',\ - '\nPICSAR::CurrentDeposition.*',\ - '\nPICSAR::FieldGather.*',\ - '\nPICSAR::ParticlePush.*',\ + '\nPPC::CurrentDeposition.*',\ + '\nPPC::FieldGather.*',\ + '\nPPC::ParticlePush.*',\ '\nPPC::Evolve::Copy.*',\ '\nWarpX::EvolveEM().*',\ 'Checkpoint().*',\ |