From 3df850eb84630c4456427bdb7d3ab32568311863 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Tue, 20 Aug 2019 16:30:52 -0700 Subject: merge kernels --- Source/FieldSolver/WarpXPushFieldsEM.cpp | 64 ++++++++++++-------------------- 1 file changed, 24 insertions(+), 40 deletions(-) (limited to 'Source/FieldSolver/WarpXPushFieldsEM.cpp') diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 1df05bc0f..e390dd61a 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -159,33 +159,29 @@ WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real a_dt) auto const& Eyfab = Ey->array(mfi); auto const& Ezfab = Ez->array(mfi); if (do_nodal) { - amrex::ParallelFor(tbx, + amrex::ParallelFor(tbx, tby, tbz, [=] AMREX_GPU_DEVICE (int j, int k, int l) { warpx_push_bx_nodal(j,k,l,Bxfab,Eyfab,Ezfab,dtsdy,dtsdz); - }); - amrex::ParallelFor(tby, + }, [=] AMREX_GPU_DEVICE (int j, int k, int l) { warpx_push_by_nodal(j,k,l,Byfab,Exfab,Ezfab,dtsdx,dtsdz); - }); - amrex::ParallelFor(tbz, + }, [=] AMREX_GPU_DEVICE (int j, int k, int l) { warpx_push_bz_nodal(j,k,l,Bzfab,Exfab,Eyfab,dtsdx,dtsdy); }); } else if (WarpX::maxwell_fdtd_solver_id == 0) { - amrex::ParallelFor(tbx, + amrex::ParallelFor(tbx, tby, tbz, [=] AMREX_GPU_DEVICE (int j, int k, int l) { warpx_push_bx_yee(j,k,l,Bxfab,Eyfab,Ezfab,dtsdy,dtsdz); - }); - amrex::ParallelFor(tby, + }, [=] AMREX_GPU_DEVICE (int j, int k, int l) { warpx_push_by_yee(j,k,l,Byfab,Exfab,Ezfab,dtsdx,dtsdz); - }); - amrex::ParallelFor(tbz, + }, [=] AMREX_GPU_DEVICE (int j, int k, int l) { warpx_push_bz_yee(j,k,l,Bzfab,Exfab,Eyfab,dtsdx,dtsdy,dxinv,xmin); @@ -198,23 +194,21 @@ WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real a_dt) betaxy, betaxz, betayx, betayz, betazx, betazy, gammax, gammay, gammaz, alphax, alphay, alphaz); - amrex::ParallelFor(tbx, + amrex::ParallelFor(tbx, tby, tbz, [=] AMREX_GPU_DEVICE (int j, int k, int l) { warpx_push_bx_ckc(j,k,l,Bxfab,Eyfab,Ezfab, betaxy, betaxz, betayx, betayz, betazx, betazy, gammax, gammay, gammaz, alphax, alphay, alphaz); - }); - amrex::ParallelFor(tby, + }, [=] AMREX_GPU_DEVICE (int j, int k, int l) { warpx_push_by_ckc(j,k,l,Byfab,Exfab,Ezfab, betaxy, betaxz, betayx, betayz, betazx, betazy, gammax, gammay, gammaz, alphax, alphay, alphaz); - }); - amrex::ParallelFor(tbz, + }, [=] AMREX_GPU_DEVICE (int j, int k, int l) { warpx_push_bz_ckc(j,k,l,Bzfab,Exfab,Eyfab, @@ -356,33 +350,29 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) auto const& jzfab = jz->array(mfi); if (do_nodal) { - amrex::ParallelFor(tex, + amrex::ParallelFor(tex, tey, tez, [=] AMREX_GPU_DEVICE (int j, int k, int l) { warpx_push_ex_nodal(j,k,l,Exfab,Byfab,Bzfab,jxfab,mu_c2_dt,dtsdy_c2,dtsdz_c2); - }); - amrex::ParallelFor(tey, + }, [=] AMREX_GPU_DEVICE (int j, int k, int l) { warpx_push_ey_nodal(j,k,l,Eyfab,Bxfab,Bzfab,jyfab,mu_c2_dt,dtsdx_c2,dtsdz_c2); - }); - amrex::ParallelFor(tez, + }, [=] AMREX_GPU_DEVICE (int j, int k, int l) { warpx_push_ez_nodal(j,k,l,Ezfab,Bxfab,Byfab,jzfab,mu_c2_dt,dtsdx_c2,dtsdy_c2); }); } else { - amrex::ParallelFor(tex, + amrex::ParallelFor(tex, tey, tez, [=] AMREX_GPU_DEVICE (int j, int k, int l) { warpx_push_ex_yee(j,k,l,Exfab,Byfab,Bzfab,jxfab,mu_c2_dt,dtsdy_c2,dtsdz_c2); - }); - amrex::ParallelFor(tey, + }, [=] AMREX_GPU_DEVICE (int j, int k, int l) { warpx_push_ey_yee(j,k,l,Eyfab,Bxfab,Bzfab,jyfab,mu_c2_dt,dtsdx_c2,dtsdz_c2,xmin); - }); - amrex::ParallelFor(tez, + }, [=] AMREX_GPU_DEVICE (int j, int k, int l) { warpx_push_ez_yee(j,k,l,Ezfab,Bxfab,Byfab,jzfab,mu_c2_dt,dtsdx_c2,dtsdy_c2,dxinv,xmin); @@ -393,17 +383,15 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) { auto const& Ffab = F->array(mfi); if (WarpX::maxwell_fdtd_solver_id == 0) { - amrex::ParallelFor(tex, + amrex::ParallelFor(tex, tey, tez, [=] AMREX_GPU_DEVICE (int j, int k, int l) { warpx_push_ex_f_yee(j,k,l,Exfab,Ffab,dtsdx_c2); - }); - amrex::ParallelFor(tey, + }, [=] AMREX_GPU_DEVICE (int j, int k, int l) { warpx_push_ey_f_yee(j,k,l,Eyfab,Ffab,dtsdy_c2); - }); - amrex::ParallelFor(tez, + }, [=] AMREX_GPU_DEVICE (int j, int k, int l) { warpx_push_ez_f_yee(j,k,l,Ezfab,Ffab,dtsdz_c2); @@ -417,23 +405,21 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) betaxy, betaxz, betayx, betayz, betazx, betazy, gammax, gammay, gammaz, alphax, alphay, alphaz); - amrex::ParallelFor(tex, + amrex::ParallelFor(tex, tey, tez, [=] AMREX_GPU_DEVICE (int j, int k, int l) { warpx_push_ex_f_ckc(j,k,l,Exfab,Ffab, betaxy, betaxz, betayx, betayz, betazx, betazy, gammax, gammay, gammaz, alphax, alphay, alphaz); - }); - amrex::ParallelFor(tey, + }, [=] AMREX_GPU_DEVICE (int j, int k, int l) { warpx_push_ey_f_ckc(j,k,l,Eyfab,Ffab, betaxy, betaxz, betayx, betayz, betazx, betazy, gammax, gammay, gammaz, alphax, alphay, alphaz); - }); - amrex::ParallelFor(tez, + }, [=] AMREX_GPU_DEVICE (int j, int k, int l) { warpx_push_ez_f_ckc(j,k,l,Ezfab,Ffab, @@ -616,7 +602,7 @@ WarpX::ApplyInverseVolumeScalingToCurrentDensity (MultiFab* Jx, MultiFab* Jy, Mu // Rescale current in r-z mode since the inverse volume factor was not // included in the current deposition. - amrex::ParallelFor(tbr, + amrex::ParallelFor(tbr, tbt, tbz, [=] AMREX_GPU_DEVICE (int i, int j, int k) { // Wrap the current density deposited in the guard cells around @@ -630,8 +616,7 @@ WarpX::ApplyInverseVolumeScalingToCurrentDensity (MultiFab* Jx, MultiFab* Jy, Mu // between on axis and off-axis factors const amrex::Real r = std::abs(rmin + (i - irmin + 0.5)*dr); Jr_arr(i,j,0) /= (2.*MathConst::pi*r); - }); - amrex::ParallelFor(tbt, + }, [=] AMREX_GPU_DEVICE (int i, int j, int k) { // Wrap the current density deposited in the guard cells around @@ -649,8 +634,7 @@ WarpX::ApplyInverseVolumeScalingToCurrentDensity (MultiFab* Jx, MultiFab* Jy, Mu } else { Jt_arr(i,j,0) /= (2.*MathConst::pi*r); } - }); - amrex::ParallelFor(tbz, + }, [=] AMREX_GPU_DEVICE (int i, int j, int k) { // Wrap the current density deposited in the guard cells around -- cgit v1.2.3 From d51022db75ab54898d68451fc4e15bdfa373e2a8 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 29 Aug 2019 20:15:29 -0700 Subject: Avoid compilation of PICSAR spectral solver --- GNUmakefile | 1 + Source/FieldSolver/Make.package | 2 +- Source/FieldSolver/WarpXPushFieldsEM.cpp | 2 +- Source/Make.WarpX | 28 +++++++++++++++------------- 4 files changed, 18 insertions(+), 15 deletions(-) (limited to 'Source/FieldSolver/WarpXPushFieldsEM.cpp') diff --git a/GNUmakefile b/GNUmakefile index cd5776cf5..84dcc195a 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -30,6 +30,7 @@ USE_ASCENT_INSITU = FALSE WarpxBinDir = Bin USE_PSATD = FALSE +USE_PSATD_PICSAR = FALSE USE_RZ = FALSE DO_ELECTROSTATIC = FALSE diff --git a/Source/FieldSolver/Make.package b/Source/FieldSolver/Make.package index 25e21c146..076be41bb 100644 --- a/Source/FieldSolver/Make.package +++ b/Source/FieldSolver/Make.package @@ -3,7 +3,7 @@ CEXE_headers += WarpX_FDTD.H CEXE_sources += WarpXPushFieldsEM.cpp ifeq ($(USE_PSATD),TRUE) include $(WARPX_HOME)/Source/FieldSolver/SpectralSolver/Make.package - ifeq ($(USE_CUDA),FALSE) + ifeq ($(USE_PSATD_PICSAR),TRUE) include $(WARPX_HOME)/Source/FieldSolver/PicsarHybridSpectralSolver/Make.package endif endif diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index e390dd61a..84654a88d 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -59,7 +59,7 @@ WarpX::PushPSATD (amrex::Real a_dt) for (int lev = 0; lev <= finest_level; ++lev) { AMREX_ALWAYS_ASSERT_WITH_MESSAGE(dt[lev] == a_dt, "dt must be consistent"); if (fft_hybrid_mpi_decomposition){ -#ifndef AMREX_USE_CUDA // Only available on CPU +#ifdef WARPX_USE_PSATD_HYBRID PushPSATD_hybridFFT(lev, a_dt); #endif } else { diff --git a/Source/Make.WarpX b/Source/Make.WarpX index 1970c8fc6..ea06f8e06 100644 --- a/Source/Make.WarpX +++ b/Source/Make.WarpX @@ -116,7 +116,7 @@ ifeq ($(USE_OPENPMD), TRUE) endif DEFINES += -DWARPX_USE_OPENPMD endif - + ifeq ($(USE_PSATD),TRUE) USERSuffix := $(USERSuffix).PSATD @@ -124,15 +124,17 @@ ifeq ($(USE_PSATD),TRUE) ifeq ($(USE_CUDA),FALSE) # Running on CPU # Use FFTW libraries += -lfftw3_mpi -lfftw3 -lfftw3_threads - FFTW_HOME ?= NOT_SET - ifneq ($(FFTW_HOME),NOT_SET) - VPATH_LOCATIONS += $(FFTW_HOME)/include - INCLUDE_LOCATIONS += $(FFTW_HOME)/include - LIBRARY_LOCATIONS += $(FFTW_HOME)/lib + ifeq ($(USE_PSATD_PICSAR),TRUE) + FFTW_HOME ?= NOT_SET + ifneq ($(FFTW_HOME),NOT_SET) + VPATH_LOCATIONS += $(FFTW_HOME)/include + INCLUDE_LOCATIONS += $(FFTW_HOME)/include + LIBRARY_LOCATIONS += $(FFTW_HOME)/lib + endif + # Compile with PICSAR's Hybrid-MPI PSATD + DEFINES += -DWARPX_USE_PSATD_HYBRID + DEFINES += -DFFTW # PICSAR uses it endif - # Compile with PICSAR's Hybrid-MPI PSATD - DEFINES += -DWARPX_USE_PSATD_HYBRID - DEFINES += -DFFTW # PICSAR uses it else # Use cuFFT libraries += -lcufft @@ -159,7 +161,7 @@ ifeq ($(USE_HDF5),TRUE) endif DEFINES += -DWARPX_USE_HDF5 libraries += -lhdf5 -lz -endif +endif # job_info support CEXE_sources += AMReX_buildInfo.cpp @@ -206,7 +208,7 @@ else ifdef WarpxBinDir -all: $(executable) +all: $(executable) $(SILENT) $(RM) AMReX_buildInfo.cpp @if [ ! -d $(WarpxBinDir) ]; then mkdir -p $(WarpxBinDir); fi $(SILENT) mv -f $(executable) $(WarpxBinDir)/ @@ -215,7 +217,7 @@ all: $(executable) else -all: $(executable) +all: $(executable) $(SILENT) $(RM) AMReX_buildInfo.cpp @echo SUCCESS @@ -223,7 +225,7 @@ endif endif -AMReX_buildInfo.cpp: +AMReX_buildInfo.cpp: $(AMREX_HOME)/Tools/C_scripts/makebuildinfo_C.py \ --amrex_home "$(AMREX_HOME)" \ --COMP "$(COMP)" --COMP_VERSION "$(COMP_VERSION)" \ -- cgit v1.2.3