From 1d69260f5dc015d392efa46e26424a50df126a7b Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 1 May 2019 10:26:34 -0700 Subject: Implement WarpXSumGuardCells --- Source/Parallelization/WarpXComm.cpp | 63 +++++++++++++++++------------------- 1 file changed, 30 insertions(+), 33 deletions(-) (limited to 'Source/Parallelization/WarpXComm.cpp') diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 28971eb0c..348b31c8b 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -1,5 +1,6 @@ #include #include +#include #include @@ -355,7 +356,8 @@ WarpX::SyncCurrent () { BL_PROFILE("SyncCurrent()"); - // Restrict fine patch current onto the coarse patch, before fine patch SumBoundary + // Restrict fine patch current onto the coarse patch, before + // summing the guard cells of the fine patch for (int lev = 1; lev <= finest_level; ++lev) { current_cp[lev][0]->setVal(0.0); @@ -391,8 +393,9 @@ WarpX::SyncCurrent () bilinear_filter.ApplyStencil(*j_fp[lev][idim], *current_fp[lev][idim]); // Then swap j_fp and current_fp std::swap(j_fp[lev][idim], current_fp[lev][idim]); - // At this point, current_fp may have false values close to the - // edges of each FAB. This will be solved with a SumBoundary later. + // At this point, current_fp may have false values close to the + // edges of each FAB. This will be solved with when summing + // the guard cells later. // j_fp contains the exact MultiFab current_fp before this step. } } @@ -427,11 +430,11 @@ WarpX::SyncCurrent () { const auto& period = Geom(lev).periodicity(); // When using a bilinear filter with many passes, current_fp has - // temporarily more ghost cells here, so that its value inside + // temporarily more ghost cells here, so that its value inside // the domain is correct at the end of this stage. - current_fp[lev][0]->SumBoundary(period); - current_fp[lev][1]->SumBoundary(period); - current_fp[lev][2]->SumBoundary(period); + WarpXSumGuardCells(*(current_fp[lev][0]),period); + WarpXSumGuardCells(*(current_fp[lev][1]),period); + WarpXSumGuardCells(*(current_fp[lev][2]),period); } // Add fine level's coarse patch to coarse level's fine patch @@ -461,9 +464,9 @@ WarpX::SyncCurrent () for (int lev = 1; lev <= finest_level; ++lev) { const auto& cperiod = Geom(lev-1).periodicity(); - current_cp[lev][0]->SumBoundary(cperiod); - current_cp[lev][1]->SumBoundary(cperiod); - current_cp[lev][2]->SumBoundary(cperiod); + WarpXSumGuardCells(*(current_cp[lev][0]),cperiod); + WarpXSumGuardCells(*(current_cp[lev][1]),cperiod); + WarpXSumGuardCells(*(current_cp[lev][2]),cperiod); } if (WarpX::use_filter) { @@ -476,7 +479,7 @@ WarpX::SyncCurrent () std::swap(j_fp[lev][idim], current_fp[lev][idim]); // Then copy the interior of j_fp to current_fp. MultiFab::Copy(*current_fp[lev][idim], *j_fp[lev][idim], 0, 0, 1, 0); - // current_fp has right number of ghost cells and + // current_fp has right number of ghost cells and // correct filtered values here. } } @@ -556,7 +559,8 @@ WarpX::SyncRho (amrex::Vector >& rhof, { if (!rhof[0]) return; - // Restrict fine patch onto the coarse patch, before fine patch SumBoundary + // Restrict fine patch onto the coarse patch, + // before summing the guard cells of the fine patch for (int lev = 1; lev <= finest_level; ++lev) { rhoc[lev]->setVal(0.0); @@ -607,7 +611,7 @@ WarpX::SyncRho (amrex::Vector >& rhof, for (int lev = 0; lev <= finest_level; ++lev) { const auto& period = Geom(lev).periodicity(); - rhof[lev]->SumBoundary(period); + WarpXSumGuardCells( *(rhof[lev]), period, 0, rhof[lev]->nComp() ); } // Add fine level's coarse patch to coarse level's fine patch @@ -631,7 +635,7 @@ WarpX::SyncRho (amrex::Vector >& rhof, for (int lev = 1; lev <= finest_level; ++lev) { const auto& cperiod = Geom(lev-1).periodicity(); - rhoc[lev]->SumBoundary(cperiod); + WarpXSumGuardCells( *(rhoc[lev]), cperiod, 0, rhoc[lev]->nComp() ); } if (WarpX::use_filter) { @@ -729,10 +733,9 @@ WarpX::ApplyFilterandSumBoundaryJ (int lev, PatchType patch_type) ng += bilinear_filter.stencil_length_each_dir-1; MultiFab jf(j[idim]->boxArray(), j[idim]->DistributionMap(), 1, ng); bilinear_filter.ApplyStencil(jf, *j[idim]); - jf.SumBoundary(period); - MultiFab::Copy(*j[idim], jf, 0, 0, 1, 0); + WarpXSumGuardCells(*(j[idim]), jf, period); } else { - j[idim]->SumBoundary(period); + WarpXSumGuardCells(*(j[idim]), period); } } } @@ -780,8 +783,7 @@ WarpX::AddCurrentFromFineLevelandSumBoundary (int lev) MultiFab::Add(jfb, jfc, 0, 0, 1, ng); mf.ParallelAdd(jfb, 0, 0, 1, ng, IntVect::TheZeroVector(), period); - jfc.SumBoundary(period); - MultiFab::Copy(*current_cp[lev+1][idim], jfc, 0, 0, 1, 0); + WarpXSumGuardCells(*current_cp[lev+1][idim], jfc, period); } else if (use_filter) // but no buffer { @@ -792,8 +794,7 @@ WarpX::AddCurrentFromFineLevelandSumBoundary (int lev) current_cp[lev+1][idim]->DistributionMap(), 1, ng); bilinear_filter.ApplyStencil(jf, *current_cp[lev+1][idim]); mf.ParallelAdd(jf, 0, 0, 1, ng, IntVect::TheZeroVector(), period); - jf.SumBoundary(period); - MultiFab::Copy(*current_cp[lev+1][idim], jf, 0, 0, 1, 0); + WarpXSumGuardCells(*current_cp[lev+1][idim], jf, period); } else if (current_buf[lev+1][idim]) // but no filter { @@ -803,14 +804,14 @@ WarpX::AddCurrentFromFineLevelandSumBoundary (int lev) mf.ParallelAdd(*current_buf[lev+1][idim], 0, 0, 1, current_buf[lev+1][idim]->nGrowVect(), IntVect::TheZeroVector(), period); - current_cp[lev+1][idim]->SumBoundary(period); + WarpXSumGuardCells(*(current_cp[lev+1][idim]), period); } else // no filter, no buffer { mf.ParallelAdd(*current_cp[lev+1][idim], 0, 0, 1, current_cp[lev+1][idim]->nGrowVect(), IntVect::TheZeroVector(), period); - current_cp[lev+1][idim]->SumBoundary(period); + WarpXSumGuardCells(*(current_cp[lev+1][idim]), period); } MultiFab::Add(*current_fp[lev][idim], mf, 0, 0, 1, 0); } @@ -840,10 +841,9 @@ WarpX::ApplyFilterandSumBoundaryRho (int lev, PatchType patch_type, int icomp, i ng += bilinear_filter.stencil_length_each_dir-1; MultiFab rf(r->boxArray(), r->DistributionMap(), ncomp, ng); bilinear_filter.ApplyStencil(rf, *r, icomp, 0, ncomp); - rf.SumBoundary(period); - MultiFab::Copy(*r, rf, 0, icomp, ncomp, 0); + WarpXSumGuardCells(*r, rf, period, icomp, ncomp ); } else { - r->SumBoundary(icomp, ncomp, period); + WarpXSumGuardCells(*r, period, icomp, ncomp); } } @@ -885,9 +885,7 @@ WarpX::AddRhoFromFineLevelandSumBoundary(int lev, int icomp, int ncomp) MultiFab::Add(rhofb, rhofc, 0, 0, ncomp, ng); mf.ParallelAdd(rhofb, 0, 0, ncomp, ng, IntVect::TheZeroVector(), period); - - rhofc.SumBoundary(period); - MultiFab::Copy(*rho_cp[lev+1], rhofc, 0, 0, ncomp, 0); + WarpXSumGuardCells( *rho_cp[lev+1], rhofc, period, icomp, ncomp ); } else if (use_filter) // but no buffer { @@ -896,8 +894,7 @@ WarpX::AddRhoFromFineLevelandSumBoundary(int lev, int icomp, int ncomp) MultiFab rf(rho_cp[lev+1]->boxArray(), rho_cp[lev+1]->DistributionMap(), ncomp, ng); bilinear_filter.ApplyStencil(rf, *rho_cp[lev+1], icomp, 0, ncomp); mf.ParallelAdd(rf, 0, 0, ncomp, ng, IntVect::TheZeroVector(), period); - rf.SumBoundary(0, ncomp, period); - MultiFab::Copy(*rho_cp[lev+1], rf, 0, icomp, ncomp, 0); + WarpXSumGuardCells( *rho_cp[lev+1], rf, period, icomp, ncomp ); } else if (charge_buf[lev+1]) // but no filter { @@ -908,14 +905,14 @@ WarpX::AddRhoFromFineLevelandSumBoundary(int lev, int icomp, int ncomp) ncomp, charge_buf[lev+1]->nGrowVect(), IntVect::TheZeroVector(), period); - rho_cp[lev+1]->SumBoundary(icomp, ncomp, period); + WarpXSumGuardCells(*(rho_cp[lev+1]), period, icomp, ncomp); } else // no filter, no buffer { mf.ParallelAdd(*rho_cp[lev+1], icomp, 0, ncomp, rho_cp[lev+1]->nGrowVect(), IntVect::TheZeroVector(), period); - rho_cp[lev+1]->SumBoundary(icomp, ncomp, period); + WarpXSumGuardCells(*(rho_cp[lev+1]), period, icomp, ncomp); } ApplyFilterandSumBoundaryRho(lev, PatchType::fine, icomp, ncomp); MultiFab::Add(*rho_fp[lev], mf, 0, icomp, ncomp, 0); -- cgit v1.2.3 From eb76cadbc5e76111679f6189e6728e462af41f66 Mon Sep 17 00:00:00 2001 From: Revathi Jambunathan Date: Wed, 8 May 2019 16:21:18 -0400 Subject: temporary print files for debugging --- GNUmakefile | 2 +- Source/Evolve/WarpXEvolveEM.cpp | 58 +++--------------- .../SpectralSolver/SpectralFieldData.cpp | 52 ++++++---------- Source/FieldSolver/WarpXFFT.cpp | 70 +++++++--------------- Source/Initialization/WarpXInitData.cpp | 20 ++++--- Source/Parallelization/WarpXComm.cpp | 7 ++- Source/Particles/WarpXParticleContainer.cpp | 2 +- Source/WarpX.cpp | 2 + Source/main.cpp | 3 + 9 files changed, 72 insertions(+), 144 deletions(-) (limited to 'Source/Parallelization/WarpXComm.cpp') diff --git a/GNUmakefile b/GNUmakefile index c662281b3..e3ed55d80 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -29,7 +29,7 @@ USE_ASCENT_INSITU = FALSE WarpxBinDir = Bin -USE_PSATD = TRUE +USE_PSATD = FALSE USE_RZ = FALSE DO_ELECTROSTATIC = FALSE diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index 025ff993a..1c5dc0104 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -69,27 +69,20 @@ WarpX::EvolveEM (int numsteps) // At the beginning, we have B^{n} and E^{n}. // Particles have p^{n} and x^{n}. // is_synchronized is true. + amrex::Print() << " in evolve before fill boundary \n"; if (is_synchronized) { FillBoundaryE(); FillBoundaryB(); UpdateAuxilaryData(); // on first step, push p by -0.5*dt + amrex::Print() << " in evolve before pushP \n"; for (int lev = 0; lev <= finest_level; ++lev) { mypc->PushP(lev, -0.5*dt[lev], *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); } is_synchronized = false; - - for (MFIter mfi(*rho_fp[0]); mfi.isValid(); ++mfi) - { - MultiFab &mf = *rho_fp[0]; - Box realspace_bx = mf[mfi].box(); // Copy the box - Array4 mf_arr = mf[mfi].array(); - amrex::Print() << " at if synchronized rho " << mf_arr(0,0,0,0) ; - amrex::Print() << " new rho " << mf_arr(0,0,0,1) << "\n"; - } - + amrex::Print() << " in evolve after pushP \n"; } else { // Beyond one step, we have E^{n} and B^{n}. @@ -98,15 +91,8 @@ WarpX::EvolveEM (int numsteps) FillBoundaryB(); UpdateAuxilaryData(); - for (MFIter mfi(*rho_fp[0]); mfi.isValid(); ++mfi) - { - MultiFab &mf = *rho_fp[0]; - Box realspace_bx = mf[mfi].box(); // Copy the box - Array4 mf_arr = mf[mfi].array(); - amrex::Print() << " at if synchronized rho " << mf_arr(0,0,0,0) ; - amrex::Print() << " new rho " << mf_arr(0,0,0,1) << "\n"; - } } + amrex::Print() << " in evolve after fill boundary \n"; if (do_subcycling == 0 || finest_level == 0) { OneStep_nosub(cur_time); @@ -116,6 +102,7 @@ WarpX::EvolveEM (int numsteps) amrex::Print() << "Error: do_subcycling = " << do_subcycling << std::endl; amrex::Abort("Unsupported do_subcycling type"); } + amrex::Print() << " in evolve after onestep no sub \n"; #ifdef WARPX_USE_PY if (warpx_py_beforeEsolve) warpx_py_beforeEsolve(); @@ -279,52 +266,23 @@ WarpX::OneStep_nosub (Real cur_time) if (warpx_py_beforedeposition) warpx_py_beforedeposition(); #endif - for (MFIter mfi(*rho_fp[0]); mfi.isValid(); ++mfi) - { - MultiFab &mf = *rho_fp[0]; - Box realspace_bx = mf[mfi].box(); // Copy the box - Array4 mf_arr = mf[mfi].array(); - amrex::Print() << " at before pushf particles and depose rho " << mf_arr(0,0,0,0) ; - amrex::Print() << " new rho " << mf_arr(0,0,0,1) << "\n"; - } PushParticlesandDepose(cur_time); - for (MFIter mfi(*rho_fp[0]); mfi.isValid(); ++mfi) - { - MultiFab &mf = *rho_fp[0]; - Box realspace_bx = mf[mfi].box(); // Copy the box - Array4 mf_arr = mf[mfi].array(); - amrex::Print() << " at after pushf particles and depose rho " << mf_arr(0,0,0,0) ; - amrex::Print() << " new rho " << mf_arr(0,0,0,1) << "\n"; - } + #ifdef WARPX_USE_PY if (warpx_py_afterdeposition) warpx_py_afterdeposition(); #endif SyncCurrent(); - for (MFIter mfi(*rho_fp[0]); mfi.isValid(); ++mfi) - { - MultiFab &mf = *rho_fp[0]; - Box realspace_bx = mf[mfi].box(); // Copy the box - Array4 mf_arr = mf[mfi].array(); - amrex::Print() << " before sync rho " << mf_arr(0,0,0,0) ; - amrex::Print() << " new rho " << mf_arr(0,0,0,1) << "\n"; - } SyncRho(rho_fp, rho_cp); - for (MFIter mfi(*rho_fp[0]); mfi.isValid(); ++mfi) - { - MultiFab &mf = *rho_fp[0]; - Box realspace_bx = mf[mfi].box(); // Copy the box - Array4 mf_arr = mf[mfi].array(); - amrex::Print() << " after sync rho " << mf_arr(0,0,0,0) ; - amrex::Print() << " new rho " << mf_arr(0,0,0,1) << "\n"; - } // Push E and B from {n} to {n+1} // (And update guard cells immediately afterwards) #ifdef WARPX_USE_PSATD PushPSATD(dt[0]); + amrex::Print() << " before fill bndry E \n"; FillBoundaryE(); + amrex::Print() << " before fill bndry B \n"; FillBoundaryB(); #else EvolveF(0.5*dt[0], DtType::FirstHalf); diff --git a/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp b/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp index 7c2061f8d..5998bdd2b 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp @@ -55,15 +55,13 @@ SpectralFieldData::SpectralFieldData( const BoxArray& realspace_ba, #ifdef AMREX_USE_GPU // Add cuFFT-specific code // Creating 3D plan for real to complex -- double precision + cudaDeviceSynchronize(); cufftResult result; result = cufftPlan3d( &forward_plan[mfi], fft_size[2], fft_size[1],fft_size[0], CUFFT_D2Z); if ( result != CUFFT_SUCCESS ) { amrex::Print() << " cufftplan3d forward failed! \n"; } - if ( result == CUFFT_SUCCESS ) { - amrex::Print() << " created cufft forward plan\n"; - } // Add 2D cuFFT-spacific code for D2Z // Note that D2Z is inherently forward plan @@ -73,9 +71,7 @@ SpectralFieldData::SpectralFieldData( const BoxArray& realspace_ba, if ( result != CUFFT_SUCCESS ) { amrex::Print() << " cufftplan3d backward failed! \n"; } - if ( result == CUFFT_SUCCESS ) { - amrex::Print() << " created cufft backward plan\n"; - } + cudaDeviceSynchronize(); #else // Create FFTW plans @@ -156,15 +152,9 @@ SpectralFieldData::ForwardTransform( const MultiFab& mf, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { tmp_arr(i,j,k) = mf_arr(i,j,k,i_comp); }); -#ifdef AMREX_USE_GPU - cudaDeviceSynchronize(); -#endif - amrex::Print() << " in forward trans icomp " << i_comp << " " << tmp_arr(0,0,0) << " mf arr " ; - amrex::Print() << " " << mf_arr(0,0,0,0); - amrex::Print() << " " << mf_arr(15,15,15,0); - amrex::Print() << " " << mf_arr(0,0,0,1); - amrex::Print() << " " << mf_arr(15,15,15,1); - amrex::Print() << "\n"; +//#ifdef AMREX_USE_GPU +// cudaDeviceSynchronize(); +//#endif } // Perform Fourier transform from `tmpRealField` to `tmpSpectralField` @@ -173,8 +163,9 @@ SpectralFieldData::ForwardTransform( const MultiFab& mf, // GPU stream as the above copy cudaDeviceSynchronize(); cufftResult result; - //cudaStream_t stream = amrex::Cuda::Device::cudaStream(); - //cufftSetStream ( forward_plan[mfi], stream); + cudaStream_t stream = amrex::Cuda::Device::cudaStream(); + amrex::Print() << " stream is " << stream << "\n"; + cufftSetStream ( forward_plan[mfi], stream); result = cufftExecD2Z( forward_plan[mfi], tmpRealField[mfi].dataPtr(), reinterpret_cast( @@ -182,13 +173,9 @@ SpectralFieldData::ForwardTransform( const MultiFab& mf, if ( result != CUFFT_SUCCESS ) { amrex::Print() << " cufftplan3d execute failed ! \n"; } - if ( result == CUFFT_SUCCESS ) { - amrex::Print() << " created cufft forward transform\n"; - } cudaDeviceSynchronize(); #else fftw_execute( forward_plan[mfi] ); - amrex::Print() << " forward fft on cpu\n"; #endif // Copy the spectral-space field `tmpSpectralField` to the appropriate @@ -219,8 +206,6 @@ SpectralFieldData::ForwardTransform( const MultiFab& mf, // Copy field into the right index fields_arr(i,j,k,field_index) = spectral_field_value; }); -// amrex::Print() << " in forward trans after D2Z" << fields_arr(0,0,0,0) ; - amrex::Print() << "\n"; } } } @@ -279,10 +264,11 @@ SpectralFieldData::BackwardTransform( MultiFab& mf, #ifdef AMREX_USE_GPU // Add cuFFT-specific code ; make sure that this is done on the same // GPU stream as the above copy - cudaDeviceSynchronize(); + cudaDeviceSynchronize(); cufftResult result; - //cudaStream_t stream = amrex::Cuda::Device::cudaStream(); - //cufftSetStream ( backward_plan[mfi], stream); + cudaStream_t stream = amrex::Cuda::Device::cudaStream(); + amrex::Print() << " stream is " << stream << "\n"; + cufftSetStream ( backward_plan[mfi], stream); result = cufftExecZ2D( backward_plan[mfi], reinterpret_cast( tmpSpectralField[mfi].dataPtr()), @@ -293,10 +279,9 @@ SpectralFieldData::BackwardTransform( MultiFab& mf, if ( result == CUFFT_SUCCESS ) { amrex::Print() << " created cufft inverse transform\n"; } - cudaDeviceSynchronize(); + cudaDeviceSynchronize(); #else fftw_execute( backward_plan[mfi] ); - amrex::Print() << " cpu inverse done\n"; #endif // Copy the temporary field `tmpRealField` to the real-space field `mf` @@ -313,15 +298,14 @@ SpectralFieldData::BackwardTransform( MultiFab& mf, // Copy and normalize field mf_arr(i,j,k,i_comp) = inv_N*tmp_arr(i,j,k); }); +//#ifdef AMREX_USE_GPU +// cudaDeviceSynchronize(); +//#endif + #ifdef AMREX_USE_GPU cudaDeviceSynchronize(); #endif - amrex::Print() << " after backward plan in real space 0,0,0 " << mf_arr(0,0,0,0) << " tmp " << tmp_arr(0,0,0) << "\n"; - amrex::Print() << " after backward plan in real space 15, 15, 15 " << mf_arr(15,15,15,0) << " tmp " << tmp_arr(0,0,0) << "\n"; - amrex::Print() << "\n"; -#ifdef AMREX_USE_GPU - cudaDeviceSynchronize(); -#endif + amrex::Print() << " divided by 1/N \n"; } } } diff --git a/Source/FieldSolver/WarpXFFT.cpp b/Source/FieldSolver/WarpXFFT.cpp index ffeb237c0..c4e0461f9 100644 --- a/Source/FieldSolver/WarpXFFT.cpp +++ b/Source/FieldSolver/WarpXFFT.cpp @@ -405,20 +405,6 @@ WarpX::PushPSATD (int lev, amrex::Real /* dt */) rho_fp_fft[lev]->ParallelCopy(*rho_fp[lev], 0, 0, 2, 0, 0, period_fp); BL_PROFILE_VAR_STOP(blp_copy); - - for (MFIter mfi(*rho_fp_fft[lev]); mfi.isValid(); ++mfi) - { - MultiFab &mf = *rho_fp_fft[lev]; - Box realspace_bx = mf[mfi].box(); // Copy the box - Array4 mf_arr = mf[mfi].array(); - amrex::Print() << " at begin Push PSATD rho " << mf_arr(0,0,0,0) ; - amrex::Print() << " new rho " << mf_arr(0,0,0,1) << "\n"; - MultiFab &mf_orig = *rho_fp[lev]; - Array4 mforig_arr = mf_orig[mfi].array(); - amrex::Print() << " at begin Push PSATD rho " << mforig_arr(0,0,0,0) ; - amrex::Print() << " new rho " << mforig_arr(0,0,0,1) << "\n"; - } - BL_PROFILE_VAR_START(blp_push_eb); if (fft_hybrid_mpi_decomposition){ if (Efield_fp_fft[lev][0]->local_size() == 1) @@ -465,44 +451,29 @@ WarpX::PushPSATD (int lev, amrex::Real /* dt */) // Not using the hybrid decomposition auto& solver = *spectral_solver_fp[lev]; - //// Perform forward Fourier transform - //amrex::Print() << " FTT of Ex \n"; - //solver.ForwardTransform(*Efield_fp_fft[lev][0], SpectralFieldIndex::Ex); - //amrex::Print() << " FTT of Ey \n"; - //solver.ForwardTransform(*Efield_fp_fft[lev][1], SpectralFieldIndex::Ey); - //amrex::Print() << " FTT of Ez \n"; - //solver.ForwardTransform(*Efield_fp_fft[lev][2], SpectralFieldIndex::Ez); - //amrex::Print() << " FTT of Bx \n"; - //solver.ForwardTransform(*Bfield_fp_fft[lev][0], SpectralFieldIndex::Bx); - //amrex::Print() << " FTT of By \n"; - //solver.ForwardTransform(*Bfield_fp_fft[lev][1], SpectralFieldIndex::By); - //amrex::Print() << " FTT of Bz \n"; - //solver.ForwardTransform(*Bfield_fp_fft[lev][2], SpectralFieldIndex::Bz); - //amrex::Print() << " FTT of Jx \n"; - //solver.ForwardTransform(*current_fp_fft[lev][0], SpectralFieldIndex::Jx); - //amrex::Print() << " FTT of Jy \n"; - //solver.ForwardTransform(*current_fp_fft[lev][1], SpectralFieldIndex::Jy); - //amrex::Print() << " FTT of Jz \n"; - //solver.ForwardTransform(*current_fp_fft[lev][2], SpectralFieldIndex::Jz); + // Perform forward Fourier transform + solver.ForwardTransform(*Efield_fp_fft[lev][0], SpectralFieldIndex::Ex); + solver.ForwardTransform(*Efield_fp_fft[lev][1], SpectralFieldIndex::Ey); + solver.ForwardTransform(*Efield_fp_fft[lev][2], SpectralFieldIndex::Ez); + solver.ForwardTransform(*Bfield_fp_fft[lev][0], SpectralFieldIndex::Bx); + solver.ForwardTransform(*Bfield_fp_fft[lev][1], SpectralFieldIndex::By); + solver.ForwardTransform(*Bfield_fp_fft[lev][2], SpectralFieldIndex::Bz); + solver.ForwardTransform(*current_fp_fft[lev][0], SpectralFieldIndex::Jx); + solver.ForwardTransform(*current_fp_fft[lev][1], SpectralFieldIndex::Jy); + solver.ForwardTransform(*current_fp_fft[lev][2], SpectralFieldIndex::Jz); solver.ForwardTransform(*rho_fp_fft[lev], SpectralFieldIndex::rho_old, 0); solver.ForwardTransform(*rho_fp_fft[lev], SpectralFieldIndex::rho_new, 1); - ////// Advance fields in spectral space - //solver.pushSpectralFields(); - - ////// Perform backward Fourier Transform - //amrex::Print() << " BT of Ex \n"; - //solver.BackwardTransform(*Efield_fp_fft[lev][0], SpectralFieldIndex::Ex); - //amrex::Print() << " BT of Ey \n"; - //solver.BackwardTransform(*Efield_fp_fft[lev][1], SpectralFieldIndex::Ey); - //amrex::Print() << " BT of Ez \n"; - //solver.BackwardTransform(*Efield_fp_fft[lev][2], SpectralFieldIndex::Ez); - //amrex::Print() << " BT of Bx \n"; - //solver.BackwardTransform(*Bfield_fp_fft[lev][0], SpectralFieldIndex::Bx); - //amrex::Print() << " BT of By \n"; - //solver.BackwardTransform(*Bfield_fp_fft[lev][1], SpectralFieldIndex::By); - //amrex::Print() << " BT of Bz \n"; - //solver.BackwardTransform(*Bfield_fp_fft[lev][2], SpectralFieldIndex::Bz); + // Advance fields in spectral space + solver.pushSpectralFields(); + + // Perform backward Fourier Transform + solver.BackwardTransform(*Efield_fp_fft[lev][0], SpectralFieldIndex::Ex); + solver.BackwardTransform(*Efield_fp_fft[lev][1], SpectralFieldIndex::Ey); + solver.BackwardTransform(*Efield_fp_fft[lev][2], SpectralFieldIndex::Ez); + solver.BackwardTransform(*Bfield_fp_fft[lev][0], SpectralFieldIndex::Bx); + solver.BackwardTransform(*Bfield_fp_fft[lev][1], SpectralFieldIndex::By); + solver.BackwardTransform(*Bfield_fp_fft[lev][2], SpectralFieldIndex::Bz); } BL_PROFILE_VAR_STOP(blp_push_eb); @@ -519,4 +490,5 @@ WarpX::PushPSATD (int lev, amrex::Real /* dt */) { amrex::Abort("WarpX::PushPSATD: TODO"); } + amrex::Print() << " coped data from fft to valid \n"; } diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 589ee168c..b9f27f07e 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -318,16 +318,28 @@ WarpX::InitLevelData (int lev, Real time) void WarpX::InitLevelDataFFT (int lev, Real time) { + + amrex::Print() << " print out pointer " << &Efield_fp_fft[lev][0] << "\n"; Efield_fp_fft[lev][0]->setVal(0.0); + amrex::Print() << " ex set \n"; Efield_fp_fft[lev][1]->setVal(0.0); + amrex::Print() << " ey set \n"; Efield_fp_fft[lev][2]->setVal(0.0); + amrex::Print() << " ez set \n"; Bfield_fp_fft[lev][0]->setVal(0.0); + amrex::Print() << " bx set \n"; Bfield_fp_fft[lev][1]->setVal(0.0); + amrex::Print() << " by set \n"; Bfield_fp_fft[lev][2]->setVal(0.0); + amrex::Print() << " bz set \n"; current_fp_fft[lev][0]->setVal(0.0); + amrex::Print() << " jx set \n"; current_fp_fft[lev][1]->setVal(0.0); + amrex::Print() << " jy set \n"; current_fp_fft[lev][2]->setVal(0.0); + amrex::Print() << " jz set \n"; rho_fp_fft[lev]->setVal(0.0); + amrex::Print() << " rhofp set \n"; if (lev > 0) { @@ -343,14 +355,6 @@ WarpX::InitLevelDataFFT (int lev, Real time) rho_cp_fft[lev]->setVal(0.0); } - for (MFIter mfi(*rho_fp_fft[lev]); mfi.isValid(); ++mfi) - { - MultiFab &mf = *rho_fp_fft[lev]; - Box realspace_bx = mf[mfi].box(); // Copy the box - Array4 mf_arr = mf[mfi].array(); - amrex::Print() << " at initialization rho " << mf_arr(0,0,0,0) ; - amrex::Print() << " new rho " << mf_arr(0,0,0,1) << "\n"; - } } #endif diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 28971eb0c..23d97133b 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -221,7 +221,9 @@ WarpX::FillBoundaryE () { for (int lev = 0; lev <= finest_level; ++lev) { + amrex::Print() << " lev " << lev << " in fill bndry E \n"; FillBoundaryE(lev); + amrex::Print() << " lev " << lev << " after fill bndry E \n"; } } @@ -245,7 +247,7 @@ void WarpX::FillBoundaryE (int lev, PatchType patch_type) { if (patch_type == PatchType::fine) - { + { if (do_pml && pml[lev]->ok()) { pml[lev]->ExchangeE(patch_type, @@ -270,9 +272,12 @@ WarpX::FillBoundaryE (int lev, PatchType patch_type) pml[lev]->FillBoundaryE(patch_type); } + amrex::Print() << " before defining multifab \n"; const auto& cperiod = Geom(lev-1).periodicity(); Vector mf{Efield_cp[lev][0].get(),Efield_cp[lev][1].get(),Efield_cp[lev][2].get()}; + amrex::Print() << " before amrex fill bndry \n"; amrex::FillBoundary(mf, cperiod); + amrex::Print() << " after amrex fill bndry \n"; } } diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 695faaa62..94b58ca07 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -660,7 +660,7 @@ WarpXParticleContainer::DepositCharge ( WarpXParIter& pti, RealVector& wp, const std::array& cxyzmin_tile = WarpX::LowerCorner(ctilebox, lev-1); #ifdef AMREX_USE_GPU - data_ptr = (*crhomf)[pti].dataPtr(); + data_ptr = (*crhomf)[pti].dataPtr(icomp); auto rholen = (*crhomf)[pti].length(); #else tile_box = amrex::convert(ctilebox, IntVect::TheUnitVector()); diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 532858556..5814d6829 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -558,7 +558,9 @@ WarpX::MakeNewLevelFromScratch (int lev, Real time, const BoxArray& new_grids, InitLevelData(lev, time); #ifdef WARPX_USE_PSATD + amrex::Print() << " alloc level data fft \n"; AllocLevelDataFFT(lev); + amrex::Print() << " init level data fft \n"; InitLevelDataFFT(lev, time); #endif } diff --git a/Source/main.cpp b/Source/main.cpp index d89e89e47..2e1a5e9cf 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -30,10 +30,13 @@ int main(int argc, char* argv[]) const Real strt_total = amrex::second(); { + amrex::Print() << " about to construct warpx \n"; WarpX warpx; + amrex::Print() << " call warpx init data \n"; warpx.InitData(); + amrex::Print() << " call warpx evolve \n"; warpx.Evolve(); Real end_total = amrex::second() - strt_total; -- cgit v1.2.3 From 5308cbefa8fbde30bed8a88943d58ec646731d9e Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Thu, 23 May 2019 15:27:17 -0700 Subject: fix a bunch of unused variable / parameter shadowing warnings --- Source/Diagnostics/BoostedFrameDiagnostic.H | 2 +- Source/Diagnostics/BoostedFrameDiagnostic.cpp | 5 --- Source/Diagnostics/FieldIO.cpp | 1 - Source/Evolve/WarpXEvolveEM.cpp | 28 +++++++------- Source/FieldSolver/WarpXPushFieldsEM.cpp | 48 ++++++++++++------------ Source/FortranInterface/WarpX_picsar.F90 | 7 +++- Source/Laser/LaserParticleContainer.cpp | 6 +-- Source/Parallelization/WarpXComm.cpp | 52 +++++++++++++------------- Source/Particles/PhysicalParticleContainer.cpp | 3 -- Source/Particles/WarpXParticleContainer.cpp | 44 +++++++++++----------- Source/Utils/WarpXMovingWindow.cpp | 6 +-- Source/WarpX.cpp | 1 - 12 files changed, 96 insertions(+), 107 deletions(-) (limited to 'Source/Parallelization/WarpXComm.cpp') diff --git a/Source/Diagnostics/BoostedFrameDiagnostic.H b/Source/Diagnostics/BoostedFrameDiagnostic.H index 55a124c51..3a09259b0 100644 --- a/Source/Diagnostics/BoostedFrameDiagnostic.H +++ b/Source/Diagnostics/BoostedFrameDiagnostic.H @@ -40,8 +40,8 @@ class BoostedFrameDiagnostic { amrex::Real current_z_lab; amrex::Real current_z_boost; - std::vector mesh_field_names_; int ncomp_to_dump_; + std::vector mesh_field_names_; int file_num; int initial_i; diff --git a/Source/Diagnostics/BoostedFrameDiagnostic.cpp b/Source/Diagnostics/BoostedFrameDiagnostic.cpp index f62369091..57872790f 100644 --- a/Source/Diagnostics/BoostedFrameDiagnostic.cpp +++ b/Source/Diagnostics/BoostedFrameDiagnostic.cpp @@ -497,8 +497,6 @@ LorentzTransformZ(MultiFab& data, Real gamma_boost, Real beta_boost, int ncomp) // Transform the transverse E and B fields. Note that ez and bz are not // changed by the tranform. Real e_lab, b_lab, j_lab, r_lab; - int i0 = 0; - int i4 = 4; e_lab = gamma_boost * (arr(i, j, k, 0) + beta_boost*clight*arr(i, j, k, 4)); b_lab = gamma_boost * (arr(i, j, k, 4) + beta_boost*arr(i, j, k, 0)/clight); @@ -718,7 +716,6 @@ writeLabFrameData(const MultiFab* cell_centered_data, if (buff_counter_[i] == 0) { // ... reset fields buffer data_buffer_[i] if (WarpX::do_boosted_frame_fields) { - const int ncomp = cell_centered_data->nComp(); Box buff_box = geom.Domain(); buff_box.setSmall(boost_direction_, i_lab - num_buffer_ + 1); buff_box.setBig(boost_direction_, i_lab); @@ -726,7 +723,6 @@ writeLabFrameData(const MultiFab* cell_centered_data, buff_ba.maxSize(max_box_size_); DistributionMapping buff_dm(buff_ba); data_buffer_[i].reset( new MultiFab(buff_ba, buff_dm, ncomp_to_dump, 0) ); - // data_buffer_[i].reset( new MultiFab(buff_ba, buff_dm, ncomp, 0) ); } // ... reset particle buffer particles_buffer_[i] if (WarpX::do_boosted_frame_particles) @@ -961,7 +957,6 @@ LabSnapShot(Real t_lab_in, Real t_boost, RealBox prob_domain_lab, my_bfd(bfd) { Real zmin_lab = prob_domain_lab_.lo(AMREX_SPACEDIM-1); - Real zmax_lab = prob_domain_lab_.hi(AMREX_SPACEDIM-1); current_z_lab = 0.0; current_z_boost = 0.0; updateCurrentZPositions(t_boost, my_bfd.inv_gamma_boost_, my_bfd.inv_beta_boost_); diff --git a/Source/Diagnostics/FieldIO.cpp b/Source/Diagnostics/FieldIO.cpp index ced1f8ea3..e3d44d1fc 100644 --- a/Source/Diagnostics/FieldIO.cpp +++ b/Source/Diagnostics/FieldIO.cpp @@ -660,7 +660,6 @@ getInterpolatedScalar( interpolated_F->setVal(0.); // Loop through the boxes and interpolate the values from the _cp data - const int use_limiter = 0; #ifdef _OPEMP #pragma omp parallel #endif diff --git a/Source/Evolve/WarpXEvolveEM.cpp b/Source/Evolve/WarpXEvolveEM.cpp index dab58f95b..ad7c7d840 100644 --- a/Source/Evolve/WarpXEvolveEM.cpp +++ b/Source/Evolve/WarpXEvolveEM.cpp @@ -514,7 +514,7 @@ WarpX::ComputeDt () * simulation box passes input parameter zmax_plasma_to_compute_max_step. */ void -WarpX::computeMaxStepBoostAccelerator(amrex::Geometry geom){ +WarpX::computeMaxStepBoostAccelerator(amrex::Geometry a_geom){ // Sanity checks: can use zmax_plasma_to_compute_max_step only if // the moving window and the boost are all in z direction. AMREX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -534,7 +534,7 @@ WarpX::computeMaxStepBoostAccelerator(amrex::Geometry geom){ // Lower end of the simulation domain. All quantities are given in boosted // frame except zmax_plasma_to_compute_max_step. - const Real zmin_domain_boost = geom.ProbLo(AMREX_SPACEDIM-1); + const Real zmin_domain_boost = a_geom.ProbLo(AMREX_SPACEDIM-1); // End of the plasma: Transform input argument // zmax_plasma_to_compute_max_step to boosted frame. const Real len_plasma_boost = zmax_plasma_to_compute_max_step/gamma_boost; @@ -591,19 +591,19 @@ WarpX::applyMirrors(Real time){ NullifyMF(Bz, lev, z_min, z_max); if (lev>0){ // Get coarse patch field MultiFabs - MultiFab& Ex = *Efield_cp[lev][0].get(); - MultiFab& Ey = *Efield_cp[lev][1].get(); - MultiFab& Ez = *Efield_cp[lev][2].get(); - MultiFab& Bx = *Bfield_cp[lev][0].get(); - MultiFab& By = *Bfield_cp[lev][1].get(); - MultiFab& Bz = *Bfield_cp[lev][2].get(); + MultiFab& cEx = *Efield_cp[lev][0].get(); + MultiFab& cEy = *Efield_cp[lev][1].get(); + MultiFab& cEz = *Efield_cp[lev][2].get(); + MultiFab& cBx = *Bfield_cp[lev][0].get(); + MultiFab& cBy = *Bfield_cp[lev][1].get(); + MultiFab& cBz = *Bfield_cp[lev][2].get(); // Set each field to zero between z_min and z_max - NullifyMF(Ex, lev, z_min, z_max); - NullifyMF(Ey, lev, z_min, z_max); - NullifyMF(Ez, lev, z_min, z_max); - NullifyMF(Bx, lev, z_min, z_max); - NullifyMF(By, lev, z_min, z_max); - NullifyMF(Bz, lev, z_min, z_max); + NullifyMF(cEx, lev, z_min, z_max); + NullifyMF(cEy, lev, z_min, z_max); + NullifyMF(cEz, lev, z_min, z_max); + NullifyMF(cBx, lev, z_min, z_max); + NullifyMF(cBy, lev, z_min, z_max); + NullifyMF(cBz, lev, z_min, z_max); } } } diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index a66d14980..c53e13f8f 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -17,30 +17,30 @@ using namespace amrex; void -WarpX::EvolveB (Real dt) +WarpX::EvolveB (Real a_dt) { for (int lev = 0; lev <= finest_level; ++lev) { - EvolveB(lev, dt); + EvolveB(lev, a_dt); } } void -WarpX::EvolveB (int lev, Real dt) +WarpX::EvolveB (int lev, Real a_dt) { BL_PROFILE("WarpX::EvolveB()"); - EvolveB(lev, PatchType::fine, dt); + EvolveB(lev, PatchType::fine, a_dt); if (lev > 0) { - EvolveB(lev, PatchType::coarse, dt); + EvolveB(lev, PatchType::coarse, a_dt); } } void -WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real dt) +WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real a_dt) { const int patch_level = (patch_type == PatchType::fine) ? lev : lev-1; const std::array& dx = WarpX::CellSize(patch_level); - Real dtsdx = dt/dx[0], dtsdy = dt/dx[1], dtsdz = dt/dx[2]; + Real dtsdx = a_dt/dx[0], dtsdy = a_dt/dx[1], dtsdz = a_dt/dx[2]; MultiFab *Ex, *Ey, *Ez, *Bx, *By, *Bz; if (patch_type == PatchType::fine) @@ -164,30 +164,30 @@ WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real dt) } void -WarpX::EvolveE (Real dt) +WarpX::EvolveE (Real a_dt) { for (int lev = 0; lev <= finest_level; ++lev) { - EvolveE(lev, dt); + EvolveE(lev, a_dt); } } void -WarpX::EvolveE (int lev, Real dt) +WarpX::EvolveE (int lev, Real a_dt) { BL_PROFILE("WarpX::EvolveE()"); - EvolveE(lev, PatchType::fine, dt); + EvolveE(lev, PatchType::fine, a_dt); if (lev > 0) { - EvolveE(lev, PatchType::coarse, dt); + EvolveE(lev, PatchType::coarse, a_dt); } } void -WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real dt) +WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) { - const Real mu_c2_dt = (PhysConst::mu0*PhysConst::c*PhysConst::c) * dt; - const Real c2dt = (PhysConst::c*PhysConst::c) * dt; + const Real mu_c2_dt = (PhysConst::mu0*PhysConst::c*PhysConst::c) * a_dt; + const Real c2dt = (PhysConst::c*PhysConst::c) * a_dt; int patch_level = (patch_type == PatchType::fine) ? lev : lev-1; const std::array& dx = WarpX::CellSize(patch_level); @@ -358,27 +358,27 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real dt) } void -WarpX::EvolveF (Real dt, DtType dt_type) +WarpX::EvolveF (Real a_dt, DtType a_dt_type) { if (!do_dive_cleaning) return; for (int lev = 0; lev <= finest_level; ++lev) { - EvolveF(lev, dt, dt_type); + EvolveF(lev, a_dt, a_dt_type); } } void -WarpX::EvolveF (int lev, Real dt, DtType dt_type) +WarpX::EvolveF (int lev, Real a_dt, DtType a_dt_type) { if (!do_dive_cleaning) return; - EvolveF(lev, PatchType::fine, dt, dt_type); - if (lev > 0) EvolveF(lev, PatchType::coarse, dt, dt_type); + EvolveF(lev, PatchType::fine, a_dt, a_dt_type); + if (lev > 0) EvolveF(lev, PatchType::coarse, a_dt, a_dt_type); } void -WarpX::EvolveF (int lev, PatchType patch_type, Real dt, DtType dt_type) +WarpX::EvolveF (int lev, PatchType patch_type, Real a_dt, DtType a_dt_type) { if (!do_dive_cleaning) return; @@ -388,7 +388,7 @@ WarpX::EvolveF (int lev, PatchType patch_type, Real dt, DtType dt_type) int patch_level = (patch_type == PatchType::fine) ? lev : lev-1; const auto& dx = WarpX::CellSize(patch_level); - const std::array dtsdx {dt/dx[0], dt/dx[1], dt/dx[2]}; + const std::array dtsdx {a_dt/dx[0], a_dt/dx[1], a_dt/dx[2]}; MultiFab *Ex, *Ey, *Ez, *rho, *F; if (patch_type == PatchType::fine) @@ -408,12 +408,12 @@ WarpX::EvolveF (int lev, PatchType patch_type, Real dt, DtType dt_type) F = F_cp[lev].get(); } - const int rhocomp = (dt_type == DtType::FirstHalf) ? 0 : 1; + const int rhocomp = (a_dt_type == DtType::FirstHalf) ? 0 : 1; MultiFab src(rho->boxArray(), rho->DistributionMap(), 1, 0); ComputeDivE(src, 0, {Ex,Ey,Ez}, dx); MultiFab::Saxpy(src, -mu_c2, *rho, rhocomp, 0, 1, 0); - MultiFab::Saxpy(*F, dt, src, 0, 0, 1, 0); + MultiFab::Saxpy(*F, a_dt, src, 0, 0, 1, 0); if (do_pml && pml[lev]->ok()) { diff --git a/Source/FortranInterface/WarpX_picsar.F90 b/Source/FortranInterface/WarpX_picsar.F90 index a4cc99926..c17e8861b 100644 --- a/Source/FortranInterface/WarpX_picsar.F90 +++ b/Source/FortranInterface/WarpX_picsar.F90 @@ -265,8 +265,10 @@ subroutine warpx_charge_deposition(rho,np,xp,yp,zp,w,q,xmin,ymin,zmin,dx,dy,dz,n real(amrex_real), intent(IN OUT):: rho(*) real(amrex_real), intent(IN) :: rmin, dr +#ifdef WARPX_RZ integer(c_long) :: type_rz_depose = 1 - +#endif + ! Compute the number of valid cells and guard cells integer(c_long) :: rho_nvalid(AMREX_SPACEDIM), rho_nguards(AMREX_SPACEDIM) rho_nvalid = rho_ntot - 2*rho_ng @@ -385,8 +387,9 @@ subroutine warpx_charge_deposition(rho,np,xp,yp,zp,w,q,xmin,ymin,zmin,dx,dy,dz,n real(amrex_real), intent(IN OUT):: jx(*), jy(*), jz(*) real(amrex_real), intent(IN) :: rmin, dr +#ifdef WARPX_RZ integer(c_long) :: type_rz_depose = 1 - +#endif ! Compute the number of valid cells and guard cells integer(c_long) :: jx_nvalid(AMREX_SPACEDIM), jy_nvalid(AMREX_SPACEDIM), jz_nvalid(AMREX_SPACEDIM), & jx_nguards(AMREX_SPACEDIM), jy_nguards(AMREX_SPACEDIM), jz_nguards(AMREX_SPACEDIM) diff --git a/Source/Laser/LaserParticleContainer.cpp b/Source/Laser/LaserParticleContainer.cpp index 52d506bb8..2f964b6f3 100644 --- a/Source/Laser/LaserParticleContainer.cpp +++ b/Source/Laser/LaserParticleContainer.cpp @@ -78,14 +78,14 @@ LaserParticleContainer::LaserParticleContainer (AmrCore* amr_core, int ispecies, parser.define(field_function); parser.registerVariables({"X","Y","t"}); - ParmParse pp("my_constants"); + ParmParse ppc("my_constants"); std::set symbols = parser.symbols(); symbols.erase("X"); symbols.erase("Y"); symbols.erase("t"); // after removing variables, we are left with constants for (auto it = symbols.begin(); it != symbols.end(); ) { Real v; - if (pp.query(it->c_str(), v)) { + if (ppc.query(it->c_str(), v)) { parser.setConstant(*it, v); it = symbols.erase(it); } else { @@ -429,8 +429,6 @@ LaserParticleContainer::Evolve (int lev, { Real wt = amrex::second(); - const Box& box = pti.validbox(); - auto& attribs = pti.GetAttribs(); auto& wp = attribs[PIdx::w ]; diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 348b31c8b..1cf347420 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -79,7 +79,7 @@ WarpX::UpdateAuxilaryData () MultiFab::Subtract(dBz, *Bfield_cp[lev][2], 0, 0, 1, ng); const Real* dx = Geom(lev-1).CellSize(); - const int ref_ratio = refRatio(lev-1)[0]; + const int rr = refRatio(lev-1)[0]; #ifdef _OPENMP #pragma omp parallel #endif @@ -89,7 +89,7 @@ WarpX::UpdateAuxilaryData () { Box ccbx = mfi.fabbox(); ccbx.enclosedCells(); - ccbx.coarsen(ref_ratio).refine(ref_ratio); // so that ccbx is coarsenable + ccbx.coarsen(rr).refine(rr); // so that ccbx is coarsenable const FArrayBox& cxfab = dBx[mfi]; const FArrayBox& cyfab = dBy[mfi]; @@ -106,18 +106,18 @@ WarpX::UpdateAuxilaryData () BL_TO_FORTRAN_ANYD(cxfab), BL_TO_FORTRAN_ANYD(cyfab), BL_TO_FORTRAN_ANYD(czfab), - dx, &ref_ratio,&use_limiter); + dx, &rr,&use_limiter); #else amrex_interp_div_free_bfield(ccbx.loVect(), ccbx.hiVect(), BL_TO_FORTRAN_ANYD(bfab[0]), BL_TO_FORTRAN_ANYD(bfab[2]), BL_TO_FORTRAN_ANYD(cxfab), BL_TO_FORTRAN_ANYD(czfab), - dx, &ref_ratio,&use_limiter); + dx, &rr,&use_limiter); amrex_interp_cc_bfield(ccbx.loVect(), ccbx.hiVect(), BL_TO_FORTRAN_ANYD(bfab[1]), BL_TO_FORTRAN_ANYD(cyfab), - &ref_ratio,&use_limiter); + &rr,&use_limiter); #endif for (int idim = 0; idim < 3; ++idim) @@ -153,7 +153,7 @@ WarpX::UpdateAuxilaryData () MultiFab::Subtract(dEy, *Efield_cp[lev][1], 0, 0, 1, ng); MultiFab::Subtract(dEz, *Efield_cp[lev][2], 0, 0, 1, ng); - const int ref_ratio = refRatio(lev-1)[0]; + const int rr = refRatio(lev-1)[0]; #ifdef _OPEMP #pragma omp parallel #endif @@ -163,7 +163,7 @@ WarpX::UpdateAuxilaryData () { Box ccbx = mfi.fabbox(); ccbx.enclosedCells(); - ccbx.coarsen(ref_ratio).refine(ref_ratio); // so that ccbx is coarsenable + ccbx.coarsen(rr).refine(rr); // so that ccbx is coarsenable const FArrayBox& cxfab = dEx[mfi]; const FArrayBox& cyfab = dEy[mfi]; @@ -180,18 +180,18 @@ WarpX::UpdateAuxilaryData () BL_TO_FORTRAN_ANYD(cxfab), BL_TO_FORTRAN_ANYD(cyfab), BL_TO_FORTRAN_ANYD(czfab), - &ref_ratio,&use_limiter); + &rr,&use_limiter); #else amrex_interp_efield(ccbx.loVect(), ccbx.hiVect(), BL_TO_FORTRAN_ANYD(efab[0]), BL_TO_FORTRAN_ANYD(efab[2]), BL_TO_FORTRAN_ANYD(cxfab), BL_TO_FORTRAN_ANYD(czfab), - &ref_ratio,&use_limiter); + &rr,&use_limiter); amrex_interp_nd_efield(ccbx.loVect(), ccbx.hiVect(), BL_TO_FORTRAN_ANYD(efab[1]), BL_TO_FORTRAN_ANYD(cyfab), - &ref_ratio); + &rr); #endif for (int idim = 0; idim < 3; ++idim) @@ -364,7 +364,7 @@ WarpX::SyncCurrent () current_cp[lev][1]->setVal(0.0); current_cp[lev][2]->setVal(0.0); - const IntVect& ref_ratio = refRatio(lev-1); + const IntVect& rr = refRatio(lev-1); std::array fine { current_fp[lev][0].get(), current_fp[lev][1].get(), @@ -372,7 +372,7 @@ WarpX::SyncCurrent () std::array< MultiFab*,3> crse { current_cp[lev][0].get(), current_cp[lev][1].get(), current_cp[lev][2].get() }; - SyncCurrent(fine, crse, ref_ratio[0]); + SyncCurrent(fine, crse, rr[0]); } Vector,3> > j_fp(finest_level+1); @@ -524,10 +524,10 @@ WarpX::SyncCurrent () void WarpX::SyncCurrent (const std::array& fine, const std::array< amrex::MultiFab*,3>& crse, - int ref_ratio) + int rr) { - BL_ASSERT(ref_ratio == 2); - const IntVect& ng = (fine[0]->nGrowVect() + 1) /ref_ratio; + BL_ASSERT(rr == 2); + const IntVect& ng = (fine[0]->nGrowVect() + 1) /rr; #ifdef _OPEMP #pragma omp parallel @@ -539,7 +539,7 @@ WarpX::SyncCurrent (const std::array& fine, for (MFIter mfi(*crse[idim],true); mfi.isValid(); ++mfi) { const Box& bx = mfi.growntilebox(ng); - Box fbx = amrex::grow(amrex::refine(bx,ref_ratio),1); + Box fbx = amrex::grow(amrex::refine(bx,rr),1); ffab.resize(fbx); fbx &= (*fine[idim])[mfi].box(); ffab.setVal(0.0); @@ -564,8 +564,8 @@ WarpX::SyncRho (amrex::Vector >& rhof, for (int lev = 1; lev <= finest_level; ++lev) { rhoc[lev]->setVal(0.0); - const IntVect& ref_ratio = refRatio(lev-1); - SyncRho(*rhof[lev], *rhoc[lev], ref_ratio[0]); + const IntVect& rr = refRatio(lev-1); + SyncRho(*rhof[lev], *rhoc[lev], rr[0]); } Vector > rho_f_g(finest_level+1); @@ -673,10 +673,10 @@ WarpX::SyncRho (amrex::Vector >& rhof, * averaging the values of the charge density of the fine patch (on the same level). */ void -WarpX::SyncRho (const MultiFab& fine, MultiFab& crse, int ref_ratio) +WarpX::SyncRho (const MultiFab& fine, MultiFab& crse, int rr) { - BL_ASSERT(ref_ratio == 2); - const IntVect& ng = (fine.nGrowVect()+1)/ref_ratio; + BL_ASSERT(rr == 2); + const IntVect& ng = (fine.nGrowVect()+1)/rr; const int nc = fine.nComp(); #ifdef _OPEMP @@ -687,7 +687,7 @@ WarpX::SyncRho (const MultiFab& fine, MultiFab& crse, int ref_ratio) for (MFIter mfi(crse,true); mfi.isValid(); ++mfi) { const Box& bx = mfi.growntilebox(ng); - Box fbx = amrex::grow(amrex::refine(bx,ref_ratio),1); + Box fbx = amrex::grow(amrex::refine(bx,rr),1); ffab.resize(fbx, nc); fbx &= fine[mfi].box(); ffab.setVal(0.0); @@ -710,7 +710,7 @@ WarpX::RestrictCurrentFromFineToCoarsePatch (int lev) current_cp[lev][1]->setVal(0.0); current_cp[lev][2]->setVal(0.0); - const IntVect& ref_ratio = refRatio(lev-1); + const IntVect& rr = refRatio(lev-1); std::array fine { current_fp[lev][0].get(), current_fp[lev][1].get(), @@ -718,7 +718,7 @@ WarpX::RestrictCurrentFromFineToCoarsePatch (int lev) std::array< MultiFab*,3> crse { current_cp[lev][0].get(), current_cp[lev][1].get(), current_cp[lev][2].get() }; - SyncCurrent(fine, crse, ref_ratio[0]); + SyncCurrent(fine, crse, rr[0]); } void @@ -824,8 +824,8 @@ WarpX::RestrictRhoFromFineToCoarsePatch (int lev) { if (rho_fp[lev]) { rho_cp[lev]->setVal(0.0); - const IntVect& ref_ratio = refRatio(lev-1); - SyncRho(*rho_fp[lev], *rho_cp[lev], ref_ratio[0]); + const IntVect& rr = refRatio(lev-1); + SyncRho(*rho_fp[lev], *rho_cp[lev], rr[0]); } } diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 37c136a3d..212084e64 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1024,9 +1024,6 @@ PhysicalParticleContainer::FieldGather (int lev, { const std::array& dx = WarpX::CellSize(lev); - // WarpX assumes the same number of guard cells for Ex, Ey, Ez, Bx, By, Bz - long ng = Ex.nGrow(); - BL_ASSERT(OnSameGrids(lev,Ex)); MultiFab* cost = WarpX::getCosts(lev); diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index c7ffe956b..9791eee80 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -237,10 +237,10 @@ WarpXParticleContainer::AddNParticles (int lev, if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) { - auto& particle_tile = DefineAndReturnParticleTile(0, 0, 0); - particle_tile.push_back_real(particle_comps["xold"], x[i]); - particle_tile.push_back_real(particle_comps["yold"], y[i]); - particle_tile.push_back_real(particle_comps["zold"], z[i]); + auto& ptile = DefineAndReturnParticleTile(0, 0, 0); + ptile.push_back_real(particle_comps["xold"], x[i]); + ptile.push_back_real(particle_comps["yold"], y[i]); + ptile.push_back_real(particle_comps["zold"], z[i]); } particle_tile.push_back(p); @@ -255,10 +255,10 @@ WarpXParticleContainer::AddNParticles (int lev, if (WarpX::do_boosted_frame_diagnostic && do_boosted_frame_diags) { - auto& particle_tile = DefineAndReturnParticleTile(0, 0, 0); - particle_tile.push_back_real(particle_comps["uxold"], vx + ibegin, vx + iend); - particle_tile.push_back_real(particle_comps["uyold"], vy + ibegin, vy + iend); - particle_tile.push_back_real(particle_comps["uzold"], vz + ibegin, vz + iend); + auto& ptile = DefineAndReturnParticleTile(0, 0, 0); + ptile.push_back_real(particle_comps["uxold"], vx + ibegin, vx + iend); + ptile.push_back_real(particle_comps["uyold"], vy + ibegin, vy + iend); + ptile.push_back_real(particle_comps["uzold"], vz + ibegin, vz + iend); } for (int comp = PIdx::uz+1; comp < PIdx::nattribs; ++comp) @@ -737,7 +737,6 @@ WarpXParticleContainer::DepositCharge (Vector >& rho, const auto& gm = m_gdb->Geom(lev); const auto& ba = m_gdb->ParticleBoxArray(lev); - const auto& dm = m_gdb->DistributionMap(lev); const Real* dx = gm.CellSize(); const Real* plo = gm.ProbLo(); @@ -807,36 +806,36 @@ WarpXParticleContainer::GetChargeDensity (int lev, bool local) #ifdef _OPENMP #pragma omp parallel -#endif { +#endif Cuda::ManagedDeviceVector xp, yp, zp; - FArrayBox local_rho; +#ifdef _OPENMP + FArrayBox rho_loc; +#endif for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { - const Box& box = pti.validbox(); - auto& wp = pti.GetAttribs(PIdx::w); const long np = pti.numParticles(); pti.GetPosition(xp, yp, zp); - const std::array& xyzmin_tile = WarpX::LowerCorner(pti.tilebox(), lev); - const std::array& xyzmin_grid = WarpX::LowerCorner(box, lev); - // Data on the grid Real* data_ptr; FArrayBox& rhofab = (*rho)[pti]; #ifdef _OPENMP + const std::array& xyzmin_tile = WarpX::LowerCorner(pti.tilebox(), lev); Box tile_box = convert(pti.tilebox(), IntVect::TheUnitVector()); const std::array& xyzmin = xyzmin_tile; tile_box.grow(ng); - local_rho.resize(tile_box); - local_rho = 0.0; - data_ptr = local_rho.dataPtr(); - auto rholen = local_rho.length(); + rho_loc.resize(tile_box); + rho_loc = 0.0; + data_ptr = rho_loc.dataPtr(); + auto rholen = rho_loc.length(); #else + const Box& box = pti.validbox(); + const std::array& xyzmin_grid = WarpX::LowerCorner(box, lev); const std::array& xyzmin = xyzmin_grid; data_ptr = rhofab.dataPtr(); auto rholen = rhofab.length(); @@ -874,10 +873,9 @@ WarpXParticleContainer::GetChargeDensity (int lev, bool local) #endif #ifdef _OPENMP - rhofab.atomicAdd(local_rho); -#endif + rhofab.atomicAdd(rho_loc); } - +#endif } if (!local) rho->SumBoundary(gm.periodicity()); diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 5f9e36bf6..a0ab1f26f 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -5,7 +5,7 @@ using namespace amrex; void -WarpX::UpdatePlasmaInjectionPosition (Real dt) +WarpX::UpdatePlasmaInjectionPosition (Real a_dt) { int dir = moving_window_dir; // Continuously inject plasma in new cells (by default only on level 0) @@ -14,12 +14,12 @@ WarpX::UpdatePlasmaInjectionPosition (Real dt) // call to this function, and injection position needs to be updated current_injection_position -= WarpX::beta_boost * #if ( AMREX_SPACEDIM == 3 ) - WarpX::boost_direction[dir] * PhysConst::c * dt; + WarpX::boost_direction[dir] * PhysConst::c * a_dt; #elif ( AMREX_SPACEDIM == 2 ) // In 2D, dir=0 corresponds to x and dir=1 corresponds to z // This needs to be converted in order to index `boost_direction` // which has 3 components, for both 2D and 3D simulations. - WarpX::boost_direction[2*dir] * PhysConst::c * dt; + WarpX::boost_direction[2*dir] * PhysConst::c * a_dt; #endif } } diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 6b6752bf1..3d7f7dcc5 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -1001,7 +1001,6 @@ WarpX::ComputeDivE (MultiFab& divE, int dcomp, void WarpX::BuildBufferMasks () { - int ngbuffer = std::max(n_field_gather_buffer, n_current_deposition_buffer); for (int lev = 1; lev <= maxLevel(); ++lev) { for (int ipass = 0; ipass < 2; ++ipass) -- cgit v1.2.3 From 5e4b588cb50ba09e1506af81c188245475bec1c0 Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Thu, 23 May 2019 17:25:51 -0700 Subject: change rr to a more descriptive name --- Source/Parallelization/WarpXComm.cpp | 48 ++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 24 deletions(-) (limited to 'Source/Parallelization/WarpXComm.cpp') diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 1cf347420..932d6905a 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -79,7 +79,7 @@ WarpX::UpdateAuxilaryData () MultiFab::Subtract(dBz, *Bfield_cp[lev][2], 0, 0, 1, ng); const Real* dx = Geom(lev-1).CellSize(); - const int rr = refRatio(lev-1)[0]; + const int refinement_ratio = refRatio(lev-1)[0]; #ifdef _OPENMP #pragma omp parallel #endif @@ -89,7 +89,7 @@ WarpX::UpdateAuxilaryData () { Box ccbx = mfi.fabbox(); ccbx.enclosedCells(); - ccbx.coarsen(rr).refine(rr); // so that ccbx is coarsenable + ccbx.coarsen(refinement_ratio).refine(refinement_ratio); // so that ccbx is coarsenable const FArrayBox& cxfab = dBx[mfi]; const FArrayBox& cyfab = dBy[mfi]; @@ -106,18 +106,18 @@ WarpX::UpdateAuxilaryData () BL_TO_FORTRAN_ANYD(cxfab), BL_TO_FORTRAN_ANYD(cyfab), BL_TO_FORTRAN_ANYD(czfab), - dx, &rr,&use_limiter); + dx, &refinement_ratio,&use_limiter); #else amrex_interp_div_free_bfield(ccbx.loVect(), ccbx.hiVect(), BL_TO_FORTRAN_ANYD(bfab[0]), BL_TO_FORTRAN_ANYD(bfab[2]), BL_TO_FORTRAN_ANYD(cxfab), BL_TO_FORTRAN_ANYD(czfab), - dx, &rr,&use_limiter); + dx, &refinement_ratio,&use_limiter); amrex_interp_cc_bfield(ccbx.loVect(), ccbx.hiVect(), BL_TO_FORTRAN_ANYD(bfab[1]), BL_TO_FORTRAN_ANYD(cyfab), - &rr,&use_limiter); + &refinement_ratio,&use_limiter); #endif for (int idim = 0; idim < 3; ++idim) @@ -153,7 +153,7 @@ WarpX::UpdateAuxilaryData () MultiFab::Subtract(dEy, *Efield_cp[lev][1], 0, 0, 1, ng); MultiFab::Subtract(dEz, *Efield_cp[lev][2], 0, 0, 1, ng); - const int rr = refRatio(lev-1)[0]; + const int refinement_ratio = refRatio(lev-1)[0]; #ifdef _OPEMP #pragma omp parallel #endif @@ -163,7 +163,7 @@ WarpX::UpdateAuxilaryData () { Box ccbx = mfi.fabbox(); ccbx.enclosedCells(); - ccbx.coarsen(rr).refine(rr); // so that ccbx is coarsenable + ccbx.coarsen(refinement_ratio).refine(refinement_ratio); // so that ccbx is coarsenable const FArrayBox& cxfab = dEx[mfi]; const FArrayBox& cyfab = dEy[mfi]; @@ -180,18 +180,18 @@ WarpX::UpdateAuxilaryData () BL_TO_FORTRAN_ANYD(cxfab), BL_TO_FORTRAN_ANYD(cyfab), BL_TO_FORTRAN_ANYD(czfab), - &rr,&use_limiter); + &refinement_ratio,&use_limiter); #else amrex_interp_efield(ccbx.loVect(), ccbx.hiVect(), BL_TO_FORTRAN_ANYD(efab[0]), BL_TO_FORTRAN_ANYD(efab[2]), BL_TO_FORTRAN_ANYD(cxfab), BL_TO_FORTRAN_ANYD(czfab), - &rr,&use_limiter); + &refinement_ratio,&use_limiter); amrex_interp_nd_efield(ccbx.loVect(), ccbx.hiVect(), BL_TO_FORTRAN_ANYD(efab[1]), BL_TO_FORTRAN_ANYD(cyfab), - &rr); + &refinement_ratio); #endif for (int idim = 0; idim < 3; ++idim) @@ -364,7 +364,7 @@ WarpX::SyncCurrent () current_cp[lev][1]->setVal(0.0); current_cp[lev][2]->setVal(0.0); - const IntVect& rr = refRatio(lev-1); + const IntVect& refinement_ratio = refRatio(lev-1); std::array fine { current_fp[lev][0].get(), current_fp[lev][1].get(), @@ -372,7 +372,7 @@ WarpX::SyncCurrent () std::array< MultiFab*,3> crse { current_cp[lev][0].get(), current_cp[lev][1].get(), current_cp[lev][2].get() }; - SyncCurrent(fine, crse, rr[0]); + SyncCurrent(fine, crse, refinement_ratio[0]); } Vector,3> > j_fp(finest_level+1); @@ -524,10 +524,10 @@ WarpX::SyncCurrent () void WarpX::SyncCurrent (const std::array& fine, const std::array< amrex::MultiFab*,3>& crse, - int rr) + int refinement_ratio) { BL_ASSERT(rr == 2); - const IntVect& ng = (fine[0]->nGrowVect() + 1) /rr; + const IntVect& ng = (fine[0]->nGrowVect() + 1) /refinement_ratio; #ifdef _OPEMP #pragma omp parallel @@ -539,7 +539,7 @@ WarpX::SyncCurrent (const std::array& fine, for (MFIter mfi(*crse[idim],true); mfi.isValid(); ++mfi) { const Box& bx = mfi.growntilebox(ng); - Box fbx = amrex::grow(amrex::refine(bx,rr),1); + Box fbx = amrex::grow(amrex::refine(bx,refinement_ratio),1); ffab.resize(fbx); fbx &= (*fine[idim])[mfi].box(); ffab.setVal(0.0); @@ -564,8 +564,8 @@ WarpX::SyncRho (amrex::Vector >& rhof, for (int lev = 1; lev <= finest_level; ++lev) { rhoc[lev]->setVal(0.0); - const IntVect& rr = refRatio(lev-1); - SyncRho(*rhof[lev], *rhoc[lev], rr[0]); + const IntVect& refinement_ratio = refRatio(lev-1); + SyncRho(*rhof[lev], *rhoc[lev], refinement_ratio[0]); } Vector > rho_f_g(finest_level+1); @@ -673,10 +673,10 @@ WarpX::SyncRho (amrex::Vector >& rhof, * averaging the values of the charge density of the fine patch (on the same level). */ void -WarpX::SyncRho (const MultiFab& fine, MultiFab& crse, int rr) +WarpX::SyncRho (const MultiFab& fine, MultiFab& crse, int refinement_ratio) { BL_ASSERT(rr == 2); - const IntVect& ng = (fine.nGrowVect()+1)/rr; + const IntVect& ng = (fine.nGrowVect()+1)/refinement_ratio; const int nc = fine.nComp(); #ifdef _OPEMP @@ -687,7 +687,7 @@ WarpX::SyncRho (const MultiFab& fine, MultiFab& crse, int rr) for (MFIter mfi(crse,true); mfi.isValid(); ++mfi) { const Box& bx = mfi.growntilebox(ng); - Box fbx = amrex::grow(amrex::refine(bx,rr),1); + Box fbx = amrex::grow(amrex::refine(bx,refinement_ratio),1); ffab.resize(fbx, nc); fbx &= fine[mfi].box(); ffab.setVal(0.0); @@ -710,7 +710,7 @@ WarpX::RestrictCurrentFromFineToCoarsePatch (int lev) current_cp[lev][1]->setVal(0.0); current_cp[lev][2]->setVal(0.0); - const IntVect& rr = refRatio(lev-1); + const IntVect& refinement_ratio = refRatio(lev-1); std::array fine { current_fp[lev][0].get(), current_fp[lev][1].get(), @@ -718,7 +718,7 @@ WarpX::RestrictCurrentFromFineToCoarsePatch (int lev) std::array< MultiFab*,3> crse { current_cp[lev][0].get(), current_cp[lev][1].get(), current_cp[lev][2].get() }; - SyncCurrent(fine, crse, rr[0]); + SyncCurrent(fine, crse, refinement_ratio[0]); } void @@ -824,8 +824,8 @@ WarpX::RestrictRhoFromFineToCoarsePatch (int lev) { if (rho_fp[lev]) { rho_cp[lev]->setVal(0.0); - const IntVect& rr = refRatio(lev-1); - SyncRho(*rho_fp[lev], *rho_cp[lev], rr[0]); + const IntVect& refinement_ratio = refRatio(lev-1); + SyncRho(*rho_fp[lev], *rho_cp[lev], refinement_ratio[0]); } } -- cgit v1.2.3 From fe37f932e428846473d75d1cad5962546f3fecac Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Thu, 23 May 2019 18:15:32 -0700 Subject: missed a couple --- Source/Parallelization/WarpXComm.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Source/Parallelization/WarpXComm.cpp') diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 932d6905a..5c9fa144f 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -526,7 +526,7 @@ WarpX::SyncCurrent (const std::array& fine, const std::array< amrex::MultiFab*,3>& crse, int refinement_ratio) { - BL_ASSERT(rr == 2); + BL_ASSERT(refinement_ratio == 2); const IntVect& ng = (fine[0]->nGrowVect() + 1) /refinement_ratio; #ifdef _OPEMP @@ -675,7 +675,7 @@ WarpX::SyncRho (amrex::Vector >& rhof, void WarpX::SyncRho (const MultiFab& fine, MultiFab& crse, int refinement_ratio) { - BL_ASSERT(rr == 2); + BL_ASSERT(refinement_ratio == 2); const IntVect& ng = (fine.nGrowVect()+1)/refinement_ratio; const int nc = fine.nComp(); -- cgit v1.2.3 From e7bc3c34fc838c4ab6ee845de2e3f6354c3cbdec Mon Sep 17 00:00:00 2001 From: Revathi Jambunathan Date: Thu, 6 Jun 2019 17:23:22 -0400 Subject: adding 2D cufft plans --- .../SpectralSolver/SpectralFieldData.cpp | 21 +++++++++++++-------- Source/Parallelization/WarpXComm.cpp | 5 ----- Source/Particles/PhysicalParticleContainer.cpp | 2 -- Source/Particles/WarpXParticleContainer.cpp | 2 -- 4 files changed, 13 insertions(+), 17 deletions(-) (limited to 'Source/Parallelization/WarpXComm.cpp') diff --git a/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp b/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp index cb26b19b7..6eeb266e7 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp @@ -55,7 +55,7 @@ SpectralFieldData::SpectralFieldData( const BoxArray& realspace_ba, #ifdef AMREX_USE_GPU // Add cuFFT-specific code // Creating 3D plan for real to complex -- double precision - + // Assuming CUDA is used for programming GPU cufftResult result; #if (AMREX_SPACEDIM == 3) result = cufftPlan3d( &forward_plan[mfi], fft_size[2], @@ -63,21 +63,26 @@ SpectralFieldData::SpectralFieldData( const BoxArray& realspace_ba, if ( result != CUFFT_SUCCESS ) { amrex::Print() << " cufftplan3d forward failed! \n"; } -#else - // Add 2D cuFFT-spacific code for D2Z - // Note that D2Z is inherently forward plan -#endif -#if (AMREX_SPACEDIM == 3) result = cufftPlan3d( &backward_plan[mfi], fft_size[2], fft_size[1], fft_size[0], CUFFT_Z2D); if ( result != CUFFT_SUCCESS ) { amrex::Print() << " cufftplan3d backward failed! \n"; } #else - // Add 2D cuFFT-specific code for Z2D - // Note that Z2D is inherently backward plan + // Add 2D cuFFT-spacific code for D2Z + // Note that D2Z is inherently forward plan + result = cufftPlan2d( &forward_plan[mfi], fft_size[1], + fft_size[0], CUFFT_D2Z ); + if ( result != CUFFT_SUCCESS ) { + amrex::Print() << " cufftplan2d forward failed! \n"; + } + result = cufftPlan2d( &backward_plan[mfi], fft_size[1], + fft_size[0], CUFFT_Z2D ); + if ( result != CUFFT_SUCCESS ) { + amrex::Print() << " cufftplan2d backward failed! \n"; + } #endif #else diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 998fc4de1..00dcb85d0 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -222,9 +222,7 @@ WarpX::FillBoundaryE () { for (int lev = 0; lev <= finest_level; ++lev) { - amrex::Print() << " lev " << lev << " in fill bndry E \n"; FillBoundaryE(lev); - amrex::Print() << " lev " << lev << " after fill bndry E \n"; } } @@ -273,12 +271,9 @@ WarpX::FillBoundaryE (int lev, PatchType patch_type) pml[lev]->FillBoundaryE(patch_type); } - amrex::Print() << " before defining multifab \n"; const auto& cperiod = Geom(lev-1).periodicity(); Vector mf{Efield_cp[lev][0].get(),Efield_cp[lev][1].get(),Efield_cp[lev][2].get()}; - amrex::Print() << " before amrex fill bndry \n"; amrex::FillBoundary(mf, cperiod); - amrex::Print() << " after amrex fill bndry \n"; } } diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 0826605ec..1f517fccb 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1388,9 +1388,7 @@ PhysicalParticleContainer::Evolve (int lev, pti.GetPosition(m_xp[thread_num], m_yp[thread_num], m_zp[thread_num]); BL_PROFILE_VAR_STOP(blp_copy); - amrex::Print() << " before deposit chage \n"; if (rho) DepositCharge(pti, wp, rho, crho, 0, np_current, np, thread_num, lev); - amrex::Print() << " after deposit chage \n"; if (! do_not_push) { diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 576c53319..47d57294d 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -612,7 +612,6 @@ WarpXParticleContainer::DepositCharge ( WarpXParIter& pti, RealVector& wp, const std::array& xyzmin = xyzmin_tile; #ifdef AMREX_USE_GPU - amrex::Print() << " before icomp data ptr " << icomp << "\n"; data_ptr = (*rhomf)[pti].dataPtr(icomp); auto rholen = (*rhomf)[pti].length(); #else @@ -635,7 +634,6 @@ WarpXParticleContainer::DepositCharge ( WarpXParIter& pti, RealVector& wp, const long nz = rholen[1]-1-2*ngRho; #endif BL_PROFILE_VAR_START(blp_pxr_chd); - amrex::Print() << " before warpxcharge deposition \n"; warpx_charge_deposition(data_ptr, &np_current, m_xp[thread_num].dataPtr(), m_yp[thread_num].dataPtr(), -- cgit v1.2.3