From eb07f18b380cb6b7205c05e7828f00a0fa2b8161 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Tue, 12 Feb 2019 10:42:13 -0800 Subject: re-order tree structure of source code --- Source/Initialization/WarpXInitData.cpp | 259 ++++++++++++++++++++++++++++++++ 1 file changed, 259 insertions(+) create mode 100644 Source/Initialization/WarpXInitData.cpp (limited to 'Source/Initialization/WarpXInitData.cpp') diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp new file mode 100644 index 000000000..496b14e7a --- /dev/null +++ b/Source/Initialization/WarpXInitData.cpp @@ -0,0 +1,259 @@ + +#include + +#include + +#include +#include + +#ifdef BL_USE_SENSEI_INSITU +#include +#endif + +using namespace amrex; + +void +WarpX::InitData () +{ + BL_PROFILE("WarpX::InitData()"); + + if (restart_chkfile.empty()) + { + ComputeDt(); + InitFromScratch(); + } + else + { + InitFromCheckpoint(); + if (is_synchronized) { + ComputeDt(); + } + PostRestart(); + } + + ComputePMLFactors(); + + if (WarpX::use_fdtd_nci_corr) { + WarpX::InitNCICorrector(); + } + + BuildBufferMasks(); + + InitDiagnostics(); + + if (ParallelDescriptor::IOProcessor()) { + std::cout << "\nGrids Summary:\n"; + printGridSummary(std::cout, 0, finestLevel()); + } + +#ifdef BL_USE_SENSEI_INSITU + insitu_bridge = new amrex::AmrMeshInSituBridge; + insitu_bridge->setEnabled(insitu_int > 0 ? 1 : 0); + insitu_bridge->setConfig(insitu_config); + insitu_bridge->setPinMesh(insitu_pin_mesh); + if (insitu_bridge->initialize()) + { + amrex::ErrorStream() + << "WarpX::InitData : Failed to initialize the in situ bridge." + << std::endl; + + amrex::Abort(); + } + insitu_bridge->setFrequency(1); +#endif + + if (restart_chkfile.empty()) + { + if (plot_int > 0) + WritePlotFile(); + + if (check_int > 0) + WriteCheckPointFile(); + + if ((insitu_int > 0) && (insitu_start == 0)) + UpdateInSitu(); + } +} + +void +WarpX::InitDiagnostics () { + if (do_boosted_frame_diagnostic) { + const Real* current_lo = geom[0].ProbLo(); + const Real* current_hi = geom[0].ProbHi(); + Real dt_boost = dt[0]; + + // Find the positions of the lab-frame box that corresponds to the boosted-frame box at t=0 + Real zmin_lab = current_lo[moving_window_dir]/( (1.+beta_boost)*gamma_boost ); + Real zmax_lab = current_hi[moving_window_dir]/( (1.+beta_boost)*gamma_boost ); + + myBFD.reset(new BoostedFrameDiagnostic(zmin_lab, + zmax_lab, + moving_window_v, dt_snapshots_lab, + num_snapshots_lab, gamma_boost, + t_new[0], dt_boost, + moving_window_dir)); + } +} + +void +WarpX::InitFromScratch () +{ + const Real time = 0.0; + + AmrCore::InitFromScratch(time); // This will call MakeNewLevelFromScratch + + mypc->AllocData(); + mypc->InitData(); + +#ifdef USE_OPENBC_POISSON + InitOpenbc(); +#endif + + InitPML(); + +#ifdef WARPX_DO_ELECTROSTATIC + if (do_electrostatic) { + getLevelMasks(masks); + + // the plus one is to convert from num_cells to num_nodes + getLevelMasks(gather_masks, n_buffer + 1); + } +#endif // WARPX_DO_ELECTROSTATIC +} + +void +WarpX::InitPML () +{ + if (do_pml) + { + pml[0].reset(new PML(boxArray(0), DistributionMap(0), &Geom(0), nullptr, + pml_ncell, pml_delta, 0, do_dive_cleaning, do_moving_window)); + for (int lev = 1; lev <= finest_level; ++lev) + { + pml[lev].reset(new PML(boxArray(lev), DistributionMap(lev), + &Geom(lev), &Geom(lev-1), + pml_ncell, pml_delta, refRatio(lev-1)[0], do_dive_cleaning, + do_moving_window)); + } + } +} + +void +WarpX::ComputePMLFactors () +{ + if (do_pml) + { + for (int lev = 0; lev <= finest_level; ++lev) + { + pml[lev]->ComputePMLFactors(dt[lev]); + } + } +} + +void +WarpX::InitNCICorrector () +{ + if (WarpX::use_fdtd_nci_corr) + { + mypc->fdtd_nci_stencilz_ex.resize(max_level+1); + mypc->fdtd_nci_stencilz_by.resize(max_level+1); + for (int lev = 0; lev <= max_level; ++lev) + { + const Geometry& gm = Geom(lev); + const Real* dx = gm.CellSize(); + amrex::Real dz, cdtodz; + if (AMREX_SPACEDIM == 3){ + dz = dx[2]; + }else{ + dz = dx[1]; + } + cdtodz = PhysConst::c * dt[lev] / dz; + WRPX_PXR_NCI_CORR_INIT( (mypc->fdtd_nci_stencilz_ex)[lev].data(), + (mypc->fdtd_nci_stencilz_by)[lev].data(), + mypc->nstencilz_fdtd_nci_corr, cdtodz, + WarpX::l_lower_order_in_v); + } + } +} + +void +WarpX::PostRestart () +{ +#ifdef WARPX_USE_PSATD + amrex::Abort("WarpX::PostRestart: TODO for PSATD"); +#endif + mypc->PostRestart(); +} + +#ifdef USE_OPENBC_POISSON +void +WarpX::InitOpenbc () +{ +#ifndef BL_USE_MPI + static_assert(false, "must use MPI"); +#endif + + static_assert(AMREX_SPACEDIM == 3, "Openbc is 3D only"); + BL_ASSERT(finestLevel() == 0); + + const int lev = 0; + + const Geometry& gm = Geom(lev); + const Box& gbox = gm.Domain(); + int lohi[6]; + warpx_openbc_decompose(gbox.loVect(), gbox.hiVect(), lohi, lohi+3); + + int nprocs = ParallelDescriptor::NProcs(); + int myproc = ParallelDescriptor::MyProc(); + Vector alllohi(6*nprocs,100000); + + MPI_Allgather(lohi, 6, MPI_INT, alllohi.data(), 6, MPI_INT, ParallelDescriptor::Communicator()); + + BoxList bl{IndexType::TheNodeType()}; + for (int i = 0; i < nprocs; ++i) + { + bl.push_back(Box(IntVect(alllohi[6*i ],alllohi[6*i+1],alllohi[6*i+2]), + IntVect(alllohi[6*i+3],alllohi[6*i+4],alllohi[6*i+5]), + IndexType::TheNodeType())); + } + BoxArray ba{bl}; + + Vector iprocmap(nprocs+1); + std::iota(iprocmap.begin(), iprocmap.end(), 0); + iprocmap.back() = myproc; + + DistributionMapping dm{iprocmap}; + + MultiFab rho_openbc(ba, dm, 1, 0); + MultiFab phi_openbc(ba, dm, 1, 0); + + bool local = true; + const std::unique_ptr& rho = mypc->GetChargeDensity(lev, local); + + rho_openbc.setVal(0.0); + rho_openbc.copy(*rho, 0, 0, 1, rho->nGrow(), 0, gm.periodicity(), FabArrayBase::ADD); + + const Real* dx = gm.CellSize(); + + warpx_openbc_potential(rho_openbc[myproc].dataPtr(), phi_openbc[myproc].dataPtr(), dx); + + BoxArray nba = boxArray(lev); + nba.surroundingNodes(); + MultiFab phi(nba, DistributionMap(lev), 1, 0); + phi.copy(phi_openbc, gm.periodicity()); + +#ifdef _OPENMP +#pragma omp parallel +#endif + for (MFIter mfi(phi); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.validbox(); + warpx_compute_E(bx.loVect(), bx.hiVect(), + BL_TO_FORTRAN_3D(phi[mfi]), + BL_TO_FORTRAN_3D((*Efield[lev][0])[mfi]), + BL_TO_FORTRAN_3D((*Efield[lev][1])[mfi]), + BL_TO_FORTRAN_3D((*Efield[lev][2])[mfi]), + dx); + } +} +#endif -- cgit v1.2.3 From e430c94f1d3990a2063638fed97fcc1d7fbaca4a Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Tue, 19 Feb 2019 15:04:21 -0800 Subject: regroup or rename some files for consistency --- Source/BoundaryConditions/Make.package | 9 +- Source/BoundaryConditions/PML.H | 170 ++++ Source/BoundaryConditions/PML.cpp | 755 +++++++++++++++ Source/BoundaryConditions/PML_routines.F90 | 1009 ++++++++++++++++++++ Source/BoundaryConditions/WarpXPML.H | 170 ---- Source/BoundaryConditions/WarpXPML.cpp | 755 --------------- Source/BoundaryConditions/WarpX_pml.F90 | 1009 -------------------- Source/BoundaryConditions/openbc_f.F90 | 62 -- Source/Diagnostics/BoostedFrameDiagnostic.H | 98 ++ Source/Diagnostics/BoostedFrameDiagnostic.cpp | 371 +++++++ Source/Diagnostics/BoostedFrame_module.F90 | 106 ++ Source/Diagnostics/Make.package | 6 +- Source/Diagnostics/WarpXBoostedFrameDiagnostic.H | 98 -- Source/Diagnostics/WarpXBoostedFrameDiagnostic.cpp | 371 ------- Source/Diagnostics/WarpX_boosted_frame.F90 | 106 -- Source/Electrostatic/Make.package | 3 + Source/Electrostatic/openbc_poisson_solver.F90 | 62 ++ Source/Filter/Make.package | 2 +- Source/Filter/WarpX_filter.F90 | 87 -- Source/Filter/filter_module.F90 | 87 ++ Source/Initialization/Make.package | 1 - Source/Initialization/WarpXInitData.cpp | 75 ++ Source/Initialization/WarpXProb.cpp | 78 -- Source/Utils/Make.package | 2 +- Source/Utils/WarpXMove.cpp | 245 ----- Source/Utils/WarpXMovingWindow.cpp | 245 +++++ Source/WarpX.H | 4 +- 27 files changed, 2991 insertions(+), 2995 deletions(-) create mode 100644 Source/BoundaryConditions/PML.H create mode 100644 Source/BoundaryConditions/PML.cpp create mode 100644 Source/BoundaryConditions/PML_routines.F90 delete mode 100644 Source/BoundaryConditions/WarpXPML.H delete mode 100644 Source/BoundaryConditions/WarpXPML.cpp delete mode 100644 Source/BoundaryConditions/WarpX_pml.F90 delete mode 100644 Source/BoundaryConditions/openbc_f.F90 create mode 100644 Source/Diagnostics/BoostedFrameDiagnostic.H create mode 100644 Source/Diagnostics/BoostedFrameDiagnostic.cpp create mode 100644 Source/Diagnostics/BoostedFrame_module.F90 delete mode 100644 Source/Diagnostics/WarpXBoostedFrameDiagnostic.H delete mode 100644 Source/Diagnostics/WarpXBoostedFrameDiagnostic.cpp delete mode 100644 Source/Diagnostics/WarpX_boosted_frame.F90 create mode 100644 Source/Electrostatic/openbc_poisson_solver.F90 delete mode 100644 Source/Filter/WarpX_filter.F90 create mode 100644 Source/Filter/filter_module.F90 delete mode 100644 Source/Initialization/WarpXProb.cpp delete mode 100644 Source/Utils/WarpXMove.cpp create mode 100644 Source/Utils/WarpXMovingWindow.cpp (limited to 'Source/Initialization/WarpXInitData.cpp') diff --git a/Source/BoundaryConditions/Make.package b/Source/BoundaryConditions/Make.package index 931b48147..1ef0cc0d3 100644 --- a/Source/BoundaryConditions/Make.package +++ b/Source/BoundaryConditions/Make.package @@ -1,9 +1,6 @@ -CEXE_sources += WarpXPML.cpp -CEXE_headers += WarpXPML.H -F90EXE_sources += WarpX_pml.F90 -ifeq ($(USE_OPENBC_POISSON),TRUE) - F90EXE_sources += openbc_f.F90 -endif +CEXE_sources += PML.cpp +CEXE_headers += PML.H +F90EXE_sources += PML_routines.F90 INCLUDE_LOCATIONS += $(WARPX_HOME)/Source/BoundaryConditions VPATH_LOCATIONS += $(WARPX_HOME)/Source/BoundaryConditions diff --git a/Source/BoundaryConditions/PML.H b/Source/BoundaryConditions/PML.H new file mode 100644 index 000000000..7a86bf0da --- /dev/null +++ b/Source/BoundaryConditions/PML.H @@ -0,0 +1,170 @@ +#include + +#ifndef WARPX_PML_H_ +#define WARPX_PML_H_ + +#include +#include + +#if (AMREX_SPACEDIM == 3) + +#define WRPX_PML_TO_FORTRAN(x) \ + (x).sigma_fac[0].data(), (x).sigma_fac[0].m_lo, (x).sigma_fac[0].m_hi, \ + (x).sigma_fac[1].data(), (x).sigma_fac[1].m_lo, (x).sigma_fac[1].m_hi, \ + (x).sigma_fac[2].data(), (x).sigma_fac[2].m_lo, (x).sigma_fac[2].m_hi, \ + (x).sigma_star_fac[0].data(), (x).sigma_star_fac[0].m_lo, (x).sigma_star_fac[0].m_hi, \ + (x).sigma_star_fac[1].data(), (x).sigma_star_fac[1].m_lo, (x).sigma_star_fac[1].m_hi, \ + (x).sigma_star_fac[2].data(), (x).sigma_star_fac[2].m_lo, (x).sigma_star_fac[2].m_hi + +#else + +#define WRPX_PML_TO_FORTRAN(x) \ + (x).sigma_fac[0].data(), (x).sigma_fac[0].m_lo, (x).sigma_fac[0].m_hi, \ + (x).sigma_fac[1].data(), (x).sigma_fac[1].m_lo, (x).sigma_fac[1].m_hi, \ + (x).sigma_star_fac[0].data(), (x).sigma_star_fac[0].m_lo, (x).sigma_star_fac[0].m_hi, \ + (x).sigma_star_fac[1].data(), (x).sigma_star_fac[1].m_lo, (x).sigma_star_fac[1].m_hi + +#endif + +struct Sigma : amrex::Vector +{ + int lo() const { return m_lo; } + int hi() const { return m_hi; } + int m_lo, m_hi; +}; + +struct SigmaBox +{ + SigmaBox (const amrex::Box& box, const amrex::BoxArray& grids, + const amrex::Real* dx, int ncell, int delta); + + void ComputePMLFactorsB (const amrex::Real* dx, amrex::Real dt); + void ComputePMLFactorsE (const amrex::Real* dx, amrex::Real dt); + + using SigmaVect = std::array; + + SigmaVect sigma; // sigma/epsilon + SigmaVect sigma_star; // sigma_star/mu + SigmaVect sigma_fac; + SigmaVect sigma_star_fac; +}; + +namespace amrex { + template<> + class FabFactory + { + public: + FabFactory (const BoxArray& grid_ba, const Real* dx, int ncell, int delta) + : m_grids(grid_ba), m_dx(dx), m_ncell(ncell), m_delta(delta) {} + virtual SigmaBox* create (const Box& box, int ncomps, + const FabInfo& info, int box_index) const final + { return new SigmaBox(box, m_grids, m_dx, m_ncell, m_delta); } + virtual void destroy (SigmaBox* fab) const final { + delete fab; + } +#ifdef AMREX_USE_GPU + virtual SigmaBox* createHostAlias (SigmaBox const& src) const final + { + return const_cast(&src); + } + virtual void destroyHostAlias (SigmaBox* fab) const final {} +#endif + virtual FabFactory* clone () const { + return new FabFactory(*this); + } + private: + const BoxArray& m_grids; + const Real* m_dx; + int m_ncell; + int m_delta; + }; +} + +class MultiSigmaBox + : public amrex::FabArray +{ +public: + MultiSigmaBox(const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, + const amrex::BoxArray& grid_ba, const amrex::Real* dx, int ncell, int delta); + void ComputePMLFactorsB (const amrex::Real* dx, amrex::Real dt); + void ComputePMLFactorsE (const amrex::Real* dx, amrex::Real dt); +private: + amrex::Real dt_B = -1.e10; + amrex::Real dt_E = -1.e10; +}; + +enum struct PatchType : int; + +class PML +{ +public: + PML (const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, + const amrex::Geometry* geom, const amrex::Geometry* cgeom, + int ncell, int delta, int ref_ratio, int do_dive_cleaning, int do_moving_window); + + void ComputePMLFactors (amrex::Real dt); + + std::array GetE_fp (); + std::array GetB_fp (); + std::array GetE_cp (); + std::array GetB_cp (); + + amrex::MultiFab* GetF_fp (); + amrex::MultiFab* GetF_cp (); + + const MultiSigmaBox& GetMultiSigmaBox_fp () const + { return *sigba_fp; } + + const MultiSigmaBox& GetMultiSigmaBox_cp () const + { return *sigba_cp; } + + void ExchangeB (const std::array& B_fp, + const std::array& B_cp); + void ExchangeE (const std::array& E_fp, + const std::array& E_cp); + void ExchangeB (PatchType patch_type, + const std::array& Bp); + void ExchangeE (PatchType patch_type, + const std::array& Ep); + + void ExchangeF (amrex::MultiFab* F_fp, amrex::MultiFab* F_cp); + void ExchangeF (PatchType patch_type, amrex::MultiFab* Fp); + + void FillBoundary (); + void FillBoundaryE (); + void FillBoundaryB (); + void FillBoundaryF (); + void FillBoundaryE (PatchType patch_type); + void FillBoundaryB (PatchType patch_type); + void FillBoundaryF (PatchType patch_type); + + bool ok () const { return m_ok; } + + void CheckPoint (const std::string& dir) const; + void Restart (const std::string& dir); + +private: + bool m_ok; + + const amrex::Geometry* m_geom; + const amrex::Geometry* m_cgeom; + + std::array,3> pml_E_fp; + std::array,3> pml_B_fp; + + std::array,3> pml_E_cp; + std::array,3> pml_B_cp; + + std::unique_ptr pml_F_fp; + std::unique_ptr pml_F_cp; + + std::unique_ptr sigba_fp; + std::unique_ptr sigba_cp; + + static amrex::BoxArray MakeBoxArray (const amrex::Geometry& geom, + const amrex::BoxArray& grid_ba, int ncell); + + static void Exchange (amrex::MultiFab& pml, amrex::MultiFab& reg, const amrex::Geometry& geom); +}; + +#endif diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp new file mode 100644 index 000000000..03373995d --- /dev/null +++ b/Source/BoundaryConditions/PML.cpp @@ -0,0 +1,755 @@ + +#include +#include +#include + +#include +#include + +#include + +#ifdef _OPENMP +#include +#endif + +using namespace amrex; + +namespace +{ + static void FillLo (int idim, Sigma& sigma, Sigma& sigma_star, + const Box& overlap, const Box& grid, Real fac) + { + int glo = grid.smallEnd(idim); + int olo = overlap.smallEnd(idim); + int ohi = overlap.bigEnd(idim); + int slo = sigma.m_lo; + int sslo = sigma_star.m_lo; + for (int i = olo; i <= ohi+1; ++i) + { + Real offset = static_cast(glo-i); + sigma[i-slo] = fac*(offset*offset); + } + for (int i = olo; i <= ohi; ++i) + { + Real offset = static_cast(glo-i) - 0.5; + sigma_star[i-sslo] = fac*(offset*offset); + } + } + + static void FillHi (int idim, Sigma& sigma, Sigma& sigma_star, + const Box& overlap, const Box& grid, Real fac) + { + int ghi = grid.bigEnd(idim); + int olo = overlap.smallEnd(idim); + int ohi = overlap.bigEnd(idim); + int slo = sigma.m_lo; + int sslo = sigma_star.m_lo; + for (int i = olo; i <= ohi+1; ++i) + { + Real offset = static_cast(i-ghi-1); + sigma[i-slo] = fac*(offset*offset); + } + for (int i = olo; i <= ohi; ++i) + { + Real offset = static_cast(i-ghi) - 0.5; + sigma_star[i-sslo] = fac*(offset*offset); + } + } + + static void FillZero (int idim, Sigma& sigma, Sigma& sigma_star, const Box& overlap) + { + int olo = overlap.smallEnd(idim); + int ohi = overlap.bigEnd(idim); + int slo = sigma.m_lo; + int sslo = sigma_star.m_lo; + std::fill(sigma.begin()+(olo-slo), sigma.begin()+(ohi+2-slo), 0.0); + std::fill(sigma_star.begin()+(olo-sslo), sigma_star.begin()+(ohi+1-sslo), 0.0); + } +} + +SigmaBox::SigmaBox (const Box& box, const BoxArray& grids, const Real* dx, int ncell, int delta) +{ + BL_ASSERT(box.cellCentered()); + + const IntVect& sz = box.size(); + const int* lo = box.loVect(); + const int* hi = box.hiVect(); + + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) + { + sigma [idim].resize(sz[idim]+1); + sigma_star [idim].resize(sz[idim] ); + sigma_fac [idim].resize(sz[idim]+1); + sigma_star_fac[idim].resize(sz[idim] ); + + sigma [idim].m_lo = lo[idim]; + sigma [idim].m_hi = hi[idim]+1; + sigma_star [idim].m_lo = lo[idim]; + sigma_star [idim].m_hi = hi[idim]; + sigma_fac [idim].m_lo = lo[idim]; + sigma_fac [idim].m_hi = hi[idim]+1; + sigma_star_fac[idim].m_lo = lo[idim]; + sigma_star_fac[idim].m_hi = hi[idim]; + } + + Array fac; + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + fac[idim] = 4.0*PhysConst::c/(dx[idim]*static_cast(delta*delta)); + } + + const std::vector >& isects = grids.intersections(box, false, ncell); + + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) + { + int jdim = (idim+1) % AMREX_SPACEDIM; +#if (AMREX_SPACEDIM == 3) + int kdim = (idim+2) % AMREX_SPACEDIM; +#endif + + Vector direct_faces, side_faces, direct_side_edges, side_side_edges, corners; + for (const auto& kv : isects) + { + const Box& grid_box = grids[kv.first]; + + if (amrex::grow(grid_box, idim, ncell).intersects(box)) + { + direct_faces.push_back(kv.first); + } + else if (amrex::grow(grid_box, jdim, ncell).intersects(box)) + { + side_faces.push_back(kv.first); + } +#if (AMREX_SPACEDIM == 3) + else if (amrex::grow(grid_box, kdim, ncell).intersects(box)) + { + side_faces.push_back(kv.first); + } + else if (amrex::grow(amrex::grow(grid_box,idim,ncell), + jdim,ncell).intersects(box)) + { + direct_side_edges.push_back(kv.first); + } + else if (amrex::grow(amrex::grow(grid_box,idim,ncell), + kdim,ncell).intersects(box)) + { + direct_side_edges.push_back(kv.first); + } + else if (amrex::grow(amrex::grow(grid_box,jdim,ncell), + kdim,ncell).intersects(box)) + { + side_side_edges.push_back(kv.first); + } +#endif + else + { + corners.push_back(kv.first); + } + } + + for (auto gid : corners) + { + const Box& grid_box = grids[gid]; + + Box lobox = amrex::adjCellLo(grid_box, idim, ncell); + lobox.grow(jdim,ncell); +#if (AMREX_SPACEDIM == 3) + lobox.grow(kdim,ncell); +#endif + Box looverlap = lobox & box; + if (looverlap.ok()) { + FillLo(idim, sigma[idim], sigma_star[idim], looverlap, grid_box, fac[idim]); + } + + Box hibox = amrex::adjCellHi(grid_box, idim, ncell); + hibox.grow(jdim,ncell); +#if (AMREX_SPACEDIM == 3) + hibox.grow(kdim,ncell); +#endif + Box hioverlap = hibox & box; + if (hioverlap.ok()) { + FillHi(idim, sigma[idim], sigma_star[idim], hioverlap, grid_box, fac[idim]); + } + + if (!looverlap.ok() && !hioverlap.ok()) { + amrex::Abort("SigmaBox::SigmaBox(): corners, how did this happen?\n"); + } + } + +#if (AMREX_SPACEDIM == 3) + for (auto gid : side_side_edges) + { + const Box& grid_box = grids[gid]; + const Box& overlap = amrex::grow(amrex::grow(grid_box,jdim,ncell),kdim,ncell) & box; + if (overlap.ok()) { + FillZero(idim, sigma[idim], sigma_star[idim], overlap); + } else { + amrex::Abort("SigmaBox::SigmaBox(): side_side_edges, how did this happen?\n"); + } + } + + for (auto gid : direct_side_edges) + { + const Box& grid_box = grids[gid]; + + Box lobox = amrex::adjCellLo(grid_box, idim, ncell); + Box looverlap = lobox.grow(jdim,ncell).grow(kdim,ncell) & box; + if (looverlap.ok()) { + FillLo(idim, sigma[idim], sigma_star[idim], looverlap, grid_box, fac[idim]); + } + + Box hibox = amrex::adjCellHi(grid_box, idim, ncell); + Box hioverlap = hibox.grow(jdim,ncell).grow(kdim,ncell) & box; + if (hioverlap.ok()) { + FillHi(idim, sigma[idim], sigma_star[idim], hioverlap, grid_box, fac[idim]); + } + + if (!looverlap.ok() && !hioverlap.ok()) { + amrex::Abort("SigmaBox::SigmaBox(): direct_side_edges, how did this happen?\n"); + } + } +#endif + + for (auto gid : side_faces) + { + const Box& grid_box = grids[gid]; +#if (AMREX_SPACEDIM == 2) + const Box& overlap = amrex::grow(grid_box,jdim,ncell) & box; +#else + const Box& overlap = amrex::grow(amrex::grow(grid_box,jdim,ncell),kdim,ncell) & box; +#endif + if (overlap.ok()) { + FillZero(idim, sigma[idim], sigma_star[idim], overlap); + } else { + amrex::Abort("SigmaBox::SigmaBox(): side_faces, how did this happen?\n"); + } + } + + for (auto gid : direct_faces) + { + const Box& grid_box = grids[gid]; + + const Box& lobox = amrex::adjCellLo(grid_box, idim, ncell); + Box looverlap = lobox & box; + if (looverlap.ok()) { + FillLo(idim, sigma[idim], sigma_star[idim], looverlap, grid_box, fac[idim]); + } + + const Box& hibox = amrex::adjCellHi(grid_box, idim, ncell); + Box hioverlap = hibox & box; + if (hioverlap.ok()) { + FillHi(idim, sigma[idim], sigma_star[idim], hioverlap, grid_box, fac[idim]); + } + + if (!looverlap.ok() && !hioverlap.ok()) { + amrex::Abort("SigmaBox::SigmaBox(): direct faces, how did this happen?\n"); + } + } + + if (direct_faces.size() > 1) { + amrex::Abort("SigmaBox::SigmaBox(): direct_faces.size() > 1, Box gaps not wide enough?\n"); + } + } +} + +void +SigmaBox::ComputePMLFactorsB (const Real* dx, Real dt) +{ + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) + { + for (int i = 0, N = sigma_star[idim].size(); i < N; ++i) + { + if (sigma_star[idim][i] == 0.0) + { + sigma_star_fac[idim][i] = 1.0; + } + else + { + sigma_star_fac[idim][i] = std::exp(-sigma_star[idim][i]*dt); + } + } + } +} + +void +SigmaBox::ComputePMLFactorsE (const Real* dx, Real dt) +{ + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) + { + for (int i = 0, N = sigma[idim].size(); i < N; ++i) + { + if (sigma[idim][i] == 0.0) + { + sigma_fac[idim][i] = 1.0; + } + else + { + sigma_fac[idim][i] = std::exp(-sigma[idim][i]*dt); + } + } + } +} + +MultiSigmaBox::MultiSigmaBox (const BoxArray& ba, const DistributionMapping& dm, + const BoxArray& grid_ba, const Real* dx, int ncell, int delta) + : FabArray(ba,dm,1,0,MFInfo(), + FabFactory(grid_ba,dx,ncell,delta)) +{} + +void +MultiSigmaBox::ComputePMLFactorsB (const Real* dx, Real dt) +{ + if (dt == dt_B) return; + + dt_B = dt; + +#ifdef _OPENMP +#pragma omp parallel +#endif + for (MFIter mfi(*this); mfi.isValid(); ++mfi) + { + (*this)[mfi].ComputePMLFactorsB(dx, dt); + } +} + +void +MultiSigmaBox::ComputePMLFactorsE (const Real* dx, Real dt) +{ + if (dt == dt_E) return; + + dt_E = dt; + +#ifdef _OPENMP +#pragma omp parallel +#endif + for (MFIter mfi(*this); mfi.isValid(); ++mfi) + { + (*this)[mfi].ComputePMLFactorsE(dx, dt); + } +} + +PML::PML (const BoxArray& grid_ba, const DistributionMapping& grid_dm, + const Geometry* geom, const Geometry* cgeom, + int ncell, int delta, int ref_ratio, int do_dive_cleaning, int do_moving_window) + : m_geom(geom), + m_cgeom(cgeom) +{ + const BoxArray& ba = MakeBoxArray(*geom, grid_ba, ncell); + if (ba.size() == 0) { + m_ok = false; + return; + } else { + m_ok = true; + } + + DistributionMapping dm{ba}; + + int nge = 2; + int ngb = 2; + int ngf = (do_moving_window) ? 2 : 0; + if (WarpX::maxwell_fdtd_solver_id == 1) ngf = std::max( ngf, 1 ); + + pml_E_fp[0].reset(new MultiFab(amrex::convert(ba,WarpX::Ex_nodal_flag), dm, 3, nge)); + pml_E_fp[1].reset(new MultiFab(amrex::convert(ba,WarpX::Ey_nodal_flag), dm, 3, nge)); + pml_E_fp[2].reset(new MultiFab(amrex::convert(ba,WarpX::Ez_nodal_flag), dm, 3, nge)); + pml_B_fp[0].reset(new MultiFab(amrex::convert(ba,WarpX::Bx_nodal_flag), dm, 2, ngb)); + pml_B_fp[1].reset(new MultiFab(amrex::convert(ba,WarpX::By_nodal_flag), dm, 2, ngb)); + pml_B_fp[2].reset(new MultiFab(amrex::convert(ba,WarpX::Bz_nodal_flag), dm, 2, ngb)); + + pml_E_fp[0]->setVal(0.0); + pml_E_fp[1]->setVal(0.0); + pml_E_fp[2]->setVal(0.0); + pml_B_fp[0]->setVal(0.0); + pml_B_fp[1]->setVal(0.0); + pml_B_fp[2]->setVal(0.0); + + if (do_dive_cleaning) + { + pml_F_fp.reset(new MultiFab(amrex::convert(ba,IntVect::TheUnitVector()), dm, 3, ngf)); + pml_F_fp->setVal(0.0); + } + + sigba_fp.reset(new MultiSigmaBox(ba, dm, grid_ba, geom->CellSize(), ncell, delta)); + + if (cgeom) + { + + nge = 1; + ngb = 1; + + BoxArray grid_cba = grid_ba; + grid_cba.coarsen(ref_ratio); + const BoxArray& cba = MakeBoxArray(*cgeom, grid_cba, ncell); + + DistributionMapping cdm{cba}; + + pml_E_cp[0].reset(new MultiFab(amrex::convert(cba,WarpX::Ex_nodal_flag), cdm, 3, nge)); + pml_E_cp[1].reset(new MultiFab(amrex::convert(cba,WarpX::Ey_nodal_flag), cdm, 3, nge)); + pml_E_cp[2].reset(new MultiFab(amrex::convert(cba,WarpX::Ez_nodal_flag), cdm, 3, nge)); + pml_B_cp[0].reset(new MultiFab(amrex::convert(cba,WarpX::Bx_nodal_flag), cdm, 2, ngb)); + pml_B_cp[1].reset(new MultiFab(amrex::convert(cba,WarpX::By_nodal_flag), cdm, 2, ngb)); + pml_B_cp[2].reset(new MultiFab(amrex::convert(cba,WarpX::Bz_nodal_flag), cdm, 2, ngb)); + + pml_E_cp[0]->setVal(0.0); + pml_E_cp[1]->setVal(0.0); + pml_E_cp[2]->setVal(0.0); + pml_B_cp[0]->setVal(0.0); + pml_B_cp[1]->setVal(0.0); + pml_B_cp[2]->setVal(0.0); + + if (do_dive_cleaning) + { + pml_F_cp.reset(new MultiFab(amrex::convert(cba,IntVect::TheUnitVector()), cdm, 3, ngf)); + pml_F_cp->setVal(0.0); + } + + sigba_cp.reset(new MultiSigmaBox(cba, cdm, grid_cba, cgeom->CellSize(), ncell, delta)); + } + +} + +BoxArray +PML::MakeBoxArray (const amrex::Geometry& geom, const amrex::BoxArray& grid_ba, int ncell) +{ + Box domain = geom.Domain(); + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + if ( ! Geometry::isPeriodic(idim) ) { + domain.grow(idim, ncell); + } + } + + BoxList bl; + for (int i = 0, N = grid_ba.size(); i < N; ++i) + { + const Box& grid_bx = grid_ba[i]; + const IntVect& grid_bx_sz = grid_bx.size(); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(grid_bx.shortside() > ncell, + "Consider using larger amr.blocking_factor"); + + Box bx = grid_bx; + bx.grow(ncell); + bx &= domain; + + Vector bndryboxes; +#if (AMREX_SPACEDIM == 3) + int kbegin = -1, kend = 1; +#else + int kbegin = 0, kend = 0; +#endif + for (int kk = kbegin; kk <= kend; ++kk) { + for (int jj = -1; jj <= 1; ++jj) { + for (int ii = -1; ii <= 1; ++ii) { + if (ii != 0 || jj != 0 || kk != 0) { + Box b = grid_bx; + b.shift(grid_bx_sz * IntVect{AMREX_D_DECL(ii,jj,kk)}); + b &= bx; + if (b.ok()) { + bndryboxes.push_back(b); + } + } + } + } + } + + const BoxList& noncovered = grid_ba.complementIn(bx); + for (const Box& b : noncovered) { + for (const auto& bb : bndryboxes) { + Box ib = b & bb; + if (ib.ok()) { + bl.push_back(ib); + } + } + } + } + + BoxArray ba(bl); + ba.removeOverlap(false); + + return ba; +} + +void +PML::ComputePMLFactors (amrex::Real dt) +{ + if (sigba_fp) { + sigba_fp->ComputePMLFactorsB(m_geom->CellSize(), dt); + sigba_fp->ComputePMLFactorsE(m_geom->CellSize(), dt); + } + if (sigba_cp) { + sigba_cp->ComputePMLFactorsB(m_cgeom->CellSize(), dt); + sigba_cp->ComputePMLFactorsE(m_cgeom->CellSize(), dt); + } +} + +std::array +PML::GetE_fp () +{ + return {pml_E_fp[0].get(), pml_E_fp[1].get(), pml_E_fp[2].get()}; +} + +std::array +PML::GetB_fp () +{ + return {pml_B_fp[0].get(), pml_B_fp[1].get(), pml_B_fp[2].get()}; +} + +std::array +PML::GetE_cp () +{ + return {pml_E_cp[0].get(), pml_E_cp[1].get(), pml_E_cp[2].get()}; +} + +std::array +PML::GetB_cp () +{ + return {pml_B_cp[0].get(), pml_B_cp[1].get(), pml_B_cp[2].get()}; +} + +MultiFab* +PML::GetF_fp () +{ + return pml_F_fp.get(); +} + +MultiFab* +PML::GetF_cp () +{ + return pml_F_cp.get(); +} + +void +PML::ExchangeB (const std::array& B_fp, + const std::array& B_cp) +{ + ExchangeB(PatchType::fine, B_fp); + ExchangeB(PatchType::coarse, B_cp); +} + +void +PML::ExchangeB (PatchType patch_type, + const std::array& Bp) +{ + if (patch_type == PatchType::fine && pml_B_fp[0] && Bp[0]) + { + Exchange(*pml_B_fp[0], *Bp[0], *m_geom); + Exchange(*pml_B_fp[1], *Bp[1], *m_geom); + Exchange(*pml_B_fp[2], *Bp[2], *m_geom); + } + else if (patch_type == PatchType::coarse && pml_B_cp[0] && Bp[0]) + { + Exchange(*pml_B_cp[0], *Bp[0], *m_cgeom); + Exchange(*pml_B_cp[1], *Bp[1], *m_cgeom); + Exchange(*pml_B_cp[2], *Bp[2], *m_cgeom); + } +} + +void +PML::ExchangeE (const std::array& E_fp, + const std::array& E_cp) +{ + ExchangeB(PatchType::fine, E_fp); + ExchangeB(PatchType::coarse, E_cp); +} + +void +PML::ExchangeE (PatchType patch_type, + const std::array& Ep) +{ + if (patch_type == PatchType::fine && pml_E_fp[0] && Ep[0]) + { + Exchange(*pml_E_fp[0], *Ep[0], *m_geom); + Exchange(*pml_E_fp[1], *Ep[1], *m_geom); + Exchange(*pml_E_fp[2], *Ep[2], *m_geom); + } + else if (patch_type == PatchType::coarse && pml_E_cp[0] && Ep[0]) + { + Exchange(*pml_E_cp[0], *Ep[0], *m_cgeom); + Exchange(*pml_E_cp[1], *Ep[1], *m_cgeom); + Exchange(*pml_E_cp[2], *Ep[2], *m_cgeom); + } +} + +void +PML::ExchangeF (MultiFab* F_fp, MultiFab* F_cp) +{ + ExchangeF(PatchType::fine, F_fp); + ExchangeF(PatchType::coarse, F_cp); +} + +void +PML::ExchangeF (PatchType patch_type, MultiFab* Fp) +{ + if (patch_type == PatchType::fine && pml_F_fp && Fp) { + Exchange(*pml_F_fp, *Fp, *m_geom); + } else if (patch_type == PatchType::coarse && pml_F_cp && Fp) { + Exchange(*pml_F_cp, *Fp, *m_cgeom); + } +} + +void +PML::Exchange (MultiFab& pml, MultiFab& reg, const Geometry& geom) +{ + const IntVect& ngr = reg.nGrowVect(); + const IntVect& ngp = pml.nGrowVect(); + const int ncp = pml.nComp(); + const auto& period = geom.periodicity(); + + MultiFab tmpregmf(reg.boxArray(), reg.DistributionMap(), ncp, ngr); + + if (ngp.max() > 0) // Copy from pml to the ghost cells of regular data + { + MultiFab totpmlmf(pml.boxArray(), pml.DistributionMap(), 1, 0); + MultiFab::LinComb(totpmlmf, 1.0, pml, 0, 1.0, pml, 1, 0, 1, 0); + if (ncp == 3) { + MultiFab::Add(totpmlmf,pml,2,0,1,0); + } + + MultiFab::Copy(tmpregmf, reg, 0, 0, 1, ngr); + tmpregmf.ParallelCopy(totpmlmf, 0, 0, 1, IntVect(0), ngr, period); + +#ifdef _OPENMP +#pragma omp parallel +#endif + for (MFIter mfi(reg); mfi.isValid(); ++mfi) + { + const FArrayBox& src = tmpregmf[mfi]; + FArrayBox& dst = reg[mfi]; + const BoxList& bl = amrex::boxDiff(dst.box(), mfi.validbox()); + for (const Box& bx : bl) + { + dst.copy(src, bx, 0, bx, 0, 1); + } + } + } + + // Copy from regular data to PML's first component + // Zero out the second (and third) component + MultiFab::Copy(tmpregmf,reg,0,0,1,0); + tmpregmf.setVal(0.0, 1, ncp-1, 0); + pml.ParallelCopy(tmpregmf, 0, 0, ncp, IntVect(0), ngp, period); +} + +void +PML::FillBoundary () +{ + FillBoundaryE(); + FillBoundaryB(); + FillBoundaryF(); +} + +void +PML::FillBoundaryE () +{ + FillBoundaryE(PatchType::fine); + FillBoundaryE(PatchType::coarse); +} + +void +PML::FillBoundaryE (PatchType patch_type) +{ + if (patch_type == PatchType::fine && pml_E_fp[0] && pml_E_fp[0]->nGrowVect().max() > 0) + { + const auto& period = m_geom->periodicity(); + Vector mf{pml_E_fp[0].get(),pml_E_fp[1].get(),pml_E_fp[2].get()}; + amrex::FillBoundary(mf, period); + } + else if (patch_type == PatchType::coarse && pml_E_cp[0] && pml_E_cp[0]->nGrowVect().max() > 0) + { + const auto& period = m_cgeom->periodicity(); + Vector mf{pml_E_cp[0].get(),pml_E_cp[1].get(),pml_E_cp[2].get()}; + amrex::FillBoundary(mf, period); + } +} + +void +PML::FillBoundaryB () +{ + FillBoundaryB(PatchType::fine); + FillBoundaryB(PatchType::coarse); +} + +void +PML::FillBoundaryB (PatchType patch_type) +{ + if (patch_type == PatchType::fine && pml_B_fp[0]) + { + const auto& period = m_geom->periodicity(); + Vector mf{pml_B_fp[0].get(),pml_B_fp[1].get(),pml_B_fp[2].get()}; + amrex::FillBoundary(mf, period); + } + else if (patch_type == PatchType::coarse && pml_B_cp[0]) + { + const auto& period = m_cgeom->periodicity(); + Vector mf{pml_B_cp[0].get(),pml_B_cp[1].get(),pml_B_cp[2].get()}; + amrex::FillBoundary(mf, period); + } +} + +void +PML::FillBoundaryF () +{ + FillBoundaryF(PatchType::fine); + FillBoundaryF(PatchType::coarse); +} + +void +PML::FillBoundaryF (PatchType patch_type) +{ + if (patch_type == PatchType::fine && pml_F_fp && pml_F_fp->nGrowVect().max() > 0) + { + const auto& period = m_geom->periodicity(); + pml_F_fp->FillBoundary(period); + } + else if (patch_type == PatchType::coarse && pml_F_cp && pml_F_cp->nGrowVect().max() > 0) + { + const auto& period = m_cgeom->periodicity(); + pml_F_cp->FillBoundary(period); + } +} + +void +PML::CheckPoint (const std::string& dir) const +{ + if (pml_E_fp[0]) + { + VisMF::Write(*pml_E_fp[0], dir+"_Ex_fp"); + VisMF::Write(*pml_E_fp[1], dir+"_Ey_fp"); + VisMF::Write(*pml_E_fp[2], dir+"_Ez_fp"); + VisMF::Write(*pml_B_fp[0], dir+"_Bx_fp"); + VisMF::Write(*pml_B_fp[1], dir+"_By_fp"); + VisMF::Write(*pml_B_fp[2], dir+"_Bz_fp"); + } + + if (pml_E_cp[0]) + { + VisMF::Write(*pml_E_cp[0], dir+"_Ex_cp"); + VisMF::Write(*pml_E_cp[1], dir+"_Ey_cp"); + VisMF::Write(*pml_E_cp[2], dir+"_Ez_cp"); + VisMF::Write(*pml_B_cp[0], dir+"_Bx_cp"); + VisMF::Write(*pml_B_cp[1], dir+"_By_cp"); + VisMF::Write(*pml_B_cp[2], dir+"_Bz_cp"); + } +} + +void +PML::Restart (const std::string& dir) +{ + if (pml_E_fp[0]) + { + VisMF::Read(*pml_E_fp[0], dir+"_Ex_fp"); + VisMF::Read(*pml_E_fp[1], dir+"_Ey_fp"); + VisMF::Read(*pml_E_fp[2], dir+"_Ez_fp"); + VisMF::Read(*pml_B_fp[0], dir+"_Bx_fp"); + VisMF::Read(*pml_B_fp[1], dir+"_By_fp"); + VisMF::Read(*pml_B_fp[2], dir+"_Bz_fp"); + } + + if (pml_E_cp[0]) + { + VisMF::Read(*pml_E_cp[0], dir+"_Ex_cp"); + VisMF::Read(*pml_E_cp[1], dir+"_Ey_cp"); + VisMF::Read(*pml_E_cp[2], dir+"_Ez_cp"); + VisMF::Read(*pml_B_cp[0], dir+"_Bx_cp"); + VisMF::Read(*pml_B_cp[1], dir+"_By_cp"); + VisMF::Read(*pml_B_cp[2], dir+"_Bz_cp"); + } +} diff --git a/Source/BoundaryConditions/PML_routines.F90 b/Source/BoundaryConditions/PML_routines.F90 new file mode 100644 index 000000000..380e52934 --- /dev/null +++ b/Source/BoundaryConditions/PML_routines.F90 @@ -0,0 +1,1009 @@ +module warpx_pml_module + + use iso_c_binding + use amrex_fort_module, only : amrex_real + + implicit none + +contains + + subroutine warpx_push_pml_bvec_3d (xlo, xhi, ylo, yhi, zlo, zhi, & + & Ex, Exlo, Exhi, & + & Ey, Eylo, Eyhi, & + & Ez, Ezlo, Ezhi, & + & Bx, Bxlo, Bxhi, & + & By, Bylo, Byhi, & + & Bz, Bzlo, Bzhi, & + & dtsdx, dtsdy, dtsdz, solver_type) & + bind(c,name='warpx_push_pml_bvec_3d') + use amrex_constants_module, only : one, two, four, eighth + integer, intent(in) :: xlo(3), xhi(3), ylo(3), yhi(3), zlo(3), zhi(3), & + Exlo(3), Exhi(3), Eylo(3), Eyhi(3), Ezlo(3), Ezhi(3), & + Bxlo(3), Bxhi(3), Bylo(3), Byhi(3), Bzlo(3), Bzhi(3), solver_type + real(amrex_real), intent(in ) :: Ex (Exlo(1):Exhi(1),Exlo(2):Exhi(2),Exlo(3):Exhi(3),3) + real(amrex_real), intent(in ) :: Ey (Eylo(1):Eyhi(1),Eylo(2):Eyhi(2),Eylo(3):Eyhi(3),3) + real(amrex_real), intent(in ) :: Ez (Ezlo(1):Ezhi(1),Ezlo(2):Ezhi(2),Ezlo(3):Ezhi(3),3) + real(amrex_real), intent(inout) :: Bx (Bxlo(1):Bxhi(1),Bxlo(2):Bxhi(2),Bxlo(3):Bxhi(3),2) + real(amrex_real), intent(inout) :: By (Bylo(1):Byhi(1),Bylo(2):Byhi(2),Bylo(3):Byhi(3),2) + real(amrex_real), intent(inout) :: Bz (Bzlo(1):Bzhi(1),Bzlo(2):Bzhi(2),Bzlo(3):Bzhi(3),2) + real(amrex_real), intent(in) :: dtsdx, dtsdy, dtsdz + + real(amrex_real), parameter :: sixteenth = 1.d0/16.d0 + + integer :: i, j, k + + real(amrex_real) :: delta, rx, ry, rz, betaxz, betaxy, betayx, betayz, betazx, betazy + real(amrex_real) :: beta, alphax, alphay, alphaz, gammax, gammay, gammaz + + ! solver_type: 0=Yee; 1=CKC + + if (solver_type==0) then + + ! Yee push + + do k = xlo(3), xhi(3) + do j = xlo(2), xhi(2) + do i = xlo(1), xhi(1) + Bx(i,j,k,1) = Bx(i,j,k,1) - dtsdy*(Ez(i,j+1,k ,1)+Ez(i,j+1,k ,2)+Ez(i,j+1,k ,3) & + & -Ez(i,j ,k ,1)-Ez(i,j ,k ,2)-Ez(i,j ,k ,3)) + Bx(i,j,k,2) = Bx(i,j,k,2) + dtsdz*(Ey(i,j ,k+1,1)+Ey(i,j ,k+1,2)+Ey(i,j ,k+1,3) & + & -Ey(i,j ,k ,1)-Ey(i,j ,k ,2)-Ey(i,j ,k ,3)) + end do + end do + end do + + do k = ylo(3), yhi(3) + do j = ylo(2), yhi(2) + do i = ylo(1), yhi(1) + By(i,j,k,1) = By(i,j,k,1) - dtsdz*(Ex(i ,j,k+1,1)+Ex(i ,j,k+1,2)+Ex(i ,j,k+1,3) & + & -Ex(i ,j,k ,1)-Ex(i ,j,k ,2)-Ex(i ,j,k ,3)) + By(i,j,k,2) = By(i,j,k,2) + dtsdx*(Ez(i+1,j,k ,1)+Ez(i+1,j,k ,2)+Ez(i+1,j,k ,3) & + & -Ez(i ,j,k ,1)-Ez(i ,j,k ,2)-Ez(i ,j,k ,3)) + end do + end do + end do + + do k = zlo(3), zhi(3) + do j = zlo(2), zhi(2) + do i = zlo(1), zhi(1) + Bz(i,j,k,1) = Bz(i,j,k,1) - dtsdx*(Ey(i+1,j ,k,1)+Ey(i+1,j ,k,2)+Ey(i+1,j ,k,3) & + & -Ey(i ,j ,k,1)-Ey(i ,j ,k,2)-Ey(i ,j ,k,3)) + Bz(i,j,k,2) = Bz(i,j,k,2) + dtsdy*(Ex(i ,j+1,k,1)+Ex(i ,j+1,k,2)+Ex(i ,j+1,k,3) & + & -Ex(i ,j ,k,1)-Ex(i ,j ,k,2)-Ex(i ,j ,k,3)) + end do + end do + end do + + else + + ! CKC push + + ! computes coefficients according to Cowan - PRST-AB 16, 041303 (2013) + delta = max(dtsdx,dtsdy,dtsdz) + rx = (dtsdx/delta)**2 + ry = (dtsdy/delta)**2 + rz = (dtsdz/delta)**2 + beta = eighth*(one-rx*ry*rz/(ry*rz+rz*rx+rx*ry)) + betaxy = ry*beta + betaxz = rz*beta + betayx = rx*beta + betayz = rz*beta + betazx = rx*beta + betazy = ry*beta + gammax = ry*rz*(sixteenth-eighth*ry*rz/(ry*rz+rz*rx+rx*ry)) + gammay = rx*rz*(sixteenth-eighth*rx*rz/(ry*rz+rz*rx+rx*ry)) + gammaz = rx*ry*(sixteenth-eighth*rx*ry/(ry*rz+rz*rx+rx*ry)) + alphax = one - two*betaxy - two* betaxz - four*gammax + alphay = one - two*betayx - two* betayz - four*gammay + alphaz = one - two*betazx - two* betazy - four*gammaz + + betaxy = dtsdx*betaxy + betaxz = dtsdx*betaxz + betayx = dtsdy*betayx + betayz = dtsdy*betayz + betazx = dtsdz*betazx + betazy = dtsdz*betazy + alphax = dtsdx*alphax + alphay = dtsdy*alphay + alphaz = dtsdz*alphaz + gammax = dtsdx*gammax + gammay = dtsdy*gammay + gammaz = dtsdz*gammaz + + do k = xlo(3), xhi(3) + do j = xlo(2), xhi(2) + do i = xlo(1), xhi(1) + Bx(i,j,k,1) = Bx(i,j,k,1) - (alphay*(Ez(i ,j+1,k ,1)+Ez(i ,j+1,k ,2)+Ez(i ,j+1,k ,3) & + -Ez(i ,j ,k ,1)-Ez(i ,j ,k ,2)-Ez(i ,j ,k ,3)) & + +betayx*(Ez(i+1,j+1,k ,1)+Ez(i+1,j+1,k ,2)+Ez(i+1,j+1,k ,3) & + -Ez(i+1,j ,k ,1)-Ez(i+1,j ,k ,2)-Ez(i+1,j ,k ,3) & + +Ez(i-1,j+1,k ,1)+Ez(i-1,j+1,k ,2)+Ez(i-1,j+1,k ,3) & + -Ez(i-1,j ,k ,1)-Ez(i-1,j ,k ,2)-Ez(i-1,j ,k ,3)) & + +betayz*(Ez(i ,j+1,k+1,1)+Ez(i ,j+1,k+1,2)+Ez(i ,j+1,k+1,3) & + -Ez(i ,j ,k+1,1)-Ez(i ,j ,k+1,2)-Ez(i ,j ,k+1,3) & + +Ez(i ,j+1,k-1,1)+Ez(i ,j+1,k-1,2)+Ez(i ,j+1,k-1,3) & + -Ez(i ,j ,k-1,1)-Ez(i ,j ,k-1,2)-Ez(i ,j ,k-1,3)) & + +gammay*(Ez(i+1,j+1,k+1,1)+Ez(i+1,j+1,k+1,2)+Ez(i+1,j+1,k+1,3) & + -Ez(i+1,j ,k+1,1)-Ez(i+1,j ,k+1,2)-Ez(i+1,j ,k+1,3) & + +Ez(i-1,j+1,k+1,1)+Ez(i-1,j+1,k+1,2)+Ez(i-1,j+1,k+1,3) & + -Ez(i-1,j ,k+1,1)-Ez(i-1,j ,k+1,2)-Ez(i-1,j ,k+1,3) & + +Ez(i+1,j+1,k-1,1)+Ez(i+1,j+1,k-1,2)+Ez(i+1,j+1,k-1,3) & + -Ez(i+1,j ,k-1,1)-Ez(i+1,j ,k-1,2)-Ez(i+1,j ,k-1,3) & + +Ez(i-1,j+1,k-1,1)+Ez(i-1,j+1,k-1,2)+Ez(i-1,j+1,k-1,3) & + -Ez(i-1,j ,k-1,1)-Ez(i-1,j ,k-1,2)-Ez(i-1,j ,k-1,3))) + + + Bx(i,j,k,2) = Bx(i,j,k,2) + (alphaz*(Ey(i ,j ,k+1,1)+Ey(i ,j ,k+1,2)+Ey(i ,j ,k+1,3) & + -Ey(i ,j ,k ,1)-Ey(i ,j ,k ,2)-Ey(i ,j ,k ,3)) & + +betazx*(Ey(i+1,j ,k+1,1)+Ey(i+1,j ,k+1,2)+Ey(i+1,j ,k+1,3) & + -Ey(i+1,j ,k ,1)-Ey(i+1,j ,k ,2)-Ey(i+1,j ,k ,3) & + +Ey(i-1,j ,k+1,1)+Ey(i-1,j ,k+1,2)+Ey(i-1,j ,k+1,3) & + -Ey(i-1,j ,k ,1)-Ey(i-1,j ,k ,2)-Ey(i-1,j ,k ,3)) & + +betazy*(Ey(i ,j+1,k+1,1)+Ey(i ,j+1,k+1,2)+Ey(i ,j+1,k+1,3) & + -Ey(i ,j+1,k ,1)-Ey(i ,j+1,k ,2)-Ey(i ,j+1,k ,3) & + +Ey(i ,j-1,k+1,1)+Ey(i ,j-1,k+1,2)+Ey(i ,j-1,k+1,3) & + -Ey(i ,j-1,k ,1)-Ey(i ,j-1,k ,2)-Ey(i ,j-1,k ,3)) & + +gammaz*(Ey(i+1,j+1,k+1,1)+Ey(i+1,j+1,k+1,2)+Ey(i+1,j+1,k+1,3) & + -Ey(i+1,j+1,k ,1)-Ey(i+1,j+1,k ,2)-Ey(i+1,j+1,k ,3) & + +Ey(i-1,j+1,k+1,1)+Ey(i-1,j+1,k+1,2)+Ey(i-1,j+1,k+1,3) & + -Ey(i-1,j+1,k ,1)-Ey(i-1,j+1,k ,2)-Ey(i-1,j+1,k ,3) & + +Ey(i+1,j-1,k+1,1)+Ey(i+1,j-1,k+1,2)+Ey(i+1,j-1,k+1,3) & + -Ey(i+1,j-1,k ,1)-Ey(i+1,j-1,k ,2)-Ey(i+1,j-1,k ,3) & + +Ey(i-1,j-1,k+1,1)+Ey(i-1,j-1,k+1,2)+Ey(i-1,j-1,k+1,3) & + -Ey(i-1,j-1,k ,1)-Ey(i-1,j-1,k ,2)-Ey(i-1,j-1,k ,3))) + end do + end do + end do + + do k = ylo(3), yhi(3) + do j = ylo(2), yhi(2) + do i = ylo(1), yhi(1) + By(i,j,k,1) = By(i,j,k,1) - (alphaz*(Ex(i ,j ,k+1,1)+Ex(i ,j ,k+1,2)+Ex(i ,j ,k+1,3) & + -Ex(i ,j ,k ,1)-Ex(i ,j ,k ,2)-Ex(i ,j ,k ,3)) & + +betazx*(Ex(i+1,j ,k+1,1)+Ex(i+1,j ,k+1,2)+Ex(i+1,j ,k+1,3) & + -Ex(i+1,j ,k ,1)-Ex(i+1,j ,k ,2)-Ex(i+1,j ,k ,3) & + +Ex(i-1,j ,k+1,1)+Ex(i-1,j ,k+1,2)+Ex(i-1,j ,k+1,3) & + -Ex(i-1,j ,k ,1)-Ex(i-1,j ,k ,2)-Ex(i-1,j ,k ,3)) & + +betaxy*(Ex(i ,j+1,k+1,1)+Ex(i ,j+1,k+1,2)+Ex(i ,j+1,k+1,3) & + -Ex(i ,j+1,k ,1)-Ex(i ,j+1,k ,2)-Ex(i ,j+1,k ,3) & + +Ex(i ,j-1,k+1,1)+Ex(i ,j-1,k+1,2)+Ex(i ,j-1,k+1,3) & + -Ex(i ,j-1,k ,1)-Ex(i ,j-1,k ,2)-Ex(i ,j-1,k ,3)) & + +gammaz*(Ex(i+1,j+1,k+1,1)+Ex(i+1,j+1,k+1,2)+Ex(i+1,j+1,k+1,3) & + -Ex(i+1,j+1,k ,1)-Ex(i+1,j+1,k ,2)-Ex(i+1,j+1,k ,3) & + +Ex(i-1,j+1,k+1,1)+Ex(i-1,j+1,k+1,2)+Ex(i-1,j+1,k+1,3) & + -Ex(i-1,j+1,k ,1)-Ex(i-1,j+1,k ,2)-Ex(i-1,j+1,k ,3) & + +Ex(i+1,j-1,k+1,1)+Ex(i+1,j-1,k+1,2)+Ex(i+1,j-1,k+1,3) & + -Ex(i+1,j-1,k ,1)-Ex(i+1,j-1,k ,2)-Ex(i+1,j-1,k ,3) & + +Ex(i-1,j-1,k+1,1)+Ex(i-1,j-1,k+1,2)+Ex(i-1,j-1,k+1,3) & + -Ex(i-1,j-1,k ,1)-Ex(i-1,j-1,k ,2)-Ex(i-1,j-1,k ,3))) + + + By(i,j,k,2) = By(i,j,k,2) + (alphax*(Ez(i+1,j ,k ,1)+Ez(i+1,j ,k ,2)+Ez(i+1,j ,k ,3) & + -Ez(i ,j ,k ,1)-Ez(i ,j ,k ,2)-Ez(i ,j ,k ,3)) & + +betaxy*(Ez(i+1,j+1,k ,1)+Ez(i+1,j+1,k ,2)+Ez(i+1,j+1,k ,3) & + -Ez(i ,j+1,k ,1)-Ez(i ,j+1,k ,2)-Ez(i ,j+1,k ,3) & + +Ez(i+1,j-1,k ,1)+Ez(i+1,j-1,k ,2)+Ez(i+1,j-1,k ,3) & + -Ez(i ,j-1,k ,1)-Ez(i ,j-1,k ,2)-Ez(i ,j-1,k ,3)) & + +betaxz*(Ez(i+1,j ,k+1,1)+Ez(i+1,j ,k+1,2)+Ez(i+1,j ,k+1,3) & + -Ez(i ,j ,k+1,1)-Ez(i ,j ,k+1,2)-Ez(i ,j ,k+1,3) & + +Ez(i+1,j ,k-1,1)+Ez(i+1,j ,k-1,2)+Ez(i+1,j ,k-1,3) & + -Ez(i ,j ,k-1,1)-Ez(i ,j ,k-1,2)-Ez(i ,j ,k-1,3)) & + +gammax*(Ez(i+1,j+1,k+1,1)+Ez(i+1,j+1,k+1,2)+Ez(i+1,j+1,k+1,3) & + -Ez(i ,j+1,k+1,1)-Ez(i ,j+1,k+1,2)-Ez(i ,j+1,k+1,3) & + +Ez(i+1,j-1,k+1,1)+Ez(i+1,j-1,k+1,2)+Ez(i+1,j-1,k+1,3) & + -Ez(i ,j-1,k+1,1)-Ez(i ,j-1,k+1,2)-Ez(i ,j-1,k+1,3) & + +Ez(i+1,j+1,k-1,1)+Ez(i+1,j+1,k-1,2)+Ez(i+1,j+1,k-1,3) & + -Ez(i ,j+1,k-1,1)-Ez(i ,j+1,k-1,2)-Ez(i ,j+1,k-1,3) & + +Ez(i+1,j-1,k-1,1)+Ez(i+1,j-1,k-1,2)+Ez(i+1,j-1,k-1,3) & + -Ez(i ,j-1,k-1,1)-Ez(i ,j-1,k-1,2)-Ez(i ,j-1,k-1,3))) + end do + end do + end do + + do k = zlo(3), zhi(3) + do j = zlo(2), zhi(2) + do i = zlo(1), zhi(1) + Bz(i,j,k,1) = Bz(i,j,k,1) - (alphax*(Ey(i+1,j ,k ,1)+Ey(i+1,j ,k ,2)+Ey(i+1,j ,k ,3) & + -Ey(i ,j ,k ,1)-Ey(i ,j ,k ,2)-Ey(i ,j ,k ,3)) & + +betaxy*(Ey(i+1,j+1,k ,1)+Ey(i+1,j+1,k ,2)+Ey(i+1,j+1,k ,3) & + -Ey(i ,j+1,k ,1)-Ey(i ,j+1,k ,2)-Ey(i ,j+1,k ,3) & + +Ey(i+1,j-1,k ,1)+Ey(i+1,j-1,k ,2)+Ey(i+1,j-1,k ,3) & + -Ey(i ,j-1,k ,1)-Ey(i ,j-1,k ,2)-Ey(i ,j-1,k ,3)) & + +betaxz*(Ey(i+1,j ,k+1,1)+Ey(i+1,j ,k+1,2)+Ey(i+1,j ,k+1,3) & + -Ey(i ,j ,k+1,1)-Ey(i ,j ,k+1,2)-Ey(i ,j ,k+1,3) & + +Ey(i+1,j ,k-1,1)+Ey(i+1,j ,k-1,2)+Ey(i+1,j ,k-1,3) & + -Ey(i ,j ,k-1,1)-Ey(i ,j ,k-1,2)-Ey(i ,j ,k-1,3)) & + +gammax*(Ey(i+1,j+1,k+1,1)+Ey(i+1,j+1,k+1,2)+Ey(i+1,j+1,k+1,3) & + -Ey(i ,j+1,k+1,1)-Ey(i ,j+1,k+1,2)-Ey(i ,j+1,k+1,3) & + +Ey(i+1,j-1,k+1,1)+Ey(i+1,j-1,k+1,2)+Ey(i+1,j-1,k+1,3) & + -Ey(i ,j-1,k+1,1)-Ey(i ,j-1,k+1,2)-Ey(i ,j-1,k+1,3) & + +Ey(i+1,j+1,k-1,1)+Ey(i+1,j+1,k-1,2)+Ey(i+1,j+1,k-1,3) & + -Ey(i ,j+1,k-1,1)-Ey(i ,j+1,k-1,2)-Ey(i ,j+1,k-1,3) & + +Ey(i+1,j-1,k-1,1)+Ey(i+1,j-1,k-1,2)+Ey(i+1,j-1,k-1,3) & + -Ey(i ,j-1,k-1,1)-Ey(i ,j-1,k-1,2)-Ey(i ,j-1,k-1,3))) + + Bz(i,j,k,2) = Bz(i,j,k,2) + (alphay*(Ex(i ,j+1,k ,1)+Ex(i ,j+1,k ,2)+Ex(i ,j+1,k ,3) & + -Ex(i ,j ,k ,1)-Ex(i ,j ,k ,2)-Ex(i ,j ,k ,3)) & + +betayx*(Ex(i+1,j+1,k ,1)+Ex(i+1,j+1,k ,2)+Ex(i+1,j+1,k ,3) & + -Ex(i+1,j ,k ,1)-Ex(i+1,j ,k ,2)-Ex(i+1,j ,k ,3) & + +Ex(i-1,j+1,k ,1)+Ex(i-1,j+1,k ,2)+Ex(i-1,j+1,k ,3) & + -Ex(i-1,j ,k ,1)-Ex(i-1,j ,k ,2)-Ex(i-1,j ,k ,3)) & + +betayz*(Ex(i ,j+1,k+1,1)+Ex(i ,j+1,k+1,2)+Ex(i ,j+1,k+1,3) & + -Ex(i ,j ,k+1,1)-Ex(i ,j ,k+1,2)-Ex(i ,j ,k+1,3) & + +Ex(i ,j+1,k-1,1)+Ex(i ,j+1,k-1,2)+Ex(i ,j+1,k-1,3) & + -Ex(i ,j ,k-1,1)-Ex(i ,j ,k-1,2)-Ex(i ,j ,k-1,3)) & + +gammay*(Ex(i+1,j+1,k+1,1)+Ex(i+1,j+1,k+1,2)+Ex(i+1,j+1,k+1,3) & + -Ex(i+1,j ,k+1,1)-Ex(i+1,j ,k+1,2)-Ex(i+1,j ,k+1,3) & + +Ex(i-1,j+1,k+1,1)+Ex(i-1,j+1,k+1,2)+Ex(i-1,j+1,k+1,3) & + -Ex(i-1,j ,k+1,1)-Ex(i-1,j ,k+1,2)-Ex(i-1,j ,k+1,3) & + +Ex(i+1,j+1,k-1,1)+Ex(i+1,j+1,k-1,2)+Ex(i+1,j+1,k-1,3) & + -Ex(i+1,j ,k-1,1)-Ex(i+1,j ,k-1,2)-Ex(i+1,j ,k-1,3) & + +Ex(i-1,j+1,k-1,1)+Ex(i-1,j+1,k-1,2)+Ex(i-1,j+1,k-1,3) & + -Ex(i-1,j ,k-1,1)-Ex(i-1,j ,k-1,2)-Ex(i-1,j ,k-1,3))) + end do + end do + end do + + end if + + end subroutine warpx_push_pml_bvec_3d + + subroutine warpx_push_pml_evec_3d (xlo, xhi, ylo, yhi, zlo, zhi, & + & Ex, Exlo, Exhi, & + & Ey, Eylo, Eyhi, & + & Ez, Ezlo, Ezhi, & + & Bx, Bxlo, Bxhi, & + & By, Bylo, Byhi, & + & Bz, Bzlo, Bzhi, & + & dtsdx, dtsdy, dtsdz) & + bind(c,name='warpx_push_pml_evec_3d') + integer, intent(in) :: xlo(3), xhi(3), ylo(3), yhi(3), zlo(3), zhi(3), & + Exlo(3), Exhi(3), Eylo(3), Eyhi(3), Ezlo(3), Ezhi(3), & + Bxlo(3), Bxhi(3), Bylo(3), Byhi(3), Bzlo(3), Bzhi(3) + real(amrex_real), intent(in) :: dtsdx, dtsdy, dtsdz + real(amrex_real), intent(inout) :: Ex (Exlo(1):Exhi(1),Exlo(2):Exhi(2),Exlo(3):Exhi(3),2) + real(amrex_real), intent(inout) :: Ey (Eylo(1):Eyhi(1),Eylo(2):Eyhi(2),Eylo(3):Eyhi(3),2) + real(amrex_real), intent(inout) :: Ez (Ezlo(1):Ezhi(1),Ezlo(2):Ezhi(2),Ezlo(3):Ezhi(3),2) + real(amrex_real), intent(in ) :: Bx (Bxlo(1):Bxhi(1),Bxlo(2):Bxhi(2),Bxlo(3):Bxhi(3),2) + real(amrex_real), intent(in ) :: By (Bylo(1):Byhi(1),Bylo(2):Byhi(2),Bylo(3):Byhi(3),2) + real(amrex_real), intent(in ) :: Bz (Bzlo(1):Bzhi(1),Bzlo(2):Bzhi(2),Bzlo(3):Bzhi(3),2) + + integer :: i, j, k + + do k = xlo(3), xhi(3) + do j = xlo(2), xhi(2) + do i = xlo(1), xhi(1) + Ex(i,j,k,1) = Ex(i,j,k,1) + dtsdy*(Bz(i,j ,k ,1)+Bz(i,j ,k ,2) & + & -Bz(i,j-1,k ,1)-Bz(i,j-1,k ,2)) + Ex(i,j,k,2) = Ex(i,j,k,2) - dtsdz*(By(i,j ,k ,1)+By(i,j ,k ,2) & + & -By(i,j ,k-1,1)-By(i,j ,k-1,2)) + end do + end do + end do + + do k = ylo(3), yhi(3) + do j = ylo(2), yhi(2) + do i = ylo(1), yhi(1) + Ey(i,j,k,1) = Ey(i,j,k,1) + dtsdz*(Bx(i ,j,k ,1)+Bx(i ,j,k ,2) & + & -Bx(i ,j,k-1,1)-Bx(i ,j,k-1,2)) + Ey(i,j,k,2) = Ey(i,j,k,2) - dtsdx*(Bz(i ,j,k ,1)+Bz(i ,j,k ,2) & + & -Bz(i-1,j,k ,1)-Bz(i-1,j,k ,2)) + end do + end do + end do + + do k = zlo(3), zhi(3) + do j = zlo(2), zhi(2) + do i = zlo(1), zhi(1) + Ez(i,j,k,1) = Ez(i,j,k,1) + dtsdx*(By(i ,j ,k,1)+By(i ,j ,k,2) & + & -By(i-1,j ,k,1)-By(i-1,j ,k,2)) + Ez(i,j,k,2) = Ez(i,j,k,2) - dtsdy*(Bx(i ,j ,k,1)+Bx(i ,j ,k,2) & + & -Bx(i ,j-1,k,1)-Bx(i ,j-1,k,2)) + end do + end do + end do + + end subroutine warpx_push_pml_evec_3d + + subroutine warpx_push_pml_bvec_2d (xlo, xhi, ylo, yhi, zlo, zhi, & + & Ex, Exlo, Exhi, & + & Ey, Eylo, Eyhi, & + & Ez, Ezlo, Ezhi, & + & Bx, Bxlo, Bxhi, & + & By, Bylo, Byhi, & + & Bz, Bzlo, Bzhi, & + & dtsdx, dtsdy, dtsdz, solver_type) & + bind(c,name='warpx_push_pml_bvec_2d') + use amrex_constants_module, only : one, two, eighth + integer, intent(in) :: xlo(2), xhi(2), ylo(2), yhi(2), zlo(2), zhi(2), & + Exlo(2), Exhi(2), Eylo(2), Eyhi(2), Ezlo(2), Ezhi(2), & + Bxlo(2), Bxhi(2), Bylo(2), Byhi(2), Bzlo(2), Bzhi(2), solver_type + real(amrex_real), intent(in ) :: Ex (Exlo(1):Exhi(1),Exlo(2):Exhi(2),3) + real(amrex_real), intent(in ) :: Ey (Eylo(1):Eyhi(1),Eylo(2):Eyhi(2),3) + real(amrex_real), intent(in ) :: Ez (Ezlo(1):Ezhi(1),Ezlo(2):Ezhi(2),3) + real(amrex_real), intent(inout) :: Bx (Bxlo(1):Bxhi(1),Bxlo(2):Bxhi(2),2) + real(amrex_real), intent(inout) :: By (Bylo(1):Byhi(1),Bylo(2):Byhi(2),2) + real(amrex_real), intent(inout) :: Bz (Bzlo(1):Bzhi(1),Bzlo(2):Bzhi(2),2) + real(amrex_real), intent(in) :: dtsdx, dtsdy, dtsdz + + integer :: i, k + + real(amrex_real) :: delta, rx, rz, betaxz, betazx, alphax, alphaz + + + ! solver_type: 0=Yee; 1=CKC + + if (solver_type==0) then + + ! Yee push + + do k = xlo(2), xhi(2) + do i = xlo(1), xhi(1) + Bx(i,k,2) = Bx(i,k,2) + dtsdz*(Ey(i,k+1,1)+Ey(i,k+1,2)+Ey(i,k+1,3) & + & -Ey(i,k ,1)-Ey(i,k ,2)-Ey(i,k ,3)) + end do + end do + + do k = ylo(2), yhi(2) + do i = ylo(1), yhi(1) + By(i,k,1) = By(i,k,1) - dtsdz*(Ex(i ,k+1,1)+Ex(i ,k+1,2)+Ex(i ,k+1,3) & + & -Ex(i ,k ,1)-Ex(i ,k ,2)-Ex(i ,k ,3)) + By(i,k,2) = By(i,k,2) + dtsdx*(Ez(i+1,k ,1)+Ez(i+1,k ,2)+Ez(i+1,k ,3) & + & -Ez(i ,k ,1)-Ez(i ,k ,2)-Ez(i ,k ,3)) + end do + end do + + do k = zlo(2), zhi(2) + do i = zlo(1), zhi(1) + Bz(i,k,1) = Bz(i,k,1) - dtsdx*(Ey(i+1,k,1)+Ey(i+1,k,2)+Ey(i+1,k,3) & + & -Ey(i ,k,1)-Ey(i ,k,2)-Ey(i ,k,3)) + end do + end do + + else + + ! Cole-Karkkainen-Cowan push + + ! computes coefficients according to Cowan - PRST-AB 16, 041303 (2013) + delta = max(dtsdx,dtsdz) + rx = (dtsdx/delta)**2 + rz = (dtsdz/delta)**2 + betaxz = eighth*rz + betazx = eighth*rx + alphax = one - two*betaxz + alphaz = one - two*betazx + + betaxz = dtsdx*betaxz + betazx = dtsdz*betazx + alphax = dtsdx*alphax + alphaz = dtsdz*alphaz + + do k = xlo(2), xhi(2) + do i = xlo(1), xhi(1) + Bx(i,k,2) = Bx(i,k,2) + ( alphaz*(Ey(i ,k+1,1)+Ey(i ,k+1,2)+Ey(i ,k+1,3) & + -Ey(i ,k ,1)-Ey(i ,k ,2)-Ey(i ,k ,3)) & + + betazx*(Ey(i+1,k+1,1)+Ey(i+1,k+1,2)+Ey(i+1,k+1,3) & + -Ey(i+1,k ,1)-Ey(i+1,k ,2)-Ey(i+1,k ,3) & + +Ey(i-1,k+1,1)+Ey(i-1,k+1,2)+Ey(i-1,k+1,3) & + -Ey(i-1,k ,1)-Ey(i-1,k ,2)-Ey(i-1,k ,3))) + end do + end do + + do k = ylo(2), yhi(2) + do i = ylo(1), yhi(1) + By(i,k,1) = By(i,k,1) - ( alphaz*(Ex(i ,k+1,1)+Ex(i ,k+1,2)+Ex(i ,k+1,3) & + -Ex(i ,k ,1)-Ex(i ,k ,2)-Ex(i ,k ,3)) & + + betazx*(Ex(i+1,k+1,1)+Ex(i+1,k+1,2)+Ex(i+1,k+1,3) & + -Ex(i+1,k ,1)-Ex(i+1,k ,2)-Ex(i+1,k ,3) & + +Ex(i-1,k+1,1)+Ex(i-1,k+1,2)+Ex(i-1,k+1,3) & + -Ex(i-1,k ,1)-Ex(i-1,k ,2)-Ex(i-1,k ,3))) + + By(i,k,2) = By(i,k,2) + ( alphax*(Ez(i+1,k ,1)+Ez(i+1,k ,2)+Ez(i+1,k ,3) & + -Ez(i ,k ,1)-Ez(i ,k ,2)-Ez(i ,k ,3)) & + + betaxz*(Ez(i+1,k+1,1)+Ez(i+1,k+1,2)+Ez(i+1,k+1,3) & + -Ez(i ,k+1,1)-Ez(i ,k+1,2)-Ez(i ,k+1,3) & + +Ez(i+1,k-1,1)+Ez(i+1,k-1,2)+Ez(i+1,k-1,3) & + -Ez(i ,k-1,1)-Ez(i ,k-1,2)-Ez(i ,k-1,3))) + + end do + end do + + do k = zlo(2), zhi(2) + do i = zlo(1), zhi(1) + Bz(i,k,1) = Bz(i,k,1) - ( alphax*(Ey(i+1,k ,1)+Ey(i+1,k ,2)+Ey(i+1,k ,3) & + -Ey(i ,k ,1)-Ey(i ,k ,2)-Ey(i ,k ,3)) & + + betaxz*(Ey(i+1,k+1,1)+Ey(i+1,k+1,2)+Ey(i+1,k+1,3) & + -Ey(i ,k+1,1)-Ey(i ,k+1,2)-Ey(i ,k+1,3) & + +Ey(i+1,k-1,1)+Ey(i+1,k-1,2)+Ey(i+1,k-1,3) & + -Ey(i ,k-1,1)-Ey(i ,k-1,2)-Ey(i ,k-1,3))) + end do + end do + + end if + + + end subroutine warpx_push_pml_bvec_2d + + subroutine warpx_push_pml_evec_2d (xlo, xhi, ylo, yhi, zlo, zhi, & + & Ex, Exlo, Exhi, & + & Ey, Eylo, Eyhi, & + & Ez, Ezlo, Ezhi, & + & Bx, Bxlo, Bxhi, & + & By, Bylo, Byhi, & + & Bz, Bzlo, Bzhi, & + & dtsdx, dtsdy, dtsdz) & + bind(c,name='warpx_push_pml_evec_2d') + integer, intent(in) :: xlo(2), xhi(2), ylo(2), yhi(2), zlo(2), zhi(2), & + Exlo(2), Exhi(2), Eylo(2), Eyhi(2), Ezlo(2), Ezhi(2), & + Bxlo(2), Bxhi(2), Bylo(2), Byhi(2), Bzlo(2), Bzhi(2) + real(amrex_real), intent(in) :: dtsdx, dtsdy, dtsdz + real(amrex_real), intent(inout) :: Ex (Exlo(1):Exhi(1),Exlo(2):Exhi(2),2) + real(amrex_real), intent(inout) :: Ey (Eylo(1):Eyhi(1),Eylo(2):Eyhi(2),2) + real(amrex_real), intent(inout) :: Ez (Ezlo(1):Ezhi(1),Ezlo(2):Ezhi(2),2) + real(amrex_real), intent(in ) :: Bx (Bxlo(1):Bxhi(1),Bxlo(2):Bxhi(2),2) + real(amrex_real), intent(in ) :: By (Bylo(1):Byhi(1),Bylo(2):Byhi(2),2) + real(amrex_real), intent(in ) :: Bz (Bzlo(1):Bzhi(1),Bzlo(2):Bzhi(2),2) + + integer :: i, k + + do k = xlo(2), xhi(2) + do i = xlo(1), xhi(1) + Ex(i,k,2) = Ex(i,k,2) - dtsdz*(By(i,k ,1)+By(i,k ,2) & + & -By(i,k-1,1)-By(i,k-1,2)) + end do + end do + + do k = ylo(2), yhi(2) + do i = ylo(1), yhi(1) + Ey(i,k,1) = Ey(i,k,1) + dtsdz*(Bx(i ,k ,1)+Bx(i ,k ,2) & + & -Bx(i ,k-1,1)-Bx(i ,k-1,2)) + Ey(i,k,2) = Ey(i,k,2) - dtsdx*(Bz(i ,k ,1)+Bz(i ,k ,2) & + & -Bz(i-1,k ,1)-Bz(i-1,k ,2)) + end do + end do + + do k = zlo(2), zhi(2) + do i = zlo(1), zhi(1) + Ez(i,k,1) = Ez(i,k,1) + dtsdx*(By(i ,k,1)+By(i ,k,2) & + & -By(i-1,k,1)-By(i-1,k,2)) + end do + end do + + end subroutine warpx_push_pml_evec_2d + + + subroutine warpx_push_pml_f_3d (lo, hi, & + & f , flo, fhi, & + & Ex, Exlo, Exhi, & + & Ey, Eylo, Eyhi, & + & Ez, Ezlo, Ezhi, & + & dtdx, dtdy, dtdz) & + bind(c,name='warpx_push_pml_f_3d') + integer, intent(in) :: lo(3), hi(3), Exlo(3), Exhi(3), Eylo(3), Eyhi(3), Ezlo(3), Ezhi(3), & + flo(3), fhi(3) + real(amrex_real), intent(inout) :: f ( flo(1): fhi(1), flo(2): fhi(2), flo(3): fhi(3),3) + real(amrex_real), intent(in ) :: Ex (Exlo(1):Exhi(1),Exlo(2):Exhi(2),Exlo(3):Exhi(3),3) + real(amrex_real), intent(in ) :: Ey (Eylo(1):Eyhi(1),Eylo(2):Eyhi(2),Eylo(3):Eyhi(3),3) + real(amrex_real), intent(in ) :: Ez (Ezlo(1):Ezhi(1),Ezlo(2):Ezhi(2),Ezlo(3):Ezhi(3),3) + real(amrex_real), intent(in) :: dtdx, dtdy, dtdz + + integer :: i, j, k + + do k = lo(3), hi(3) + do j = lo(2), hi(2) + do i = lo(1), hi(1) + f(i,j,k,1) = f(i,j,k,1) + dtdx*((Ex(i,j,k,1)-Ex(i-1,j,k,1)) & + & + (Ex(i,j,k,2)-Ex(i-1,j,k,2)) & + & + (Ex(i,j,k,3)-Ex(i-1,j,k,3))) + f(i,j,k,2) = f(i,j,k,2) + dtdy*((Ey(i,j,k,1)-Ey(i,j-1,k,1)) & + & + (Ey(i,j,k,2)-Ey(i,j-1,k,2)) & + & + (Ey(i,j,k,3)-Ey(i,j-1,k,3))) + f(i,j,k,3) = f(i,j,k,3) + dtdz*((Ez(i,j,k,1)-Ez(i,j,k-1,1)) & + & + (Ez(i,j,k,2)-Ez(i,j,k-1,2)) & + & + (Ez(i,j,k,3)-Ez(i,j,k-1,3))) + end do + end do + end do + end subroutine warpx_push_pml_f_3d + + subroutine warpx_push_pml_f_2d (lo, hi, & + & f , flo, fhi, & + & Ex, Exlo, Exhi, & + & Ey, Eylo, Eyhi, & + & Ez, Ezlo, Ezhi, & + & dtdx, dtdy, dtdz) & + bind(c,name='warpx_push_pml_f_2d') + integer, intent(in) :: lo(2), hi(2), Exlo(2), Exhi(2), Eylo(2), Eyhi(2), Ezlo(2), Ezhi(2), & + flo(2), fhi(2) + real(amrex_real), intent(inout) :: f ( flo(1): fhi(1), flo(2): fhi(2),3) + real(amrex_real), intent(in ) :: Ex (Exlo(1):Exhi(1),Exlo(2):Exhi(2),3) + real(amrex_real), intent(in ) :: Ey (Eylo(1):Eyhi(1),Eylo(2):Eyhi(2),3) + real(amrex_real), intent(in ) :: Ez (Ezlo(1):Ezhi(1),Ezlo(2):Ezhi(2),3) + real(amrex_real), intent(in) :: dtdx, dtdy, dtdz + + integer :: i, k + + do k = lo(2), hi(2) + do i = lo(1), hi(1) + f(i,k,1) = f(i,k,1) + dtdx*((Ex(i,k,1)-Ex(i-1,k,1)) & + & + (Ex(i,k,2)-Ex(i-1,k,2)) & + & + (Ex(i,k,3)-Ex(i-1,k,3))) + f(i,k,3) = f(i,k,3) + dtdz*((Ez(i,k,1)-Ez(i,k-1,1)) & + & + (Ez(i,k,2)-Ez(i,k-1,2)) & + & + (Ez(i,k,3)-Ez(i,k-1,3))) + end do + end do + end subroutine warpx_push_pml_f_2d + + subroutine warpx_push_pml_evec_f_3d (xlo, xhi, ylo, yhi, zlo, zhi, & + & Ex, Exlo, Exhi, & + & Ey, Eylo, Eyhi, & + & Ez, Ezlo, Ezhi, & + & f, flo, fhi, & + & dtsdx, dtsdy, dtsdz, solver_type) & + bind(c,name='warpx_push_pml_evec_f_3d') + use amrex_constants_module, only : one, two, four, eighth + integer, intent(in) :: xlo(3), xhi(3), ylo(3), yhi(3), zlo(3), zhi(3), & + Exlo(3), Exhi(3), Eylo(3), Eyhi(3), Ezlo(3), Ezhi(3), flo(3), fhi(3), & + solver_type + real(amrex_real), intent(inout) :: Ex (Exlo(1):Exhi(1),Exlo(2):Exhi(2),Exlo(3):Exhi(3),3) + real(amrex_real), intent(inout) :: Ey (Eylo(1):Eyhi(1),Eylo(2):Eyhi(2),Eylo(3):Eyhi(3),3) + real(amrex_real), intent(inout) :: Ez (Ezlo(1):Ezhi(1),Ezlo(2):Ezhi(2),Ezlo(3):Ezhi(3),3) + real(amrex_real), intent(in ) :: f ( flo(1): fhi(1), flo(2): fhi(2), flo(3): fhi(3),3) + real(amrex_real), intent(in) :: dtsdx, dtsdy, dtsdz + + real(amrex_real), parameter :: sixteenth = 1.d0/16.d0 + + integer :: i, j, k + + real(amrex_real) :: delta, rx, ry, rz, betaxz, betaxy, betayx, betayz, betazx, betazy + real(amrex_real) :: beta, alphax, alphay, alphaz, gammax, gammay, gammaz + + ! solver_type: 0=Yee; 1=CKC + + if (solver_type==0) then + + do k = xlo(3), xhi(3) + do j = xlo(2), xhi(2) + do i = xlo(1), xhi(1) + Ex(i,j,k,3) = Ex(i,j,k,3) + dtsdx*((f(i+1,j,k,1)-f(i,j,k,1)) & + & + (f(i+1,j,k,2)-f(i,j,k,2)) & + & + (f(i+1,j,k,3)-f(i,j,k,3))) + end do + end do + end do + + do k = ylo(3), yhi(3) + do j = ylo(2), yhi(2) + do i = ylo(1), yhi(1) + Ey(i,j,k,3) = Ey(i,j,k,3) + dtsdx*((f(i,j+1,k,1)-f(i,j,k,1)) & + & + (f(i,j+1,k,2)-f(i,j,k,2)) & + & + (f(i,j+1,k,3)-f(i,j,k,3))) + end do + end do + end do + + do k = zlo(3), zhi(3) + do j = zlo(2), zhi(2) + do i = zlo(1), zhi(1) + Ez(i,j,k,3) = Ez(i,j,k,3) + dtsdx*((f(i,j,k+1,1)-f(i,j,k,1)) & + & + (f(i,j,k+1,2)-f(i,j,k,2)) & + & + (f(i,j,k+1,3)-f(i,j,k,3))) + end do + end do + end do + + else + + ! CKC push + + ! computes coefficients according to Cowan - PRST-AB 16, 041303 (2013) + delta = max(dtsdx,dtsdy,dtsdz) + rx = (dtsdx/delta)**2 + ry = (dtsdy/delta)**2 + rz = (dtsdz/delta)**2 + beta = eighth*(one-rx*ry*rz/(ry*rz+rz*rx+rx*ry)) + betaxy = ry*beta + betaxz = rz*beta + betayx = rx*beta + betayz = rz*beta + betazx = rx*beta + betazy = ry*beta + gammax = ry*rz*(sixteenth-eighth*ry*rz/(ry*rz+rz*rx+rx*ry)) + gammay = rx*rz*(sixteenth-eighth*rx*rz/(ry*rz+rz*rx+rx*ry)) + gammaz = rx*ry*(sixteenth-eighth*rx*ry/(ry*rz+rz*rx+rx*ry)) + alphax = one - two*betaxy - two* betaxz - four*gammax + alphay = one - two*betayx - two* betayz - four*gammay + alphaz = one - two*betazx - two* betazy - four*gammaz + + betaxy = dtsdx*betaxy + betaxz = dtsdx*betaxz + betayx = dtsdy*betayx + betayz = dtsdy*betayz + betazx = dtsdz*betazx + betazy = dtsdz*betazy + alphax = dtsdx*alphax + alphay = dtsdy*alphay + alphaz = dtsdz*alphaz + gammax = dtsdx*gammax + gammay = dtsdy*gammay + gammaz = dtsdz*gammaz + + do k = xlo(3), xhi(3) + do j = xlo(2), xhi(2) + do i = xlo(1), xhi(1) + Ex(i,j,k,3) = Ex(i,j,k,3) & + +alphax*(f(i+1,j ,k ,1)+f(i+1,j ,k ,2)+f(i+1,j ,k ,3) & + -f(i ,j ,k ,1)-f(i ,j ,k ,2)-f(i ,j ,k ,3)) & + +betaxy*(f(i+1,j+1,k ,1)+f(i+1,j+1,k ,2)+f(i+1,j+1,k ,3) & + -f(i ,j+1,k ,1)-f(i ,j+1,k ,2)-f(i ,j+1,k ,3) & + +f(i+1,j-1,k ,1)+f(i+1,j-1,k ,2)+f(i+1,j-1,k ,3) & + -f(i ,j-1,k ,1)-f(i ,j-1,k ,2)-f(i ,j-1,k ,3)) & + +betaxz*(f(i+1,j ,k+1,1)+f(i+1,j ,k+1,2)+f(i+1,j ,k+1,3) & + -f(i ,j ,k+1,1)-f(i ,j ,k+1,2)-f(i ,j ,k+1,3) & + +f(i+1,j ,k-1,1)+f(i+1,j ,k-1,2)+f(i+1,j ,k-1,3) & + -f(i ,j ,k-1,1)-f(i ,j ,k-1,2)-f(i ,j ,k-1,3)) & + +gammax*(f(i+1,j+1,k+1,1)+f(i+1,j+1,k+1,2)+f(i+1,j+1,k+1,3) & + -f(i ,j+1,k+1,1)-f(i ,j+1,k+1,2)-f(i ,j+1,k+1,3) & + +f(i+1,j-1,k+1,1)+f(i+1,j-1,k+1,2)+f(i+1,j-1,k+1,3) & + -f(i ,j-1,k+1,1)-f(i ,j-1,k+1,2)-f(i ,j-1,k+1,3) & + +f(i+1,j+1,k-1,1)+f(i+1,j+1,k-1,2)+f(i+1,j+1,k-1,3) & + -f(i ,j+1,k-1,1)-f(i ,j+1,k-1,2)-f(i ,j+1,k-1,3) & + +f(i+1,j-1,k-1,1)+f(i+1,j-1,k-1,2)+f(i+1,j-1,k-1,3) & + -f(i ,j-1,k-1,1)-f(i ,j-1,k-1,2)-f(i ,j-1,k-1,3)) + end do + end do + end do + + do k = ylo(3), yhi(3) + do j = ylo(2), yhi(2) + do i = ylo(1), yhi(1) + Ey(i,j,k,3) = Ey(i,j,k,3) & + +alphay*(f(i ,j+1,k ,1)+f(i ,j+1,k ,2)+f(i ,j+1,k ,3) & + -f(i ,j ,k ,1)-f(i ,j ,k ,2)-f(i ,j ,k ,3)) & + +betayx*(f(i+1,j+1,k ,1)+f(i+1,j+1,k ,2)+f(i+1,j+1,k ,3) & + -f(i+1,j ,k ,1)-f(i+1,j ,k ,2)-f(i+1,j ,k ,3) & + +f(i-1,j+1,k ,1)+f(i-1,j+1,k ,2)+f(i-1,j+1,k ,3) & + -f(i-1,j ,k ,1)-f(i-1,j ,k ,2)-f(i-1,j ,k ,3)) & + +betayz*(f(i ,j+1,k+1,1)+f(i ,j+1,k+1,2)+f(i ,j+1,k+1,3) & + -f(i ,j ,k+1,1)-f(i ,j ,k+1,2)-f(i ,j ,k+1,3) & + +f(i ,j+1,k-1,1)+f(i ,j+1,k-1,2)+f(i ,j+1,k-1,3) & + -f(i ,j ,k-1,1)-f(i ,j ,k-1,2)-f(i ,j ,k-1,3)) & + +gammay*(f(i+1,j+1,k+1,1)+f(i+1,j+1,k+1,2)+f(i+1,j+1,k+1,3) & + -f(i+1,j ,k+1,1)-f(i+1,j ,k+1,2)-f(i+1,j ,k+1,3) & + +f(i-1,j+1,k+1,1)+f(i-1,j+1,k+1,2)+f(i-1,j+1,k+1,3) & + -f(i-1,j ,k+1,1)-f(i-1,j ,k+1,2)-f(i-1,j ,k+1,3) & + +f(i+1,j+1,k-1,1)+f(i+1,j+1,k-1,2)+f(i+1,j+1,k-1,3) & + -f(i+1,j ,k-1,1)-f(i+1,j ,k-1,2)-f(i+1,j ,k-1,3) & + +f(i-1,j+1,k-1,1)+f(i-1,j+1,k-1,2)+f(i-1,j+1,k-1,3) & + -f(i-1,j ,k-1,1)-f(i-1,j ,k-1,2)-f(i-1,j ,k-1,3)) + end do + end do + end do + + do k = zlo(3), zhi(3) + do j = zlo(2), zhi(2) + do i = zlo(1), zhi(1) + Ez(i,j,k,3) = Ez(i,j,k,3) & + +alphaz*(f(i ,j ,k+1,1)+f(i ,j ,k+1,2)+f(i ,j ,k+1,3) & + -f(i ,j ,k ,1)-f(i ,j ,k ,2)-f(i ,j ,k ,3)) & + +betazx*(f(i+1,j ,k+1,1)+f(i+1,j ,k+1,2)+f(i+1,j ,k+1,3) & + -f(i+1,j ,k ,1)-f(i+1,j ,k ,2)-f(i+1,j ,k ,3) & + +f(i-1,j ,k+1,1)+f(i-1,j ,k+1,2)+f(i-1,j ,k+1,3) & + -f(i-1,j ,k ,1)-f(i-1,j ,k ,2)-f(i-1,j ,k ,3)) & + +betazy*(f(i ,j+1,k+1,1)+f(i ,j+1,k+1,2)+f(i ,j+1,k+1,3) & + -f(i ,j+1,k ,1)-f(i ,j+1,k ,2)-f(i ,j+1,k ,3) & + +f(i ,j-1,k+1,1)+f(i ,j-1,k+1,2)+f(i ,j-1,k+1,3) & + -f(i ,j-1,k ,1)-f(i ,j-1,k ,2)-f(i ,j-1,k ,3)) & + +gammaz*(f(i+1,j+1,k+1,1)+f(i+1,j+1,k+1,2)+f(i+1,j+1,k+1,3) & + -f(i+1,j+1,k ,1)-f(i+1,j+1,k ,2)-f(i+1,j+1,k ,3) & + +f(i-1,j+1,k+1,1)+f(i-1,j+1,k+1,2)+f(i-1,j+1,k+1,3) & + -f(i-1,j+1,k ,1)-f(i-1,j+1,k ,2)-f(i-1,j+1,k ,3) & + +f(i+1,j-1,k+1,1)+f(i+1,j-1,k+1,2)+f(i+1,j-1,k+1,3) & + -f(i+1,j-1,k ,1)-f(i+1,j-1,k ,2)-f(i+1,j-1,k ,3) & + +f(i-1,j-1,k+1,1)+f(i-1,j-1,k+1,2)+f(i-1,j-1,k+1,3) & + -f(i-1,j-1,k ,1)-f(i-1,j-1,k ,2)-f(i-1,j-1,k ,3)) + end do + end do + end do + + endif + + end subroutine warpx_push_pml_evec_f_3d + + + subroutine warpx_push_pml_evec_f_2d (xlo, xhi, ylo, yhi, zlo, zhi, & + & Ex, Exlo, Exhi, & + & Ey, Eylo, Eyhi, & + & Ez, Ezlo, Ezhi, & + & f, flo, fhi, & + & dtsdx, dtsdy, dtsdz, solver_type) & + bind(c,name='warpx_push_pml_evec_f_2d') + use amrex_constants_module, only : one, two, four, eighth + integer, intent(in) :: xlo(2), xhi(2), ylo(2), yhi(2), zlo(2), zhi(2), & + Exlo(2), Exhi(2), Eylo(2), Eyhi(2), Ezlo(2), Ezhi(2), flo(2), fhi(2), & + solver_type + real(amrex_real), intent(inout) :: Ex (Exlo(1):Exhi(1),Exlo(2):Exhi(2),3) + real(amrex_real), intent(inout) :: Ey (Eylo(1):Eyhi(1),Eylo(2):Eyhi(2),3) + real(amrex_real), intent(inout) :: Ez (Ezlo(1):Ezhi(1),Ezlo(2):Ezhi(2),3) + real(amrex_real), intent(in ) :: f ( flo(1): fhi(1), flo(2): fhi(2),3) + real(amrex_real), intent(in) :: dtsdx, dtsdy, dtsdz + + integer :: i, k + + real(amrex_real) :: delta, rx, rz, betaxz, betazx, alphax, alphaz + + ! solver_type: 0=Yee; 1=CKC + + if (solver_type==0) then + + do k = xlo(2), xhi(2) + do i = xlo(1), xhi(1) + Ex(i,k,3) = Ex(i,k,3) + dtsdx*((f(i+1,k,1)-f(i,k,1)) & + & + (f(i+1,k,2)-f(i,k,2)) & + & + (f(i+1,k,3)-f(i,k,3))) + end do + end do + + do k = zlo(2), zhi(2) + do i = zlo(1), zhi(1) + Ez(i,k,3) = Ez(i,k,3) + dtsdz*((f(i,k+1,1)-f(i,k,1)) & + & + (f(i,k+1,2)-f(i,k,2)) & + & + (f(i,k+1,3)-f(i,k,3))) + end do + end do + + else + + ! Cole-Karkkainen-Cowan push + + ! computes coefficients according to Cowan - PRST-AB 16, 041303 (2013) + delta = max(dtsdx,dtsdz) + rx = (dtsdx/delta)**2 + rz = (dtsdz/delta)**2 + betaxz = eighth*rz + betazx = eighth*rx + alphax = one - two*betaxz + alphaz = one - two*betazx + + betaxz = dtsdx*betaxz + betazx = dtsdz*betazx + alphax = dtsdx*alphax + alphaz = dtsdz*alphaz + + do k = xlo(2), xhi(2) + do i = xlo(1), xhi(1) + Ex(i,k,3) = Ex(i,k,3) & + + alphax*(f(i+1,k ,1)+f(i+1,k ,2)+f(i+1,k ,3) & + -f(i ,k ,1)-f(i ,k ,2)-f(i ,k ,3)) & + + betaxz*(f(i+1,k+1,1)+f(i+1,k+1,2)+f(i+1,k+1,3) & + -f(i ,k+1,1)-f(i ,k+1,2)-f(i ,k+1,3) & + +f(i+1,k-1,1)+f(i+1,k-1,2)+f(i+1,k-1,3) & + -f(i ,k-1,1)-f(i ,k-1,2)-f(i ,k-1,3)) + end do + end do + + do k = zlo(2), zhi(2) + do i = zlo(1), zhi(1) + Ez(i,k,3) = Ez(i,k,3) & + + alphaz*(f(i ,k+1,1)+f(i ,k+1,2)+f(i ,k+1,3) & + -f(i ,k ,1)-f(i ,k ,2)-f(i ,k ,3)) & + + betazx*(f(i+1,k+1,1)+f(i+1,k+1,2)+f(i+1,k+1,3) & + -f(i+1,k ,1)-f(i+1,k ,2)-f(i+1,k ,3) & + +f(i-1,k+1,1)+f(i-1,k+1,2)+f(i-1,k+1,3) & + -f(i-1,k ,1)-f(i-1,k ,2)-f(i-1,k ,3)) + end do + end do + + endif + + end subroutine warpx_push_pml_evec_f_2d + + + subroutine warpx_damp_pml_2d (texlo, texhi, teylo, teyhi, tezlo, tezhi, & + & tbxlo, tbxhi, tbylo, tbyhi, tbzlo, tbzhi, & + & ex, exlo, exhi, ey, eylo, eyhi, ez, ezlo, ezhi, & + & bx, bxlo, bxhi, by, bylo, byhi, bz, bzlo, bzhi, & + & sigex, sexlo, sexhi, sigez, sezlo, sezhi, & + & sigcx, scxlo, scxhi, sigcz, sczlo, sczhi) & + bind(c,name='warpx_damp_pml_2d') + integer, dimension(2), intent(in) :: texlo, texhi, teylo, teyhi, tezlo, tezhi, & + tbxlo, tbxhi, tbylo, tbyhi, tbzlo, tbzhi, & + exlo, exhi, eylo, eyhi, ezlo, ezhi, bxlo, bxhi, bylo, byhi, bzlo, bzhi + integer, intent(in), value :: sexlo, sexhi, sezlo, sezhi, & + & scxlo, scxhi, sczlo, sczhi + real(amrex_real), intent(inout) :: ex(exlo(1):exhi(1),exlo(2):exhi(2),3) + real(amrex_real), intent(inout) :: ey(eylo(1):eyhi(1),eylo(2):eyhi(2),3) + real(amrex_real), intent(inout) :: ez(ezlo(1):ezhi(1),ezlo(2):ezhi(2),3) + real(amrex_real), intent(inout) :: bx(bxlo(1):bxhi(1),bxlo(2):bxhi(2),2) + real(amrex_real), intent(inout) :: by(bylo(1):byhi(1),bylo(2):byhi(2),2) + real(amrex_real), intent(inout) :: bz(bzlo(1):bzhi(1),bzlo(2):bzhi(2),2) + real(amrex_real), intent(in) :: sigex(sexlo:sexhi) + real(amrex_real), intent(in) :: sigez(sezlo:sezhi) + real(amrex_real), intent(in) :: sigcx(scxlo:scxhi) + real(amrex_real), intent(in) :: sigcz(sczlo:sczhi) + + integer :: i,k + + do k = texlo(2), texhi(2) + do i = texlo(1), texhi(1) + ex(i,k,2) = ex(i,k,2) * sigez(k) + ex(i,k,3) = ex(i,k,3) * sigcx(i) + end do + end do + + do k = teylo(2), teyhi(2) + do i = teylo(1), teyhi(1) + ey(i,k,1) = ey(i,k,1) * sigez(k) + ey(i,k,2) = ey(i,k,2) * sigex(i) + end do + end do + + do k = tezlo(2), tezhi(2) + do i = tezlo(1), tezhi(1) + ez(i,k,1) = ez(i,k,1) * sigex(i) + ez(i,k,3) = ez(i,k,3) * sigcz(k) + end do + end do + + do k = tbxlo(2), tbxhi(2) + do i = tbxlo(1), tbxhi(1) + bx(i,k,2) = bx(i,k,2) * sigcz(k) + end do + end do + + do k = tbylo(2), tbyhi(2) + do i = tbylo(1), tbyhi(1) + by(i,k,1) = by(i,k,1) * sigcz(k) + by(i,k,2) = by(i,k,2) * sigcx(i) + end do + end do + + do k = tbzlo(2), tbzhi(2) + do i = tbzlo(1), tbzhi(1) + bz(i,k,1) = bz(i,k,1) * sigcx(i) + end do + end do + + end subroutine warpx_damp_pml_2d + + + subroutine warpx_damp_pml_3d (texlo, texhi, teylo, teyhi, tezlo, tezhi, & + & tbxlo, tbxhi, tbylo, tbyhi, tbzlo, tbzhi, & + & ex, exlo, exhi, ey, eylo, eyhi, ez, ezlo, ezhi, & + & bx, bxlo, bxhi, by, bylo, byhi, bz, bzlo, bzhi, & + & sigex, sexlo, sexhi, sigey, seylo, seyhi, sigez, sezlo, sezhi, & + & sigcx, scxlo, scxhi, sigcy, scylo, scyhi, sigcz, sczlo, sczhi) & + bind(c,name='warpx_damp_pml_3d') + integer, dimension(3), intent(in) :: texlo, texhi, teylo, teyhi, tezlo, tezhi, & + tbxlo, tbxhi, tbylo, tbyhi, tbzlo, tbzhi, & + exlo, exhi, eylo, eyhi, ezlo, ezhi, bxlo, bxhi, bylo, byhi, bzlo, bzhi + integer, intent(in), value :: sexlo, sexhi, seylo, seyhi, sezlo, sezhi, & + & scxlo, scxhi, scylo, scyhi, sczlo, sczhi + real(amrex_real), intent(inout) :: ex(exlo(1):exhi(1),exlo(2):exhi(2),exlo(3):exhi(3),3) + real(amrex_real), intent(inout) :: ey(eylo(1):eyhi(1),eylo(2):eyhi(2),eylo(3):eyhi(3),3) + real(amrex_real), intent(inout) :: ez(ezlo(1):ezhi(1),ezlo(2):ezhi(2),ezlo(3):ezhi(3),3) + real(amrex_real), intent(inout) :: bx(bxlo(1):bxhi(1),bxlo(2):bxhi(2),bxlo(3):bxhi(3),2) + real(amrex_real), intent(inout) :: by(bylo(1):byhi(1),bylo(2):byhi(2),bylo(3):byhi(3),2) + real(amrex_real), intent(inout) :: bz(bzlo(1):bzhi(1),bzlo(2):bzhi(2),bzlo(3):bzhi(3),2) + real(amrex_real), intent(in) :: sigex(sexlo:sexhi) + real(amrex_real), intent(in) :: sigey(seylo:seyhi) + real(amrex_real), intent(in) :: sigez(sezlo:sezhi) + real(amrex_real), intent(in) :: sigcx(scxlo:scxhi) + real(amrex_real), intent(in) :: sigcy(scylo:scyhi) + real(amrex_real), intent(in) :: sigcz(sczlo:sczhi) + + integer :: i,j,k + + do k = texlo(3), texhi(3) + do j = texlo(2), texhi(2) + do i = texlo(1), texhi(1) + ex(i,j,k,1) = ex(i,j,k,1) * sigey(j) + ex(i,j,k,2) = ex(i,j,k,2) * sigez(k) + ex(i,j,k,3) = ex(i,j,k,3) * sigcx(i) + end do + end do + end do + + do k = teylo(3), teyhi(3) + do j = teylo(2), teyhi(2) + do i = teylo(1), teyhi(1) + ey(i,j,k,1) = ey(i,j,k,1) * sigez(k) + ey(i,j,k,2) = ey(i,j,k,2) * sigex(i) + ey(i,j,k,3) = ey(i,j,k,3) * sigcy(j) + end do + end do + end do + + do k = tezlo(3), tezhi(3) + do j = tezlo(2), tezhi(2) + do i = tezlo(1), tezhi(1) + ez(i,j,k,1) = ez(i,j,k,1) * sigex(i) + ez(i,j,k,2) = ez(i,j,k,2) * sigey(j) + ez(i,j,k,3) = ez(i,j,k,3) * sigcz(k) + end do + end do + end do + + do k = tbxlo(3), tbxhi(3) + do j = tbxlo(2), tbxhi(2) + do i = tbxlo(1), tbxhi(1) + bx(i,j,k,1) = bx(i,j,k,1) * sigcy(j) + bx(i,j,k,2) = bx(i,j,k,2) * sigcz(k) + end do + end do + end do + + do k = tbylo(3), tbyhi(3) + do j = tbylo(2), tbyhi(2) + do i = tbylo(1), tbyhi(1) + by(i,j,k,1) = by(i,j,k,1) * sigcz(k) + by(i,j,k,2) = by(i,j,k,2) * sigcx(i) + end do + end do + end do + + do k = tbzlo(3), tbzhi(3) + do j = tbzlo(2), tbzhi(2) + do i = tbzlo(1), tbzhi(1) + bz(i,j,k,1) = bz(i,j,k,1) * sigcx(i) + bz(i,j,k,2) = bz(i,j,k,2) * sigcy(j) + end do + end do + end do + + end subroutine warpx_damp_pml_3d + + subroutine warpx_damp_pml_f_2d (tndlo, tndhi, f, flo, fhi,& + & sigex, sexlo, sexhi, sigez, sezlo, sezhi, & + & sigcx, scxlo, scxhi, sigcz, sczlo, sczhi) & + bind(c,name='warpx_damp_pml_f_2d') + integer, dimension(2), intent(in) :: tndlo, tndhi, flo, fhi + integer, intent(in), value :: sexlo, sexhi, sezlo, sezhi, & + & scxlo, scxhi, sczlo, sczhi + real(amrex_real), intent(inout) :: f ( flo(1): fhi(1), flo(2): fhi(2),3) + real(amrex_real), intent(in) :: sigex(sexlo:sexhi) + real(amrex_real), intent(in) :: sigez(sezlo:sezhi) + real(amrex_real), intent(in) :: sigcx(scxlo:scxhi) + real(amrex_real), intent(in) :: sigcz(sczlo:sczhi) + + integer :: i,k + + do k = tndlo(2), tndhi(2) + do i = tndlo(1), tndhi(1) + f(i,k,1) = f(i,k,1) * sigex(i) + f(i,k,3) = f(i,k,3) * sigez(k) + end do + end do + end subroutine warpx_damp_pml_f_2d + + + subroutine warpx_damp_pml_f_3d (tndlo, tndhi, f, flo, fhi,& + & sigex, sexlo, sexhi, sigey, seylo, seyhi, sigez, sezlo, sezhi, & + & sigcx, scxlo, scxhi, sigcy, scylo, scyhi, sigcz, sczlo, sczhi) & + bind(c,name='warpx_damp_pml_f_3d') + integer, dimension(3), intent(in) :: tndlo, tndhi, flo, fhi + integer, intent(in), value :: sexlo, sexhi, seylo, seyhi, sezlo, sezhi, & + & scxlo, scxhi, scylo, scyhi, sczlo, sczhi + real(amrex_real), intent(inout) :: f ( flo(1): fhi(1), flo(2): fhi(2), flo(3): fhi(3),3) + real(amrex_real), intent(in) :: sigex(sexlo:sexhi) + real(amrex_real), intent(in) :: sigey(seylo:seyhi) + real(amrex_real), intent(in) :: sigez(sezlo:sezhi) + real(amrex_real), intent(in) :: sigcx(scxlo:scxhi) + real(amrex_real), intent(in) :: sigcy(scylo:scyhi) + real(amrex_real), intent(in) :: sigcz(sczlo:sczhi) + + integer :: i,j,k + + do k = tndlo(3), tndhi(3) + do j = tndlo(2), tndhi(2) + do i = tndlo(1), tndhi(1) + f(i,j,k,1) = f(i,j,k,1) * sigex(i) + f(i,j,k,2) = f(i,j,k,2) * sigey(j) + f(i,j,k,3) = f(i,j,k,3) * sigez(k) + end do + end do + end do + end subroutine warpx_damp_pml_f_3d + +end module warpx_pml_module diff --git a/Source/BoundaryConditions/WarpXPML.H b/Source/BoundaryConditions/WarpXPML.H deleted file mode 100644 index 7a86bf0da..000000000 --- a/Source/BoundaryConditions/WarpXPML.H +++ /dev/null @@ -1,170 +0,0 @@ -#include - -#ifndef WARPX_PML_H_ -#define WARPX_PML_H_ - -#include -#include - -#if (AMREX_SPACEDIM == 3) - -#define WRPX_PML_TO_FORTRAN(x) \ - (x).sigma_fac[0].data(), (x).sigma_fac[0].m_lo, (x).sigma_fac[0].m_hi, \ - (x).sigma_fac[1].data(), (x).sigma_fac[1].m_lo, (x).sigma_fac[1].m_hi, \ - (x).sigma_fac[2].data(), (x).sigma_fac[2].m_lo, (x).sigma_fac[2].m_hi, \ - (x).sigma_star_fac[0].data(), (x).sigma_star_fac[0].m_lo, (x).sigma_star_fac[0].m_hi, \ - (x).sigma_star_fac[1].data(), (x).sigma_star_fac[1].m_lo, (x).sigma_star_fac[1].m_hi, \ - (x).sigma_star_fac[2].data(), (x).sigma_star_fac[2].m_lo, (x).sigma_star_fac[2].m_hi - -#else - -#define WRPX_PML_TO_FORTRAN(x) \ - (x).sigma_fac[0].data(), (x).sigma_fac[0].m_lo, (x).sigma_fac[0].m_hi, \ - (x).sigma_fac[1].data(), (x).sigma_fac[1].m_lo, (x).sigma_fac[1].m_hi, \ - (x).sigma_star_fac[0].data(), (x).sigma_star_fac[0].m_lo, (x).sigma_star_fac[0].m_hi, \ - (x).sigma_star_fac[1].data(), (x).sigma_star_fac[1].m_lo, (x).sigma_star_fac[1].m_hi - -#endif - -struct Sigma : amrex::Vector -{ - int lo() const { return m_lo; } - int hi() const { return m_hi; } - int m_lo, m_hi; -}; - -struct SigmaBox -{ - SigmaBox (const amrex::Box& box, const amrex::BoxArray& grids, - const amrex::Real* dx, int ncell, int delta); - - void ComputePMLFactorsB (const amrex::Real* dx, amrex::Real dt); - void ComputePMLFactorsE (const amrex::Real* dx, amrex::Real dt); - - using SigmaVect = std::array; - - SigmaVect sigma; // sigma/epsilon - SigmaVect sigma_star; // sigma_star/mu - SigmaVect sigma_fac; - SigmaVect sigma_star_fac; -}; - -namespace amrex { - template<> - class FabFactory - { - public: - FabFactory (const BoxArray& grid_ba, const Real* dx, int ncell, int delta) - : m_grids(grid_ba), m_dx(dx), m_ncell(ncell), m_delta(delta) {} - virtual SigmaBox* create (const Box& box, int ncomps, - const FabInfo& info, int box_index) const final - { return new SigmaBox(box, m_grids, m_dx, m_ncell, m_delta); } - virtual void destroy (SigmaBox* fab) const final { - delete fab; - } -#ifdef AMREX_USE_GPU - virtual SigmaBox* createHostAlias (SigmaBox const& src) const final - { - return const_cast(&src); - } - virtual void destroyHostAlias (SigmaBox* fab) const final {} -#endif - virtual FabFactory* clone () const { - return new FabFactory(*this); - } - private: - const BoxArray& m_grids; - const Real* m_dx; - int m_ncell; - int m_delta; - }; -} - -class MultiSigmaBox - : public amrex::FabArray -{ -public: - MultiSigmaBox(const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, - const amrex::BoxArray& grid_ba, const amrex::Real* dx, int ncell, int delta); - void ComputePMLFactorsB (const amrex::Real* dx, amrex::Real dt); - void ComputePMLFactorsE (const amrex::Real* dx, amrex::Real dt); -private: - amrex::Real dt_B = -1.e10; - amrex::Real dt_E = -1.e10; -}; - -enum struct PatchType : int; - -class PML -{ -public: - PML (const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, - const amrex::Geometry* geom, const amrex::Geometry* cgeom, - int ncell, int delta, int ref_ratio, int do_dive_cleaning, int do_moving_window); - - void ComputePMLFactors (amrex::Real dt); - - std::array GetE_fp (); - std::array GetB_fp (); - std::array GetE_cp (); - std::array GetB_cp (); - - amrex::MultiFab* GetF_fp (); - amrex::MultiFab* GetF_cp (); - - const MultiSigmaBox& GetMultiSigmaBox_fp () const - { return *sigba_fp; } - - const MultiSigmaBox& GetMultiSigmaBox_cp () const - { return *sigba_cp; } - - void ExchangeB (const std::array& B_fp, - const std::array& B_cp); - void ExchangeE (const std::array& E_fp, - const std::array& E_cp); - void ExchangeB (PatchType patch_type, - const std::array& Bp); - void ExchangeE (PatchType patch_type, - const std::array& Ep); - - void ExchangeF (amrex::MultiFab* F_fp, amrex::MultiFab* F_cp); - void ExchangeF (PatchType patch_type, amrex::MultiFab* Fp); - - void FillBoundary (); - void FillBoundaryE (); - void FillBoundaryB (); - void FillBoundaryF (); - void FillBoundaryE (PatchType patch_type); - void FillBoundaryB (PatchType patch_type); - void FillBoundaryF (PatchType patch_type); - - bool ok () const { return m_ok; } - - void CheckPoint (const std::string& dir) const; - void Restart (const std::string& dir); - -private: - bool m_ok; - - const amrex::Geometry* m_geom; - const amrex::Geometry* m_cgeom; - - std::array,3> pml_E_fp; - std::array,3> pml_B_fp; - - std::array,3> pml_E_cp; - std::array,3> pml_B_cp; - - std::unique_ptr pml_F_fp; - std::unique_ptr pml_F_cp; - - std::unique_ptr sigba_fp; - std::unique_ptr sigba_cp; - - static amrex::BoxArray MakeBoxArray (const amrex::Geometry& geom, - const amrex::BoxArray& grid_ba, int ncell); - - static void Exchange (amrex::MultiFab& pml, amrex::MultiFab& reg, const amrex::Geometry& geom); -}; - -#endif diff --git a/Source/BoundaryConditions/WarpXPML.cpp b/Source/BoundaryConditions/WarpXPML.cpp deleted file mode 100644 index b07d4164d..000000000 --- a/Source/BoundaryConditions/WarpXPML.cpp +++ /dev/null @@ -1,755 +0,0 @@ - -#include -#include -#include - -#include -#include - -#include - -#ifdef _OPENMP -#include -#endif - -using namespace amrex; - -namespace -{ - static void FillLo (int idim, Sigma& sigma, Sigma& sigma_star, - const Box& overlap, const Box& grid, Real fac) - { - int glo = grid.smallEnd(idim); - int olo = overlap.smallEnd(idim); - int ohi = overlap.bigEnd(idim); - int slo = sigma.m_lo; - int sslo = sigma_star.m_lo; - for (int i = olo; i <= ohi+1; ++i) - { - Real offset = static_cast(glo-i); - sigma[i-slo] = fac*(offset*offset); - } - for (int i = olo; i <= ohi; ++i) - { - Real offset = static_cast(glo-i) - 0.5; - sigma_star[i-sslo] = fac*(offset*offset); - } - } - - static void FillHi (int idim, Sigma& sigma, Sigma& sigma_star, - const Box& overlap, const Box& grid, Real fac) - { - int ghi = grid.bigEnd(idim); - int olo = overlap.smallEnd(idim); - int ohi = overlap.bigEnd(idim); - int slo = sigma.m_lo; - int sslo = sigma_star.m_lo; - for (int i = olo; i <= ohi+1; ++i) - { - Real offset = static_cast(i-ghi-1); - sigma[i-slo] = fac*(offset*offset); - } - for (int i = olo; i <= ohi; ++i) - { - Real offset = static_cast(i-ghi) - 0.5; - sigma_star[i-sslo] = fac*(offset*offset); - } - } - - static void FillZero (int idim, Sigma& sigma, Sigma& sigma_star, const Box& overlap) - { - int olo = overlap.smallEnd(idim); - int ohi = overlap.bigEnd(idim); - int slo = sigma.m_lo; - int sslo = sigma_star.m_lo; - std::fill(sigma.begin()+(olo-slo), sigma.begin()+(ohi+2-slo), 0.0); - std::fill(sigma_star.begin()+(olo-sslo), sigma_star.begin()+(ohi+1-sslo), 0.0); - } -} - -SigmaBox::SigmaBox (const Box& box, const BoxArray& grids, const Real* dx, int ncell, int delta) -{ - BL_ASSERT(box.cellCentered()); - - const IntVect& sz = box.size(); - const int* lo = box.loVect(); - const int* hi = box.hiVect(); - - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) - { - sigma [idim].resize(sz[idim]+1); - sigma_star [idim].resize(sz[idim] ); - sigma_fac [idim].resize(sz[idim]+1); - sigma_star_fac[idim].resize(sz[idim] ); - - sigma [idim].m_lo = lo[idim]; - sigma [idim].m_hi = hi[idim]+1; - sigma_star [idim].m_lo = lo[idim]; - sigma_star [idim].m_hi = hi[idim]; - sigma_fac [idim].m_lo = lo[idim]; - sigma_fac [idim].m_hi = hi[idim]+1; - sigma_star_fac[idim].m_lo = lo[idim]; - sigma_star_fac[idim].m_hi = hi[idim]; - } - - Array fac; - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - fac[idim] = 4.0*PhysConst::c/(dx[idim]*static_cast(delta*delta)); - } - - const std::vector >& isects = grids.intersections(box, false, ncell); - - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) - { - int jdim = (idim+1) % AMREX_SPACEDIM; -#if (AMREX_SPACEDIM == 3) - int kdim = (idim+2) % AMREX_SPACEDIM; -#endif - - Vector direct_faces, side_faces, direct_side_edges, side_side_edges, corners; - for (const auto& kv : isects) - { - const Box& grid_box = grids[kv.first]; - - if (amrex::grow(grid_box, idim, ncell).intersects(box)) - { - direct_faces.push_back(kv.first); - } - else if (amrex::grow(grid_box, jdim, ncell).intersects(box)) - { - side_faces.push_back(kv.first); - } -#if (AMREX_SPACEDIM == 3) - else if (amrex::grow(grid_box, kdim, ncell).intersects(box)) - { - side_faces.push_back(kv.first); - } - else if (amrex::grow(amrex::grow(grid_box,idim,ncell), - jdim,ncell).intersects(box)) - { - direct_side_edges.push_back(kv.first); - } - else if (amrex::grow(amrex::grow(grid_box,idim,ncell), - kdim,ncell).intersects(box)) - { - direct_side_edges.push_back(kv.first); - } - else if (amrex::grow(amrex::grow(grid_box,jdim,ncell), - kdim,ncell).intersects(box)) - { - side_side_edges.push_back(kv.first); - } -#endif - else - { - corners.push_back(kv.first); - } - } - - for (auto gid : corners) - { - const Box& grid_box = grids[gid]; - - Box lobox = amrex::adjCellLo(grid_box, idim, ncell); - lobox.grow(jdim,ncell); -#if (AMREX_SPACEDIM == 3) - lobox.grow(kdim,ncell); -#endif - Box looverlap = lobox & box; - if (looverlap.ok()) { - FillLo(idim, sigma[idim], sigma_star[idim], looverlap, grid_box, fac[idim]); - } - - Box hibox = amrex::adjCellHi(grid_box, idim, ncell); - hibox.grow(jdim,ncell); -#if (AMREX_SPACEDIM == 3) - hibox.grow(kdim,ncell); -#endif - Box hioverlap = hibox & box; - if (hioverlap.ok()) { - FillHi(idim, sigma[idim], sigma_star[idim], hioverlap, grid_box, fac[idim]); - } - - if (!looverlap.ok() && !hioverlap.ok()) { - amrex::Abort("SigmaBox::SigmaBox(): corners, how did this happen?\n"); - } - } - -#if (AMREX_SPACEDIM == 3) - for (auto gid : side_side_edges) - { - const Box& grid_box = grids[gid]; - const Box& overlap = amrex::grow(amrex::grow(grid_box,jdim,ncell),kdim,ncell) & box; - if (overlap.ok()) { - FillZero(idim, sigma[idim], sigma_star[idim], overlap); - } else { - amrex::Abort("SigmaBox::SigmaBox(): side_side_edges, how did this happen?\n"); - } - } - - for (auto gid : direct_side_edges) - { - const Box& grid_box = grids[gid]; - - Box lobox = amrex::adjCellLo(grid_box, idim, ncell); - Box looverlap = lobox.grow(jdim,ncell).grow(kdim,ncell) & box; - if (looverlap.ok()) { - FillLo(idim, sigma[idim], sigma_star[idim], looverlap, grid_box, fac[idim]); - } - - Box hibox = amrex::adjCellHi(grid_box, idim, ncell); - Box hioverlap = hibox.grow(jdim,ncell).grow(kdim,ncell) & box; - if (hioverlap.ok()) { - FillHi(idim, sigma[idim], sigma_star[idim], hioverlap, grid_box, fac[idim]); - } - - if (!looverlap.ok() && !hioverlap.ok()) { - amrex::Abort("SigmaBox::SigmaBox(): direct_side_edges, how did this happen?\n"); - } - } -#endif - - for (auto gid : side_faces) - { - const Box& grid_box = grids[gid]; -#if (AMREX_SPACEDIM == 2) - const Box& overlap = amrex::grow(grid_box,jdim,ncell) & box; -#else - const Box& overlap = amrex::grow(amrex::grow(grid_box,jdim,ncell),kdim,ncell) & box; -#endif - if (overlap.ok()) { - FillZero(idim, sigma[idim], sigma_star[idim], overlap); - } else { - amrex::Abort("SigmaBox::SigmaBox(): side_faces, how did this happen?\n"); - } - } - - for (auto gid : direct_faces) - { - const Box& grid_box = grids[gid]; - - const Box& lobox = amrex::adjCellLo(grid_box, idim, ncell); - Box looverlap = lobox & box; - if (looverlap.ok()) { - FillLo(idim, sigma[idim], sigma_star[idim], looverlap, grid_box, fac[idim]); - } - - const Box& hibox = amrex::adjCellHi(grid_box, idim, ncell); - Box hioverlap = hibox & box; - if (hioverlap.ok()) { - FillHi(idim, sigma[idim], sigma_star[idim], hioverlap, grid_box, fac[idim]); - } - - if (!looverlap.ok() && !hioverlap.ok()) { - amrex::Abort("SigmaBox::SigmaBox(): direct faces, how did this happen?\n"); - } - } - - if (direct_faces.size() > 1) { - amrex::Abort("SigmaBox::SigmaBox(): direct_faces.size() > 1, Box gaps not wide enough?\n"); - } - } -} - -void -SigmaBox::ComputePMLFactorsB (const Real* dx, Real dt) -{ - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) - { - for (int i = 0, N = sigma_star[idim].size(); i < N; ++i) - { - if (sigma_star[idim][i] == 0.0) - { - sigma_star_fac[idim][i] = 1.0; - } - else - { - sigma_star_fac[idim][i] = std::exp(-sigma_star[idim][i]*dt); - } - } - } -} - -void -SigmaBox::ComputePMLFactorsE (const Real* dx, Real dt) -{ - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) - { - for (int i = 0, N = sigma[idim].size(); i < N; ++i) - { - if (sigma[idim][i] == 0.0) - { - sigma_fac[idim][i] = 1.0; - } - else - { - sigma_fac[idim][i] = std::exp(-sigma[idim][i]*dt); - } - } - } -} - -MultiSigmaBox::MultiSigmaBox (const BoxArray& ba, const DistributionMapping& dm, - const BoxArray& grid_ba, const Real* dx, int ncell, int delta) - : FabArray(ba,dm,1,0,MFInfo(), - FabFactory(grid_ba,dx,ncell,delta)) -{} - -void -MultiSigmaBox::ComputePMLFactorsB (const Real* dx, Real dt) -{ - if (dt == dt_B) return; - - dt_B = dt; - -#ifdef _OPENMP -#pragma omp parallel -#endif - for (MFIter mfi(*this); mfi.isValid(); ++mfi) - { - (*this)[mfi].ComputePMLFactorsB(dx, dt); - } -} - -void -MultiSigmaBox::ComputePMLFactorsE (const Real* dx, Real dt) -{ - if (dt == dt_E) return; - - dt_E = dt; - -#ifdef _OPENMP -#pragma omp parallel -#endif - for (MFIter mfi(*this); mfi.isValid(); ++mfi) - { - (*this)[mfi].ComputePMLFactorsE(dx, dt); - } -} - -PML::PML (const BoxArray& grid_ba, const DistributionMapping& grid_dm, - const Geometry* geom, const Geometry* cgeom, - int ncell, int delta, int ref_ratio, int do_dive_cleaning, int do_moving_window) - : m_geom(geom), - m_cgeom(cgeom) -{ - const BoxArray& ba = MakeBoxArray(*geom, grid_ba, ncell); - if (ba.size() == 0) { - m_ok = false; - return; - } else { - m_ok = true; - } - - DistributionMapping dm{ba}; - - int nge = 2; - int ngb = 2; - int ngf = (do_moving_window) ? 2 : 0; - if (WarpX::maxwell_fdtd_solver_id == 1) ngf = std::max( ngf, 1 ); - - pml_E_fp[0].reset(new MultiFab(amrex::convert(ba,WarpX::Ex_nodal_flag), dm, 3, nge)); - pml_E_fp[1].reset(new MultiFab(amrex::convert(ba,WarpX::Ey_nodal_flag), dm, 3, nge)); - pml_E_fp[2].reset(new MultiFab(amrex::convert(ba,WarpX::Ez_nodal_flag), dm, 3, nge)); - pml_B_fp[0].reset(new MultiFab(amrex::convert(ba,WarpX::Bx_nodal_flag), dm, 2, ngb)); - pml_B_fp[1].reset(new MultiFab(amrex::convert(ba,WarpX::By_nodal_flag), dm, 2, ngb)); - pml_B_fp[2].reset(new MultiFab(amrex::convert(ba,WarpX::Bz_nodal_flag), dm, 2, ngb)); - - pml_E_fp[0]->setVal(0.0); - pml_E_fp[1]->setVal(0.0); - pml_E_fp[2]->setVal(0.0); - pml_B_fp[0]->setVal(0.0); - pml_B_fp[1]->setVal(0.0); - pml_B_fp[2]->setVal(0.0); - - if (do_dive_cleaning) - { - pml_F_fp.reset(new MultiFab(amrex::convert(ba,IntVect::TheUnitVector()), dm, 3, ngf)); - pml_F_fp->setVal(0.0); - } - - sigba_fp.reset(new MultiSigmaBox(ba, dm, grid_ba, geom->CellSize(), ncell, delta)); - - if (cgeom) - { - - nge = 1; - ngb = 1; - - BoxArray grid_cba = grid_ba; - grid_cba.coarsen(ref_ratio); - const BoxArray& cba = MakeBoxArray(*cgeom, grid_cba, ncell); - - DistributionMapping cdm{cba}; - - pml_E_cp[0].reset(new MultiFab(amrex::convert(cba,WarpX::Ex_nodal_flag), cdm, 3, nge)); - pml_E_cp[1].reset(new MultiFab(amrex::convert(cba,WarpX::Ey_nodal_flag), cdm, 3, nge)); - pml_E_cp[2].reset(new MultiFab(amrex::convert(cba,WarpX::Ez_nodal_flag), cdm, 3, nge)); - pml_B_cp[0].reset(new MultiFab(amrex::convert(cba,WarpX::Bx_nodal_flag), cdm, 2, ngb)); - pml_B_cp[1].reset(new MultiFab(amrex::convert(cba,WarpX::By_nodal_flag), cdm, 2, ngb)); - pml_B_cp[2].reset(new MultiFab(amrex::convert(cba,WarpX::Bz_nodal_flag), cdm, 2, ngb)); - - pml_E_cp[0]->setVal(0.0); - pml_E_cp[1]->setVal(0.0); - pml_E_cp[2]->setVal(0.0); - pml_B_cp[0]->setVal(0.0); - pml_B_cp[1]->setVal(0.0); - pml_B_cp[2]->setVal(0.0); - - if (do_dive_cleaning) - { - pml_F_cp.reset(new MultiFab(amrex::convert(cba,IntVect::TheUnitVector()), cdm, 3, ngf)); - pml_F_cp->setVal(0.0); - } - - sigba_cp.reset(new MultiSigmaBox(cba, cdm, grid_cba, cgeom->CellSize(), ncell, delta)); - } - -} - -BoxArray -PML::MakeBoxArray (const amrex::Geometry& geom, const amrex::BoxArray& grid_ba, int ncell) -{ - Box domain = geom.Domain(); - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if ( ! Geometry::isPeriodic(idim) ) { - domain.grow(idim, ncell); - } - } - - BoxList bl; - for (int i = 0, N = grid_ba.size(); i < N; ++i) - { - const Box& grid_bx = grid_ba[i]; - const IntVect& grid_bx_sz = grid_bx.size(); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(grid_bx.shortside() > ncell, - "Consider using larger amr.blocking_factor"); - - Box bx = grid_bx; - bx.grow(ncell); - bx &= domain; - - Vector bndryboxes; -#if (AMREX_SPACEDIM == 3) - int kbegin = -1, kend = 1; -#else - int kbegin = 0, kend = 0; -#endif - for (int kk = kbegin; kk <= kend; ++kk) { - for (int jj = -1; jj <= 1; ++jj) { - for (int ii = -1; ii <= 1; ++ii) { - if (ii != 0 || jj != 0 || kk != 0) { - Box b = grid_bx; - b.shift(grid_bx_sz * IntVect{AMREX_D_DECL(ii,jj,kk)}); - b &= bx; - if (b.ok()) { - bndryboxes.push_back(b); - } - } - } - } - } - - const BoxList& noncovered = grid_ba.complementIn(bx); - for (const Box& b : noncovered) { - for (const auto& bb : bndryboxes) { - Box ib = b & bb; - if (ib.ok()) { - bl.push_back(ib); - } - } - } - } - - BoxArray ba(bl); - ba.removeOverlap(false); - - return ba; -} - -void -PML::ComputePMLFactors (amrex::Real dt) -{ - if (sigba_fp) { - sigba_fp->ComputePMLFactorsB(m_geom->CellSize(), dt); - sigba_fp->ComputePMLFactorsE(m_geom->CellSize(), dt); - } - if (sigba_cp) { - sigba_cp->ComputePMLFactorsB(m_cgeom->CellSize(), dt); - sigba_cp->ComputePMLFactorsE(m_cgeom->CellSize(), dt); - } -} - -std::array -PML::GetE_fp () -{ - return {pml_E_fp[0].get(), pml_E_fp[1].get(), pml_E_fp[2].get()}; -} - -std::array -PML::GetB_fp () -{ - return {pml_B_fp[0].get(), pml_B_fp[1].get(), pml_B_fp[2].get()}; -} - -std::array -PML::GetE_cp () -{ - return {pml_E_cp[0].get(), pml_E_cp[1].get(), pml_E_cp[2].get()}; -} - -std::array -PML::GetB_cp () -{ - return {pml_B_cp[0].get(), pml_B_cp[1].get(), pml_B_cp[2].get()}; -} - -MultiFab* -PML::GetF_fp () -{ - return pml_F_fp.get(); -} - -MultiFab* -PML::GetF_cp () -{ - return pml_F_cp.get(); -} - -void -PML::ExchangeB (const std::array& B_fp, - const std::array& B_cp) -{ - ExchangeB(PatchType::fine, B_fp); - ExchangeB(PatchType::coarse, B_cp); -} - -void -PML::ExchangeB (PatchType patch_type, - const std::array& Bp) -{ - if (patch_type == PatchType::fine && pml_B_fp[0] && Bp[0]) - { - Exchange(*pml_B_fp[0], *Bp[0], *m_geom); - Exchange(*pml_B_fp[1], *Bp[1], *m_geom); - Exchange(*pml_B_fp[2], *Bp[2], *m_geom); - } - else if (patch_type == PatchType::coarse && pml_B_cp[0] && Bp[0]) - { - Exchange(*pml_B_cp[0], *Bp[0], *m_cgeom); - Exchange(*pml_B_cp[1], *Bp[1], *m_cgeom); - Exchange(*pml_B_cp[2], *Bp[2], *m_cgeom); - } -} - -void -PML::ExchangeE (const std::array& E_fp, - const std::array& E_cp) -{ - ExchangeB(PatchType::fine, E_fp); - ExchangeB(PatchType::coarse, E_cp); -} - -void -PML::ExchangeE (PatchType patch_type, - const std::array& Ep) -{ - if (patch_type == PatchType::fine && pml_E_fp[0] && Ep[0]) - { - Exchange(*pml_E_fp[0], *Ep[0], *m_geom); - Exchange(*pml_E_fp[1], *Ep[1], *m_geom); - Exchange(*pml_E_fp[2], *Ep[2], *m_geom); - } - else if (patch_type == PatchType::coarse && pml_E_cp[0] && Ep[0]) - { - Exchange(*pml_E_cp[0], *Ep[0], *m_cgeom); - Exchange(*pml_E_cp[1], *Ep[1], *m_cgeom); - Exchange(*pml_E_cp[2], *Ep[2], *m_cgeom); - } -} - -void -PML::ExchangeF (MultiFab* F_fp, MultiFab* F_cp) -{ - ExchangeF(PatchType::fine, F_fp); - ExchangeF(PatchType::coarse, F_cp); -} - -void -PML::ExchangeF (PatchType patch_type, MultiFab* Fp) -{ - if (patch_type == PatchType::fine && pml_F_fp && Fp) { - Exchange(*pml_F_fp, *Fp, *m_geom); - } else if (patch_type == PatchType::coarse && pml_F_cp && Fp) { - Exchange(*pml_F_cp, *Fp, *m_cgeom); - } -} - -void -PML::Exchange (MultiFab& pml, MultiFab& reg, const Geometry& geom) -{ - const IntVect& ngr = reg.nGrowVect(); - const IntVect& ngp = pml.nGrowVect(); - const int ncp = pml.nComp(); - const auto& period = geom.periodicity(); - - MultiFab tmpregmf(reg.boxArray(), reg.DistributionMap(), ncp, ngr); - - if (ngp.max() > 0) // Copy from pml to the ghost cells of regular data - { - MultiFab totpmlmf(pml.boxArray(), pml.DistributionMap(), 1, 0); - MultiFab::LinComb(totpmlmf, 1.0, pml, 0, 1.0, pml, 1, 0, 1, 0); - if (ncp == 3) { - MultiFab::Add(totpmlmf,pml,2,0,1,0); - } - - MultiFab::Copy(tmpregmf, reg, 0, 0, 1, ngr); - tmpregmf.ParallelCopy(totpmlmf, 0, 0, 1, IntVect(0), ngr, period); - -#ifdef _OPENMP -#pragma omp parallel -#endif - for (MFIter mfi(reg); mfi.isValid(); ++mfi) - { - const FArrayBox& src = tmpregmf[mfi]; - FArrayBox& dst = reg[mfi]; - const BoxList& bl = amrex::boxDiff(dst.box(), mfi.validbox()); - for (const Box& bx : bl) - { - dst.copy(src, bx, 0, bx, 0, 1); - } - } - } - - // Copy from regular data to PML's first component - // Zero out the second (and third) component - MultiFab::Copy(tmpregmf,reg,0,0,1,0); - tmpregmf.setVal(0.0, 1, ncp-1, 0); - pml.ParallelCopy(tmpregmf, 0, 0, ncp, IntVect(0), ngp, period); -} - -void -PML::FillBoundary () -{ - FillBoundaryE(); - FillBoundaryB(); - FillBoundaryF(); -} - -void -PML::FillBoundaryE () -{ - FillBoundaryE(PatchType::fine); - FillBoundaryE(PatchType::coarse); -} - -void -PML::FillBoundaryE (PatchType patch_type) -{ - if (patch_type == PatchType::fine && pml_E_fp[0] && pml_E_fp[0]->nGrowVect().max() > 0) - { - const auto& period = m_geom->periodicity(); - Vector mf{pml_E_fp[0].get(),pml_E_fp[1].get(),pml_E_fp[2].get()}; - amrex::FillBoundary(mf, period); - } - else if (patch_type == PatchType::coarse && pml_E_cp[0] && pml_E_cp[0]->nGrowVect().max() > 0) - { - const auto& period = m_cgeom->periodicity(); - Vector mf{pml_E_cp[0].get(),pml_E_cp[1].get(),pml_E_cp[2].get()}; - amrex::FillBoundary(mf, period); - } -} - -void -PML::FillBoundaryB () -{ - FillBoundaryB(PatchType::fine); - FillBoundaryB(PatchType::coarse); -} - -void -PML::FillBoundaryB (PatchType patch_type) -{ - if (patch_type == PatchType::fine && pml_B_fp[0]) - { - const auto& period = m_geom->periodicity(); - Vector mf{pml_B_fp[0].get(),pml_B_fp[1].get(),pml_B_fp[2].get()}; - amrex::FillBoundary(mf, period); - } - else if (patch_type == PatchType::coarse && pml_B_cp[0]) - { - const auto& period = m_cgeom->periodicity(); - Vector mf{pml_B_cp[0].get(),pml_B_cp[1].get(),pml_B_cp[2].get()}; - amrex::FillBoundary(mf, period); - } -} - -void -PML::FillBoundaryF () -{ - FillBoundaryF(PatchType::fine); - FillBoundaryF(PatchType::coarse); -} - -void -PML::FillBoundaryF (PatchType patch_type) -{ - if (patch_type == PatchType::fine && pml_F_fp && pml_F_fp->nGrowVect().max() > 0) - { - const auto& period = m_geom->periodicity(); - pml_F_fp->FillBoundary(period); - } - else if (patch_type == PatchType::coarse && pml_F_cp && pml_F_cp->nGrowVect().max() > 0) - { - const auto& period = m_cgeom->periodicity(); - pml_F_cp->FillBoundary(period); - } -} - -void -PML::CheckPoint (const std::string& dir) const -{ - if (pml_E_fp[0]) - { - VisMF::Write(*pml_E_fp[0], dir+"_Ex_fp"); - VisMF::Write(*pml_E_fp[1], dir+"_Ey_fp"); - VisMF::Write(*pml_E_fp[2], dir+"_Ez_fp"); - VisMF::Write(*pml_B_fp[0], dir+"_Bx_fp"); - VisMF::Write(*pml_B_fp[1], dir+"_By_fp"); - VisMF::Write(*pml_B_fp[2], dir+"_Bz_fp"); - } - - if (pml_E_cp[0]) - { - VisMF::Write(*pml_E_cp[0], dir+"_Ex_cp"); - VisMF::Write(*pml_E_cp[1], dir+"_Ey_cp"); - VisMF::Write(*pml_E_cp[2], dir+"_Ez_cp"); - VisMF::Write(*pml_B_cp[0], dir+"_Bx_cp"); - VisMF::Write(*pml_B_cp[1], dir+"_By_cp"); - VisMF::Write(*pml_B_cp[2], dir+"_Bz_cp"); - } -} - -void -PML::Restart (const std::string& dir) -{ - if (pml_E_fp[0]) - { - VisMF::Read(*pml_E_fp[0], dir+"_Ex_fp"); - VisMF::Read(*pml_E_fp[1], dir+"_Ey_fp"); - VisMF::Read(*pml_E_fp[2], dir+"_Ez_fp"); - VisMF::Read(*pml_B_fp[0], dir+"_Bx_fp"); - VisMF::Read(*pml_B_fp[1], dir+"_By_fp"); - VisMF::Read(*pml_B_fp[2], dir+"_Bz_fp"); - } - - if (pml_E_cp[0]) - { - VisMF::Read(*pml_E_cp[0], dir+"_Ex_cp"); - VisMF::Read(*pml_E_cp[1], dir+"_Ey_cp"); - VisMF::Read(*pml_E_cp[2], dir+"_Ez_cp"); - VisMF::Read(*pml_B_cp[0], dir+"_Bx_cp"); - VisMF::Read(*pml_B_cp[1], dir+"_By_cp"); - VisMF::Read(*pml_B_cp[2], dir+"_Bz_cp"); - } -} diff --git a/Source/BoundaryConditions/WarpX_pml.F90 b/Source/BoundaryConditions/WarpX_pml.F90 deleted file mode 100644 index 380e52934..000000000 --- a/Source/BoundaryConditions/WarpX_pml.F90 +++ /dev/null @@ -1,1009 +0,0 @@ -module warpx_pml_module - - use iso_c_binding - use amrex_fort_module, only : amrex_real - - implicit none - -contains - - subroutine warpx_push_pml_bvec_3d (xlo, xhi, ylo, yhi, zlo, zhi, & - & Ex, Exlo, Exhi, & - & Ey, Eylo, Eyhi, & - & Ez, Ezlo, Ezhi, & - & Bx, Bxlo, Bxhi, & - & By, Bylo, Byhi, & - & Bz, Bzlo, Bzhi, & - & dtsdx, dtsdy, dtsdz, solver_type) & - bind(c,name='warpx_push_pml_bvec_3d') - use amrex_constants_module, only : one, two, four, eighth - integer, intent(in) :: xlo(3), xhi(3), ylo(3), yhi(3), zlo(3), zhi(3), & - Exlo(3), Exhi(3), Eylo(3), Eyhi(3), Ezlo(3), Ezhi(3), & - Bxlo(3), Bxhi(3), Bylo(3), Byhi(3), Bzlo(3), Bzhi(3), solver_type - real(amrex_real), intent(in ) :: Ex (Exlo(1):Exhi(1),Exlo(2):Exhi(2),Exlo(3):Exhi(3),3) - real(amrex_real), intent(in ) :: Ey (Eylo(1):Eyhi(1),Eylo(2):Eyhi(2),Eylo(3):Eyhi(3),3) - real(amrex_real), intent(in ) :: Ez (Ezlo(1):Ezhi(1),Ezlo(2):Ezhi(2),Ezlo(3):Ezhi(3),3) - real(amrex_real), intent(inout) :: Bx (Bxlo(1):Bxhi(1),Bxlo(2):Bxhi(2),Bxlo(3):Bxhi(3),2) - real(amrex_real), intent(inout) :: By (Bylo(1):Byhi(1),Bylo(2):Byhi(2),Bylo(3):Byhi(3),2) - real(amrex_real), intent(inout) :: Bz (Bzlo(1):Bzhi(1),Bzlo(2):Bzhi(2),Bzlo(3):Bzhi(3),2) - real(amrex_real), intent(in) :: dtsdx, dtsdy, dtsdz - - real(amrex_real), parameter :: sixteenth = 1.d0/16.d0 - - integer :: i, j, k - - real(amrex_real) :: delta, rx, ry, rz, betaxz, betaxy, betayx, betayz, betazx, betazy - real(amrex_real) :: beta, alphax, alphay, alphaz, gammax, gammay, gammaz - - ! solver_type: 0=Yee; 1=CKC - - if (solver_type==0) then - - ! Yee push - - do k = xlo(3), xhi(3) - do j = xlo(2), xhi(2) - do i = xlo(1), xhi(1) - Bx(i,j,k,1) = Bx(i,j,k,1) - dtsdy*(Ez(i,j+1,k ,1)+Ez(i,j+1,k ,2)+Ez(i,j+1,k ,3) & - & -Ez(i,j ,k ,1)-Ez(i,j ,k ,2)-Ez(i,j ,k ,3)) - Bx(i,j,k,2) = Bx(i,j,k,2) + dtsdz*(Ey(i,j ,k+1,1)+Ey(i,j ,k+1,2)+Ey(i,j ,k+1,3) & - & -Ey(i,j ,k ,1)-Ey(i,j ,k ,2)-Ey(i,j ,k ,3)) - end do - end do - end do - - do k = ylo(3), yhi(3) - do j = ylo(2), yhi(2) - do i = ylo(1), yhi(1) - By(i,j,k,1) = By(i,j,k,1) - dtsdz*(Ex(i ,j,k+1,1)+Ex(i ,j,k+1,2)+Ex(i ,j,k+1,3) & - & -Ex(i ,j,k ,1)-Ex(i ,j,k ,2)-Ex(i ,j,k ,3)) - By(i,j,k,2) = By(i,j,k,2) + dtsdx*(Ez(i+1,j,k ,1)+Ez(i+1,j,k ,2)+Ez(i+1,j,k ,3) & - & -Ez(i ,j,k ,1)-Ez(i ,j,k ,2)-Ez(i ,j,k ,3)) - end do - end do - end do - - do k = zlo(3), zhi(3) - do j = zlo(2), zhi(2) - do i = zlo(1), zhi(1) - Bz(i,j,k,1) = Bz(i,j,k,1) - dtsdx*(Ey(i+1,j ,k,1)+Ey(i+1,j ,k,2)+Ey(i+1,j ,k,3) & - & -Ey(i ,j ,k,1)-Ey(i ,j ,k,2)-Ey(i ,j ,k,3)) - Bz(i,j,k,2) = Bz(i,j,k,2) + dtsdy*(Ex(i ,j+1,k,1)+Ex(i ,j+1,k,2)+Ex(i ,j+1,k,3) & - & -Ex(i ,j ,k,1)-Ex(i ,j ,k,2)-Ex(i ,j ,k,3)) - end do - end do - end do - - else - - ! CKC push - - ! computes coefficients according to Cowan - PRST-AB 16, 041303 (2013) - delta = max(dtsdx,dtsdy,dtsdz) - rx = (dtsdx/delta)**2 - ry = (dtsdy/delta)**2 - rz = (dtsdz/delta)**2 - beta = eighth*(one-rx*ry*rz/(ry*rz+rz*rx+rx*ry)) - betaxy = ry*beta - betaxz = rz*beta - betayx = rx*beta - betayz = rz*beta - betazx = rx*beta - betazy = ry*beta - gammax = ry*rz*(sixteenth-eighth*ry*rz/(ry*rz+rz*rx+rx*ry)) - gammay = rx*rz*(sixteenth-eighth*rx*rz/(ry*rz+rz*rx+rx*ry)) - gammaz = rx*ry*(sixteenth-eighth*rx*ry/(ry*rz+rz*rx+rx*ry)) - alphax = one - two*betaxy - two* betaxz - four*gammax - alphay = one - two*betayx - two* betayz - four*gammay - alphaz = one - two*betazx - two* betazy - four*gammaz - - betaxy = dtsdx*betaxy - betaxz = dtsdx*betaxz - betayx = dtsdy*betayx - betayz = dtsdy*betayz - betazx = dtsdz*betazx - betazy = dtsdz*betazy - alphax = dtsdx*alphax - alphay = dtsdy*alphay - alphaz = dtsdz*alphaz - gammax = dtsdx*gammax - gammay = dtsdy*gammay - gammaz = dtsdz*gammaz - - do k = xlo(3), xhi(3) - do j = xlo(2), xhi(2) - do i = xlo(1), xhi(1) - Bx(i,j,k,1) = Bx(i,j,k,1) - (alphay*(Ez(i ,j+1,k ,1)+Ez(i ,j+1,k ,2)+Ez(i ,j+1,k ,3) & - -Ez(i ,j ,k ,1)-Ez(i ,j ,k ,2)-Ez(i ,j ,k ,3)) & - +betayx*(Ez(i+1,j+1,k ,1)+Ez(i+1,j+1,k ,2)+Ez(i+1,j+1,k ,3) & - -Ez(i+1,j ,k ,1)-Ez(i+1,j ,k ,2)-Ez(i+1,j ,k ,3) & - +Ez(i-1,j+1,k ,1)+Ez(i-1,j+1,k ,2)+Ez(i-1,j+1,k ,3) & - -Ez(i-1,j ,k ,1)-Ez(i-1,j ,k ,2)-Ez(i-1,j ,k ,3)) & - +betayz*(Ez(i ,j+1,k+1,1)+Ez(i ,j+1,k+1,2)+Ez(i ,j+1,k+1,3) & - -Ez(i ,j ,k+1,1)-Ez(i ,j ,k+1,2)-Ez(i ,j ,k+1,3) & - +Ez(i ,j+1,k-1,1)+Ez(i ,j+1,k-1,2)+Ez(i ,j+1,k-1,3) & - -Ez(i ,j ,k-1,1)-Ez(i ,j ,k-1,2)-Ez(i ,j ,k-1,3)) & - +gammay*(Ez(i+1,j+1,k+1,1)+Ez(i+1,j+1,k+1,2)+Ez(i+1,j+1,k+1,3) & - -Ez(i+1,j ,k+1,1)-Ez(i+1,j ,k+1,2)-Ez(i+1,j ,k+1,3) & - +Ez(i-1,j+1,k+1,1)+Ez(i-1,j+1,k+1,2)+Ez(i-1,j+1,k+1,3) & - -Ez(i-1,j ,k+1,1)-Ez(i-1,j ,k+1,2)-Ez(i-1,j ,k+1,3) & - +Ez(i+1,j+1,k-1,1)+Ez(i+1,j+1,k-1,2)+Ez(i+1,j+1,k-1,3) & - -Ez(i+1,j ,k-1,1)-Ez(i+1,j ,k-1,2)-Ez(i+1,j ,k-1,3) & - +Ez(i-1,j+1,k-1,1)+Ez(i-1,j+1,k-1,2)+Ez(i-1,j+1,k-1,3) & - -Ez(i-1,j ,k-1,1)-Ez(i-1,j ,k-1,2)-Ez(i-1,j ,k-1,3))) - - - Bx(i,j,k,2) = Bx(i,j,k,2) + (alphaz*(Ey(i ,j ,k+1,1)+Ey(i ,j ,k+1,2)+Ey(i ,j ,k+1,3) & - -Ey(i ,j ,k ,1)-Ey(i ,j ,k ,2)-Ey(i ,j ,k ,3)) & - +betazx*(Ey(i+1,j ,k+1,1)+Ey(i+1,j ,k+1,2)+Ey(i+1,j ,k+1,3) & - -Ey(i+1,j ,k ,1)-Ey(i+1,j ,k ,2)-Ey(i+1,j ,k ,3) & - +Ey(i-1,j ,k+1,1)+Ey(i-1,j ,k+1,2)+Ey(i-1,j ,k+1,3) & - -Ey(i-1,j ,k ,1)-Ey(i-1,j ,k ,2)-Ey(i-1,j ,k ,3)) & - +betazy*(Ey(i ,j+1,k+1,1)+Ey(i ,j+1,k+1,2)+Ey(i ,j+1,k+1,3) & - -Ey(i ,j+1,k ,1)-Ey(i ,j+1,k ,2)-Ey(i ,j+1,k ,3) & - +Ey(i ,j-1,k+1,1)+Ey(i ,j-1,k+1,2)+Ey(i ,j-1,k+1,3) & - -Ey(i ,j-1,k ,1)-Ey(i ,j-1,k ,2)-Ey(i ,j-1,k ,3)) & - +gammaz*(Ey(i+1,j+1,k+1,1)+Ey(i+1,j+1,k+1,2)+Ey(i+1,j+1,k+1,3) & - -Ey(i+1,j+1,k ,1)-Ey(i+1,j+1,k ,2)-Ey(i+1,j+1,k ,3) & - +Ey(i-1,j+1,k+1,1)+Ey(i-1,j+1,k+1,2)+Ey(i-1,j+1,k+1,3) & - -Ey(i-1,j+1,k ,1)-Ey(i-1,j+1,k ,2)-Ey(i-1,j+1,k ,3) & - +Ey(i+1,j-1,k+1,1)+Ey(i+1,j-1,k+1,2)+Ey(i+1,j-1,k+1,3) & - -Ey(i+1,j-1,k ,1)-Ey(i+1,j-1,k ,2)-Ey(i+1,j-1,k ,3) & - +Ey(i-1,j-1,k+1,1)+Ey(i-1,j-1,k+1,2)+Ey(i-1,j-1,k+1,3) & - -Ey(i-1,j-1,k ,1)-Ey(i-1,j-1,k ,2)-Ey(i-1,j-1,k ,3))) - end do - end do - end do - - do k = ylo(3), yhi(3) - do j = ylo(2), yhi(2) - do i = ylo(1), yhi(1) - By(i,j,k,1) = By(i,j,k,1) - (alphaz*(Ex(i ,j ,k+1,1)+Ex(i ,j ,k+1,2)+Ex(i ,j ,k+1,3) & - -Ex(i ,j ,k ,1)-Ex(i ,j ,k ,2)-Ex(i ,j ,k ,3)) & - +betazx*(Ex(i+1,j ,k+1,1)+Ex(i+1,j ,k+1,2)+Ex(i+1,j ,k+1,3) & - -Ex(i+1,j ,k ,1)-Ex(i+1,j ,k ,2)-Ex(i+1,j ,k ,3) & - +Ex(i-1,j ,k+1,1)+Ex(i-1,j ,k+1,2)+Ex(i-1,j ,k+1,3) & - -Ex(i-1,j ,k ,1)-Ex(i-1,j ,k ,2)-Ex(i-1,j ,k ,3)) & - +betaxy*(Ex(i ,j+1,k+1,1)+Ex(i ,j+1,k+1,2)+Ex(i ,j+1,k+1,3) & - -Ex(i ,j+1,k ,1)-Ex(i ,j+1,k ,2)-Ex(i ,j+1,k ,3) & - +Ex(i ,j-1,k+1,1)+Ex(i ,j-1,k+1,2)+Ex(i ,j-1,k+1,3) & - -Ex(i ,j-1,k ,1)-Ex(i ,j-1,k ,2)-Ex(i ,j-1,k ,3)) & - +gammaz*(Ex(i+1,j+1,k+1,1)+Ex(i+1,j+1,k+1,2)+Ex(i+1,j+1,k+1,3) & - -Ex(i+1,j+1,k ,1)-Ex(i+1,j+1,k ,2)-Ex(i+1,j+1,k ,3) & - +Ex(i-1,j+1,k+1,1)+Ex(i-1,j+1,k+1,2)+Ex(i-1,j+1,k+1,3) & - -Ex(i-1,j+1,k ,1)-Ex(i-1,j+1,k ,2)-Ex(i-1,j+1,k ,3) & - +Ex(i+1,j-1,k+1,1)+Ex(i+1,j-1,k+1,2)+Ex(i+1,j-1,k+1,3) & - -Ex(i+1,j-1,k ,1)-Ex(i+1,j-1,k ,2)-Ex(i+1,j-1,k ,3) & - +Ex(i-1,j-1,k+1,1)+Ex(i-1,j-1,k+1,2)+Ex(i-1,j-1,k+1,3) & - -Ex(i-1,j-1,k ,1)-Ex(i-1,j-1,k ,2)-Ex(i-1,j-1,k ,3))) - - - By(i,j,k,2) = By(i,j,k,2) + (alphax*(Ez(i+1,j ,k ,1)+Ez(i+1,j ,k ,2)+Ez(i+1,j ,k ,3) & - -Ez(i ,j ,k ,1)-Ez(i ,j ,k ,2)-Ez(i ,j ,k ,3)) & - +betaxy*(Ez(i+1,j+1,k ,1)+Ez(i+1,j+1,k ,2)+Ez(i+1,j+1,k ,3) & - -Ez(i ,j+1,k ,1)-Ez(i ,j+1,k ,2)-Ez(i ,j+1,k ,3) & - +Ez(i+1,j-1,k ,1)+Ez(i+1,j-1,k ,2)+Ez(i+1,j-1,k ,3) & - -Ez(i ,j-1,k ,1)-Ez(i ,j-1,k ,2)-Ez(i ,j-1,k ,3)) & - +betaxz*(Ez(i+1,j ,k+1,1)+Ez(i+1,j ,k+1,2)+Ez(i+1,j ,k+1,3) & - -Ez(i ,j ,k+1,1)-Ez(i ,j ,k+1,2)-Ez(i ,j ,k+1,3) & - +Ez(i+1,j ,k-1,1)+Ez(i+1,j ,k-1,2)+Ez(i+1,j ,k-1,3) & - -Ez(i ,j ,k-1,1)-Ez(i ,j ,k-1,2)-Ez(i ,j ,k-1,3)) & - +gammax*(Ez(i+1,j+1,k+1,1)+Ez(i+1,j+1,k+1,2)+Ez(i+1,j+1,k+1,3) & - -Ez(i ,j+1,k+1,1)-Ez(i ,j+1,k+1,2)-Ez(i ,j+1,k+1,3) & - +Ez(i+1,j-1,k+1,1)+Ez(i+1,j-1,k+1,2)+Ez(i+1,j-1,k+1,3) & - -Ez(i ,j-1,k+1,1)-Ez(i ,j-1,k+1,2)-Ez(i ,j-1,k+1,3) & - +Ez(i+1,j+1,k-1,1)+Ez(i+1,j+1,k-1,2)+Ez(i+1,j+1,k-1,3) & - -Ez(i ,j+1,k-1,1)-Ez(i ,j+1,k-1,2)-Ez(i ,j+1,k-1,3) & - +Ez(i+1,j-1,k-1,1)+Ez(i+1,j-1,k-1,2)+Ez(i+1,j-1,k-1,3) & - -Ez(i ,j-1,k-1,1)-Ez(i ,j-1,k-1,2)-Ez(i ,j-1,k-1,3))) - end do - end do - end do - - do k = zlo(3), zhi(3) - do j = zlo(2), zhi(2) - do i = zlo(1), zhi(1) - Bz(i,j,k,1) = Bz(i,j,k,1) - (alphax*(Ey(i+1,j ,k ,1)+Ey(i+1,j ,k ,2)+Ey(i+1,j ,k ,3) & - -Ey(i ,j ,k ,1)-Ey(i ,j ,k ,2)-Ey(i ,j ,k ,3)) & - +betaxy*(Ey(i+1,j+1,k ,1)+Ey(i+1,j+1,k ,2)+Ey(i+1,j+1,k ,3) & - -Ey(i ,j+1,k ,1)-Ey(i ,j+1,k ,2)-Ey(i ,j+1,k ,3) & - +Ey(i+1,j-1,k ,1)+Ey(i+1,j-1,k ,2)+Ey(i+1,j-1,k ,3) & - -Ey(i ,j-1,k ,1)-Ey(i ,j-1,k ,2)-Ey(i ,j-1,k ,3)) & - +betaxz*(Ey(i+1,j ,k+1,1)+Ey(i+1,j ,k+1,2)+Ey(i+1,j ,k+1,3) & - -Ey(i ,j ,k+1,1)-Ey(i ,j ,k+1,2)-Ey(i ,j ,k+1,3) & - +Ey(i+1,j ,k-1,1)+Ey(i+1,j ,k-1,2)+Ey(i+1,j ,k-1,3) & - -Ey(i ,j ,k-1,1)-Ey(i ,j ,k-1,2)-Ey(i ,j ,k-1,3)) & - +gammax*(Ey(i+1,j+1,k+1,1)+Ey(i+1,j+1,k+1,2)+Ey(i+1,j+1,k+1,3) & - -Ey(i ,j+1,k+1,1)-Ey(i ,j+1,k+1,2)-Ey(i ,j+1,k+1,3) & - +Ey(i+1,j-1,k+1,1)+Ey(i+1,j-1,k+1,2)+Ey(i+1,j-1,k+1,3) & - -Ey(i ,j-1,k+1,1)-Ey(i ,j-1,k+1,2)-Ey(i ,j-1,k+1,3) & - +Ey(i+1,j+1,k-1,1)+Ey(i+1,j+1,k-1,2)+Ey(i+1,j+1,k-1,3) & - -Ey(i ,j+1,k-1,1)-Ey(i ,j+1,k-1,2)-Ey(i ,j+1,k-1,3) & - +Ey(i+1,j-1,k-1,1)+Ey(i+1,j-1,k-1,2)+Ey(i+1,j-1,k-1,3) & - -Ey(i ,j-1,k-1,1)-Ey(i ,j-1,k-1,2)-Ey(i ,j-1,k-1,3))) - - Bz(i,j,k,2) = Bz(i,j,k,2) + (alphay*(Ex(i ,j+1,k ,1)+Ex(i ,j+1,k ,2)+Ex(i ,j+1,k ,3) & - -Ex(i ,j ,k ,1)-Ex(i ,j ,k ,2)-Ex(i ,j ,k ,3)) & - +betayx*(Ex(i+1,j+1,k ,1)+Ex(i+1,j+1,k ,2)+Ex(i+1,j+1,k ,3) & - -Ex(i+1,j ,k ,1)-Ex(i+1,j ,k ,2)-Ex(i+1,j ,k ,3) & - +Ex(i-1,j+1,k ,1)+Ex(i-1,j+1,k ,2)+Ex(i-1,j+1,k ,3) & - -Ex(i-1,j ,k ,1)-Ex(i-1,j ,k ,2)-Ex(i-1,j ,k ,3)) & - +betayz*(Ex(i ,j+1,k+1,1)+Ex(i ,j+1,k+1,2)+Ex(i ,j+1,k+1,3) & - -Ex(i ,j ,k+1,1)-Ex(i ,j ,k+1,2)-Ex(i ,j ,k+1,3) & - +Ex(i ,j+1,k-1,1)+Ex(i ,j+1,k-1,2)+Ex(i ,j+1,k-1,3) & - -Ex(i ,j ,k-1,1)-Ex(i ,j ,k-1,2)-Ex(i ,j ,k-1,3)) & - +gammay*(Ex(i+1,j+1,k+1,1)+Ex(i+1,j+1,k+1,2)+Ex(i+1,j+1,k+1,3) & - -Ex(i+1,j ,k+1,1)-Ex(i+1,j ,k+1,2)-Ex(i+1,j ,k+1,3) & - +Ex(i-1,j+1,k+1,1)+Ex(i-1,j+1,k+1,2)+Ex(i-1,j+1,k+1,3) & - -Ex(i-1,j ,k+1,1)-Ex(i-1,j ,k+1,2)-Ex(i-1,j ,k+1,3) & - +Ex(i+1,j+1,k-1,1)+Ex(i+1,j+1,k-1,2)+Ex(i+1,j+1,k-1,3) & - -Ex(i+1,j ,k-1,1)-Ex(i+1,j ,k-1,2)-Ex(i+1,j ,k-1,3) & - +Ex(i-1,j+1,k-1,1)+Ex(i-1,j+1,k-1,2)+Ex(i-1,j+1,k-1,3) & - -Ex(i-1,j ,k-1,1)-Ex(i-1,j ,k-1,2)-Ex(i-1,j ,k-1,3))) - end do - end do - end do - - end if - - end subroutine warpx_push_pml_bvec_3d - - subroutine warpx_push_pml_evec_3d (xlo, xhi, ylo, yhi, zlo, zhi, & - & Ex, Exlo, Exhi, & - & Ey, Eylo, Eyhi, & - & Ez, Ezlo, Ezhi, & - & Bx, Bxlo, Bxhi, & - & By, Bylo, Byhi, & - & Bz, Bzlo, Bzhi, & - & dtsdx, dtsdy, dtsdz) & - bind(c,name='warpx_push_pml_evec_3d') - integer, intent(in) :: xlo(3), xhi(3), ylo(3), yhi(3), zlo(3), zhi(3), & - Exlo(3), Exhi(3), Eylo(3), Eyhi(3), Ezlo(3), Ezhi(3), & - Bxlo(3), Bxhi(3), Bylo(3), Byhi(3), Bzlo(3), Bzhi(3) - real(amrex_real), intent(in) :: dtsdx, dtsdy, dtsdz - real(amrex_real), intent(inout) :: Ex (Exlo(1):Exhi(1),Exlo(2):Exhi(2),Exlo(3):Exhi(3),2) - real(amrex_real), intent(inout) :: Ey (Eylo(1):Eyhi(1),Eylo(2):Eyhi(2),Eylo(3):Eyhi(3),2) - real(amrex_real), intent(inout) :: Ez (Ezlo(1):Ezhi(1),Ezlo(2):Ezhi(2),Ezlo(3):Ezhi(3),2) - real(amrex_real), intent(in ) :: Bx (Bxlo(1):Bxhi(1),Bxlo(2):Bxhi(2),Bxlo(3):Bxhi(3),2) - real(amrex_real), intent(in ) :: By (Bylo(1):Byhi(1),Bylo(2):Byhi(2),Bylo(3):Byhi(3),2) - real(amrex_real), intent(in ) :: Bz (Bzlo(1):Bzhi(1),Bzlo(2):Bzhi(2),Bzlo(3):Bzhi(3),2) - - integer :: i, j, k - - do k = xlo(3), xhi(3) - do j = xlo(2), xhi(2) - do i = xlo(1), xhi(1) - Ex(i,j,k,1) = Ex(i,j,k,1) + dtsdy*(Bz(i,j ,k ,1)+Bz(i,j ,k ,2) & - & -Bz(i,j-1,k ,1)-Bz(i,j-1,k ,2)) - Ex(i,j,k,2) = Ex(i,j,k,2) - dtsdz*(By(i,j ,k ,1)+By(i,j ,k ,2) & - & -By(i,j ,k-1,1)-By(i,j ,k-1,2)) - end do - end do - end do - - do k = ylo(3), yhi(3) - do j = ylo(2), yhi(2) - do i = ylo(1), yhi(1) - Ey(i,j,k,1) = Ey(i,j,k,1) + dtsdz*(Bx(i ,j,k ,1)+Bx(i ,j,k ,2) & - & -Bx(i ,j,k-1,1)-Bx(i ,j,k-1,2)) - Ey(i,j,k,2) = Ey(i,j,k,2) - dtsdx*(Bz(i ,j,k ,1)+Bz(i ,j,k ,2) & - & -Bz(i-1,j,k ,1)-Bz(i-1,j,k ,2)) - end do - end do - end do - - do k = zlo(3), zhi(3) - do j = zlo(2), zhi(2) - do i = zlo(1), zhi(1) - Ez(i,j,k,1) = Ez(i,j,k,1) + dtsdx*(By(i ,j ,k,1)+By(i ,j ,k,2) & - & -By(i-1,j ,k,1)-By(i-1,j ,k,2)) - Ez(i,j,k,2) = Ez(i,j,k,2) - dtsdy*(Bx(i ,j ,k,1)+Bx(i ,j ,k,2) & - & -Bx(i ,j-1,k,1)-Bx(i ,j-1,k,2)) - end do - end do - end do - - end subroutine warpx_push_pml_evec_3d - - subroutine warpx_push_pml_bvec_2d (xlo, xhi, ylo, yhi, zlo, zhi, & - & Ex, Exlo, Exhi, & - & Ey, Eylo, Eyhi, & - & Ez, Ezlo, Ezhi, & - & Bx, Bxlo, Bxhi, & - & By, Bylo, Byhi, & - & Bz, Bzlo, Bzhi, & - & dtsdx, dtsdy, dtsdz, solver_type) & - bind(c,name='warpx_push_pml_bvec_2d') - use amrex_constants_module, only : one, two, eighth - integer, intent(in) :: xlo(2), xhi(2), ylo(2), yhi(2), zlo(2), zhi(2), & - Exlo(2), Exhi(2), Eylo(2), Eyhi(2), Ezlo(2), Ezhi(2), & - Bxlo(2), Bxhi(2), Bylo(2), Byhi(2), Bzlo(2), Bzhi(2), solver_type - real(amrex_real), intent(in ) :: Ex (Exlo(1):Exhi(1),Exlo(2):Exhi(2),3) - real(amrex_real), intent(in ) :: Ey (Eylo(1):Eyhi(1),Eylo(2):Eyhi(2),3) - real(amrex_real), intent(in ) :: Ez (Ezlo(1):Ezhi(1),Ezlo(2):Ezhi(2),3) - real(amrex_real), intent(inout) :: Bx (Bxlo(1):Bxhi(1),Bxlo(2):Bxhi(2),2) - real(amrex_real), intent(inout) :: By (Bylo(1):Byhi(1),Bylo(2):Byhi(2),2) - real(amrex_real), intent(inout) :: Bz (Bzlo(1):Bzhi(1),Bzlo(2):Bzhi(2),2) - real(amrex_real), intent(in) :: dtsdx, dtsdy, dtsdz - - integer :: i, k - - real(amrex_real) :: delta, rx, rz, betaxz, betazx, alphax, alphaz - - - ! solver_type: 0=Yee; 1=CKC - - if (solver_type==0) then - - ! Yee push - - do k = xlo(2), xhi(2) - do i = xlo(1), xhi(1) - Bx(i,k,2) = Bx(i,k,2) + dtsdz*(Ey(i,k+1,1)+Ey(i,k+1,2)+Ey(i,k+1,3) & - & -Ey(i,k ,1)-Ey(i,k ,2)-Ey(i,k ,3)) - end do - end do - - do k = ylo(2), yhi(2) - do i = ylo(1), yhi(1) - By(i,k,1) = By(i,k,1) - dtsdz*(Ex(i ,k+1,1)+Ex(i ,k+1,2)+Ex(i ,k+1,3) & - & -Ex(i ,k ,1)-Ex(i ,k ,2)-Ex(i ,k ,3)) - By(i,k,2) = By(i,k,2) + dtsdx*(Ez(i+1,k ,1)+Ez(i+1,k ,2)+Ez(i+1,k ,3) & - & -Ez(i ,k ,1)-Ez(i ,k ,2)-Ez(i ,k ,3)) - end do - end do - - do k = zlo(2), zhi(2) - do i = zlo(1), zhi(1) - Bz(i,k,1) = Bz(i,k,1) - dtsdx*(Ey(i+1,k,1)+Ey(i+1,k,2)+Ey(i+1,k,3) & - & -Ey(i ,k,1)-Ey(i ,k,2)-Ey(i ,k,3)) - end do - end do - - else - - ! Cole-Karkkainen-Cowan push - - ! computes coefficients according to Cowan - PRST-AB 16, 041303 (2013) - delta = max(dtsdx,dtsdz) - rx = (dtsdx/delta)**2 - rz = (dtsdz/delta)**2 - betaxz = eighth*rz - betazx = eighth*rx - alphax = one - two*betaxz - alphaz = one - two*betazx - - betaxz = dtsdx*betaxz - betazx = dtsdz*betazx - alphax = dtsdx*alphax - alphaz = dtsdz*alphaz - - do k = xlo(2), xhi(2) - do i = xlo(1), xhi(1) - Bx(i,k,2) = Bx(i,k,2) + ( alphaz*(Ey(i ,k+1,1)+Ey(i ,k+1,2)+Ey(i ,k+1,3) & - -Ey(i ,k ,1)-Ey(i ,k ,2)-Ey(i ,k ,3)) & - + betazx*(Ey(i+1,k+1,1)+Ey(i+1,k+1,2)+Ey(i+1,k+1,3) & - -Ey(i+1,k ,1)-Ey(i+1,k ,2)-Ey(i+1,k ,3) & - +Ey(i-1,k+1,1)+Ey(i-1,k+1,2)+Ey(i-1,k+1,3) & - -Ey(i-1,k ,1)-Ey(i-1,k ,2)-Ey(i-1,k ,3))) - end do - end do - - do k = ylo(2), yhi(2) - do i = ylo(1), yhi(1) - By(i,k,1) = By(i,k,1) - ( alphaz*(Ex(i ,k+1,1)+Ex(i ,k+1,2)+Ex(i ,k+1,3) & - -Ex(i ,k ,1)-Ex(i ,k ,2)-Ex(i ,k ,3)) & - + betazx*(Ex(i+1,k+1,1)+Ex(i+1,k+1,2)+Ex(i+1,k+1,3) & - -Ex(i+1,k ,1)-Ex(i+1,k ,2)-Ex(i+1,k ,3) & - +Ex(i-1,k+1,1)+Ex(i-1,k+1,2)+Ex(i-1,k+1,3) & - -Ex(i-1,k ,1)-Ex(i-1,k ,2)-Ex(i-1,k ,3))) - - By(i,k,2) = By(i,k,2) + ( alphax*(Ez(i+1,k ,1)+Ez(i+1,k ,2)+Ez(i+1,k ,3) & - -Ez(i ,k ,1)-Ez(i ,k ,2)-Ez(i ,k ,3)) & - + betaxz*(Ez(i+1,k+1,1)+Ez(i+1,k+1,2)+Ez(i+1,k+1,3) & - -Ez(i ,k+1,1)-Ez(i ,k+1,2)-Ez(i ,k+1,3) & - +Ez(i+1,k-1,1)+Ez(i+1,k-1,2)+Ez(i+1,k-1,3) & - -Ez(i ,k-1,1)-Ez(i ,k-1,2)-Ez(i ,k-1,3))) - - end do - end do - - do k = zlo(2), zhi(2) - do i = zlo(1), zhi(1) - Bz(i,k,1) = Bz(i,k,1) - ( alphax*(Ey(i+1,k ,1)+Ey(i+1,k ,2)+Ey(i+1,k ,3) & - -Ey(i ,k ,1)-Ey(i ,k ,2)-Ey(i ,k ,3)) & - + betaxz*(Ey(i+1,k+1,1)+Ey(i+1,k+1,2)+Ey(i+1,k+1,3) & - -Ey(i ,k+1,1)-Ey(i ,k+1,2)-Ey(i ,k+1,3) & - +Ey(i+1,k-1,1)+Ey(i+1,k-1,2)+Ey(i+1,k-1,3) & - -Ey(i ,k-1,1)-Ey(i ,k-1,2)-Ey(i ,k-1,3))) - end do - end do - - end if - - - end subroutine warpx_push_pml_bvec_2d - - subroutine warpx_push_pml_evec_2d (xlo, xhi, ylo, yhi, zlo, zhi, & - & Ex, Exlo, Exhi, & - & Ey, Eylo, Eyhi, & - & Ez, Ezlo, Ezhi, & - & Bx, Bxlo, Bxhi, & - & By, Bylo, Byhi, & - & Bz, Bzlo, Bzhi, & - & dtsdx, dtsdy, dtsdz) & - bind(c,name='warpx_push_pml_evec_2d') - integer, intent(in) :: xlo(2), xhi(2), ylo(2), yhi(2), zlo(2), zhi(2), & - Exlo(2), Exhi(2), Eylo(2), Eyhi(2), Ezlo(2), Ezhi(2), & - Bxlo(2), Bxhi(2), Bylo(2), Byhi(2), Bzlo(2), Bzhi(2) - real(amrex_real), intent(in) :: dtsdx, dtsdy, dtsdz - real(amrex_real), intent(inout) :: Ex (Exlo(1):Exhi(1),Exlo(2):Exhi(2),2) - real(amrex_real), intent(inout) :: Ey (Eylo(1):Eyhi(1),Eylo(2):Eyhi(2),2) - real(amrex_real), intent(inout) :: Ez (Ezlo(1):Ezhi(1),Ezlo(2):Ezhi(2),2) - real(amrex_real), intent(in ) :: Bx (Bxlo(1):Bxhi(1),Bxlo(2):Bxhi(2),2) - real(amrex_real), intent(in ) :: By (Bylo(1):Byhi(1),Bylo(2):Byhi(2),2) - real(amrex_real), intent(in ) :: Bz (Bzlo(1):Bzhi(1),Bzlo(2):Bzhi(2),2) - - integer :: i, k - - do k = xlo(2), xhi(2) - do i = xlo(1), xhi(1) - Ex(i,k,2) = Ex(i,k,2) - dtsdz*(By(i,k ,1)+By(i,k ,2) & - & -By(i,k-1,1)-By(i,k-1,2)) - end do - end do - - do k = ylo(2), yhi(2) - do i = ylo(1), yhi(1) - Ey(i,k,1) = Ey(i,k,1) + dtsdz*(Bx(i ,k ,1)+Bx(i ,k ,2) & - & -Bx(i ,k-1,1)-Bx(i ,k-1,2)) - Ey(i,k,2) = Ey(i,k,2) - dtsdx*(Bz(i ,k ,1)+Bz(i ,k ,2) & - & -Bz(i-1,k ,1)-Bz(i-1,k ,2)) - end do - end do - - do k = zlo(2), zhi(2) - do i = zlo(1), zhi(1) - Ez(i,k,1) = Ez(i,k,1) + dtsdx*(By(i ,k,1)+By(i ,k,2) & - & -By(i-1,k,1)-By(i-1,k,2)) - end do - end do - - end subroutine warpx_push_pml_evec_2d - - - subroutine warpx_push_pml_f_3d (lo, hi, & - & f , flo, fhi, & - & Ex, Exlo, Exhi, & - & Ey, Eylo, Eyhi, & - & Ez, Ezlo, Ezhi, & - & dtdx, dtdy, dtdz) & - bind(c,name='warpx_push_pml_f_3d') - integer, intent(in) :: lo(3), hi(3), Exlo(3), Exhi(3), Eylo(3), Eyhi(3), Ezlo(3), Ezhi(3), & - flo(3), fhi(3) - real(amrex_real), intent(inout) :: f ( flo(1): fhi(1), flo(2): fhi(2), flo(3): fhi(3),3) - real(amrex_real), intent(in ) :: Ex (Exlo(1):Exhi(1),Exlo(2):Exhi(2),Exlo(3):Exhi(3),3) - real(amrex_real), intent(in ) :: Ey (Eylo(1):Eyhi(1),Eylo(2):Eyhi(2),Eylo(3):Eyhi(3),3) - real(amrex_real), intent(in ) :: Ez (Ezlo(1):Ezhi(1),Ezlo(2):Ezhi(2),Ezlo(3):Ezhi(3),3) - real(amrex_real), intent(in) :: dtdx, dtdy, dtdz - - integer :: i, j, k - - do k = lo(3), hi(3) - do j = lo(2), hi(2) - do i = lo(1), hi(1) - f(i,j,k,1) = f(i,j,k,1) + dtdx*((Ex(i,j,k,1)-Ex(i-1,j,k,1)) & - & + (Ex(i,j,k,2)-Ex(i-1,j,k,2)) & - & + (Ex(i,j,k,3)-Ex(i-1,j,k,3))) - f(i,j,k,2) = f(i,j,k,2) + dtdy*((Ey(i,j,k,1)-Ey(i,j-1,k,1)) & - & + (Ey(i,j,k,2)-Ey(i,j-1,k,2)) & - & + (Ey(i,j,k,3)-Ey(i,j-1,k,3))) - f(i,j,k,3) = f(i,j,k,3) + dtdz*((Ez(i,j,k,1)-Ez(i,j,k-1,1)) & - & + (Ez(i,j,k,2)-Ez(i,j,k-1,2)) & - & + (Ez(i,j,k,3)-Ez(i,j,k-1,3))) - end do - end do - end do - end subroutine warpx_push_pml_f_3d - - subroutine warpx_push_pml_f_2d (lo, hi, & - & f , flo, fhi, & - & Ex, Exlo, Exhi, & - & Ey, Eylo, Eyhi, & - & Ez, Ezlo, Ezhi, & - & dtdx, dtdy, dtdz) & - bind(c,name='warpx_push_pml_f_2d') - integer, intent(in) :: lo(2), hi(2), Exlo(2), Exhi(2), Eylo(2), Eyhi(2), Ezlo(2), Ezhi(2), & - flo(2), fhi(2) - real(amrex_real), intent(inout) :: f ( flo(1): fhi(1), flo(2): fhi(2),3) - real(amrex_real), intent(in ) :: Ex (Exlo(1):Exhi(1),Exlo(2):Exhi(2),3) - real(amrex_real), intent(in ) :: Ey (Eylo(1):Eyhi(1),Eylo(2):Eyhi(2),3) - real(amrex_real), intent(in ) :: Ez (Ezlo(1):Ezhi(1),Ezlo(2):Ezhi(2),3) - real(amrex_real), intent(in) :: dtdx, dtdy, dtdz - - integer :: i, k - - do k = lo(2), hi(2) - do i = lo(1), hi(1) - f(i,k,1) = f(i,k,1) + dtdx*((Ex(i,k,1)-Ex(i-1,k,1)) & - & + (Ex(i,k,2)-Ex(i-1,k,2)) & - & + (Ex(i,k,3)-Ex(i-1,k,3))) - f(i,k,3) = f(i,k,3) + dtdz*((Ez(i,k,1)-Ez(i,k-1,1)) & - & + (Ez(i,k,2)-Ez(i,k-1,2)) & - & + (Ez(i,k,3)-Ez(i,k-1,3))) - end do - end do - end subroutine warpx_push_pml_f_2d - - subroutine warpx_push_pml_evec_f_3d (xlo, xhi, ylo, yhi, zlo, zhi, & - & Ex, Exlo, Exhi, & - & Ey, Eylo, Eyhi, & - & Ez, Ezlo, Ezhi, & - & f, flo, fhi, & - & dtsdx, dtsdy, dtsdz, solver_type) & - bind(c,name='warpx_push_pml_evec_f_3d') - use amrex_constants_module, only : one, two, four, eighth - integer, intent(in) :: xlo(3), xhi(3), ylo(3), yhi(3), zlo(3), zhi(3), & - Exlo(3), Exhi(3), Eylo(3), Eyhi(3), Ezlo(3), Ezhi(3), flo(3), fhi(3), & - solver_type - real(amrex_real), intent(inout) :: Ex (Exlo(1):Exhi(1),Exlo(2):Exhi(2),Exlo(3):Exhi(3),3) - real(amrex_real), intent(inout) :: Ey (Eylo(1):Eyhi(1),Eylo(2):Eyhi(2),Eylo(3):Eyhi(3),3) - real(amrex_real), intent(inout) :: Ez (Ezlo(1):Ezhi(1),Ezlo(2):Ezhi(2),Ezlo(3):Ezhi(3),3) - real(amrex_real), intent(in ) :: f ( flo(1): fhi(1), flo(2): fhi(2), flo(3): fhi(3),3) - real(amrex_real), intent(in) :: dtsdx, dtsdy, dtsdz - - real(amrex_real), parameter :: sixteenth = 1.d0/16.d0 - - integer :: i, j, k - - real(amrex_real) :: delta, rx, ry, rz, betaxz, betaxy, betayx, betayz, betazx, betazy - real(amrex_real) :: beta, alphax, alphay, alphaz, gammax, gammay, gammaz - - ! solver_type: 0=Yee; 1=CKC - - if (solver_type==0) then - - do k = xlo(3), xhi(3) - do j = xlo(2), xhi(2) - do i = xlo(1), xhi(1) - Ex(i,j,k,3) = Ex(i,j,k,3) + dtsdx*((f(i+1,j,k,1)-f(i,j,k,1)) & - & + (f(i+1,j,k,2)-f(i,j,k,2)) & - & + (f(i+1,j,k,3)-f(i,j,k,3))) - end do - end do - end do - - do k = ylo(3), yhi(3) - do j = ylo(2), yhi(2) - do i = ylo(1), yhi(1) - Ey(i,j,k,3) = Ey(i,j,k,3) + dtsdx*((f(i,j+1,k,1)-f(i,j,k,1)) & - & + (f(i,j+1,k,2)-f(i,j,k,2)) & - & + (f(i,j+1,k,3)-f(i,j,k,3))) - end do - end do - end do - - do k = zlo(3), zhi(3) - do j = zlo(2), zhi(2) - do i = zlo(1), zhi(1) - Ez(i,j,k,3) = Ez(i,j,k,3) + dtsdx*((f(i,j,k+1,1)-f(i,j,k,1)) & - & + (f(i,j,k+1,2)-f(i,j,k,2)) & - & + (f(i,j,k+1,3)-f(i,j,k,3))) - end do - end do - end do - - else - - ! CKC push - - ! computes coefficients according to Cowan - PRST-AB 16, 041303 (2013) - delta = max(dtsdx,dtsdy,dtsdz) - rx = (dtsdx/delta)**2 - ry = (dtsdy/delta)**2 - rz = (dtsdz/delta)**2 - beta = eighth*(one-rx*ry*rz/(ry*rz+rz*rx+rx*ry)) - betaxy = ry*beta - betaxz = rz*beta - betayx = rx*beta - betayz = rz*beta - betazx = rx*beta - betazy = ry*beta - gammax = ry*rz*(sixteenth-eighth*ry*rz/(ry*rz+rz*rx+rx*ry)) - gammay = rx*rz*(sixteenth-eighth*rx*rz/(ry*rz+rz*rx+rx*ry)) - gammaz = rx*ry*(sixteenth-eighth*rx*ry/(ry*rz+rz*rx+rx*ry)) - alphax = one - two*betaxy - two* betaxz - four*gammax - alphay = one - two*betayx - two* betayz - four*gammay - alphaz = one - two*betazx - two* betazy - four*gammaz - - betaxy = dtsdx*betaxy - betaxz = dtsdx*betaxz - betayx = dtsdy*betayx - betayz = dtsdy*betayz - betazx = dtsdz*betazx - betazy = dtsdz*betazy - alphax = dtsdx*alphax - alphay = dtsdy*alphay - alphaz = dtsdz*alphaz - gammax = dtsdx*gammax - gammay = dtsdy*gammay - gammaz = dtsdz*gammaz - - do k = xlo(3), xhi(3) - do j = xlo(2), xhi(2) - do i = xlo(1), xhi(1) - Ex(i,j,k,3) = Ex(i,j,k,3) & - +alphax*(f(i+1,j ,k ,1)+f(i+1,j ,k ,2)+f(i+1,j ,k ,3) & - -f(i ,j ,k ,1)-f(i ,j ,k ,2)-f(i ,j ,k ,3)) & - +betaxy*(f(i+1,j+1,k ,1)+f(i+1,j+1,k ,2)+f(i+1,j+1,k ,3) & - -f(i ,j+1,k ,1)-f(i ,j+1,k ,2)-f(i ,j+1,k ,3) & - +f(i+1,j-1,k ,1)+f(i+1,j-1,k ,2)+f(i+1,j-1,k ,3) & - -f(i ,j-1,k ,1)-f(i ,j-1,k ,2)-f(i ,j-1,k ,3)) & - +betaxz*(f(i+1,j ,k+1,1)+f(i+1,j ,k+1,2)+f(i+1,j ,k+1,3) & - -f(i ,j ,k+1,1)-f(i ,j ,k+1,2)-f(i ,j ,k+1,3) & - +f(i+1,j ,k-1,1)+f(i+1,j ,k-1,2)+f(i+1,j ,k-1,3) & - -f(i ,j ,k-1,1)-f(i ,j ,k-1,2)-f(i ,j ,k-1,3)) & - +gammax*(f(i+1,j+1,k+1,1)+f(i+1,j+1,k+1,2)+f(i+1,j+1,k+1,3) & - -f(i ,j+1,k+1,1)-f(i ,j+1,k+1,2)-f(i ,j+1,k+1,3) & - +f(i+1,j-1,k+1,1)+f(i+1,j-1,k+1,2)+f(i+1,j-1,k+1,3) & - -f(i ,j-1,k+1,1)-f(i ,j-1,k+1,2)-f(i ,j-1,k+1,3) & - +f(i+1,j+1,k-1,1)+f(i+1,j+1,k-1,2)+f(i+1,j+1,k-1,3) & - -f(i ,j+1,k-1,1)-f(i ,j+1,k-1,2)-f(i ,j+1,k-1,3) & - +f(i+1,j-1,k-1,1)+f(i+1,j-1,k-1,2)+f(i+1,j-1,k-1,3) & - -f(i ,j-1,k-1,1)-f(i ,j-1,k-1,2)-f(i ,j-1,k-1,3)) - end do - end do - end do - - do k = ylo(3), yhi(3) - do j = ylo(2), yhi(2) - do i = ylo(1), yhi(1) - Ey(i,j,k,3) = Ey(i,j,k,3) & - +alphay*(f(i ,j+1,k ,1)+f(i ,j+1,k ,2)+f(i ,j+1,k ,3) & - -f(i ,j ,k ,1)-f(i ,j ,k ,2)-f(i ,j ,k ,3)) & - +betayx*(f(i+1,j+1,k ,1)+f(i+1,j+1,k ,2)+f(i+1,j+1,k ,3) & - -f(i+1,j ,k ,1)-f(i+1,j ,k ,2)-f(i+1,j ,k ,3) & - +f(i-1,j+1,k ,1)+f(i-1,j+1,k ,2)+f(i-1,j+1,k ,3) & - -f(i-1,j ,k ,1)-f(i-1,j ,k ,2)-f(i-1,j ,k ,3)) & - +betayz*(f(i ,j+1,k+1,1)+f(i ,j+1,k+1,2)+f(i ,j+1,k+1,3) & - -f(i ,j ,k+1,1)-f(i ,j ,k+1,2)-f(i ,j ,k+1,3) & - +f(i ,j+1,k-1,1)+f(i ,j+1,k-1,2)+f(i ,j+1,k-1,3) & - -f(i ,j ,k-1,1)-f(i ,j ,k-1,2)-f(i ,j ,k-1,3)) & - +gammay*(f(i+1,j+1,k+1,1)+f(i+1,j+1,k+1,2)+f(i+1,j+1,k+1,3) & - -f(i+1,j ,k+1,1)-f(i+1,j ,k+1,2)-f(i+1,j ,k+1,3) & - +f(i-1,j+1,k+1,1)+f(i-1,j+1,k+1,2)+f(i-1,j+1,k+1,3) & - -f(i-1,j ,k+1,1)-f(i-1,j ,k+1,2)-f(i-1,j ,k+1,3) & - +f(i+1,j+1,k-1,1)+f(i+1,j+1,k-1,2)+f(i+1,j+1,k-1,3) & - -f(i+1,j ,k-1,1)-f(i+1,j ,k-1,2)-f(i+1,j ,k-1,3) & - +f(i-1,j+1,k-1,1)+f(i-1,j+1,k-1,2)+f(i-1,j+1,k-1,3) & - -f(i-1,j ,k-1,1)-f(i-1,j ,k-1,2)-f(i-1,j ,k-1,3)) - end do - end do - end do - - do k = zlo(3), zhi(3) - do j = zlo(2), zhi(2) - do i = zlo(1), zhi(1) - Ez(i,j,k,3) = Ez(i,j,k,3) & - +alphaz*(f(i ,j ,k+1,1)+f(i ,j ,k+1,2)+f(i ,j ,k+1,3) & - -f(i ,j ,k ,1)-f(i ,j ,k ,2)-f(i ,j ,k ,3)) & - +betazx*(f(i+1,j ,k+1,1)+f(i+1,j ,k+1,2)+f(i+1,j ,k+1,3) & - -f(i+1,j ,k ,1)-f(i+1,j ,k ,2)-f(i+1,j ,k ,3) & - +f(i-1,j ,k+1,1)+f(i-1,j ,k+1,2)+f(i-1,j ,k+1,3) & - -f(i-1,j ,k ,1)-f(i-1,j ,k ,2)-f(i-1,j ,k ,3)) & - +betazy*(f(i ,j+1,k+1,1)+f(i ,j+1,k+1,2)+f(i ,j+1,k+1,3) & - -f(i ,j+1,k ,1)-f(i ,j+1,k ,2)-f(i ,j+1,k ,3) & - +f(i ,j-1,k+1,1)+f(i ,j-1,k+1,2)+f(i ,j-1,k+1,3) & - -f(i ,j-1,k ,1)-f(i ,j-1,k ,2)-f(i ,j-1,k ,3)) & - +gammaz*(f(i+1,j+1,k+1,1)+f(i+1,j+1,k+1,2)+f(i+1,j+1,k+1,3) & - -f(i+1,j+1,k ,1)-f(i+1,j+1,k ,2)-f(i+1,j+1,k ,3) & - +f(i-1,j+1,k+1,1)+f(i-1,j+1,k+1,2)+f(i-1,j+1,k+1,3) & - -f(i-1,j+1,k ,1)-f(i-1,j+1,k ,2)-f(i-1,j+1,k ,3) & - +f(i+1,j-1,k+1,1)+f(i+1,j-1,k+1,2)+f(i+1,j-1,k+1,3) & - -f(i+1,j-1,k ,1)-f(i+1,j-1,k ,2)-f(i+1,j-1,k ,3) & - +f(i-1,j-1,k+1,1)+f(i-1,j-1,k+1,2)+f(i-1,j-1,k+1,3) & - -f(i-1,j-1,k ,1)-f(i-1,j-1,k ,2)-f(i-1,j-1,k ,3)) - end do - end do - end do - - endif - - end subroutine warpx_push_pml_evec_f_3d - - - subroutine warpx_push_pml_evec_f_2d (xlo, xhi, ylo, yhi, zlo, zhi, & - & Ex, Exlo, Exhi, & - & Ey, Eylo, Eyhi, & - & Ez, Ezlo, Ezhi, & - & f, flo, fhi, & - & dtsdx, dtsdy, dtsdz, solver_type) & - bind(c,name='warpx_push_pml_evec_f_2d') - use amrex_constants_module, only : one, two, four, eighth - integer, intent(in) :: xlo(2), xhi(2), ylo(2), yhi(2), zlo(2), zhi(2), & - Exlo(2), Exhi(2), Eylo(2), Eyhi(2), Ezlo(2), Ezhi(2), flo(2), fhi(2), & - solver_type - real(amrex_real), intent(inout) :: Ex (Exlo(1):Exhi(1),Exlo(2):Exhi(2),3) - real(amrex_real), intent(inout) :: Ey (Eylo(1):Eyhi(1),Eylo(2):Eyhi(2),3) - real(amrex_real), intent(inout) :: Ez (Ezlo(1):Ezhi(1),Ezlo(2):Ezhi(2),3) - real(amrex_real), intent(in ) :: f ( flo(1): fhi(1), flo(2): fhi(2),3) - real(amrex_real), intent(in) :: dtsdx, dtsdy, dtsdz - - integer :: i, k - - real(amrex_real) :: delta, rx, rz, betaxz, betazx, alphax, alphaz - - ! solver_type: 0=Yee; 1=CKC - - if (solver_type==0) then - - do k = xlo(2), xhi(2) - do i = xlo(1), xhi(1) - Ex(i,k,3) = Ex(i,k,3) + dtsdx*((f(i+1,k,1)-f(i,k,1)) & - & + (f(i+1,k,2)-f(i,k,2)) & - & + (f(i+1,k,3)-f(i,k,3))) - end do - end do - - do k = zlo(2), zhi(2) - do i = zlo(1), zhi(1) - Ez(i,k,3) = Ez(i,k,3) + dtsdz*((f(i,k+1,1)-f(i,k,1)) & - & + (f(i,k+1,2)-f(i,k,2)) & - & + (f(i,k+1,3)-f(i,k,3))) - end do - end do - - else - - ! Cole-Karkkainen-Cowan push - - ! computes coefficients according to Cowan - PRST-AB 16, 041303 (2013) - delta = max(dtsdx,dtsdz) - rx = (dtsdx/delta)**2 - rz = (dtsdz/delta)**2 - betaxz = eighth*rz - betazx = eighth*rx - alphax = one - two*betaxz - alphaz = one - two*betazx - - betaxz = dtsdx*betaxz - betazx = dtsdz*betazx - alphax = dtsdx*alphax - alphaz = dtsdz*alphaz - - do k = xlo(2), xhi(2) - do i = xlo(1), xhi(1) - Ex(i,k,3) = Ex(i,k,3) & - + alphax*(f(i+1,k ,1)+f(i+1,k ,2)+f(i+1,k ,3) & - -f(i ,k ,1)-f(i ,k ,2)-f(i ,k ,3)) & - + betaxz*(f(i+1,k+1,1)+f(i+1,k+1,2)+f(i+1,k+1,3) & - -f(i ,k+1,1)-f(i ,k+1,2)-f(i ,k+1,3) & - +f(i+1,k-1,1)+f(i+1,k-1,2)+f(i+1,k-1,3) & - -f(i ,k-1,1)-f(i ,k-1,2)-f(i ,k-1,3)) - end do - end do - - do k = zlo(2), zhi(2) - do i = zlo(1), zhi(1) - Ez(i,k,3) = Ez(i,k,3) & - + alphaz*(f(i ,k+1,1)+f(i ,k+1,2)+f(i ,k+1,3) & - -f(i ,k ,1)-f(i ,k ,2)-f(i ,k ,3)) & - + betazx*(f(i+1,k+1,1)+f(i+1,k+1,2)+f(i+1,k+1,3) & - -f(i+1,k ,1)-f(i+1,k ,2)-f(i+1,k ,3) & - +f(i-1,k+1,1)+f(i-1,k+1,2)+f(i-1,k+1,3) & - -f(i-1,k ,1)-f(i-1,k ,2)-f(i-1,k ,3)) - end do - end do - - endif - - end subroutine warpx_push_pml_evec_f_2d - - - subroutine warpx_damp_pml_2d (texlo, texhi, teylo, teyhi, tezlo, tezhi, & - & tbxlo, tbxhi, tbylo, tbyhi, tbzlo, tbzhi, & - & ex, exlo, exhi, ey, eylo, eyhi, ez, ezlo, ezhi, & - & bx, bxlo, bxhi, by, bylo, byhi, bz, bzlo, bzhi, & - & sigex, sexlo, sexhi, sigez, sezlo, sezhi, & - & sigcx, scxlo, scxhi, sigcz, sczlo, sczhi) & - bind(c,name='warpx_damp_pml_2d') - integer, dimension(2), intent(in) :: texlo, texhi, teylo, teyhi, tezlo, tezhi, & - tbxlo, tbxhi, tbylo, tbyhi, tbzlo, tbzhi, & - exlo, exhi, eylo, eyhi, ezlo, ezhi, bxlo, bxhi, bylo, byhi, bzlo, bzhi - integer, intent(in), value :: sexlo, sexhi, sezlo, sezhi, & - & scxlo, scxhi, sczlo, sczhi - real(amrex_real), intent(inout) :: ex(exlo(1):exhi(1),exlo(2):exhi(2),3) - real(amrex_real), intent(inout) :: ey(eylo(1):eyhi(1),eylo(2):eyhi(2),3) - real(amrex_real), intent(inout) :: ez(ezlo(1):ezhi(1),ezlo(2):ezhi(2),3) - real(amrex_real), intent(inout) :: bx(bxlo(1):bxhi(1),bxlo(2):bxhi(2),2) - real(amrex_real), intent(inout) :: by(bylo(1):byhi(1),bylo(2):byhi(2),2) - real(amrex_real), intent(inout) :: bz(bzlo(1):bzhi(1),bzlo(2):bzhi(2),2) - real(amrex_real), intent(in) :: sigex(sexlo:sexhi) - real(amrex_real), intent(in) :: sigez(sezlo:sezhi) - real(amrex_real), intent(in) :: sigcx(scxlo:scxhi) - real(amrex_real), intent(in) :: sigcz(sczlo:sczhi) - - integer :: i,k - - do k = texlo(2), texhi(2) - do i = texlo(1), texhi(1) - ex(i,k,2) = ex(i,k,2) * sigez(k) - ex(i,k,3) = ex(i,k,3) * sigcx(i) - end do - end do - - do k = teylo(2), teyhi(2) - do i = teylo(1), teyhi(1) - ey(i,k,1) = ey(i,k,1) * sigez(k) - ey(i,k,2) = ey(i,k,2) * sigex(i) - end do - end do - - do k = tezlo(2), tezhi(2) - do i = tezlo(1), tezhi(1) - ez(i,k,1) = ez(i,k,1) * sigex(i) - ez(i,k,3) = ez(i,k,3) * sigcz(k) - end do - end do - - do k = tbxlo(2), tbxhi(2) - do i = tbxlo(1), tbxhi(1) - bx(i,k,2) = bx(i,k,2) * sigcz(k) - end do - end do - - do k = tbylo(2), tbyhi(2) - do i = tbylo(1), tbyhi(1) - by(i,k,1) = by(i,k,1) * sigcz(k) - by(i,k,2) = by(i,k,2) * sigcx(i) - end do - end do - - do k = tbzlo(2), tbzhi(2) - do i = tbzlo(1), tbzhi(1) - bz(i,k,1) = bz(i,k,1) * sigcx(i) - end do - end do - - end subroutine warpx_damp_pml_2d - - - subroutine warpx_damp_pml_3d (texlo, texhi, teylo, teyhi, tezlo, tezhi, & - & tbxlo, tbxhi, tbylo, tbyhi, tbzlo, tbzhi, & - & ex, exlo, exhi, ey, eylo, eyhi, ez, ezlo, ezhi, & - & bx, bxlo, bxhi, by, bylo, byhi, bz, bzlo, bzhi, & - & sigex, sexlo, sexhi, sigey, seylo, seyhi, sigez, sezlo, sezhi, & - & sigcx, scxlo, scxhi, sigcy, scylo, scyhi, sigcz, sczlo, sczhi) & - bind(c,name='warpx_damp_pml_3d') - integer, dimension(3), intent(in) :: texlo, texhi, teylo, teyhi, tezlo, tezhi, & - tbxlo, tbxhi, tbylo, tbyhi, tbzlo, tbzhi, & - exlo, exhi, eylo, eyhi, ezlo, ezhi, bxlo, bxhi, bylo, byhi, bzlo, bzhi - integer, intent(in), value :: sexlo, sexhi, seylo, seyhi, sezlo, sezhi, & - & scxlo, scxhi, scylo, scyhi, sczlo, sczhi - real(amrex_real), intent(inout) :: ex(exlo(1):exhi(1),exlo(2):exhi(2),exlo(3):exhi(3),3) - real(amrex_real), intent(inout) :: ey(eylo(1):eyhi(1),eylo(2):eyhi(2),eylo(3):eyhi(3),3) - real(amrex_real), intent(inout) :: ez(ezlo(1):ezhi(1),ezlo(2):ezhi(2),ezlo(3):ezhi(3),3) - real(amrex_real), intent(inout) :: bx(bxlo(1):bxhi(1),bxlo(2):bxhi(2),bxlo(3):bxhi(3),2) - real(amrex_real), intent(inout) :: by(bylo(1):byhi(1),bylo(2):byhi(2),bylo(3):byhi(3),2) - real(amrex_real), intent(inout) :: bz(bzlo(1):bzhi(1),bzlo(2):bzhi(2),bzlo(3):bzhi(3),2) - real(amrex_real), intent(in) :: sigex(sexlo:sexhi) - real(amrex_real), intent(in) :: sigey(seylo:seyhi) - real(amrex_real), intent(in) :: sigez(sezlo:sezhi) - real(amrex_real), intent(in) :: sigcx(scxlo:scxhi) - real(amrex_real), intent(in) :: sigcy(scylo:scyhi) - real(amrex_real), intent(in) :: sigcz(sczlo:sczhi) - - integer :: i,j,k - - do k = texlo(3), texhi(3) - do j = texlo(2), texhi(2) - do i = texlo(1), texhi(1) - ex(i,j,k,1) = ex(i,j,k,1) * sigey(j) - ex(i,j,k,2) = ex(i,j,k,2) * sigez(k) - ex(i,j,k,3) = ex(i,j,k,3) * sigcx(i) - end do - end do - end do - - do k = teylo(3), teyhi(3) - do j = teylo(2), teyhi(2) - do i = teylo(1), teyhi(1) - ey(i,j,k,1) = ey(i,j,k,1) * sigez(k) - ey(i,j,k,2) = ey(i,j,k,2) * sigex(i) - ey(i,j,k,3) = ey(i,j,k,3) * sigcy(j) - end do - end do - end do - - do k = tezlo(3), tezhi(3) - do j = tezlo(2), tezhi(2) - do i = tezlo(1), tezhi(1) - ez(i,j,k,1) = ez(i,j,k,1) * sigex(i) - ez(i,j,k,2) = ez(i,j,k,2) * sigey(j) - ez(i,j,k,3) = ez(i,j,k,3) * sigcz(k) - end do - end do - end do - - do k = tbxlo(3), tbxhi(3) - do j = tbxlo(2), tbxhi(2) - do i = tbxlo(1), tbxhi(1) - bx(i,j,k,1) = bx(i,j,k,1) * sigcy(j) - bx(i,j,k,2) = bx(i,j,k,2) * sigcz(k) - end do - end do - end do - - do k = tbylo(3), tbyhi(3) - do j = tbylo(2), tbyhi(2) - do i = tbylo(1), tbyhi(1) - by(i,j,k,1) = by(i,j,k,1) * sigcz(k) - by(i,j,k,2) = by(i,j,k,2) * sigcx(i) - end do - end do - end do - - do k = tbzlo(3), tbzhi(3) - do j = tbzlo(2), tbzhi(2) - do i = tbzlo(1), tbzhi(1) - bz(i,j,k,1) = bz(i,j,k,1) * sigcx(i) - bz(i,j,k,2) = bz(i,j,k,2) * sigcy(j) - end do - end do - end do - - end subroutine warpx_damp_pml_3d - - subroutine warpx_damp_pml_f_2d (tndlo, tndhi, f, flo, fhi,& - & sigex, sexlo, sexhi, sigez, sezlo, sezhi, & - & sigcx, scxlo, scxhi, sigcz, sczlo, sczhi) & - bind(c,name='warpx_damp_pml_f_2d') - integer, dimension(2), intent(in) :: tndlo, tndhi, flo, fhi - integer, intent(in), value :: sexlo, sexhi, sezlo, sezhi, & - & scxlo, scxhi, sczlo, sczhi - real(amrex_real), intent(inout) :: f ( flo(1): fhi(1), flo(2): fhi(2),3) - real(amrex_real), intent(in) :: sigex(sexlo:sexhi) - real(amrex_real), intent(in) :: sigez(sezlo:sezhi) - real(amrex_real), intent(in) :: sigcx(scxlo:scxhi) - real(amrex_real), intent(in) :: sigcz(sczlo:sczhi) - - integer :: i,k - - do k = tndlo(2), tndhi(2) - do i = tndlo(1), tndhi(1) - f(i,k,1) = f(i,k,1) * sigex(i) - f(i,k,3) = f(i,k,3) * sigez(k) - end do - end do - end subroutine warpx_damp_pml_f_2d - - - subroutine warpx_damp_pml_f_3d (tndlo, tndhi, f, flo, fhi,& - & sigex, sexlo, sexhi, sigey, seylo, seyhi, sigez, sezlo, sezhi, & - & sigcx, scxlo, scxhi, sigcy, scylo, scyhi, sigcz, sczlo, sczhi) & - bind(c,name='warpx_damp_pml_f_3d') - integer, dimension(3), intent(in) :: tndlo, tndhi, flo, fhi - integer, intent(in), value :: sexlo, sexhi, seylo, seyhi, sezlo, sezhi, & - & scxlo, scxhi, scylo, scyhi, sczlo, sczhi - real(amrex_real), intent(inout) :: f ( flo(1): fhi(1), flo(2): fhi(2), flo(3): fhi(3),3) - real(amrex_real), intent(in) :: sigex(sexlo:sexhi) - real(amrex_real), intent(in) :: sigey(seylo:seyhi) - real(amrex_real), intent(in) :: sigez(sezlo:sezhi) - real(amrex_real), intent(in) :: sigcx(scxlo:scxhi) - real(amrex_real), intent(in) :: sigcy(scylo:scyhi) - real(amrex_real), intent(in) :: sigcz(sczlo:sczhi) - - integer :: i,j,k - - do k = tndlo(3), tndhi(3) - do j = tndlo(2), tndhi(2) - do i = tndlo(1), tndhi(1) - f(i,j,k,1) = f(i,j,k,1) * sigex(i) - f(i,j,k,2) = f(i,j,k,2) * sigey(j) - f(i,j,k,3) = f(i,j,k,3) * sigez(k) - end do - end do - end do - end subroutine warpx_damp_pml_f_3d - -end module warpx_pml_module diff --git a/Source/BoundaryConditions/openbc_f.F90 b/Source/BoundaryConditions/openbc_f.F90 deleted file mode 100644 index c18b1db24..000000000 --- a/Source/BoundaryConditions/openbc_f.F90 +++ /dev/null @@ -1,62 +0,0 @@ - -module warpx_openbc_module - - implicit none - - integer, parameter :: idecomp = 0 ! 0=xyz, 1=xy, 2=yz, 3=xz, 4=x, 5=y, 6=z. - integer, parameter :: igfflag = 1 ! =0 for ordinary 1/r Green function; - ! =1 for integrated Green function - - integer, save :: gb_lo(3), gb_hi(3) - integer, save :: lc_lo(3), lc_hi(3) - -contains - - subroutine warpx_openbc_decompose(glo, ghi, lo, hi) bind(c,name='warpx_openbc_decompose') - - use mpi - - integer, intent(in) :: glo(3), ghi(3) - integer, intent(out) :: lo(3), hi(3) - integer :: myrank, mprocs, ierr, npx, npy, npz - - call MPI_Comm_size(MPI_COMM_WORLD, mprocs, ierr); - call MPI_COMM_RANK(MPI_COMM_WORLD,myrank,ierr) - - gb_lo = glo + 1 ! +1 because AMReX's domain index starts with 0 - gb_hi = ghi + 2 ! +2 to nodalize it - - call procgriddecomp(mprocs, gb_lo(1),gb_hi(1), & - & gb_lo(2),gb_hi(2), & - & gb_lo(3),gb_hi(3), & - & idecomp, npx, npy, npz) - - call decompose(myrank,mprocs, gb_lo(1),gb_hi(1), & - & gb_lo(2),gb_hi(2), & - & gb_lo(3),gb_hi(3), & - idecomp, lc_lo(1), lc_hi(1), & - & lc_lo(2), lc_hi(2), & - & lc_lo(3), lc_hi(3)) - - lo = lc_lo - 1 ! AMReX's domain index starts with zero. - hi = lc_hi - 1 - - end subroutine warpx_openbc_decompose - - subroutine warpx_openbc_potential(rho, phi, dx) & - bind(C, name="warpx_openbc_potential") - - double precision, intent(in ) :: dx(3) - double precision, intent(in ) :: rho(lc_lo(1):lc_hi(1),lc_lo(2):lc_hi(2),lc_lo(3):lc_hi(3)) - double precision, intent(out) :: phi(lc_lo(1):lc_hi(1),lc_lo(2):lc_hi(2),lc_lo(3):lc_hi(3)) - - integer :: ierr - - call openbcpotential(rho, phi, dx(1), dx(2), dx(3), & - lc_lo(1), lc_hi(1), lc_lo(2), lc_hi(2), lc_lo(3), lc_hi(3), & - gb_lo(1), gb_hi(1), gb_lo(2), gb_hi(2), gb_lo(3), gb_hi(3), & - idecomp, igfflag, ierr) - - end subroutine warpx_openbc_potential - -end module warpx_openbc_module diff --git a/Source/Diagnostics/BoostedFrameDiagnostic.H b/Source/Diagnostics/BoostedFrameDiagnostic.H new file mode 100644 index 000000000..452062137 --- /dev/null +++ b/Source/Diagnostics/BoostedFrameDiagnostic.H @@ -0,0 +1,98 @@ +#ifndef WARPX_BoostedFrameDiagnostic_H_ +#define WARPX_BoostedFrameDiagnostic_H_ + +#include + +#include +#include +#include + +#include "ParticleContainer.H" +#include "WarpXConst.H" + +/// +/// BoostedFrameDiagnostic is for handling IO when running in a boosted +/// frame of reference. Because of the relativity of simultaneity, events that +/// are synchronized in the simulation frame are not synchronized in the +/// lab frame. Thus, at a given t_boost, we must write slices of data to +/// multiple output files, each one corresponding to a given time in the lab frame. +/// +class BoostedFrameDiagnostic { + + /// + /// LabSnapShot stores metadata corresponding to a single time + /// snapshot in the lab frame. The snapshot is written to disk + /// in the directory "file_name". zmin_lab, zmax_lab, and t_lab + /// are all constant for a given snapshot. current_z_lab and + /// current_z_boost for each snapshot are updated as the + /// simulation time in the boosted frame advances. + /// + struct LabSnapShot { + + std::string file_name; + amrex::Real t_lab; + amrex::Real zmin_lab; + amrex::Real zmax_lab; + amrex::Real current_z_lab; + amrex::Real current_z_boost; + int file_num; + + LabSnapShot(amrex::Real t_lab_in, amrex::Real zmin_lab_in, + amrex::Real zmax_lab_in, int file_num_in); + + /// + /// This snapshot is at time t_lab, and the simulation is at time t_boost. + /// The Lorentz transformation picks out one slice corresponding to both + /// of those times, at position current_z_boost and current_z_lab in the + /// boosted and lab frames, respectively. + /// + void updateCurrentZPositions(amrex::Real t_boost, amrex::Real inv_gamma, + amrex::Real inv_beta); + + /// + /// Write some useful metadata about this snapshot. + /// + void writeSnapShotHeader(); + }; + + amrex::Real gamma_boost_; + amrex::Real inv_gamma_boost_; + amrex::Real beta_boost_; + amrex::Real inv_beta_boost_; + amrex::Real dz_lab_; + amrex::Real inv_dz_lab_; + amrex::Real dt_snapshots_lab_; + amrex::Real dt_boost_; + int N_snapshots_; + int Nz_lab_; + int boost_direction_; + + amrex::Vector > data_buffer_; + amrex::Vector > particles_buffer_; + int num_buffer_ = 256; + int max_box_size_ = 256; + amrex::Vector buff_counter_; + + amrex::Vector snapshots_; + + void writeParticleData(const WarpXParticleContainer::DiagnosticParticleData& pdata, + const std::string& name, const int i_lab); + +public: + + BoostedFrameDiagnostic(amrex::Real zmin_lab, amrex::Real zmax_lab, + amrex::Real v_window_lab, amrex::Real dt_snapshots_lab, + int N_snapshots, amrex::Real gamma_boost, + amrex::Real t_boost, amrex::Real dt_boost, int boost_direction); + + void Flush(const amrex::Geometry& geom); + + void writeLabFrameData(const amrex::MultiFab* cell_centered_data, + const MultiParticleContainer& mypc, + const amrex::Geometry& geom, + const amrex::Real t_boost, const amrex::Real dt); + + void writeMetaData(); +}; + +#endif diff --git a/Source/Diagnostics/BoostedFrameDiagnostic.cpp b/Source/Diagnostics/BoostedFrameDiagnostic.cpp new file mode 100644 index 000000000..6c3e3c58a --- /dev/null +++ b/Source/Diagnostics/BoostedFrameDiagnostic.cpp @@ -0,0 +1,371 @@ +#include "BoostedFrameDiagnostic.H" +#include +#include "WarpX_f.H" +#include "WarpX.H" + +using namespace amrex; + +BoostedFrameDiagnostic:: +BoostedFrameDiagnostic(Real zmin_lab, Real zmax_lab, Real v_window_lab, + Real dt_snapshots_lab, int N_snapshots, + Real gamma_boost, Real t_boost, Real dt_boost, + int boost_direction) + : gamma_boost_(gamma_boost), + dt_snapshots_lab_(dt_snapshots_lab), + dt_boost_(dt_boost), + N_snapshots_(N_snapshots), + boost_direction_(boost_direction) +{ + + BL_PROFILE("BoostedFrameDiagnostic::BoostedFrameDiagnostic"); + + AMREX_ALWAYS_ASSERT(WarpX::do_boosted_frame_fields or + WarpX::do_boosted_frame_particles); + + inv_gamma_boost_ = 1.0 / gamma_boost_; + beta_boost_ = std::sqrt(1.0 - inv_gamma_boost_*inv_gamma_boost_); + inv_beta_boost_ = 1.0 / beta_boost_; + + dz_lab_ = PhysConst::c * dt_boost_ * inv_beta_boost_ * inv_gamma_boost_; + inv_dz_lab_ = 1.0 / dz_lab_; + Nz_lab_ = static_cast((zmax_lab - zmin_lab) * inv_dz_lab_); + + writeMetaData(); + + if (WarpX::do_boosted_frame_fields) data_buffer_.resize(N_snapshots); + if (WarpX::do_boosted_frame_particles) particles_buffer_.resize(N_snapshots); + for (int i = 0; i < N_snapshots; ++i) { + Real t_lab = i * dt_snapshots_lab_; + LabSnapShot snapshot(t_lab, zmin_lab + v_window_lab * t_lab, + zmax_lab + v_window_lab * t_lab, i); + snapshot.updateCurrentZPositions(t_boost, inv_gamma_boost_, inv_beta_boost_); + snapshots_.push_back(snapshot); + buff_counter_.push_back(0); + if (WarpX::do_boosted_frame_fields) data_buffer_[i].reset( nullptr ); + } + + AMREX_ALWAYS_ASSERT(max_box_size_ >= num_buffer_); +} + +void BoostedFrameDiagnostic::Flush(const Geometry& geom) +{ + BL_PROFILE("BoostedFrameDiagnostic::Flush"); + + VisMF::Header::Version current_version = VisMF::GetHeaderVersion(); + VisMF::SetHeaderVersion(amrex::VisMF::Header::NoFabHeader_v1); + + auto & mypc = WarpX::GetInstance().GetPartContainer(); + const std::vector species_names = mypc.GetSpeciesNames(); + + for (int i = 0; i < N_snapshots_; ++i) { + + int i_lab = (snapshots_[i].current_z_lab - snapshots_[i].zmin_lab) / dz_lab_; + + if (buff_counter_[i] != 0) { + if (WarpX::do_boosted_frame_fields) { + const BoxArray& ba = data_buffer_[i]->boxArray(); + const int hi = ba[0].bigEnd(boost_direction_); + const int lo = hi - buff_counter_[i] + 1; + + Box buff_box = geom.Domain(); + buff_box.setSmall(boost_direction_, lo); + buff_box.setBig(boost_direction_, hi); + + BoxArray buff_ba(buff_box); + buff_ba.maxSize(max_box_size_); + DistributionMapping buff_dm(buff_ba); + + const int ncomp = data_buffer_[i]->nComp(); + + MultiFab tmp(buff_ba, buff_dm, ncomp, 0); + + tmp.copy(*data_buffer_[i], 0, 0, ncomp); + + std::stringstream ss; + ss << snapshots_[i].file_name << "/Level_0/" << Concatenate("buffer", i_lab, 5); + VisMF::Write(tmp, ss.str()); + } + + if (WarpX::do_boosted_frame_particles) { + for (int j = 0; j < mypc.nSpecies(); ++j) { + std::stringstream part_ss; + part_ss << snapshots_[i].file_name + "/" + species_names[j] + "/"; + writeParticleData(particles_buffer_[i][j], part_ss.str(), i_lab); + } + particles_buffer_[i].clear(); + } + buff_counter_[i] = 0; + } + } + + VisMF::SetHeaderVersion(current_version); +} + +void +BoostedFrameDiagnostic:: +writeLabFrameData(const MultiFab* cell_centered_data, + const MultiParticleContainer& mypc, + const Geometry& geom, const Real t_boost, const Real dt) { + + BL_PROFILE("BoostedFrameDiagnostic::writeLabFrameData"); + + VisMF::Header::Version current_version = VisMF::GetHeaderVersion(); + VisMF::SetHeaderVersion(amrex::VisMF::Header::NoFabHeader_v1); + + const RealBox& domain_z_boost = geom.ProbDomain(); + const Real zlo_boost = domain_z_boost.lo(boost_direction_); + const Real zhi_boost = domain_z_boost.hi(boost_direction_); + + const std::vector species_names = mypc.GetSpeciesNames(); + + for (int i = 0; i < N_snapshots_; ++i) { + const Real old_z_boost = snapshots_[i].current_z_boost; + snapshots_[i].updateCurrentZPositions(t_boost, + inv_gamma_boost_, + inv_beta_boost_); + + if ( (snapshots_[i].current_z_boost < zlo_boost) or + (snapshots_[i].current_z_boost > zhi_boost) or + (snapshots_[i].current_z_lab < snapshots_[i].zmin_lab) or + (snapshots_[i].current_z_lab > snapshots_[i].zmax_lab) ) continue; + + int i_lab = (snapshots_[i].current_z_lab - snapshots_[i].zmin_lab) / dz_lab_; + + if (buff_counter_[i] == 0) { + if (WarpX::do_boosted_frame_fields) { + const int ncomp = cell_centered_data->nComp(); + Box buff_box = geom.Domain(); + buff_box.setSmall(boost_direction_, i_lab - num_buffer_ + 1); + buff_box.setBig(boost_direction_, i_lab); + BoxArray buff_ba(buff_box); + buff_ba.maxSize(max_box_size_); + DistributionMapping buff_dm(buff_ba); + data_buffer_[i].reset( new MultiFab(buff_ba, buff_dm, ncomp, 0) ); + } + if (WarpX::do_boosted_frame_particles) particles_buffer_[i].resize(mypc.nSpecies()); + } + + if (WarpX::do_boosted_frame_fields) { + const int ncomp = cell_centered_data->nComp(); + const int start_comp = 0; + const bool interpolate = true; + std::unique_ptr slice = amrex::get_slice_data(boost_direction_, + snapshots_[i].current_z_boost, + *cell_centered_data, geom, + start_comp, ncomp, interpolate); + + // transform it to the lab frame + for (MFIter mfi(*slice); mfi.isValid(); ++mfi) { + const Box& tile_box = mfi.tilebox(); + WRPX_LORENTZ_TRANSFORM_Z(BL_TO_FORTRAN_ANYD((*slice)[mfi]), + BL_TO_FORTRAN_BOX(tile_box), + &gamma_boost_, &beta_boost_); + } + + Real dx = geom.CellSize(boost_direction_); + int i_boost = (snapshots_[i].current_z_boost - geom.ProbLo(boost_direction_))/dx; + Box slice_box = geom.Domain(); + slice_box.setSmall(boost_direction_, i_boost); + slice_box.setBig(boost_direction_, i_boost); + BoxArray slice_ba(slice_box); + slice_ba.maxSize(max_box_size_); + MultiFab tmp(slice_ba, data_buffer_[i]->DistributionMap(), ncomp, 0); + tmp.copy(*slice, 0, 0, ncomp); + +#ifdef _OPENMP +#pragma omp parallel +#endif + for (MFIter mfi(tmp, true); mfi.isValid(); ++mfi) { + const Box& tile_box = mfi.tilebox(); + WRPX_COPY_SLICE(BL_TO_FORTRAN_BOX(tile_box), + BL_TO_FORTRAN_ANYD(tmp[mfi]), + BL_TO_FORTRAN_ANYD((*data_buffer_[i])[mfi]), + &ncomp, &i_boost, &i_lab); + } + } + + if (WarpX::do_boosted_frame_particles) { + mypc.GetLabFrameData(snapshots_[i].file_name, i_lab, boost_direction_, + old_z_boost, snapshots_[i].current_z_boost, + t_boost, snapshots_[i].t_lab, dt, particles_buffer_[i]); + } + + + ++buff_counter_[i]; + + if (buff_counter_[i] == num_buffer_) { + + if (WarpX::do_boosted_frame_fields) { + std::stringstream mesh_ss; + mesh_ss << snapshots_[i].file_name << "/Level_0/" << Concatenate("buffer", i_lab, 5); + VisMF::Write(*data_buffer_[i], mesh_ss.str()); + } + + if (WarpX::do_boosted_frame_particles) { + for (int j = 0; j < mypc.nSpecies(); ++j) { + std::stringstream part_ss; + part_ss << snapshots_[i].file_name + "/" + species_names[j] + "/"; + writeParticleData(particles_buffer_[i][j], part_ss.str(), i_lab); + } + particles_buffer_[i].clear(); + } + buff_counter_[i] = 0; + } + } + + VisMF::SetHeaderVersion(current_version); +} + +void +BoostedFrameDiagnostic:: +writeParticleData(const WarpXParticleContainer::DiagnosticParticleData& pdata, const std::string& name, + const int i_lab) +{ + BL_PROFILE("BoostedFrameDiagnostic::writeParticleData"); + + std::string field_name; + std::ofstream ofs; + + const int MyProc = ParallelDescriptor::MyProc(); + auto np = pdata.GetRealData(DiagIdx::w).size(); + + if (np == 0) return; + + field_name = name + Concatenate("w_", i_lab, 5) + "_" + std::to_string(MyProc); + ofs.open(field_name.c_str(), std::ios::out|std::ios::binary); + writeRealData(pdata.GetRealData(DiagIdx::w).data(), np, ofs); + ofs.close(); + + field_name = name + Concatenate("x_", i_lab, 5) + "_" + std::to_string(MyProc); + ofs.open(field_name.c_str(), std::ios::out|std::ios::binary); + writeRealData(pdata.GetRealData(DiagIdx::x).data(), np, ofs); + ofs.close(); + + field_name = name + Concatenate("y_", i_lab, 5) + "_" + std::to_string(MyProc); + ofs.open(field_name.c_str(), std::ios::out|std::ios::binary); + writeRealData(pdata.GetRealData(DiagIdx::y).data(), np, ofs); + ofs.close(); + + field_name = name + Concatenate("z_", i_lab, 5) + "_" + std::to_string(MyProc); + ofs.open(field_name.c_str(), std::ios::out|std::ios::binary); + writeRealData(pdata.GetRealData(DiagIdx::z).data(), np, ofs); + ofs.close(); + + field_name = name + Concatenate("ux_", i_lab, 5) + "_" + std::to_string(MyProc); + ofs.open(field_name.c_str(), std::ios::out|std::ios::binary); + writeRealData(pdata.GetRealData(DiagIdx::ux).data(), np, ofs); + ofs.close(); + + field_name = name + Concatenate("uy_", i_lab, 5) + "_" + std::to_string(MyProc); + ofs.open(field_name.c_str(), std::ios::out|std::ios::binary); + writeRealData(pdata.GetRealData(DiagIdx::uy).data(), np, ofs); + ofs.close(); + + field_name = name + Concatenate("uz_", i_lab, 5) + "_" + std::to_string(MyProc); + ofs.open(field_name.c_str(), std::ios::out|std::ios::binary); + writeRealData(pdata.GetRealData(DiagIdx::uz).data(), np, ofs); + ofs.close(); +} + +void +BoostedFrameDiagnostic:: +writeMetaData() +{ + BL_PROFILE("BoostedFrameDiagnostic::writeMetaData"); + + if (ParallelDescriptor::IOProcessor()) { + std::string DiagnosticDirectory = "lab_frame_data"; + + if (!UtilCreateDirectory(DiagnosticDirectory, 0755)) + CreateDirectoryFailed(DiagnosticDirectory); + + VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size); + std::ofstream HeaderFile; + HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size()); + std::string HeaderFileName(DiagnosticDirectory + "/Header"); + HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out | + std::ofstream::trunc | + std::ofstream::binary); + if(!HeaderFile.good()) + FileOpenFailed(HeaderFileName); + + HeaderFile.precision(17); + + HeaderFile << N_snapshots_ << "\n"; + HeaderFile << dt_snapshots_lab_ << "\n"; + HeaderFile << gamma_boost_ << "\n"; + HeaderFile << beta_boost_ << "\n"; + HeaderFile << dz_lab_ << "\n"; + HeaderFile << Nz_lab_ << "\n"; + } +} + +BoostedFrameDiagnostic::LabSnapShot:: +LabSnapShot(Real t_lab_in, Real zmin_lab_in, + Real zmax_lab_in, int file_num_in) + : t_lab(t_lab_in), + zmin_lab(zmin_lab_in), + zmax_lab(zmax_lab_in), + file_num(file_num_in) +{ + current_z_lab = 0.0; + current_z_boost = 0.0; + file_name = Concatenate("lab_frame_data/snapshot", file_num, 5); + + if (ParallelDescriptor::IOProcessor()) { + + if (!UtilCreateDirectory(file_name, 0755)) + CreateDirectoryFailed(file_name); + + const int nlevels = 1; + for(int i = 0; i < nlevels; ++i) { + const std::string &fullpath = LevelFullPath(i, file_name); + if (!UtilCreateDirectory(fullpath, 0755)) + CreateDirectoryFailed(fullpath); + } + + auto & mypc = WarpX::GetInstance().GetPartContainer(); + const std::vector species_names = mypc.GetSpeciesNames(); + int nspecies = mypc.nSpecies(); + + const std::string particles_prefix = "particle"; + for(int i = 0; i < nspecies; ++i) { + const std::string fullpath = file_name + "/" + species_names[i]; + if (!UtilCreateDirectory(fullpath, 0755)) + CreateDirectoryFailed(fullpath); + } + } + + ParallelDescriptor::Barrier(); + + if (ParallelDescriptor::IOProcessor()) writeSnapShotHeader(); +} + +void +BoostedFrameDiagnostic::LabSnapShot:: +updateCurrentZPositions(Real t_boost, Real inv_gamma, Real inv_beta) { + current_z_boost = (t_lab*inv_gamma - t_boost)*PhysConst::c*inv_beta; + current_z_lab = (t_lab - t_boost*inv_gamma)*PhysConst::c*inv_beta; +} + +void +BoostedFrameDiagnostic::LabSnapShot:: +writeSnapShotHeader() { + if (ParallelDescriptor::IOProcessor()) { + VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size); + std::ofstream HeaderFile; + HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size()); + std::string HeaderFileName(file_name + "/Header"); + HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out | + std::ofstream::trunc | + std::ofstream::binary); + if(!HeaderFile.good()) + FileOpenFailed(HeaderFileName); + + HeaderFile.precision(17); + + HeaderFile << t_lab << "\n"; + HeaderFile << zmin_lab << "\n"; + HeaderFile << zmax_lab << "\n"; + } +} diff --git a/Source/Diagnostics/BoostedFrame_module.F90 b/Source/Diagnostics/BoostedFrame_module.F90 new file mode 100644 index 000000000..3bce1817e --- /dev/null +++ b/Source/Diagnostics/BoostedFrame_module.F90 @@ -0,0 +1,106 @@ + +module warpx_boosted_frame_module + + use iso_c_binding + use amrex_fort_module, only : amrex_real + use constants, only : clight + + implicit none + +contains + +! +! Given cell-centered data in the boosted reference frame of the simulation, +! this transforms E and B in place so that the multifab now contains values +! in the lab frame. This routine assumes that the simulation frame is moving +! in the positive z direction with respect to the lab frame. +! + subroutine warpx_lorentz_transform_z(data, dlo, dhi, tlo, thi, gamma_boost, beta_boost) & + bind(C, name="warpx_lorentz_transform_z") + + integer(c_int), intent(in) :: dlo(3), dhi(3) + integer(c_int), intent(in) :: tlo(3), thi(3) + real(amrex_real), intent(inout) :: data(dlo(1):dhi(1),dlo(2):dhi(2),dlo(3):dhi(3), 10) + real(amrex_real), intent(in) :: gamma_boost, beta_boost + + integer i, j, k + real(amrex_real) e_lab, b_lab, j_lab, r_lab + + do k = tlo(3), thi(3) + do j = tlo(2), thi(2) + do i = tlo(1), thi(1) + + ! Transform the transverse E and B fields. Note that ez and bz are not + ! changed by the tranform. + e_lab = gamma_boost * (data(i, j, k, 1) + beta_boost*clight*data(i, j, k, 5)) + b_lab = gamma_boost * (data(i, j, k, 5) + beta_boost*data(i, j, k, 1)/clight) + + data(i, j, k, 1) = e_lab + data(i, j, k, 5) = b_lab + + e_lab = gamma_boost * (data(i, j, k, 2) - beta_boost*clight*data(i, j, k, 4)) + b_lab = gamma_boost * (data(i, j, k, 4) - beta_boost*data(i, j, k, 2)/clight) + + data(i, j, k, 2) = e_lab + data(i, j, k, 4) = b_lab + + ! Transform the charge and current density. Only the z component of j is affected. + j_lab = gamma_boost*(data(i, j, k, 9) + beta_boost*clight*data(i, j, k, 10)) + r_lab = gamma_boost*(data(i, j, k, 10) + beta_boost*data(i, j, k, 9)/clight) + + data(i, j, k, 9) = j_lab + data(i, j, k, 10) = r_lab + + end do + end do + end do + + end subroutine warpx_lorentz_transform_z + + subroutine warpx_copy_slice_3d(lo, hi, tmp, tlo, thi, & + buf, blo, bhi, ncomp, & + i_boost, i_lab) & + bind(C, name="warpx_copy_slice_3d") + + integer , intent(in) :: ncomp, i_boost, i_lab + integer , intent(in) :: lo(3), hi(3) + integer , intent(in) :: tlo(3), thi(3) + integer , intent(in) :: blo(3), bhi(3) + real(amrex_real), intent(inout) :: tmp(tlo(1):thi(1),tlo(2):thi(2),tlo(3):thi(3),ncomp) + real(amrex_real), intent(inout) :: buf(blo(1):bhi(1),blo(2):bhi(2),blo(3):bhi(3),ncomp) + + integer n, i, j, k + + do n = 1, ncomp + do j = lo(2), hi(2) + do i = lo(1), hi(1) + buf(i, j, i_lab, n) = tmp(i, j, i_boost, n) + end do + end do + end do + + end subroutine warpx_copy_slice_3d + + subroutine warpx_copy_slice_2d(lo, hi, tmp, tlo, thi, & + buf, blo, bhi, ncomp, & + i_boost, i_lab) & + bind(C, name="warpx_copy_slice_2d") + + integer , intent(in) :: ncomp, i_boost, i_lab + integer , intent(in) :: lo(2), hi(2) + integer , intent(in) :: tlo(2), thi(2) + integer , intent(in) :: blo(2), bhi(2) + real(amrex_real), intent(inout) :: tmp(tlo(1):thi(1),tlo(2):thi(2),ncomp) + real(amrex_real), intent(inout) :: buf(blo(1):bhi(1),blo(2):bhi(2),ncomp) + + integer n, i, j, k + + do n = 1, ncomp + do i = lo(1), hi(1) + buf(i, i_lab, n) = tmp(i, i_boost, n) + end do + end do + + end subroutine warpx_copy_slice_2d + +end module warpx_boosted_frame_module diff --git a/Source/Diagnostics/Make.package b/Source/Diagnostics/Make.package index ac5f033f5..eb7bfe231 100644 --- a/Source/Diagnostics/Make.package +++ b/Source/Diagnostics/Make.package @@ -1,8 +1,8 @@ CEXE_sources += WarpXIO.cpp -CEXE_sources += WarpXBoostedFrameDiagnostic.cpp +CEXE_sources += BoostedFrameDiagnostic.cpp CEXE_sources += ParticleIO.cpp -CEXE_headers += WarpXBoostedFrameDiagnostic.H -F90EXE_sources += WarpX_boosted_frame.F90 +CEXE_headers += BoostedFrameDiagnostic.H +F90EXE_sources += BoostedFrame_module.F90 INCLUDE_LOCATIONS += $(WARPX_HOME)/Source/Diagnostics VPATH_LOCATIONS += $(WARPX_HOME)/Source/Diagnostics diff --git a/Source/Diagnostics/WarpXBoostedFrameDiagnostic.H b/Source/Diagnostics/WarpXBoostedFrameDiagnostic.H deleted file mode 100644 index 452062137..000000000 --- a/Source/Diagnostics/WarpXBoostedFrameDiagnostic.H +++ /dev/null @@ -1,98 +0,0 @@ -#ifndef WARPX_BoostedFrameDiagnostic_H_ -#define WARPX_BoostedFrameDiagnostic_H_ - -#include - -#include -#include -#include - -#include "ParticleContainer.H" -#include "WarpXConst.H" - -/// -/// BoostedFrameDiagnostic is for handling IO when running in a boosted -/// frame of reference. Because of the relativity of simultaneity, events that -/// are synchronized in the simulation frame are not synchronized in the -/// lab frame. Thus, at a given t_boost, we must write slices of data to -/// multiple output files, each one corresponding to a given time in the lab frame. -/// -class BoostedFrameDiagnostic { - - /// - /// LabSnapShot stores metadata corresponding to a single time - /// snapshot in the lab frame. The snapshot is written to disk - /// in the directory "file_name". zmin_lab, zmax_lab, and t_lab - /// are all constant for a given snapshot. current_z_lab and - /// current_z_boost for each snapshot are updated as the - /// simulation time in the boosted frame advances. - /// - struct LabSnapShot { - - std::string file_name; - amrex::Real t_lab; - amrex::Real zmin_lab; - amrex::Real zmax_lab; - amrex::Real current_z_lab; - amrex::Real current_z_boost; - int file_num; - - LabSnapShot(amrex::Real t_lab_in, amrex::Real zmin_lab_in, - amrex::Real zmax_lab_in, int file_num_in); - - /// - /// This snapshot is at time t_lab, and the simulation is at time t_boost. - /// The Lorentz transformation picks out one slice corresponding to both - /// of those times, at position current_z_boost and current_z_lab in the - /// boosted and lab frames, respectively. - /// - void updateCurrentZPositions(amrex::Real t_boost, amrex::Real inv_gamma, - amrex::Real inv_beta); - - /// - /// Write some useful metadata about this snapshot. - /// - void writeSnapShotHeader(); - }; - - amrex::Real gamma_boost_; - amrex::Real inv_gamma_boost_; - amrex::Real beta_boost_; - amrex::Real inv_beta_boost_; - amrex::Real dz_lab_; - amrex::Real inv_dz_lab_; - amrex::Real dt_snapshots_lab_; - amrex::Real dt_boost_; - int N_snapshots_; - int Nz_lab_; - int boost_direction_; - - amrex::Vector > data_buffer_; - amrex::Vector > particles_buffer_; - int num_buffer_ = 256; - int max_box_size_ = 256; - amrex::Vector buff_counter_; - - amrex::Vector snapshots_; - - void writeParticleData(const WarpXParticleContainer::DiagnosticParticleData& pdata, - const std::string& name, const int i_lab); - -public: - - BoostedFrameDiagnostic(amrex::Real zmin_lab, amrex::Real zmax_lab, - amrex::Real v_window_lab, amrex::Real dt_snapshots_lab, - int N_snapshots, amrex::Real gamma_boost, - amrex::Real t_boost, amrex::Real dt_boost, int boost_direction); - - void Flush(const amrex::Geometry& geom); - - void writeLabFrameData(const amrex::MultiFab* cell_centered_data, - const MultiParticleContainer& mypc, - const amrex::Geometry& geom, - const amrex::Real t_boost, const amrex::Real dt); - - void writeMetaData(); -}; - -#endif diff --git a/Source/Diagnostics/WarpXBoostedFrameDiagnostic.cpp b/Source/Diagnostics/WarpXBoostedFrameDiagnostic.cpp deleted file mode 100644 index 96903f992..000000000 --- a/Source/Diagnostics/WarpXBoostedFrameDiagnostic.cpp +++ /dev/null @@ -1,371 +0,0 @@ -#include "WarpXBoostedFrameDiagnostic.H" -#include -#include "WarpX_f.H" -#include "WarpX.H" - -using namespace amrex; - -BoostedFrameDiagnostic:: -BoostedFrameDiagnostic(Real zmin_lab, Real zmax_lab, Real v_window_lab, - Real dt_snapshots_lab, int N_snapshots, - Real gamma_boost, Real t_boost, Real dt_boost, - int boost_direction) - : gamma_boost_(gamma_boost), - dt_snapshots_lab_(dt_snapshots_lab), - dt_boost_(dt_boost), - N_snapshots_(N_snapshots), - boost_direction_(boost_direction) -{ - - BL_PROFILE("BoostedFrameDiagnostic::BoostedFrameDiagnostic"); - - AMREX_ALWAYS_ASSERT(WarpX::do_boosted_frame_fields or - WarpX::do_boosted_frame_particles); - - inv_gamma_boost_ = 1.0 / gamma_boost_; - beta_boost_ = std::sqrt(1.0 - inv_gamma_boost_*inv_gamma_boost_); - inv_beta_boost_ = 1.0 / beta_boost_; - - dz_lab_ = PhysConst::c * dt_boost_ * inv_beta_boost_ * inv_gamma_boost_; - inv_dz_lab_ = 1.0 / dz_lab_; - Nz_lab_ = static_cast((zmax_lab - zmin_lab) * inv_dz_lab_); - - writeMetaData(); - - if (WarpX::do_boosted_frame_fields) data_buffer_.resize(N_snapshots); - if (WarpX::do_boosted_frame_particles) particles_buffer_.resize(N_snapshots); - for (int i = 0; i < N_snapshots; ++i) { - Real t_lab = i * dt_snapshots_lab_; - LabSnapShot snapshot(t_lab, zmin_lab + v_window_lab * t_lab, - zmax_lab + v_window_lab * t_lab, i); - snapshot.updateCurrentZPositions(t_boost, inv_gamma_boost_, inv_beta_boost_); - snapshots_.push_back(snapshot); - buff_counter_.push_back(0); - if (WarpX::do_boosted_frame_fields) data_buffer_[i].reset( nullptr ); - } - - AMREX_ALWAYS_ASSERT(max_box_size_ >= num_buffer_); -} - -void BoostedFrameDiagnostic::Flush(const Geometry& geom) -{ - BL_PROFILE("BoostedFrameDiagnostic::Flush"); - - VisMF::Header::Version current_version = VisMF::GetHeaderVersion(); - VisMF::SetHeaderVersion(amrex::VisMF::Header::NoFabHeader_v1); - - auto & mypc = WarpX::GetInstance().GetPartContainer(); - const std::vector species_names = mypc.GetSpeciesNames(); - - for (int i = 0; i < N_snapshots_; ++i) { - - int i_lab = (snapshots_[i].current_z_lab - snapshots_[i].zmin_lab) / dz_lab_; - - if (buff_counter_[i] != 0) { - if (WarpX::do_boosted_frame_fields) { - const BoxArray& ba = data_buffer_[i]->boxArray(); - const int hi = ba[0].bigEnd(boost_direction_); - const int lo = hi - buff_counter_[i] + 1; - - Box buff_box = geom.Domain(); - buff_box.setSmall(boost_direction_, lo); - buff_box.setBig(boost_direction_, hi); - - BoxArray buff_ba(buff_box); - buff_ba.maxSize(max_box_size_); - DistributionMapping buff_dm(buff_ba); - - const int ncomp = data_buffer_[i]->nComp(); - - MultiFab tmp(buff_ba, buff_dm, ncomp, 0); - - tmp.copy(*data_buffer_[i], 0, 0, ncomp); - - std::stringstream ss; - ss << snapshots_[i].file_name << "/Level_0/" << Concatenate("buffer", i_lab, 5); - VisMF::Write(tmp, ss.str()); - } - - if (WarpX::do_boosted_frame_particles) { - for (int j = 0; j < mypc.nSpecies(); ++j) { - std::stringstream part_ss; - part_ss << snapshots_[i].file_name + "/" + species_names[j] + "/"; - writeParticleData(particles_buffer_[i][j], part_ss.str(), i_lab); - } - particles_buffer_[i].clear(); - } - buff_counter_[i] = 0; - } - } - - VisMF::SetHeaderVersion(current_version); -} - -void -BoostedFrameDiagnostic:: -writeLabFrameData(const MultiFab* cell_centered_data, - const MultiParticleContainer& mypc, - const Geometry& geom, const Real t_boost, const Real dt) { - - BL_PROFILE("BoostedFrameDiagnostic::writeLabFrameData"); - - VisMF::Header::Version current_version = VisMF::GetHeaderVersion(); - VisMF::SetHeaderVersion(amrex::VisMF::Header::NoFabHeader_v1); - - const RealBox& domain_z_boost = geom.ProbDomain(); - const Real zlo_boost = domain_z_boost.lo(boost_direction_); - const Real zhi_boost = domain_z_boost.hi(boost_direction_); - - const std::vector species_names = mypc.GetSpeciesNames(); - - for (int i = 0; i < N_snapshots_; ++i) { - const Real old_z_boost = snapshots_[i].current_z_boost; - snapshots_[i].updateCurrentZPositions(t_boost, - inv_gamma_boost_, - inv_beta_boost_); - - if ( (snapshots_[i].current_z_boost < zlo_boost) or - (snapshots_[i].current_z_boost > zhi_boost) or - (snapshots_[i].current_z_lab < snapshots_[i].zmin_lab) or - (snapshots_[i].current_z_lab > snapshots_[i].zmax_lab) ) continue; - - int i_lab = (snapshots_[i].current_z_lab - snapshots_[i].zmin_lab) / dz_lab_; - - if (buff_counter_[i] == 0) { - if (WarpX::do_boosted_frame_fields) { - const int ncomp = cell_centered_data->nComp(); - Box buff_box = geom.Domain(); - buff_box.setSmall(boost_direction_, i_lab - num_buffer_ + 1); - buff_box.setBig(boost_direction_, i_lab); - BoxArray buff_ba(buff_box); - buff_ba.maxSize(max_box_size_); - DistributionMapping buff_dm(buff_ba); - data_buffer_[i].reset( new MultiFab(buff_ba, buff_dm, ncomp, 0) ); - } - if (WarpX::do_boosted_frame_particles) particles_buffer_[i].resize(mypc.nSpecies()); - } - - if (WarpX::do_boosted_frame_fields) { - const int ncomp = cell_centered_data->nComp(); - const int start_comp = 0; - const bool interpolate = true; - std::unique_ptr slice = amrex::get_slice_data(boost_direction_, - snapshots_[i].current_z_boost, - *cell_centered_data, geom, - start_comp, ncomp, interpolate); - - // transform it to the lab frame - for (MFIter mfi(*slice); mfi.isValid(); ++mfi) { - const Box& tile_box = mfi.tilebox(); - WRPX_LORENTZ_TRANSFORM_Z(BL_TO_FORTRAN_ANYD((*slice)[mfi]), - BL_TO_FORTRAN_BOX(tile_box), - &gamma_boost_, &beta_boost_); - } - - Real dx = geom.CellSize(boost_direction_); - int i_boost = (snapshots_[i].current_z_boost - geom.ProbLo(boost_direction_))/dx; - Box slice_box = geom.Domain(); - slice_box.setSmall(boost_direction_, i_boost); - slice_box.setBig(boost_direction_, i_boost); - BoxArray slice_ba(slice_box); - slice_ba.maxSize(max_box_size_); - MultiFab tmp(slice_ba, data_buffer_[i]->DistributionMap(), ncomp, 0); - tmp.copy(*slice, 0, 0, ncomp); - -#ifdef _OPENMP -#pragma omp parallel -#endif - for (MFIter mfi(tmp, true); mfi.isValid(); ++mfi) { - const Box& tile_box = mfi.tilebox(); - WRPX_COPY_SLICE(BL_TO_FORTRAN_BOX(tile_box), - BL_TO_FORTRAN_ANYD(tmp[mfi]), - BL_TO_FORTRAN_ANYD((*data_buffer_[i])[mfi]), - &ncomp, &i_boost, &i_lab); - } - } - - if (WarpX::do_boosted_frame_particles) { - mypc.GetLabFrameData(snapshots_[i].file_name, i_lab, boost_direction_, - old_z_boost, snapshots_[i].current_z_boost, - t_boost, snapshots_[i].t_lab, dt, particles_buffer_[i]); - } - - - ++buff_counter_[i]; - - if (buff_counter_[i] == num_buffer_) { - - if (WarpX::do_boosted_frame_fields) { - std::stringstream mesh_ss; - mesh_ss << snapshots_[i].file_name << "/Level_0/" << Concatenate("buffer", i_lab, 5); - VisMF::Write(*data_buffer_[i], mesh_ss.str()); - } - - if (WarpX::do_boosted_frame_particles) { - for (int j = 0; j < mypc.nSpecies(); ++j) { - std::stringstream part_ss; - part_ss << snapshots_[i].file_name + "/" + species_names[j] + "/"; - writeParticleData(particles_buffer_[i][j], part_ss.str(), i_lab); - } - particles_buffer_[i].clear(); - } - buff_counter_[i] = 0; - } - } - - VisMF::SetHeaderVersion(current_version); -} - -void -BoostedFrameDiagnostic:: -writeParticleData(const WarpXParticleContainer::DiagnosticParticleData& pdata, const std::string& name, - const int i_lab) -{ - BL_PROFILE("BoostedFrameDiagnostic::writeParticleData"); - - std::string field_name; - std::ofstream ofs; - - const int MyProc = ParallelDescriptor::MyProc(); - auto np = pdata.GetRealData(DiagIdx::w).size(); - - if (np == 0) return; - - field_name = name + Concatenate("w_", i_lab, 5) + "_" + std::to_string(MyProc); - ofs.open(field_name.c_str(), std::ios::out|std::ios::binary); - writeRealData(pdata.GetRealData(DiagIdx::w).data(), np, ofs); - ofs.close(); - - field_name = name + Concatenate("x_", i_lab, 5) + "_" + std::to_string(MyProc); - ofs.open(field_name.c_str(), std::ios::out|std::ios::binary); - writeRealData(pdata.GetRealData(DiagIdx::x).data(), np, ofs); - ofs.close(); - - field_name = name + Concatenate("y_", i_lab, 5) + "_" + std::to_string(MyProc); - ofs.open(field_name.c_str(), std::ios::out|std::ios::binary); - writeRealData(pdata.GetRealData(DiagIdx::y).data(), np, ofs); - ofs.close(); - - field_name = name + Concatenate("z_", i_lab, 5) + "_" + std::to_string(MyProc); - ofs.open(field_name.c_str(), std::ios::out|std::ios::binary); - writeRealData(pdata.GetRealData(DiagIdx::z).data(), np, ofs); - ofs.close(); - - field_name = name + Concatenate("ux_", i_lab, 5) + "_" + std::to_string(MyProc); - ofs.open(field_name.c_str(), std::ios::out|std::ios::binary); - writeRealData(pdata.GetRealData(DiagIdx::ux).data(), np, ofs); - ofs.close(); - - field_name = name + Concatenate("uy_", i_lab, 5) + "_" + std::to_string(MyProc); - ofs.open(field_name.c_str(), std::ios::out|std::ios::binary); - writeRealData(pdata.GetRealData(DiagIdx::uy).data(), np, ofs); - ofs.close(); - - field_name = name + Concatenate("uz_", i_lab, 5) + "_" + std::to_string(MyProc); - ofs.open(field_name.c_str(), std::ios::out|std::ios::binary); - writeRealData(pdata.GetRealData(DiagIdx::uz).data(), np, ofs); - ofs.close(); -} - -void -BoostedFrameDiagnostic:: -writeMetaData() -{ - BL_PROFILE("BoostedFrameDiagnostic::writeMetaData"); - - if (ParallelDescriptor::IOProcessor()) { - std::string DiagnosticDirectory = "lab_frame_data"; - - if (!UtilCreateDirectory(DiagnosticDirectory, 0755)) - CreateDirectoryFailed(DiagnosticDirectory); - - VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size); - std::ofstream HeaderFile; - HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size()); - std::string HeaderFileName(DiagnosticDirectory + "/Header"); - HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out | - std::ofstream::trunc | - std::ofstream::binary); - if(!HeaderFile.good()) - FileOpenFailed(HeaderFileName); - - HeaderFile.precision(17); - - HeaderFile << N_snapshots_ << "\n"; - HeaderFile << dt_snapshots_lab_ << "\n"; - HeaderFile << gamma_boost_ << "\n"; - HeaderFile << beta_boost_ << "\n"; - HeaderFile << dz_lab_ << "\n"; - HeaderFile << Nz_lab_ << "\n"; - } -} - -BoostedFrameDiagnostic::LabSnapShot:: -LabSnapShot(Real t_lab_in, Real zmin_lab_in, - Real zmax_lab_in, int file_num_in) - : t_lab(t_lab_in), - zmin_lab(zmin_lab_in), - zmax_lab(zmax_lab_in), - file_num(file_num_in) -{ - current_z_lab = 0.0; - current_z_boost = 0.0; - file_name = Concatenate("lab_frame_data/snapshot", file_num, 5); - - if (ParallelDescriptor::IOProcessor()) { - - if (!UtilCreateDirectory(file_name, 0755)) - CreateDirectoryFailed(file_name); - - const int nlevels = 1; - for(int i = 0; i < nlevels; ++i) { - const std::string &fullpath = LevelFullPath(i, file_name); - if (!UtilCreateDirectory(fullpath, 0755)) - CreateDirectoryFailed(fullpath); - } - - auto & mypc = WarpX::GetInstance().GetPartContainer(); - const std::vector species_names = mypc.GetSpeciesNames(); - int nspecies = mypc.nSpecies(); - - const std::string particles_prefix = "particle"; - for(int i = 0; i < nspecies; ++i) { - const std::string fullpath = file_name + "/" + species_names[i]; - if (!UtilCreateDirectory(fullpath, 0755)) - CreateDirectoryFailed(fullpath); - } - } - - ParallelDescriptor::Barrier(); - - if (ParallelDescriptor::IOProcessor()) writeSnapShotHeader(); -} - -void -BoostedFrameDiagnostic::LabSnapShot:: -updateCurrentZPositions(Real t_boost, Real inv_gamma, Real inv_beta) { - current_z_boost = (t_lab*inv_gamma - t_boost)*PhysConst::c*inv_beta; - current_z_lab = (t_lab - t_boost*inv_gamma)*PhysConst::c*inv_beta; -} - -void -BoostedFrameDiagnostic::LabSnapShot:: -writeSnapShotHeader() { - if (ParallelDescriptor::IOProcessor()) { - VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size); - std::ofstream HeaderFile; - HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size()); - std::string HeaderFileName(file_name + "/Header"); - HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out | - std::ofstream::trunc | - std::ofstream::binary); - if(!HeaderFile.good()) - FileOpenFailed(HeaderFileName); - - HeaderFile.precision(17); - - HeaderFile << t_lab << "\n"; - HeaderFile << zmin_lab << "\n"; - HeaderFile << zmax_lab << "\n"; - } -} diff --git a/Source/Diagnostics/WarpX_boosted_frame.F90 b/Source/Diagnostics/WarpX_boosted_frame.F90 deleted file mode 100644 index 3bce1817e..000000000 --- a/Source/Diagnostics/WarpX_boosted_frame.F90 +++ /dev/null @@ -1,106 +0,0 @@ - -module warpx_boosted_frame_module - - use iso_c_binding - use amrex_fort_module, only : amrex_real - use constants, only : clight - - implicit none - -contains - -! -! Given cell-centered data in the boosted reference frame of the simulation, -! this transforms E and B in place so that the multifab now contains values -! in the lab frame. This routine assumes that the simulation frame is moving -! in the positive z direction with respect to the lab frame. -! - subroutine warpx_lorentz_transform_z(data, dlo, dhi, tlo, thi, gamma_boost, beta_boost) & - bind(C, name="warpx_lorentz_transform_z") - - integer(c_int), intent(in) :: dlo(3), dhi(3) - integer(c_int), intent(in) :: tlo(3), thi(3) - real(amrex_real), intent(inout) :: data(dlo(1):dhi(1),dlo(2):dhi(2),dlo(3):dhi(3), 10) - real(amrex_real), intent(in) :: gamma_boost, beta_boost - - integer i, j, k - real(amrex_real) e_lab, b_lab, j_lab, r_lab - - do k = tlo(3), thi(3) - do j = tlo(2), thi(2) - do i = tlo(1), thi(1) - - ! Transform the transverse E and B fields. Note that ez and bz are not - ! changed by the tranform. - e_lab = gamma_boost * (data(i, j, k, 1) + beta_boost*clight*data(i, j, k, 5)) - b_lab = gamma_boost * (data(i, j, k, 5) + beta_boost*data(i, j, k, 1)/clight) - - data(i, j, k, 1) = e_lab - data(i, j, k, 5) = b_lab - - e_lab = gamma_boost * (data(i, j, k, 2) - beta_boost*clight*data(i, j, k, 4)) - b_lab = gamma_boost * (data(i, j, k, 4) - beta_boost*data(i, j, k, 2)/clight) - - data(i, j, k, 2) = e_lab - data(i, j, k, 4) = b_lab - - ! Transform the charge and current density. Only the z component of j is affected. - j_lab = gamma_boost*(data(i, j, k, 9) + beta_boost*clight*data(i, j, k, 10)) - r_lab = gamma_boost*(data(i, j, k, 10) + beta_boost*data(i, j, k, 9)/clight) - - data(i, j, k, 9) = j_lab - data(i, j, k, 10) = r_lab - - end do - end do - end do - - end subroutine warpx_lorentz_transform_z - - subroutine warpx_copy_slice_3d(lo, hi, tmp, tlo, thi, & - buf, blo, bhi, ncomp, & - i_boost, i_lab) & - bind(C, name="warpx_copy_slice_3d") - - integer , intent(in) :: ncomp, i_boost, i_lab - integer , intent(in) :: lo(3), hi(3) - integer , intent(in) :: tlo(3), thi(3) - integer , intent(in) :: blo(3), bhi(3) - real(amrex_real), intent(inout) :: tmp(tlo(1):thi(1),tlo(2):thi(2),tlo(3):thi(3),ncomp) - real(amrex_real), intent(inout) :: buf(blo(1):bhi(1),blo(2):bhi(2),blo(3):bhi(3),ncomp) - - integer n, i, j, k - - do n = 1, ncomp - do j = lo(2), hi(2) - do i = lo(1), hi(1) - buf(i, j, i_lab, n) = tmp(i, j, i_boost, n) - end do - end do - end do - - end subroutine warpx_copy_slice_3d - - subroutine warpx_copy_slice_2d(lo, hi, tmp, tlo, thi, & - buf, blo, bhi, ncomp, & - i_boost, i_lab) & - bind(C, name="warpx_copy_slice_2d") - - integer , intent(in) :: ncomp, i_boost, i_lab - integer , intent(in) :: lo(2), hi(2) - integer , intent(in) :: tlo(2), thi(2) - integer , intent(in) :: blo(2), bhi(2) - real(amrex_real), intent(inout) :: tmp(tlo(1):thi(1),tlo(2):thi(2),ncomp) - real(amrex_real), intent(inout) :: buf(blo(1):bhi(1),blo(2):bhi(2),ncomp) - - integer n, i, j, k - - do n = 1, ncomp - do i = lo(1), hi(1) - buf(i, i_lab, n) = tmp(i, i_boost, n) - end do - end do - - end subroutine warpx_copy_slice_2d - -end module warpx_boosted_frame_module diff --git a/Source/Electrostatic/Make.package b/Source/Electrostatic/Make.package index 5640c3940..e8b2f3669 100644 --- a/Source/Electrostatic/Make.package +++ b/Source/Electrostatic/Make.package @@ -2,6 +2,9 @@ F90EXE_sources += WarpX_electrostatic.F90 ifeq ($(DO_ELECTROSTATIC),TRUE) CEXE_sources += WarpXElectrostatic.cpp endif +ifeq ($(USE_OPENBC_POISSON),TRUE) + F90EXE_sources += openbc_poisson_solver.F90 +endif INCLUDE_LOCATIONS += $(WARPX_HOME)/Source/Electrostatic VPATH_LOCATIONS += $(WARPX_HOME)/Source/Electrostatic diff --git a/Source/Electrostatic/openbc_poisson_solver.F90 b/Source/Electrostatic/openbc_poisson_solver.F90 new file mode 100644 index 000000000..c18b1db24 --- /dev/null +++ b/Source/Electrostatic/openbc_poisson_solver.F90 @@ -0,0 +1,62 @@ + +module warpx_openbc_module + + implicit none + + integer, parameter :: idecomp = 0 ! 0=xyz, 1=xy, 2=yz, 3=xz, 4=x, 5=y, 6=z. + integer, parameter :: igfflag = 1 ! =0 for ordinary 1/r Green function; + ! =1 for integrated Green function + + integer, save :: gb_lo(3), gb_hi(3) + integer, save :: lc_lo(3), lc_hi(3) + +contains + + subroutine warpx_openbc_decompose(glo, ghi, lo, hi) bind(c,name='warpx_openbc_decompose') + + use mpi + + integer, intent(in) :: glo(3), ghi(3) + integer, intent(out) :: lo(3), hi(3) + integer :: myrank, mprocs, ierr, npx, npy, npz + + call MPI_Comm_size(MPI_COMM_WORLD, mprocs, ierr); + call MPI_COMM_RANK(MPI_COMM_WORLD,myrank,ierr) + + gb_lo = glo + 1 ! +1 because AMReX's domain index starts with 0 + gb_hi = ghi + 2 ! +2 to nodalize it + + call procgriddecomp(mprocs, gb_lo(1),gb_hi(1), & + & gb_lo(2),gb_hi(2), & + & gb_lo(3),gb_hi(3), & + & idecomp, npx, npy, npz) + + call decompose(myrank,mprocs, gb_lo(1),gb_hi(1), & + & gb_lo(2),gb_hi(2), & + & gb_lo(3),gb_hi(3), & + idecomp, lc_lo(1), lc_hi(1), & + & lc_lo(2), lc_hi(2), & + & lc_lo(3), lc_hi(3)) + + lo = lc_lo - 1 ! AMReX's domain index starts with zero. + hi = lc_hi - 1 + + end subroutine warpx_openbc_decompose + + subroutine warpx_openbc_potential(rho, phi, dx) & + bind(C, name="warpx_openbc_potential") + + double precision, intent(in ) :: dx(3) + double precision, intent(in ) :: rho(lc_lo(1):lc_hi(1),lc_lo(2):lc_hi(2),lc_lo(3):lc_hi(3)) + double precision, intent(out) :: phi(lc_lo(1):lc_hi(1),lc_lo(2):lc_hi(2),lc_lo(3):lc_hi(3)) + + integer :: ierr + + call openbcpotential(rho, phi, dx(1), dx(2), dx(3), & + lc_lo(1), lc_hi(1), lc_lo(2), lc_hi(2), lc_lo(3), lc_hi(3), & + gb_lo(1), gb_hi(1), gb_lo(2), gb_hi(2), gb_lo(3), gb_hi(3), & + idecomp, igfflag, ierr) + + end subroutine warpx_openbc_potential + +end module warpx_openbc_module diff --git a/Source/Filter/Make.package b/Source/Filter/Make.package index 404e8daa4..56c0e7852 100644 --- a/Source/Filter/Make.package +++ b/Source/Filter/Make.package @@ -1,4 +1,4 @@ -F90EXE_sources += WarpX_filter.F90 +F90EXE_sources += filter_module.F90 INCLUDE_LOCATIONS += $(WARPX_HOME)/Source/Filter VPATH_LOCATIONS += $(WARPX_HOME)/Source/Filter diff --git a/Source/Filter/WarpX_filter.F90 b/Source/Filter/WarpX_filter.F90 deleted file mode 100644 index ea87be7ca..000000000 --- a/Source/Filter/WarpX_filter.F90 +++ /dev/null @@ -1,87 +0,0 @@ -module warpx_filter_module - - use iso_c_binding - use amrex_fort_module, only : rt => amrex_real - use amrex_constants_module, only : fourth, eighth - - implicit none - -contains - - subroutine warpx_filter_3d(lo, hi, src, slo, shi, dst, dlo, dhi, nc) & - bind(c, name='warpx_filter_3d') - integer, intent(in), value :: nc - integer, dimension(3), intent(in) :: lo, hi, dlo, dhi, slo, shi - real(rt), intent(inout) :: dst(dlo(1):dhi(1), dlo(2):dhi(2), dlo(3):dhi(3),nc) - real(rt), intent(in ) :: src(slo(1):shi(1), slo(2):shi(2), slo(3):shi(3),nc) - - real(rt), parameter :: sixteenth = 1.d0/16.d0 - real(rt), parameter :: thirtysecond = 1.d0/32.d0 - real(rt), parameter :: sixtyfourth = 1.d0/64.d0 - integer :: i,j,k,c - - do c = 1, nc - do k = lo(3), hi(3) - do j = lo(2), hi(2) - do i = lo(1), hi(1) - dst(i,j,k,c) = sixtyfourth*(src(i-1,j-1,k-1,c) & - & +src(i+1,j-1,k-1,c) & - & +src(i-1,j+1,k-1,c) & - & +src(i+1,j+1,k-1,c) & - & +src(i-1,j-1,k+1,c) & - & +src(i+1,j-1,k+1,c) & - & +src(i-1,j+1,k+1,c) & - & +src(i+1,j+1,k+1,c)) & - + thirtysecond*(src(i-1,j-1,k ,c) & - & +src(i+1,j-1,k ,c) & - & +src(i-1,j+1,k ,c) & - & +src(i+1,j+1,k ,c) & - & +src(i-1,j ,k-1,c) & - & +src(i+1,j ,k-1,c) & - & +src(i-1,j ,k+1,c) & - & +src(i+1,j ,k+1,c) & - & +src(i ,j-1,k-1,c) & - & +src(i ,j+1,k-1,c) & - & +src(i ,j-1,k+1,c) & - & +src(i ,j+1,k+1,c)) & - + sixteenth *(src(i-1,j ,k ,c) & - & +src(i+1,j ,k ,c) & - & +src(i ,j-1,k ,c) & - & +src(i ,j+1,k ,c) & - & +src(i ,j ,k-1,c) & - & +src(i ,j ,k+1,c)) & - + eighth * src(i ,j ,k ,c) - end do - end do - end do - end do - end subroutine warpx_filter_3d - - subroutine warpx_filter_2d(lo, hi, src, slo, shi, dst, dlo, dhi, nc) & - bind(c, name='warpx_filter_2d') - integer, intent(in), value :: nc - integer, dimension(2), intent(in) :: lo, hi, dlo, dhi, slo, shi - real(rt), intent(inout) :: dst(dlo(1):dhi(1), dlo(2):dhi(2), nc) - real(rt), intent(in ) :: src(slo(1):shi(1), slo(2):shi(2), nc) - - real(rt), parameter :: sixteenth = 1.d0/16.d0 - integer :: i,j,comp - - do comp = 1, nc - do j = lo(2), hi(2) - do i = lo(1), hi(1) - dst(i, j, comp) = sixteenth*(src(i-1,j-1,comp) & - & +src(i+1,j-1,comp) & - & +src(i-1,j+1,comp) & - & +src(i+1,j+1,comp)) & - + eighth *(src(i ,j-1,comp) & - & +src(i ,j+1,comp) & - & +src(i-1,j ,comp) & - & +src(i+1,j ,comp)) & - + fourth * src(i ,j ,comp) - end do - end do - end do - end subroutine warpx_filter_2d - -end module warpx_filter_module diff --git a/Source/Filter/filter_module.F90 b/Source/Filter/filter_module.F90 new file mode 100644 index 000000000..ea87be7ca --- /dev/null +++ b/Source/Filter/filter_module.F90 @@ -0,0 +1,87 @@ +module warpx_filter_module + + use iso_c_binding + use amrex_fort_module, only : rt => amrex_real + use amrex_constants_module, only : fourth, eighth + + implicit none + +contains + + subroutine warpx_filter_3d(lo, hi, src, slo, shi, dst, dlo, dhi, nc) & + bind(c, name='warpx_filter_3d') + integer, intent(in), value :: nc + integer, dimension(3), intent(in) :: lo, hi, dlo, dhi, slo, shi + real(rt), intent(inout) :: dst(dlo(1):dhi(1), dlo(2):dhi(2), dlo(3):dhi(3),nc) + real(rt), intent(in ) :: src(slo(1):shi(1), slo(2):shi(2), slo(3):shi(3),nc) + + real(rt), parameter :: sixteenth = 1.d0/16.d0 + real(rt), parameter :: thirtysecond = 1.d0/32.d0 + real(rt), parameter :: sixtyfourth = 1.d0/64.d0 + integer :: i,j,k,c + + do c = 1, nc + do k = lo(3), hi(3) + do j = lo(2), hi(2) + do i = lo(1), hi(1) + dst(i,j,k,c) = sixtyfourth*(src(i-1,j-1,k-1,c) & + & +src(i+1,j-1,k-1,c) & + & +src(i-1,j+1,k-1,c) & + & +src(i+1,j+1,k-1,c) & + & +src(i-1,j-1,k+1,c) & + & +src(i+1,j-1,k+1,c) & + & +src(i-1,j+1,k+1,c) & + & +src(i+1,j+1,k+1,c)) & + + thirtysecond*(src(i-1,j-1,k ,c) & + & +src(i+1,j-1,k ,c) & + & +src(i-1,j+1,k ,c) & + & +src(i+1,j+1,k ,c) & + & +src(i-1,j ,k-1,c) & + & +src(i+1,j ,k-1,c) & + & +src(i-1,j ,k+1,c) & + & +src(i+1,j ,k+1,c) & + & +src(i ,j-1,k-1,c) & + & +src(i ,j+1,k-1,c) & + & +src(i ,j-1,k+1,c) & + & +src(i ,j+1,k+1,c)) & + + sixteenth *(src(i-1,j ,k ,c) & + & +src(i+1,j ,k ,c) & + & +src(i ,j-1,k ,c) & + & +src(i ,j+1,k ,c) & + & +src(i ,j ,k-1,c) & + & +src(i ,j ,k+1,c)) & + + eighth * src(i ,j ,k ,c) + end do + end do + end do + end do + end subroutine warpx_filter_3d + + subroutine warpx_filter_2d(lo, hi, src, slo, shi, dst, dlo, dhi, nc) & + bind(c, name='warpx_filter_2d') + integer, intent(in), value :: nc + integer, dimension(2), intent(in) :: lo, hi, dlo, dhi, slo, shi + real(rt), intent(inout) :: dst(dlo(1):dhi(1), dlo(2):dhi(2), nc) + real(rt), intent(in ) :: src(slo(1):shi(1), slo(2):shi(2), nc) + + real(rt), parameter :: sixteenth = 1.d0/16.d0 + integer :: i,j,comp + + do comp = 1, nc + do j = lo(2), hi(2) + do i = lo(1), hi(1) + dst(i, j, comp) = sixteenth*(src(i-1,j-1,comp) & + & +src(i+1,j-1,comp) & + & +src(i-1,j+1,comp) & + & +src(i+1,j+1,comp)) & + + eighth *(src(i ,j-1,comp) & + & +src(i ,j+1,comp) & + & +src(i-1,j ,comp) & + & +src(i+1,j ,comp)) & + + fourth * src(i ,j ,comp) + end do + end do + end do + end subroutine warpx_filter_2d + +end module warpx_filter_module diff --git a/Source/Initialization/Make.package b/Source/Initialization/Make.package index 32f2932a0..829b9a91c 100644 --- a/Source/Initialization/Make.package +++ b/Source/Initialization/Make.package @@ -2,7 +2,6 @@ CEXE_sources += CustomDensityProb.cpp CEXE_sources += WarpXInitData.cpp CEXE_sources += CustomMomentumProb.cpp CEXE_sources += PlasmaInjector.cpp -CEXE_sources += WarpXProb.cpp CEXE_headers += PlasmaInjector.H F90EXE_sources += WarpX_parser.F90 diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 496b14e7a..ff5442b00 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -257,3 +258,77 @@ WarpX::InitOpenbc () } } #endif + +void +WarpX::InitLevelData (int lev, Real time) +{ + for (int i = 0; i < 3; ++i) { + current_fp[lev][i]->setVal(0.0); + Efield_fp[lev][i]->setVal(0.0); + Bfield_fp[lev][i]->setVal(0.0); + } + + if (lev > 0) { + for (int i = 0; i < 3; ++i) { + Efield_aux[lev][i]->setVal(0.0); + Bfield_aux[lev][i]->setVal(0.0); + + current_cp[lev][i]->setVal(0.0); + Efield_cp[lev][i]->setVal(0.0); + Bfield_cp[lev][i]->setVal(0.0); + } + } + + if (F_fp[lev]) { + F_fp[lev]->setVal(0.0); + } + + if (rho_fp[lev]) { + rho_fp[lev]->setVal(0.0); + } + + if (F_cp[lev]) { + F_cp[lev]->setVal(0.0); + } + + if (rho_cp[lev]) { + rho_cp[lev]->setVal(0.0); + } + + if (costs[lev]) { + costs[lev]->setVal(0.0); + } +} + +#ifdef WARPX_USE_PSATD + +void +WarpX::InitLevelDataFFT (int lev, Real time) +{ + Efield_fp_fft[lev][0]->setVal(0.0); + Efield_fp_fft[lev][1]->setVal(0.0); + Efield_fp_fft[lev][2]->setVal(0.0); + Bfield_fp_fft[lev][0]->setVal(0.0); + Bfield_fp_fft[lev][1]->setVal(0.0); + Bfield_fp_fft[lev][2]->setVal(0.0); + current_fp_fft[lev][0]->setVal(0.0); + current_fp_fft[lev][1]->setVal(0.0); + current_fp_fft[lev][2]->setVal(0.0); + rho_fp_fft[lev]->setVal(0.0); + + if (lev > 0) + { + Efield_cp_fft[lev][0]->setVal(0.0); + Efield_cp_fft[lev][1]->setVal(0.0); + Efield_cp_fft[lev][2]->setVal(0.0); + Bfield_cp_fft[lev][0]->setVal(0.0); + Bfield_cp_fft[lev][1]->setVal(0.0); + Bfield_cp_fft[lev][2]->setVal(0.0); + current_cp_fft[lev][0]->setVal(0.0); + current_cp_fft[lev][1]->setVal(0.0); + current_cp_fft[lev][2]->setVal(0.0); + rho_cp_fft[lev]->setVal(0.0); + } +} + +#endif diff --git a/Source/Initialization/WarpXProb.cpp b/Source/Initialization/WarpXProb.cpp deleted file mode 100644 index ecbf1afaf..000000000 --- a/Source/Initialization/WarpXProb.cpp +++ /dev/null @@ -1,78 +0,0 @@ -#include -#include - -using namespace amrex; - -void -WarpX::InitLevelData (int lev, Real time) -{ - for (int i = 0; i < 3; ++i) { - current_fp[lev][i]->setVal(0.0); - Efield_fp[lev][i]->setVal(0.0); - Bfield_fp[lev][i]->setVal(0.0); - } - - if (lev > 0) { - for (int i = 0; i < 3; ++i) { - Efield_aux[lev][i]->setVal(0.0); - Bfield_aux[lev][i]->setVal(0.0); - - current_cp[lev][i]->setVal(0.0); - Efield_cp[lev][i]->setVal(0.0); - Bfield_cp[lev][i]->setVal(0.0); - } - } - - if (F_fp[lev]) { - F_fp[lev]->setVal(0.0); - } - - if (rho_fp[lev]) { - rho_fp[lev]->setVal(0.0); - } - - if (F_cp[lev]) { - F_cp[lev]->setVal(0.0); - } - - if (rho_cp[lev]) { - rho_cp[lev]->setVal(0.0); - } - - if (costs[lev]) { - costs[lev]->setVal(0.0); - } -} - -#ifdef WARPX_USE_PSATD - -void -WarpX::InitLevelDataFFT (int lev, Real time) -{ - Efield_fp_fft[lev][0]->setVal(0.0); - Efield_fp_fft[lev][1]->setVal(0.0); - Efield_fp_fft[lev][2]->setVal(0.0); - Bfield_fp_fft[lev][0]->setVal(0.0); - Bfield_fp_fft[lev][1]->setVal(0.0); - Bfield_fp_fft[lev][2]->setVal(0.0); - current_fp_fft[lev][0]->setVal(0.0); - current_fp_fft[lev][1]->setVal(0.0); - current_fp_fft[lev][2]->setVal(0.0); - rho_fp_fft[lev]->setVal(0.0); - - if (lev > 0) - { - Efield_cp_fft[lev][0]->setVal(0.0); - Efield_cp_fft[lev][1]->setVal(0.0); - Efield_cp_fft[lev][2]->setVal(0.0); - Bfield_cp_fft[lev][0]->setVal(0.0); - Bfield_cp_fft[lev][1]->setVal(0.0); - Bfield_cp_fft[lev][2]->setVal(0.0); - current_cp_fft[lev][0]->setVal(0.0); - current_cp_fft[lev][1]->setVal(0.0); - current_cp_fft[lev][2]->setVal(0.0); - rho_cp_fft[lev]->setVal(0.0); - } -} - -#endif diff --git a/Source/Utils/Make.package b/Source/Utils/Make.package index 81ef76e51..d8fe6fc1b 100644 --- a/Source/Utils/Make.package +++ b/Source/Utils/Make.package @@ -1,4 +1,4 @@ -CEXE_sources += WarpXMove.cpp +CEXE_sources += WarpXMovingWindow.cpp CEXE_sources += WarpXWrappers.cpp CEXE_sources += WarpXConst.cpp CEXE_sources += WarpXTagging.cpp diff --git a/Source/Utils/WarpXMove.cpp b/Source/Utils/WarpXMove.cpp deleted file mode 100644 index 908c70573..000000000 --- a/Source/Utils/WarpXMove.cpp +++ /dev/null @@ -1,245 +0,0 @@ - -#include -#include - -using namespace amrex; - -void -WarpX::UpdatePlasmaInjectionPosition (Real dt) -{ - int dir = moving_window_dir; - // Continuously inject plasma in new cells (by default only on level 0) - if (WarpX::do_plasma_injection and (WarpX::gamma_boost > 1)){ - // In boosted-frame simulations, the plasma has moved since the last - // call to this function, and injection position needs to be updated - current_injection_position -= WarpX::beta_boost * -#if ( AMREX_SPACEDIM == 3 ) - WarpX::boost_direction[dir] * PhysConst::c * dt; -#elif ( AMREX_SPACEDIM == 2 ) - // In 2D, dir=0 corresponds to x and dir=1 corresponds to z - // This needs to be converted in order to index `boost_direction` - // which has 3 components, for both 2D and 3D simulations. - WarpX::boost_direction[2*dir] * PhysConst::c * dt; -#endif - } -} - -int -WarpX::MoveWindow (bool move_j) -{ - if (do_moving_window == 0) return 0; - - // Update the continuous position of the moving window, - // and of the plasma injection - moving_window_x += moving_window_v * dt[0]; - int dir = moving_window_dir; - UpdatePlasmaInjectionPosition( dt[0] ); - - // compute the number of cells to shift on the base level - Real new_lo[AMREX_SPACEDIM]; - Real new_hi[AMREX_SPACEDIM]; - const Real* current_lo = geom[0].ProbLo(); - const Real* current_hi = geom[0].ProbHi(); - const Real* cdx = geom[0].CellSize(); - int num_shift_base = static_cast((moving_window_x - current_lo[dir]) / cdx[dir]); - - if (num_shift_base == 0) return 0; - - // update the problem domain. Note the we only do this on the base level because - // amrex::Geometry objects share the same, static RealBox. - for (int i=0; i 0) { - num_shift_crse = num_shift; - num_shift *= refRatio(lev-1)[dir]; - } - - // Shift each component of vector fields (E, B, j) - for (int dim = 0; dim < 3; ++dim) { - - // Fine grid - shiftMF(*Bfield_fp[lev][dim], geom[lev], num_shift, dir); - shiftMF(*Efield_fp[lev][dim], geom[lev], num_shift, dir); - if (move_j) { - shiftMF(*current_fp[lev][dim], geom[lev], num_shift, dir); - } - if (do_pml && pml[lev]->ok()) { - const std::array& pml_B = pml[lev]->GetB_fp(); - const std::array& pml_E = pml[lev]->GetE_fp(); - shiftMF(*pml_B[dim], geom[lev], num_shift, dir); - shiftMF(*pml_E[dim], geom[lev], num_shift, dir); - } - - if (lev > 0) { - // Coarse grid - shiftMF(*Bfield_cp[lev][dim], geom[lev-1], num_shift_crse, dir); - shiftMF(*Efield_cp[lev][dim], geom[lev-1], num_shift_crse, dir); - shiftMF(*Bfield_aux[lev][dim], geom[lev], num_shift, dir); - shiftMF(*Efield_aux[lev][dim], geom[lev], num_shift, dir); - if (move_j) { - shiftMF(*current_cp[lev][dim], geom[lev-1], num_shift_crse, dir); - } - if (do_pml && pml[lev]->ok()) { - const std::array& pml_B = pml[lev]->GetB_cp(); - const std::array& pml_E = pml[lev]->GetE_cp(); - shiftMF(*pml_B[dim], geom[lev-1], num_shift_crse, dir); - shiftMF(*pml_E[dim], geom[lev-1], num_shift_crse, dir); - } - } - } - - // Shift scalar component F for dive cleaning - if (do_dive_cleaning) { - // Fine grid - shiftMF(*F_fp[lev], geom[lev], num_shift, dir); - if (do_pml && pml[lev]->ok()) { - MultiFab* pml_F = pml[lev]->GetF_fp(); - shiftMF(*pml_F, geom[lev], num_shift, dir); - } - if (lev > 0) { - // Coarse grid - shiftMF(*F_cp[lev], geom[lev-1], num_shift_crse, dir); - if (do_pml && pml[lev]->ok()) { - MultiFab* pml_F = pml[lev]->GetF_cp(); - shiftMF(*pml_F, geom[lev-1], num_shift_crse, dir); - } - shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir); - } - } - - // Shift scalar component rho - if (move_j) { - if (rho_fp[lev]){ - // Fine grid - shiftMF(*rho_fp[lev], geom[lev], num_shift, dir); - if (lev > 0){ - // Coarse grid - shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir); - } - } - } - } - - // Continuously inject plasma in new cells (by default only on level 0) - if (WarpX::do_plasma_injection) { - - const int lev = 0; - - // particleBox encloses the cells where we generate particles - // (only injects particles in an integer number of cells, - // for correct particle spacing) - RealBox particleBox = geom[lev].ProbDomain(); - Real new_injection_position; - if (moving_window_v >= 0){ - // Forward-moving window - Real dx = geom[lev].CellSize(dir); - new_injection_position = current_injection_position + - std::floor( (geom[lev].ProbHi(dir) - current_injection_position)/dx ) * dx; - } else { - // Backward-moving window - Real dx = geom[lev].CellSize(dir); - new_injection_position = current_injection_position - - std::floor( (current_injection_position - geom[lev].ProbLo(dir))/dx) * dx; - } - // Modify the corresponding bounds of the particleBox - if (moving_window_v >= 0) { - particleBox.setLo( dir, current_injection_position ); - particleBox.setHi( dir, new_injection_position ); - } else { - particleBox.setLo( dir, new_injection_position ); - particleBox.setHi( dir, current_injection_position ); - } - // Perform the injection of new particles in particleBox - if (particleBox.ok() and (current_injection_position != new_injection_position)){ - for (int i = 0; i < num_injected_species; ++i) { - int ispecies = injected_plasma_species[i]; - WarpXParticleContainer& pc = mypc->GetParticleContainer(ispecies); - auto& ppc = dynamic_cast(pc); - ppc.AddPlasma(lev, particleBox); - } - // Update the injection position - current_injection_position = new_injection_position; - } - } - - return num_shift_base; -} - -void -WarpX::shiftMF (MultiFab& mf, const Geometry& geom, int num_shift, int dir) -{ - BL_PROFILE("WarpX::shiftMF()"); - const BoxArray& ba = mf.boxArray(); - const DistributionMapping& dm = mf.DistributionMap(); - const int nc = mf.nComp(); - const IntVect& ng = mf.nGrowVect(); - - AMREX_ALWAYS_ASSERT(ng.min() >= num_shift); - - MultiFab tmpmf(ba, dm, nc, ng); - MultiFab::Copy(tmpmf, mf, 0, 0, nc, ng); - tmpmf.FillBoundary(geom.periodicity()); - - // Make a box that covers the region that the window moved into - const IndexType& typ = ba.ixType(); - const Box& domainBox = geom.Domain(); - Box adjBox; - if (num_shift > 0) { - adjBox = adjCellHi(domainBox, dir, ng[dir]); - } else { - adjBox = adjCellLo(domainBox, dir, ng[dir]); - } - adjBox = amrex::convert(adjBox, typ); - - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (idim == dir and typ.nodeCentered(dir)) { - if (num_shift > 0) { - adjBox.growLo(idim, -1); - } else { - adjBox.growHi(idim, -1); - } - } else if (idim != dir) { - adjBox.growLo(idim, ng[idim]); - adjBox.growHi(idim, ng[idim]); - } - } - -#ifdef _OPENMP -#pragma omp parallel -#endif - for (MFIter mfi(tmpmf); mfi.isValid(); ++mfi ) - { - FArrayBox* dstfab = mf.fabPtr(mfi); - FArrayBox* srcfab = tmpmf.fabPtr(mfi); - Box outbox = mfi.fabbox(); - outbox &= adjBox; - AMREX_LAUNCH_HOST_DEVICE_LAMBDA(outbox, toutbox, - { - srcfab->setVal(0.0, toutbox, 0, nc); - }); - Box dstBox = mf[mfi].box(); - if (num_shift > 0) { - dstBox.growHi(dir, -num_shift); - } else { - dstBox.growLo(dir, num_shift); - } - AMREX_LAUNCH_HOST_DEVICE_LAMBDA(dstBox, tdstBox, - { - dstfab->setVal(0.0, tdstBox, 0, nc); - dstfab->copy(*srcfab, amrex::shift(tdstBox,dir,num_shift), 0, tdstBox, 0, nc); - }); - } -} diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp new file mode 100644 index 000000000..908c70573 --- /dev/null +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -0,0 +1,245 @@ + +#include +#include + +using namespace amrex; + +void +WarpX::UpdatePlasmaInjectionPosition (Real dt) +{ + int dir = moving_window_dir; + // Continuously inject plasma in new cells (by default only on level 0) + if (WarpX::do_plasma_injection and (WarpX::gamma_boost > 1)){ + // In boosted-frame simulations, the plasma has moved since the last + // call to this function, and injection position needs to be updated + current_injection_position -= WarpX::beta_boost * +#if ( AMREX_SPACEDIM == 3 ) + WarpX::boost_direction[dir] * PhysConst::c * dt; +#elif ( AMREX_SPACEDIM == 2 ) + // In 2D, dir=0 corresponds to x and dir=1 corresponds to z + // This needs to be converted in order to index `boost_direction` + // which has 3 components, for both 2D and 3D simulations. + WarpX::boost_direction[2*dir] * PhysConst::c * dt; +#endif + } +} + +int +WarpX::MoveWindow (bool move_j) +{ + if (do_moving_window == 0) return 0; + + // Update the continuous position of the moving window, + // and of the plasma injection + moving_window_x += moving_window_v * dt[0]; + int dir = moving_window_dir; + UpdatePlasmaInjectionPosition( dt[0] ); + + // compute the number of cells to shift on the base level + Real new_lo[AMREX_SPACEDIM]; + Real new_hi[AMREX_SPACEDIM]; + const Real* current_lo = geom[0].ProbLo(); + const Real* current_hi = geom[0].ProbHi(); + const Real* cdx = geom[0].CellSize(); + int num_shift_base = static_cast((moving_window_x - current_lo[dir]) / cdx[dir]); + + if (num_shift_base == 0) return 0; + + // update the problem domain. Note the we only do this on the base level because + // amrex::Geometry objects share the same, static RealBox. + for (int i=0; i 0) { + num_shift_crse = num_shift; + num_shift *= refRatio(lev-1)[dir]; + } + + // Shift each component of vector fields (E, B, j) + for (int dim = 0; dim < 3; ++dim) { + + // Fine grid + shiftMF(*Bfield_fp[lev][dim], geom[lev], num_shift, dir); + shiftMF(*Efield_fp[lev][dim], geom[lev], num_shift, dir); + if (move_j) { + shiftMF(*current_fp[lev][dim], geom[lev], num_shift, dir); + } + if (do_pml && pml[lev]->ok()) { + const std::array& pml_B = pml[lev]->GetB_fp(); + const std::array& pml_E = pml[lev]->GetE_fp(); + shiftMF(*pml_B[dim], geom[lev], num_shift, dir); + shiftMF(*pml_E[dim], geom[lev], num_shift, dir); + } + + if (lev > 0) { + // Coarse grid + shiftMF(*Bfield_cp[lev][dim], geom[lev-1], num_shift_crse, dir); + shiftMF(*Efield_cp[lev][dim], geom[lev-1], num_shift_crse, dir); + shiftMF(*Bfield_aux[lev][dim], geom[lev], num_shift, dir); + shiftMF(*Efield_aux[lev][dim], geom[lev], num_shift, dir); + if (move_j) { + shiftMF(*current_cp[lev][dim], geom[lev-1], num_shift_crse, dir); + } + if (do_pml && pml[lev]->ok()) { + const std::array& pml_B = pml[lev]->GetB_cp(); + const std::array& pml_E = pml[lev]->GetE_cp(); + shiftMF(*pml_B[dim], geom[lev-1], num_shift_crse, dir); + shiftMF(*pml_E[dim], geom[lev-1], num_shift_crse, dir); + } + } + } + + // Shift scalar component F for dive cleaning + if (do_dive_cleaning) { + // Fine grid + shiftMF(*F_fp[lev], geom[lev], num_shift, dir); + if (do_pml && pml[lev]->ok()) { + MultiFab* pml_F = pml[lev]->GetF_fp(); + shiftMF(*pml_F, geom[lev], num_shift, dir); + } + if (lev > 0) { + // Coarse grid + shiftMF(*F_cp[lev], geom[lev-1], num_shift_crse, dir); + if (do_pml && pml[lev]->ok()) { + MultiFab* pml_F = pml[lev]->GetF_cp(); + shiftMF(*pml_F, geom[lev-1], num_shift_crse, dir); + } + shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir); + } + } + + // Shift scalar component rho + if (move_j) { + if (rho_fp[lev]){ + // Fine grid + shiftMF(*rho_fp[lev], geom[lev], num_shift, dir); + if (lev > 0){ + // Coarse grid + shiftMF(*rho_cp[lev], geom[lev-1], num_shift_crse, dir); + } + } + } + } + + // Continuously inject plasma in new cells (by default only on level 0) + if (WarpX::do_plasma_injection) { + + const int lev = 0; + + // particleBox encloses the cells where we generate particles + // (only injects particles in an integer number of cells, + // for correct particle spacing) + RealBox particleBox = geom[lev].ProbDomain(); + Real new_injection_position; + if (moving_window_v >= 0){ + // Forward-moving window + Real dx = geom[lev].CellSize(dir); + new_injection_position = current_injection_position + + std::floor( (geom[lev].ProbHi(dir) - current_injection_position)/dx ) * dx; + } else { + // Backward-moving window + Real dx = geom[lev].CellSize(dir); + new_injection_position = current_injection_position - + std::floor( (current_injection_position - geom[lev].ProbLo(dir))/dx) * dx; + } + // Modify the corresponding bounds of the particleBox + if (moving_window_v >= 0) { + particleBox.setLo( dir, current_injection_position ); + particleBox.setHi( dir, new_injection_position ); + } else { + particleBox.setLo( dir, new_injection_position ); + particleBox.setHi( dir, current_injection_position ); + } + // Perform the injection of new particles in particleBox + if (particleBox.ok() and (current_injection_position != new_injection_position)){ + for (int i = 0; i < num_injected_species; ++i) { + int ispecies = injected_plasma_species[i]; + WarpXParticleContainer& pc = mypc->GetParticleContainer(ispecies); + auto& ppc = dynamic_cast(pc); + ppc.AddPlasma(lev, particleBox); + } + // Update the injection position + current_injection_position = new_injection_position; + } + } + + return num_shift_base; +} + +void +WarpX::shiftMF (MultiFab& mf, const Geometry& geom, int num_shift, int dir) +{ + BL_PROFILE("WarpX::shiftMF()"); + const BoxArray& ba = mf.boxArray(); + const DistributionMapping& dm = mf.DistributionMap(); + const int nc = mf.nComp(); + const IntVect& ng = mf.nGrowVect(); + + AMREX_ALWAYS_ASSERT(ng.min() >= num_shift); + + MultiFab tmpmf(ba, dm, nc, ng); + MultiFab::Copy(tmpmf, mf, 0, 0, nc, ng); + tmpmf.FillBoundary(geom.periodicity()); + + // Make a box that covers the region that the window moved into + const IndexType& typ = ba.ixType(); + const Box& domainBox = geom.Domain(); + Box adjBox; + if (num_shift > 0) { + adjBox = adjCellHi(domainBox, dir, ng[dir]); + } else { + adjBox = adjCellLo(domainBox, dir, ng[dir]); + } + adjBox = amrex::convert(adjBox, typ); + + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + if (idim == dir and typ.nodeCentered(dir)) { + if (num_shift > 0) { + adjBox.growLo(idim, -1); + } else { + adjBox.growHi(idim, -1); + } + } else if (idim != dir) { + adjBox.growLo(idim, ng[idim]); + adjBox.growHi(idim, ng[idim]); + } + } + +#ifdef _OPENMP +#pragma omp parallel +#endif + for (MFIter mfi(tmpmf); mfi.isValid(); ++mfi ) + { + FArrayBox* dstfab = mf.fabPtr(mfi); + FArrayBox* srcfab = tmpmf.fabPtr(mfi); + Box outbox = mfi.fabbox(); + outbox &= adjBox; + AMREX_LAUNCH_HOST_DEVICE_LAMBDA(outbox, toutbox, + { + srcfab->setVal(0.0, toutbox, 0, nc); + }); + Box dstBox = mf[mfi].box(); + if (num_shift > 0) { + dstBox.growHi(dir, -num_shift); + } else { + dstBox.growLo(dir, num_shift); + } + AMREX_LAUNCH_HOST_DEVICE_LAMBDA(dstBox, tdstBox, + { + dstfab->setVal(0.0, tdstBox, 0, nc); + dstfab->copy(*srcfab, amrex::shift(tdstBox,dir,num_shift), 0, tdstBox, 0, nc); + }); + } +} diff --git a/Source/WarpX.H b/Source/WarpX.H index bf82bceca..152dcf3e0 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -20,8 +20,8 @@ #include #include -#include -#include +#include +#include #ifdef WARPX_USE_PSATD #include -- cgit v1.2.3