diff options
-rw-r--r-- | .gitignore | 3 | ||||
-rw-r--r-- | Docs/source/latex_theory/Boosted_frame/Boosted_frame.tex | 3 | ||||
-rw-r--r-- | Docs/source/visualization/sensei.rst | 77 | ||||
-rw-r--r-- | Docs/source/visualization/visualization.rst | 1 | ||||
-rw-r--r-- | GNUmakefile | 2 | ||||
-rw-r--r-- | Source/Make.WarpX | 5 | ||||
-rw-r--r-- | Source/WarpX.H | 14 | ||||
-rw-r--r-- | Source/WarpX.cpp | 23 | ||||
-rw-r--r-- | Source/WarpXEvolve.cpp | 38 | ||||
-rw-r--r-- | Source/WarpXIO.cpp | 275 | ||||
-rw-r--r-- | Source/WarpXInitData.cpp | 31 |
11 files changed, 460 insertions, 12 deletions
diff --git a/.gitignore b/.gitignore index 2bbdbeaf3..de0eb96d4 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ o/ tmp_build_dir/ AMReX_buildInfo.cpp Backtrace.* +*.vth +*.sw[opqrs] +*.png diff --git a/Docs/source/latex_theory/Boosted_frame/Boosted_frame.tex b/Docs/source/latex_theory/Boosted_frame/Boosted_frame.tex index b4497ec22..c3cb08ca2 100644 --- a/Docs/source/latex_theory/Boosted_frame/Boosted_frame.tex +++ b/Docs/source/latex_theory/Boosted_frame/Boosted_frame.tex @@ -142,8 +142,7 @@ physics at stake for a given resolution. For instance, NCI-specific corrections include periodically smoothing the electromagnetic field components \cite{Martinscpc10}, using a special time step \cite{VayAAC2010,Vayjcp2011} or -applying a wide-band smoothing of the current components \cite{ - VayAAC2010,Vayjcp2011,VayPOPL2011}. Another set of mitigation methods +applying a wide-band smoothing of the current components \cite{VayAAC2010,Vayjcp2011,VayPOPL2011}. Another set of mitigation methods involve scaling the deposited currents by a carefully-designed wavenumber-dependent factor \cite{GodfreyJCP2014_FDTD,GodfreyIEEE2014} or slightly modifying the diff --git a/Docs/source/visualization/sensei.rst b/Docs/source/visualization/sensei.rst new file mode 100644 index 000000000..4c71035cf --- /dev/null +++ b/Docs/source/visualization/sensei.rst @@ -0,0 +1,77 @@ +In situ Visualization with SENSEI +================================= +SENSEI is a light weight framework for in situ data analysis. SENSEI's data +model and API provide uniform access to and run time selection of a diverse set +of visualization and analysis back ends including VisIt Libsim, ParaView +Catalyst, VTK-m, Ascent, ADIOS, Yt, and Python. + +SENSEI uses an XML file to select and configure one or more back ends at run +time. Run time selection of the back end via XML means one user can access +Catalyst, another Libsim, yet another Python with no changes to the code. + +Compiling with GNU Make +----------------------- +For codes making use of AMReX's build system add the following variable to the +code's main :code:`GNUmakefile`. + +.. code-block:: bash + + USE_SENSEI_INSITU = TRUE + +When set, AMReX's make files will query environment variables for the lists of +compiler and linker flags, include directories, and link libraries. These lists +can be quite elaborate when using more sophisticated back ends, and are best +set automatically using the :code:`sensei_config` command line tool that should +be installed with SENSEI. Prior to invoking make use the following command to +set these variables: + +.. code-block:: bash + + source sensei_config + +Typically, the :code:`sensei_config` tool is in the users PATH after loading +the desired SENSEI module. After configuring the build environment with +:code:`sensei_config`, proceed as usual. + +.. code-block:: bash + + make -j4 -f GNUmakefile + +ParmParse Configuration +----------------------- +Once an AMReX code has been compiled with SENSEI features enabled, it will need +to be enabled and configured at runtime. This is done using ParmParse input file. +The following 3 ParmParse parameters are used: + +.. code-block:: python + + insitu.int = 2 + insitu.start = 0 + insitu.config = render_iso_catalyst_2d.xml + +:code:`insitu.int` turns in situ processing on or off and controls how often +data is processed. :code:`insitu.start` controls when in situ processing +starts. :code:`insitu.config` points to the SENSEI XML file which selects and +configures the desired back end. + +Obtaining SENSEI +----------------- +SENSEI is hosted on Kitware's Gitlab site at https://gitlab.kitware.com/sensei/sensei +It's best to checkout the latest release rather than working on the master branch. + +To ease the burden of wrangling back end installs SENSEI provides two platforms +with all dependencies pre-installed, a VirtualBox VM, and a NERSC Cori +deployment. New users are encouraged to experiment with one of these. + + +SENSEI VM +~~~~~~~~~ +The SENSEI VM comes with all of SENSEI's dependencies and the major back ends +such as VisIt and ParaView installed. The VM is the easiest way to test things +out. It also can be used to see how installs were done and the environment +configured. + +NERSC Cori +~~~~~~~~~~ +SENSEI is deployed at NERSC on Cori. The NERSC deployment includes the major +back ends such as ParaView Catalyst, VisIt Libsim, and Python. diff --git a/Docs/source/visualization/visualization.rst b/Docs/source/visualization/visualization.rst index 0681082a8..67da7e047 100644 --- a/Docs/source/visualization/visualization.rst +++ b/Docs/source/visualization/visualization.rst @@ -7,3 +7,4 @@ Visualizing the simulation results yt visit picviewer + sensei diff --git a/GNUmakefile b/GNUmakefile index 036b50b48..01b9f03ef 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -24,6 +24,8 @@ EBASE = main USE_PYTHON_MAIN = FALSE +USE_SENSEI_INSITU = FALSE + WarpxBinDir = Bin USE_PSATD = FALSE diff --git a/Source/Make.WarpX b/Source/Make.WarpX index 44001eeaa..9dfe667e0 100644 --- a/Source/Make.WarpX +++ b/Source/Make.WarpX @@ -31,6 +31,11 @@ include $(AMREX_HOME)/Src/Particle/Make.package include $(AMREX_HOME)/Src/Boundary/Make.package include $(AMREX_HOME)/Src/AmrCore/Make.package +ifeq ($(USE_SENSEI_INSITU),TRUE) + include $(AMREX_HOME)/Src/Amr/Make.package + include $(AMREX_HOME)/Src/Extern/SENSEI/Make.package +endif + include $(PICSAR_HOME)/src/Make.package DEFINES += -DPICSAR_NO_ASSUMED_ALIGNMENT diff --git a/Source/WarpX.H b/Source/WarpX.H index 32df63e19..b3c382cc7 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -27,6 +27,12 @@ #include <fftw3.h> #endif +#if defined(BL_USE_SENSEI_INSITU) +namespace amrex { +class AmrMeshInSituBridge; +} +#endif + enum struct DtType : int { Full = 0, @@ -170,6 +176,7 @@ public: void WriteCheckPointFile () const; void WritePlotFile () const; + void UpdateInSitu () const; void WritePlotFileES(const amrex::Vector<std::unique_ptr<amrex::MultiFab> >& rho, const amrex::Vector<std::unique_ptr<amrex::MultiFab> >& phi, @@ -568,6 +575,13 @@ private: void PushPSATD (int lev, amrex::Real dt); #endif + +#if defined(BL_USE_SENSEI_INSITU) + amrex::AmrMeshInSituBridge *insitu_bridge; +#endif + int insitu_int; + int insitu_start; + std::string insitu_config; }; #endif diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index ae39f073c..44eff9cd4 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -18,6 +18,10 @@ #include <WarpXWrappers.h> #include <WarpXUtil.H> +#ifdef BL_USE_SENSEI_INSITU +#include <AMReX_AmrMeshInSituBridge.H> +#endif + using namespace amrex; Vector<Real> WarpX::B_external(3, 0.0); @@ -197,6 +201,10 @@ WarpX::WarpX () comm_fft.resize(nlevs_max,MPI_COMM_NULL); color_fft.resize(nlevs_max,-1); #endif + +#ifdef BL_USE_SENSEI_INSITU + insitu_bridge = nullptr; +#endif } WarpX::~WarpX () @@ -205,6 +213,10 @@ WarpX::~WarpX () for (int lev = 0; lev < nlevs_max; ++lev) { ClearLevel(lev); } + +#ifdef BL_USE_SENSEI_INSITU + delete insitu_bridge; +#endif } void @@ -433,6 +445,17 @@ WarpX::ReadParameters () pp.query("noz", noz_fft); } #endif + + { + insitu_start = 0; + insitu_int = 0; + insitu_config = ""; + + ParmParse pp("insitu"); + pp.query("int", insitu_int); + pp.query("start", insitu_start); + pp.query("config", insitu_config); + } } // This is a virtual function. diff --git a/Source/WarpXEvolve.cpp b/Source/WarpXEvolve.cpp index 092d5adfd..4a77fde43 100644 --- a/Source/WarpXEvolve.cpp +++ b/Source/WarpXEvolve.cpp @@ -9,6 +9,10 @@ #include <WarpX_py.H> #endif +#ifdef BL_USE_SENSEI_INSITU +#include <AMReX_AmrMeshInSituBridge.H> +#endif + using namespace amrex; void @@ -35,6 +39,7 @@ WarpX::EvolveEM (int numsteps) Real cur_time = t_new[0]; static int last_plot_file_step = 0; static int last_check_file_step = 0; + static int last_insitu_step = 0; int numsteps_max; if (numsteps < 0) { // Note that the default argument is numsteps = -1 @@ -133,7 +138,10 @@ WarpX::EvolveEM (int numsteps) bool to_make_plot = (plot_int > 0) && ((step+1) % plot_int == 0); - bool move_j = is_synchronized || to_make_plot; + bool do_insitu = ((step+1) >= insitu_start) && + (insitu_int > 0) && ((step+1) % insitu_int == 0); + + bool move_j = is_synchronized || to_make_plot || do_insitu; // If is_synchronized we need to shift j too so that next step we can evolve E by dt/2. // We might need to move j because we are going to make a plotfile. MoveWindow(move_j); @@ -166,7 +174,7 @@ WarpX::EvolveEM (int numsteps) myBFD->writeLabFrameData(cell_centered_data.get(), *mypc, geom[0], cur_time, dt[0]); } - if (to_make_plot) + if (to_make_plot || do_insitu) { FillBoundaryE(); FillBoundaryB(); @@ -179,9 +187,16 @@ WarpX::EvolveEM (int numsteps) } last_plot_file_step = step+1; - WritePlotFile(); + last_insitu_step = step+1; + + if (to_make_plot) + WritePlotFile(); + + if (do_insitu) + UpdateInSitu(); } + if (check_int > 0 && (step+1) % check_int == 0) { last_check_file_step = step+1; WriteCheckPointFile(); @@ -198,7 +213,12 @@ WarpX::EvolveEM (int numsteps) // End loop on time steps } - if (plot_int > 0 && istep[0] > last_plot_file_step && (max_time_reached || istep[0] >= max_step)) + bool write_plot_file = plot_int > 0 && istep[0] > last_plot_file_step && (max_time_reached || istep[0] >= max_step); + + bool do_insitu = (insitu_start >= istep[0]) && (insitu_int > 0) && + (istep[0] > last_insitu_step) && (max_time_reached || istep[0] >= max_step); + + if (write_plot_file || do_insitu) { FillBoundaryE(); FillBoundaryB(); @@ -210,7 +230,11 @@ WarpX::EvolveEM (int numsteps) *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); } - WritePlotFile(); + if (write_plot_file) + WritePlotFile(); + + if (do_insitu) + UpdateInSitu(); } if (check_int > 0 && istep[0] > last_check_file_step && (max_time_reached || istep[0] >= max_step)) { @@ -220,6 +244,10 @@ WarpX::EvolveEM (int numsteps) if (do_boosted_frame_diagnostic) { myBFD->Flush(geom[0]); } + +#ifdef BL_USE_SENSEI_INSITU + insitu_bridge->finalize(); +#endif } /* /brief Perform one PIC iteration, without subcycling diff --git a/Source/WarpXIO.cpp b/Source/WarpXIO.cpp index d59720a33..4c673b1f5 100644 --- a/Source/WarpXIO.cpp +++ b/Source/WarpXIO.cpp @@ -7,6 +7,10 @@ #include "AMReX_buildInfo.H" +#ifdef BL_USE_SENSEI_INSITU +#include <AMReX_AmrMeshInSituBridge.H> +#endif + using namespace amrex; namespace @@ -454,6 +458,277 @@ WarpX::GetCellCenteredData() { } void +WarpX::UpdateInSitu () const +{ +#ifdef BL_USE_SENSEI_INSITU + BL_PROFILE("WarpX::UpdateInSitu()"); + + int numLevels = finest_level + 1; + Vector<std::string> varnames; + Vector<MultiFab> mf; + + const int ncomp = 3*3 + + static_cast<int>(plot_part_per_cell) + + static_cast<int>(plot_part_per_grid) + + static_cast<int>(plot_part_per_proc) + + static_cast<int>(plot_proc_number) + + static_cast<int>(plot_divb) + + static_cast<int>(plot_dive) + + static_cast<int>(plot_finepatch)*6 + + static_cast<int>(plot_crsepatch)*6 + + static_cast<int>(costs[0] != nullptr); + + for (int lev = 0; lev <= finest_level; ++lev) + { + const int ngrow = 1; + mf.push_back(MultiFab(grids[lev], dmap[lev], ncomp, ngrow)); + + Vector<const MultiFab*> srcmf(AMREX_SPACEDIM); + PackPlotDataPtrs(srcmf, current_fp[lev]); + int dcomp = 0; + amrex::average_edge_to_cellcenter(mf[lev], dcomp, srcmf); + +#if (AMREX_SPACEDIM == 2) + MultiFab::Copy(mf[lev], mf[lev], dcomp+1, dcomp+2, 1, ngrow); + amrex::average_node_to_cellcenter(mf[lev], dcomp+1, *current_fp[lev][1], 0, 1); +#endif + if (lev == 0) + { + varnames.push_back("jx"); + varnames.push_back("jy"); + varnames.push_back("jz"); + } + dcomp += 3; + + PackPlotDataPtrs(srcmf, Efield_aux[lev]); + amrex::average_edge_to_cellcenter(mf[lev], dcomp, srcmf); +#if (AMREX_SPACEDIM == 2) + MultiFab::Copy(mf[lev], mf[lev], dcomp+1, dcomp+2, 1, ngrow); + amrex::average_node_to_cellcenter(mf[lev], dcomp+1, *Efield_aux[lev][1], 0, 1); +#endif + if (lev == 0) + { + varnames.push_back("Ex"); + varnames.push_back("Ey"); + varnames.push_back("Ez"); + } + dcomp += 3; + + PackPlotDataPtrs(srcmf, Bfield_aux[lev]); + amrex::average_face_to_cellcenter(mf[lev], dcomp, srcmf); +#if (AMREX_SPACEDIM == 2) + MultiFab::Copy(mf[lev], mf[lev], dcomp+1, dcomp+2, 1, ngrow); + MultiFab::Copy(mf[lev], *Bfield_aux[lev][1], 0, dcomp+1, 1, ngrow); +#endif + if (lev == 0) + { + varnames.push_back("Bx"); + varnames.push_back("By"); + varnames.push_back("Bz"); + } + dcomp += 3; + + if (plot_part_per_cell) + { + MultiFab temp_dat(grids[lev],mf[lev].DistributionMap(),1,0); + temp_dat.setVal(0); + + // MultiFab containing number of particles in each cell + mypc->Increment(temp_dat, lev); + MultiFab::Copy(mf[lev], temp_dat, 0, dcomp, 1, 0); + if (lev == 0) + { + varnames.push_back("part_per_cell"); + } + dcomp += 1; + } + + if (plot_part_per_grid || plot_part_per_proc) + { + const Vector<long>& npart_in_grid = mypc->NumberOfParticlesInGrid(lev); + + if (plot_part_per_grid) + { +#ifdef _OPENMP +#pragma omp parallel +#endif + for (MFIter mfi(mf[lev]); mfi.isValid(); ++mfi) { + (mf[lev])[mfi].setVal(static_cast<Real>(npart_in_grid[mfi.index()]), dcomp); + } + if (lev == 0) + { + varnames.push_back("part_per_grid"); + } + dcomp += 1; + } + + if (plot_part_per_proc) + { + long n_per_proc = 0; +#ifdef _OPENMP +#pragma omp parallel reduction(+:n_per_proc) +#endif + for (MFIter mfi(mf[lev]); mfi.isValid(); ++mfi) { + n_per_proc += npart_in_grid[mfi.index()]; + } + mf[lev].setVal(static_cast<Real>(n_per_proc), dcomp,1); + if (lev == 0) + { + varnames.push_back("part_per_proc"); + } + dcomp += 1; + } + } + + if (plot_proc_number) + { +#ifdef _OPENMP +#pragma omp parallel +#endif + for (MFIter mfi(mf[lev]); mfi.isValid(); ++mfi) { + (mf[lev])[mfi].setVal(static_cast<Real>(ParallelDescriptor::MyProc()), dcomp); + } + if (lev == 0) + { + varnames.push_back("proc_number"); + } + dcomp += 1; + } + + if (plot_divb) + { + ComputeDivB(mf[lev], dcomp, + {Bfield_aux[lev][0].get(),Bfield_aux[lev][1].get(),Bfield_aux[lev][2].get()}, + WarpX::CellSize(lev)); + if (lev == 0) + { + varnames.push_back("divB"); + } + dcomp += 1; + } + + if (plot_dive) + { + const BoxArray& ba = amrex::convert(boxArray(lev),IntVect::TheUnitVector()); + MultiFab dive(ba,DistributionMap(lev),1,0); + ComputeDivE(dive, 0, + {Efield_aux[lev][0].get(), Efield_aux[lev][1].get(), Efield_aux[lev][2].get()}, + WarpX::CellSize(lev)); + amrex::average_node_to_cellcenter(mf[lev], dcomp, dive, 0, 1); + if (lev == 0) + { + varnames.push_back("divE"); + } + dcomp += 1; + } + + if (plot_finepatch) + { + PackPlotDataPtrs(srcmf, Efield_fp[lev]); + amrex::average_edge_to_cellcenter(mf[lev], dcomp, srcmf); +#if (AMREX_SPACEDIM == 2) + MultiFab::Copy(mf[lev], mf[lev], dcomp+1, dcomp+2, 1, ngrow); + amrex::average_node_to_cellcenter(mf[lev], dcomp+1, *Efield_fp[lev][1], 0, 1); +#endif + if (lev == 0) + { + varnames.push_back("Ex_fp"); + varnames.push_back("Ey_fp"); + varnames.push_back("Ez_fp"); + } + dcomp += 3; + + PackPlotDataPtrs(srcmf, Bfield_fp[lev]); + amrex::average_face_to_cellcenter(mf[lev], dcomp, srcmf); +#if (AMREX_SPACEDIM == 2) + MultiFab::Copy(mf[lev], mf[lev], dcomp+1, dcomp+2, 1, ngrow); + MultiFab::Copy(mf[lev], *Bfield_fp[lev][1], 0, dcomp+1, 1, ngrow); +#endif + if (lev == 0) + { + varnames.push_back("Bx_fp"); + varnames.push_back("By_fp"); + varnames.push_back("Bz_fp"); + } + dcomp += 3; + } + + if (plot_crsepatch) + { + // First the electric field + if (lev == 0) + { + mf[lev].setVal(0.0, dcomp, 3, ngrow); + } + else + { + std::array<std::unique_ptr<MultiFab>, 3> E = getInterpolatedE(lev); + PackPlotDataPtrs(srcmf, E); + amrex::average_edge_to_cellcenter(mf[lev], dcomp, srcmf); +#if (AMREX_SPACEDIM == 2) + MultiFab::Copy(mf[lev], mf[lev], dcomp+1, dcomp+2, 1, ngrow); + amrex::average_node_to_cellcenter(mf[lev], dcomp+1, *E[1], 0, 1); +#endif + } + if (lev == 0) + { + varnames.push_back("Ex_cp"); + varnames.push_back("Ey_cp"); + varnames.push_back("Ez_cp"); + } + dcomp += 3; + + // now the magnetic field + if (lev == 0) + { + mf[lev].setVal(0.0, dcomp, 3, ngrow); + } + else + { + std::array<std::unique_ptr<MultiFab>, 3> B = getInterpolatedB(lev); + PackPlotDataPtrs(srcmf, B); + amrex::average_face_to_cellcenter(mf[lev], dcomp, srcmf); +#if (AMREX_SPACEDIM == 2) + MultiFab::Copy(mf[lev], mf[lev], dcomp+1, dcomp+2, 1, ngrow); + MultiFab::Copy(mf[lev], *B[1], 0, dcomp+1, 1, ngrow); +#endif + } + if (lev == 0) + { + varnames.push_back("Bx_cp"); + varnames.push_back("By_cp"); + varnames.push_back("Bz_cp"); + } + dcomp += 3; + } + + if (costs[0] != nullptr) + { + MultiFab::Copy(mf[lev], *costs[lev], 0, dcomp, 1, ngrow); + if (lev == 0) + { + varnames.push_back("costs"); + } + dcomp += 1; + } + + BL_ASSERT(dcomp == ncomp); + } + + if (insitu_bridge->update(istep[0], t_new[0], + dynamic_cast<amrex::AmrMesh*>(const_cast<WarpX*>(this)), + {&mf}, {varnames})) + { + amrex::ErrorStream() + << "WarpXIO::UpdateInSitu : Failed to update the in situ bridge." + << std::endl; + + amrex::Abort(); + } +#endif +} + +void WarpX::WritePlotFile () const { BL_PROFILE("WarpX::WritePlotFile()"); diff --git a/Source/WarpXInitData.cpp b/Source/WarpXInitData.cpp index 769df87f5..a6b63a92d 100644 --- a/Source/WarpXInitData.cpp +++ b/Source/WarpXInitData.cpp @@ -7,6 +7,10 @@ #include <WarpX_f.H> #include <WarpXWrappers.h> +#ifdef BL_USE_SENSEI_INSITU +#include <AMReX_AmrMeshInSituBridge.H> +#endif + using namespace amrex; void @@ -43,14 +47,31 @@ WarpX::InitData () printGridSummary(std::cout, 0, finestLevel()); } +#ifdef BL_USE_SENSEI_INSITU + insitu_bridge = new amrex::AmrMeshInSituBridge; + insitu_bridge->setEnabled(insitu_int > 0 ? 1 : 0); + insitu_bridge->setConfig(insitu_config); + if (insitu_bridge->initialize()) + { + amrex::ErrorStream() + << "WarpX::InitData : Failed to initialize the in situ bridge." + << std::endl; + + amrex::Abort(); + } + insitu_bridge->setFrequency(1); +#endif + if (restart_chkfile.empty()) { - if (plot_int > 0) { + if (plot_int > 0) WritePlotFile(); - } - if (check_int > 0) { - WriteCheckPointFile(); - } + + if (check_int > 0) + WriteCheckPointFile(); + + if ((insitu_int > 0) && (insitu_start == 0)) + UpdateInSitu(); } } |