diff options
Diffstat (limited to 'Source')
| -rw-r--r-- | Source/Diagnostics/BTDiagnostics.H | 1 | ||||
| -rw-r--r-- | Source/Diagnostics/BTDiagnostics.cpp | 182 | ||||
| -rw-r--r-- | Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H | 3 | ||||
| -rw-r--r-- | Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp | 2 |
4 files changed, 183 insertions, 5 deletions
diff --git a/Source/Diagnostics/BTDiagnostics.H b/Source/Diagnostics/BTDiagnostics.H index 5815be63f..3033c86f0 100644 --- a/Source/Diagnostics/BTDiagnostics.H +++ b/Source/Diagnostics/BTDiagnostics.H @@ -55,7 +55,6 @@ private: // boost parameters amrex::Real m_gamma_boost; amrex::Real m_beta_boost; - int m_boost_direction; int m_moving_window_dir; /** Number of back-transformed snapshots in the lab-frame requested by the user */ diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 371a3b1cb..5782cf58e 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -3,6 +3,8 @@ #include "Utils/WarpXConst.H" #include "ComputeDiagFunctors/ComputeDiagFunctor.H" #include "ComputeDiagFunctors/CellCenterFunctor.H" +#include "ComputeDiagFunctors/BackTransformFunctor.H" +#include "Utils/CoarsenIO.H" #include <AMReX_ParallelDescriptor.H> #include <AMReX_PlotFileUtil.H> #include <AMReX_VisMF.H> @@ -55,8 +57,8 @@ void BTDiagnostics::DerivedInitData () m_file_name[i] = amrex::Concatenate(m_file_prefix +"/snapshots/snapshot",i,5); } for (int lev = 0; lev < nmax_lev; ++lev) { - // Define cell-centered multifab over the whole domain with user-defined crse_ratio - // for nlevels + // Define cell-centered multifab over the whole domain with + // user-defined crse_ratio for nlevels DefineCellCenteredMultiFab(lev); } @@ -113,6 +115,28 @@ BTDiagnostics::TMP_writeMetaData () // This function will have the same functionality as writeMetaData in // previously used BackTransformedDiagnostics class to write // back-transformed data in a customized format + + if (amrex::ParallelDescriptor::IOProcessor()) { + const std::string fullpath = m_file_prefix + "/snapshots"; + if (!amrex::UtilCreateDirectory(fullpath, 0755)) amrex::CreateDirectoryFailed(fullpath); + + amrex::VisMF::IO_Buffer io_buffer(amrex::VisMF::IO_Buffer_Size); + std::ofstream HeaderFile; + HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size()); + std::string HeaderFileName(m_file_prefix + "/snapshots/Header"); + HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out | + std::ofstream::trunc | + std::ofstream::binary); + if (!HeaderFile.good()) amrex::FileOpenFailed( HeaderFileName ); + + HeaderFile.precision(17); + HeaderFile << m_num_snapshots_lab << "\n"; + HeaderFile << m_dt_snapshots_lab << "\n"; + HeaderFile << m_gamma_boost << "\n"; + HeaderFile << m_beta_boost << "\n"; + + } + } bool @@ -139,17 +163,107 @@ BTDiagnostics::InitializeFieldBufferData ( int i_buffer , int lev) { auto & warpx = WarpX::GetInstance(); // 1. Lab-frame time for the i^th snapshot + m_t_lab.at(i_buffer) = i_buffer * m_dt_snapshots_lab; // 2. Define domain in boosted frame at level, lev + amrex::RealBox diag_dom; + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim ) { + // Setting lo-coordinate for the diag domain by taking the max of user-defined + // lo-cordinate and lo-coordinat of the simulation domain at level, lev + diag_dom.setLo(idim, std::max(m_lo[idim],warpx.Geom(lev).ProbLo(idim)) ); + // Setting hi-coordinate for the diag domain by taking the max of user-defined + // hi-cordinate and hi-coordinate of the simulation domain at level, lev + diag_dom.setHi(idim, std::min(m_hi[idim],warpx.Geom(lev).ProbHi(idim)) ); + } // 3. Initializing the m_buffer_box for the i^th snapshot. // At initialization, the Box has the same index space as the boosted-frame // As time-progresses, the z-dimension indices will be modified based on // current_z_lab + amrex::IntVect lo(0); + amrex::IntVect hi(-1); + for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { + // lo index with same cell-size as simulation at level, lev. + const int lo_index = static_cast<int>( floor( + ( diag_dom.lo(idim) - warpx.Geom(lev).ProbLo(idim) ) / + warpx.Geom(lev).CellSize(idim) ) ); + // Taking max of (0,lo_index) because lo_index must always be >=0 + lo[idim] = std::max( 0, lo_index ); + // hi index with same cell-size as simulation at level, lev. + const int hi_index = static_cast<int>( ceil( + ( diag_dom.hi(idim) - warpx.Geom(lev).ProbLo(idim) ) / + warpx.Geom(lev).CellSize(idim) ) ); + // Taking max of (0,hi_index) because hi_index must always be >=0 + // Subtracting by 1 because lo,hi indices are set to cell-centered staggering. + hi[idim] = std::max( 0, hi_index) - 1; + // if hi<=lo, then hi = lo + 1, to ensure one cell in that dimension + if ( hi[idim] <= lo[idim] ) { + hi[idim] = lo[idim] + 1; + AMREX_ALWAYS_ASSERT_WITH_MESSAGE( + m_crse_ratio[idim]==1, "coarsening ratio in reduced dimension must be 1." + ); + } + } + amrex::Box diag_box( lo, hi ); + // The box is not coarsened yet. Should this be coarsened here or when + // the buffer multifab is initialized? + m_buffer_box[i_buffer] = diag_box; + // 4. Define buffer_domain in lab-frame for the i^th snapshot. // Replace z-dimension with lab-frame co-ordinates. + amrex::Real zmin_lab = diag_dom.lo(m_moving_window_dir) + / ( (1.0_rt + m_beta_boost) * m_gamma_boost); + amrex::Real zmax_lab = diag_dom.hi(m_moving_window_dir) + / ( (1.0_rt + m_beta_boost) * m_gamma_boost); + + + m_buffer_domain_lab[i_buffer] = warpx.Geom(lev).ProbDomain(); + m_buffer_domain_lab[i_buffer].setLo(m_moving_window_dir, zmin_lab + warpx.moving_window_v * m_t_lab[i_buffer]); + m_buffer_domain_lab[i_buffer].setHi(m_moving_window_dir, zmin_lab + warpx.moving_window_v * m_t_lab[i_buffer]); + // 5. Initialize buffer counter and z-positions of the i^th snapshot in // boosted-frame and lab-frame + m_buffer_counter[i_buffer] = 0; + m_current_z_lab[i_buffer] = 0.0_rt; + m_current_z_boost[i_buffer] = 0.0_rt; + // Now Update Current Z Positions + m_current_z_boost[i_buffer] = UpdateCurrentZBoostCoordinate(m_t_lab[i_buffer], + warpx.gett_new(lev) ); + m_current_z_lab[i_buffer] = UpdateCurrentZLabCoordinate(m_t_lab[i_buffer], + warpx.gett_new(lev) ); + + // 6. Compute ncells_lab required for writing Header file and potentially to generate // Back-Transform geometry to ensure compatibility with plotfiles // + // For the z-dimension, number of cells in the lab-frame is + // computed using the coarsened cell-size in the lab-frame obtained using + // the ref_ratio at level, lev. + amrex::IntVect ref_ratio = WarpX::RefRatio(lev); + // Number of lab-frame cells in z-direction at level, lev + const int num_zcells_lab = static_cast<int>( floor ( + ( zmax_lab - zmin_lab) + / dz_lab(warpx.getdt(lev), ref_ratio[AMREX_SPACEDIM-1] ) ) ); + // Take the max of 0 and num_zcells_lab + int Nz_lab = std::max( 0, num_zcells_lab ); + // Number of lab-frame cells in x-direction at level, lev + const int num_xcells_lab = static_cast<int>( floor ( + ( diag_dom.hi(0) - diag_dom.lo(0) ) + / warpx.Geom(lev).CellSize(0) + ) ); + // Take the max of 0 and num_ycells_lab + int Nx_lab = std::max( 0, num_xcells_lab); +#if (AMREX_SPACEDIM == 3) + // Number of lab-frame cells in the y-direction at level, lev + const int num_ycells_lab = static_cast<int>( floor ( + ( diag_dom.hi(1) - diag_dom.lo(1) ) + / warpx.Geom(lev).CellSize(1) + ) ); + // Take the max of 0 and num_xcells_lab + int Ny_lab = std::max( 0, num_ycells_lab ); + m_buffer_ncells_lab[i_buffer] = {Nx_lab, Ny_lab, Nz_lab}; +#else + int Ny_lab = 0; + m_buffer_ncells_lab[i_buffer] = {Nx_lab, Nz_lab}; +#endif + // 7. Call funtion to create directories for customized output format TMP_createLabFrameDirectories(i_buffer, lev); } @@ -161,6 +275,12 @@ BTDiagnostics::DefineCellCenteredMultiFab(int lev) // This MultiFab will store all the user-requested fields in the boosted-frame auto & warpx = WarpX::GetInstance(); // The BoxArray is coarsened based on the user-defined coarsening ratio + amrex::BoxArray ba = warpx.boxArray(lev); + ba.coarsen(m_crse_ratio); + amrex::DistributionMapping dmap = warpx.DistributionMap(lev); + int ngrow = 1; + m_cell_centered_data[lev].reset( new amrex::MultiFab(ba, dmap, m_varnames.size(), ngrow) ); + } void @@ -177,10 +297,47 @@ BTDiagnostics::InitializeFieldFunctors (int lev) m_cell_center_functors[lev].clear(); m_cell_center_functors[lev].resize( m_varnames.size() ); // 1. create an object of class BackTransformFunctor + for (int i = 0; i < num_BT_functors; ++i) + { + // coarsening ratio is not provided since the source MultiFab, m_cell_centered_data + // is coarsened based on the user-defined m_crse_ratio + m_all_field_functors[lev][i] = std::make_unique<BackTransformFunctor>( + m_cell_centered_data[lev].get(), lev, m_varnames.size() ); + } // 2. Define all cell-centered functors required to compute cell-centere data // Fill vector of cell-center functors for all components // Only E,B, j, and rho are included in the cell-center functors for BackTransform Diags + for (int comp=0, n=m_cell_center_functors[lev].size(); comp<n; comp++){ + if ( m_varnames[comp] == "Ex" ){ + m_cell_center_functors[lev][comp] = std::make_unique<CellCenterFunctor>(warpx.get_pointer_Efield_aux(lev, 0), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "Ey" ){ + m_cell_center_functors[lev][comp] = std::make_unique<CellCenterFunctor>(warpx.get_pointer_Efield_aux(lev, 1), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "Ez" ){ + m_cell_center_functors[lev][comp] = std::make_unique<CellCenterFunctor>(warpx.get_pointer_Efield_aux(lev, 2), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "Bx" ){ + m_cell_center_functors[lev][comp] = std::make_unique<CellCenterFunctor>(warpx.get_pointer_Bfield_aux(lev, 0), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "By" ){ + m_cell_center_functors[lev][comp] = std::make_unique<CellCenterFunctor>(warpx.get_pointer_Bfield_aux(lev, 1), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "Bz" ){ + m_cell_center_functors[lev][comp] = std::make_unique<CellCenterFunctor>(warpx.get_pointer_Bfield_aux(lev, 2), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "jx" ){ + m_cell_center_functors[lev][comp] = std::make_unique<CellCenterFunctor>(warpx.get_pointer_current_fp(lev, 0), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "jy" ){ + m_cell_center_functors[lev][comp] = std::make_unique<CellCenterFunctor>(warpx.get_pointer_current_fp(lev, 1), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "jz" ){ + m_cell_center_functors[lev][comp] = std::make_unique<CellCenterFunctor>(warpx.get_pointer_current_fp(lev, 2), lev, m_crse_ratio); + } else if ( m_varnames[comp] == "rho" ){ + // rho_new is stored in component 1 of rho_fp when using PSATD +#ifdef WARPX_USE_PSATD + amrex::MultiFab* rho_new = new amrex::MultiFab(*warpx.get_pointer_rho_fp(lev), amrex::make_alias, 1, 1); + m_cell_center_functors[lev][comp] = std::make_unique<CellCenterFunctor>(rho_new, lev, m_crse_ratio); +#else + m_cell_center_functors[lev][comp] = std::make_unique<CellCenterFunctor>(warpx.get_pointer_rho_fp(lev), lev, m_crse_ratio); +#endif + } + } + } // Temporary function only to debug the current implementation. @@ -197,9 +354,30 @@ BTDiagnostics::TMP_createLabFrameDirectories(int i_buffer, int lev) void BTDiagnostics::PrepareFieldDataForOutput () { + auto & warpx = WarpX::GetInstance(); // In this function, we will get cell-centered data for every level, lev, // using the cell-center functors and their respective opeators() // Call m_cell_center_functors->operator + for (int lev = 0; lev < nmax_lev; ++lev) { + int icomp_dst = 0; + for (int icomp = 0, n=m_cell_center_functors[0].size(); icomp<n; ++icomp) { + // Call all the cell-center functors in m_cell_center_functors. + // Each of them computes cell-centered data for a field and + // stores it in cell-centered MultiFab, m_cell_centered_data[lev]. + m_cell_center_functors[lev][icomp]->operator()(*m_cell_centered_data[lev], icomp_dst); + icomp_dst += m_cell_center_functors[lev][icomp]->nComp(); + } + // Check that the proper number of user-requested components are cell-centered + AMREX_ALWAYS_ASSERT( icomp_dst == m_varnames.size() ); + // fill boundary call is required to average_down (flatten) data to + // the coarsest level. + m_cell_centered_data[lev]->FillBoundary(warpx.Geom(lev).periodicity() ); + } + // Flattening out MF over levels + + for (int lev = warpx.finestLevel(); lev > 0; --lev) { + CoarsenIO::Coarsen( *m_cell_centered_data[lev-1], *m_cell_centered_data[lev], 0, 0, m_varnames.size(), 0, WarpX::RefRatio(lev-1) ); + } } diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H index 6903c9e45..b8702e715 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H @@ -33,8 +33,9 @@ public: // initializing crse_ratio with 1, to accurately initialize ComputeDiagFunctor BackTransformFunctor( const amrex::MultiFab * const mf_src, const int lev, - const int ncomp, amrex::IntVect crse_ratio= amrex::IntVect(1)); + const int ncomp, const amrex::IntVect crse_ratio= amrex::IntVect(1)); + void operator()(amrex::MultiFab& mf_dst, int dcomp) const override {}; private: /** pointer to source multifab (cell-centered multi-component multifab) */ amrex::MultiFab const * const m_mf_src = nullptr; diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp index 5b049a112..2d8651efe 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp @@ -1,7 +1,7 @@ #include "BackTransformFunctor.H" BackTransformFunctor::BackTransformFunctor(amrex::MultiFab const * mf_src, int lev, - int ncomp, amrex::IntVect crse_ratio) + const int ncomp, const amrex::IntVect crse_ratio) : ComputeDiagFunctor(ncomp, crse_ratio), m_mf_src(mf_src), m_lev(lev) { // Get the right slice of each field in the CC MultiFab, transform it and store it in the output. |