From 0d91a3d9f2cb70ad0ffffca2c614ecd63005fa46 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Thu, 18 Nov 2021 16:31:10 -0800 Subject: Bug fixes and cleanup in load balancing (#2563) * added helper function to rebuild MultiFabs and iMultiFabs during load balancing and included rebuilding of EB multifabs * added redistribute call for the particle boundary buffer during load balancing * consistently use DistribtionMap rather than dmap in ElectrostaticSolver.cpp * applied suggested changes from code review by Phil Miller * removed default argument for redistribute in RemakeMultiFab * removed RemakeMultiFab() as a member of WarpX * Only remake EB multifabs if they are used Co-authored-by: Lorenzo Giacomel <47607756+lgiacome@users.noreply.github.com> * adapted existing particle scraping test (PICMI version) to also cover the redistribution of particle buffers from load balancing * added redeclaring of m_borrowing * Move redeclaring of m_borrow inside if statement for ECT solver algorihtm Co-authored-by: Lorenzo Giacomel <47607756+lgiacome@users.noreply.github.com> * added calls to MarkCells and ComputeFaceExtensions * fixed issue causing CI test to fail and copied conditionals from WarpXInitData.cpp to recompute EB quantities * Guard cells communication for EB data when re-gridding (#105) * Add 2D circle EB test (#2538) * Added embedded_circle test * Add embedded_circle test files * Removed diag files * removed PICMI input file * Update to use default regression analysis * Added line breaks for spacing * Added description * Fixed benchmark file * Added load balancing to test * Commented out load_balancing portion of test. This will be added back in once load balancing is fixed. Co-authored-by: Axel Huebl * Added guard cells communication for EB data in regridding Co-authored-by: Kevin Z. Zhu <86268612+KZhu-ME@users.noreply.github.com> Co-authored-by: Axel Huebl * moved all EB grid data calculations to a new function InitializeEBGridData() which is now called by both WarpX::InitLevelData and WarpX::RemakeLevel * Fix typo in doc string. Co-authored-by: Phil Miller Co-authored-by: Lorenzo Giacomel <47607756+lgiacome@users.noreply.github.com> Co-authored-by: Kevin Z. Zhu <86268612+KZhu-ME@users.noreply.github.com> Co-authored-by: Axel Huebl Co-authored-by: Phil Miller --- Source/Initialization/WarpXInitData.cpp | 77 ++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 36 deletions(-) (limited to 'Source/Initialization/WarpXInitData.cpp') diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 59b7d1f25..41fda5567 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -485,42 +485,7 @@ WarpX::InitLevelData (int lev, Real /*time*/) } #ifdef AMREX_USE_EB - if(lev==maxLevel()) { - if (WarpX::maxwell_solver_id == MaxwellSolverAlgo::Yee || - WarpX::maxwell_solver_id == MaxwellSolverAlgo::CKC || - WarpX::maxwell_solver_id == MaxwellSolverAlgo::ECT) { - - ComputeEdgeLengths(); - ComputeFaceAreas(); - ScaleEdges(); - ScaleAreas(); - - const auto &period = Geom(lev).periodicity(); - WarpXCommUtil::FillBoundary(*m_edge_lengths[lev][0], guard_cells.ng_alloc_EB, period); - WarpXCommUtil::FillBoundary(*m_edge_lengths[lev][1], guard_cells.ng_alloc_EB, period); - WarpXCommUtil::FillBoundary(*m_edge_lengths[lev][2], guard_cells.ng_alloc_EB, period); - WarpXCommUtil::FillBoundary(*m_face_areas[lev][0], guard_cells.ng_alloc_EB, period); - WarpXCommUtil::FillBoundary(*m_face_areas[lev][1], guard_cells.ng_alloc_EB, period); - WarpXCommUtil::FillBoundary(*m_face_areas[lev][2], guard_cells.ng_alloc_EB, period); - - if (WarpX::maxwell_solver_id == MaxwellSolverAlgo::ECT) { - WarpXCommUtil::FillBoundary(*m_area_mod[lev][0], guard_cells.ng_alloc_EB, period); - WarpXCommUtil::FillBoundary(*m_area_mod[lev][1], guard_cells.ng_alloc_EB, period); - WarpXCommUtil::FillBoundary(*m_area_mod[lev][2], guard_cells.ng_alloc_EB, period); - MarkCells(); - WarpXCommUtil::FillBoundary(*m_flag_info_face[lev][0], guard_cells.ng_alloc_EB, period); - WarpXCommUtil::FillBoundary(*m_flag_info_face[lev][1], guard_cells.ng_alloc_EB, period); - WarpXCommUtil::FillBoundary(*m_flag_info_face[lev][2], guard_cells.ng_alloc_EB, period); - WarpXCommUtil::FillBoundary(*m_flag_ext_face[lev][0], guard_cells.ng_alloc_EB, period); - WarpXCommUtil::FillBoundary(*m_flag_ext_face[lev][1], guard_cells.ng_alloc_EB, period); - WarpXCommUtil::FillBoundary(*m_flag_ext_face[lev][2], guard_cells.ng_alloc_EB, period); - ComputeFaceExtensions(); - } - } - - ComputeDistanceToEB(); - - } + InitializeEBGridData(lev); #endif // if the input string for the B-field is "parse_b_ext_grid_function", @@ -869,3 +834,43 @@ void WarpX::CheckGuardCells(amrex::MultiFab const& mf) } } } + +void WarpX::InitializeEBGridData(int lev) +{ + if(lev==maxLevel()) { + if (WarpX::maxwell_solver_id == MaxwellSolverAlgo::Yee || + WarpX::maxwell_solver_id == MaxwellSolverAlgo::CKC || + WarpX::maxwell_solver_id == MaxwellSolverAlgo::ECT) { + + ComputeEdgeLengths(); + ComputeFaceAreas(); + ScaleEdges(); + ScaleAreas(); + + const auto &period = Geom(lev).periodicity(); + WarpXCommUtil::FillBoundary(*m_edge_lengths[lev][0], guard_cells.ng_alloc_EB, period); + WarpXCommUtil::FillBoundary(*m_edge_lengths[lev][1], guard_cells.ng_alloc_EB, period); + WarpXCommUtil::FillBoundary(*m_edge_lengths[lev][2], guard_cells.ng_alloc_EB, period); + WarpXCommUtil::FillBoundary(*m_face_areas[lev][0], guard_cells.ng_alloc_EB, period); + WarpXCommUtil::FillBoundary(*m_face_areas[lev][1], guard_cells.ng_alloc_EB, period); + WarpXCommUtil::FillBoundary(*m_face_areas[lev][2], guard_cells.ng_alloc_EB, period); + + if (WarpX::maxwell_solver_id == MaxwellSolverAlgo::ECT) { + WarpXCommUtil::FillBoundary(*m_area_mod[lev][0], guard_cells.ng_alloc_EB, period); + WarpXCommUtil::FillBoundary(*m_area_mod[lev][1], guard_cells.ng_alloc_EB, period); + WarpXCommUtil::FillBoundary(*m_area_mod[lev][2], guard_cells.ng_alloc_EB, period); + MarkCells(); + WarpXCommUtil::FillBoundary(*m_flag_info_face[lev][0], guard_cells.ng_alloc_EB, period); + WarpXCommUtil::FillBoundary(*m_flag_info_face[lev][1], guard_cells.ng_alloc_EB, period); + WarpXCommUtil::FillBoundary(*m_flag_info_face[lev][2], guard_cells.ng_alloc_EB, period); + WarpXCommUtil::FillBoundary(*m_flag_ext_face[lev][0], guard_cells.ng_alloc_EB, period); + WarpXCommUtil::FillBoundary(*m_flag_ext_face[lev][1], guard_cells.ng_alloc_EB, period); + WarpXCommUtil::FillBoundary(*m_flag_ext_face[lev][2], guard_cells.ng_alloc_EB, period); + ComputeFaceExtensions(); + } + } + + ComputeDistanceToEB(); + + } +} -- cgit v1.2.3