Bug fixes and cleanup in load balancing (#2563)

* added helper function to rebuild MultiFabs and iMultiFabs during load balancing and included rebuilding of EB multifabs

* added redistribute call for the particle boundary buffer during load balancing

* consistently use DistribtionMap rather than dmap in ElectrostaticSolver.cpp

* applied suggested changes from code review by Phil Miller

* removed default argument for redistribute in RemakeMultiFab

* removed RemakeMultiFab() as a member of WarpX

* Only remake EB multifabs if they are used

Co-authored-by: Lorenzo Giacomel <47607756+lgiacome@users.noreply.github.com>

* adapted existing particle scraping test (PICMI version) to also cover the redistribution of particle buffers from load balancing

* added redeclaring of m_borrowing

* Move redeclaring of m_borrow inside if statement for ECT solver algorihtm

Co-authored-by: Lorenzo Giacomel <47607756+lgiacome@users.noreply.github.com>

* added calls to MarkCells and ComputeFaceExtensions

* fixed issue causing CI test to fail and copied conditionals from WarpXInitData.cpp to recompute EB quantities

* Guard cells communication for EB data when re-gridding (#105)

* Add 2D circle EB test (#2538)

* Added embedded_circle test

* Add embedded_circle test files

* Removed diag files

* removed PICMI input file

* Update to use default regression analysis

* Added line breaks for spacing

* Added description

* Fixed benchmark file

* Added load balancing to test

* Commented out load_balancing portion of test.
This will be added back in once load balancing is fixed.

Co-authored-by: Axel Huebl <axel.huebl@plasma.ninja>

* Added guard cells communication for EB data in regridding

Co-authored-by: Kevin Z. Zhu <86268612+KZhu-ME@users.noreply.github.com>
Co-authored-by: Axel Huebl <axel.huebl@plasma.ninja>

* moved all EB grid data calculations to a new function InitializeEBGridData() which is now called by both WarpX::InitLevelData and WarpX::RemakeLevel

* Fix typo in doc string.

Co-authored-by: Phil Miller <unmobile+gh@gmail.com>

Co-authored-by: Lorenzo Giacomel <47607756+lgiacome@users.noreply.github.com>
Co-authored-by: Kevin Z. Zhu <86268612+KZhu-ME@users.noreply.github.com>
Co-authored-by: Axel Huebl <axel.huebl@plasma.ninja>
Co-authored-by: Phil Miller <unmobile+gh@gmail.com>

1 files changed, 41 insertions, 36 deletions

diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp
index 59b7d1f25..41fda5567 100644
--- a/Source/Initialization/WarpXInitData.cpp
+++ b/Source/Initialization/WarpXInitData.cpp
@@ -485,42 +485,7 @@ WarpX::InitLevelData (int lev, Real /*time*/)
     }
 
 #ifdef AMREX_USE_EB
-    if(lev==maxLevel()) {
-        if (WarpX::maxwell_solver_id == MaxwellSolverAlgo::Yee ||
-            WarpX::maxwell_solver_id == MaxwellSolverAlgo::CKC ||
-            WarpX::maxwell_solver_id == MaxwellSolverAlgo::ECT) {
-
-            ComputeEdgeLengths();
-            ComputeFaceAreas();
-            ScaleEdges();
-            ScaleAreas();
-
-            const auto &period = Geom(lev).periodicity();
-            WarpXCommUtil::FillBoundary(*m_edge_lengths[lev][0], guard_cells.ng_alloc_EB, period);
-            WarpXCommUtil::FillBoundary(*m_edge_lengths[lev][1], guard_cells.ng_alloc_EB, period);
-            WarpXCommUtil::FillBoundary(*m_edge_lengths[lev][2], guard_cells.ng_alloc_EB, period);
-            WarpXCommUtil::FillBoundary(*m_face_areas[lev][0], guard_cells.ng_alloc_EB, period);
-            WarpXCommUtil::FillBoundary(*m_face_areas[lev][1], guard_cells.ng_alloc_EB, period);
-            WarpXCommUtil::FillBoundary(*m_face_areas[lev][2], guard_cells.ng_alloc_EB, period);
-
-            if (WarpX::maxwell_solver_id == MaxwellSolverAlgo::ECT) {
-                WarpXCommUtil::FillBoundary(*m_area_mod[lev][0], guard_cells.ng_alloc_EB, period);
-                WarpXCommUtil::FillBoundary(*m_area_mod[lev][1], guard_cells.ng_alloc_EB, period);
-                WarpXCommUtil::FillBoundary(*m_area_mod[lev][2], guard_cells.ng_alloc_EB, period);
-                MarkCells();
-                WarpXCommUtil::FillBoundary(*m_flag_info_face[lev][0], guard_cells.ng_alloc_EB, period);
-                WarpXCommUtil::FillBoundary(*m_flag_info_face[lev][1], guard_cells.ng_alloc_EB, period);
-                WarpXCommUtil::FillBoundary(*m_flag_info_face[lev][2], guard_cells.ng_alloc_EB, period);
-                WarpXCommUtil::FillBoundary(*m_flag_ext_face[lev][0], guard_cells.ng_alloc_EB, period);
-                WarpXCommUtil::FillBoundary(*m_flag_ext_face[lev][1], guard_cells.ng_alloc_EB, period);
-                WarpXCommUtil::FillBoundary(*m_flag_ext_face[lev][2], guard_cells.ng_alloc_EB, period);
-                ComputeFaceExtensions();
-            }
-        }
-
-        ComputeDistanceToEB();
-
-    }
+    InitializeEBGridData(lev);
 #endif
 
     // if the input string for the B-field is "parse_b_ext_grid_function",
@@ -869,3 +834,43 @@ void WarpX::CheckGuardCells(amrex::MultiFab const& mf)
         }
     }
 }
+
+void WarpX::InitializeEBGridData(int lev)
+{
+    if(lev==maxLevel()) {
+        if (WarpX::maxwell_solver_id == MaxwellSolverAlgo::Yee ||
+            WarpX::maxwell_solver_id == MaxwellSolverAlgo::CKC ||
+            WarpX::maxwell_solver_id == MaxwellSolverAlgo::ECT) {
+
+            ComputeEdgeLengths();
+            ComputeFaceAreas();
+            ScaleEdges();
+            ScaleAreas();
+
+            const auto &period = Geom(lev).periodicity();
+            WarpXCommUtil::FillBoundary(*m_edge_lengths[lev][0], guard_cells.ng_alloc_EB, period);
+            WarpXCommUtil::FillBoundary(*m_edge_lengths[lev][1], guard_cells.ng_alloc_EB, period);
+            WarpXCommUtil::FillBoundary(*m_edge_lengths[lev][2], guard_cells.ng_alloc_EB, period);
+            WarpXCommUtil::FillBoundary(*m_face_areas[lev][0], guard_cells.ng_alloc_EB, period);
+            WarpXCommUtil::FillBoundary(*m_face_areas[lev][1], guard_cells.ng_alloc_EB, period);
+            WarpXCommUtil::FillBoundary(*m_face_areas[lev][2], guard_cells.ng_alloc_EB, period);
+
+            if (WarpX::maxwell_solver_id == MaxwellSolverAlgo::ECT) {
+                WarpXCommUtil::FillBoundary(*m_area_mod[lev][0], guard_cells.ng_alloc_EB, period);
+                WarpXCommUtil::FillBoundary(*m_area_mod[lev][1], guard_cells.ng_alloc_EB, period);
+                WarpXCommUtil::FillBoundary(*m_area_mod[lev][2], guard_cells.ng_alloc_EB, period);
+                MarkCells();
+                WarpXCommUtil::FillBoundary(*m_flag_info_face[lev][0], guard_cells.ng_alloc_EB, period);
+                WarpXCommUtil::FillBoundary(*m_flag_info_face[lev][1], guard_cells.ng_alloc_EB, period);
+                WarpXCommUtil::FillBoundary(*m_flag_info_face[lev][2], guard_cells.ng_alloc_EB, period);
+                WarpXCommUtil::FillBoundary(*m_flag_ext_face[lev][0], guard_cells.ng_alloc_EB, period);
+                WarpXCommUtil::FillBoundary(*m_flag_ext_face[lev][1], guard_cells.ng_alloc_EB, period);
+                WarpXCommUtil::FillBoundary(*m_flag_ext_face[lev][2], guard_cells.ng_alloc_EB, period);
+                ComputeFaceExtensions();
+            }
+        }
+
+        ComputeDistanceToEB();
+
+    }
+}