Silver-Mueller: Avoid Managed Memory (#2019)

Avoid relying on managed memory usage in Silver-Mueller boundary
conditions. Previously, we initialized the coefficients on the
host, copied them to device and then accidentially used the
device memory on the host again, as we calculated some constants.

This now keeps the initial host-memory around so we can use it
for host-only operations.

1 files changed, 14 insertions, 18 deletions

diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp
index 3a752ca4b..14decf5c1 100644
--- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp
+++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp
@@ -36,56 +36,52 @@ FiniteDifferenceSolver::FiniteDifferenceSolver (
     m_nmodes = WarpX::GetInstance().n_rz_azimuthal_modes;
     m_rmin = WarpX::GetInstance().Geom(0).ProbLo(0);
     if (fdtd_algo == MaxwellSolverAlgo::Yee) {
-
-        amrex::Vector<amrex::Real> stencil_coefs_r, stencil_coefs_z;
         CylindricalYeeAlgorithm::InitializeStencilCoefficients( cell_size,
-            stencil_coefs_r, stencil_coefs_z );
-        m_stencil_coefs_r.resize(stencil_coefs_r.size());
-        m_stencil_coefs_z.resize(stencil_coefs_z.size());
+            m_h_stencil_coefs_r, m_h_stencil_coefs_z );
+        m_stencil_coefs_r.resize(m_h_stencil_coefs_r.size());
+        m_stencil_coefs_z.resize(m_h_stencil_coefs_z.size());
         amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice,
-                              stencil_coefs_r.begin(), stencil_coefs_r.end(),
+                              m_h_stencil_coefs_r.begin(), m_h_stencil_coefs_r.end(),
                               m_stencil_coefs_r.begin());
         amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice,
-                              stencil_coefs_z.begin(), stencil_coefs_z.end(),
+                              m_h_stencil_coefs_z.begin(), m_h_stencil_coefs_z.end(),
                               m_stencil_coefs_z.begin());
         amrex::Gpu::synchronize();
     } else {
         amrex::Abort("FiniteDifferenceSolver: Unknown algorithm");
     }
 #else
-    amrex::Vector<amrex::Real> stencil_coefs_x, stencil_coefs_y, stencil_coefs_z;
-
     if (do_nodal) {
 
         CartesianNodalAlgorithm::InitializeStencilCoefficients( cell_size,
-            stencil_coefs_x, stencil_coefs_y, stencil_coefs_z );
+            m_h_stencil_coefs_x, m_h_stencil_coefs_y, m_h_stencil_coefs_z );
 
     } else if (fdtd_algo == MaxwellSolverAlgo::Yee) {
 
         CartesianYeeAlgorithm::InitializeStencilCoefficients( cell_size,
-            stencil_coefs_x, stencil_coefs_y, stencil_coefs_z );
+            m_h_stencil_coefs_x, m_h_stencil_coefs_y, m_h_stencil_coefs_z );
 
     } else if (fdtd_algo == MaxwellSolverAlgo::CKC) {
 
         CartesianCKCAlgorithm::InitializeStencilCoefficients( cell_size,
-            stencil_coefs_x, stencil_coefs_y, stencil_coefs_z );
+            m_h_stencil_coefs_x, m_h_stencil_coefs_y, m_h_stencil_coefs_z );
 
     } else {
         amrex::Abort("FiniteDifferenceSolver: Unknown algorithm");
     }
 
-    m_stencil_coefs_x.resize(stencil_coefs_x.size());
-    m_stencil_coefs_y.resize(stencil_coefs_y.size());
-    m_stencil_coefs_z.resize(stencil_coefs_z.size());
+    m_stencil_coefs_x.resize(m_h_stencil_coefs_x.size());
+    m_stencil_coefs_y.resize(m_h_stencil_coefs_y.size());
+    m_stencil_coefs_z.resize(m_h_stencil_coefs_z.size());
 
     amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice,
-                          stencil_coefs_x.begin(), stencil_coefs_x.end(),
+                          m_h_stencil_coefs_x.begin(), m_h_stencil_coefs_x.end(),
                           m_stencil_coefs_x.begin());
     amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice,
-                          stencil_coefs_y.begin(), stencil_coefs_y.end(),
+                          m_h_stencil_coefs_y.begin(), m_h_stencil_coefs_y.end(),
                           m_stencil_coefs_y.begin());
     amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice,
-                          stencil_coefs_z.begin(), stencil_coefs_z.end(),
+                          m_h_stencil_coefs_z.begin(), m_h_stencil_coefs_z.end(),
                           m_stencil_coefs_z.begin());
     amrex::Gpu::synchronize();
 #endif