From 3991a0e4fe8ff55b9ca0dc68b863794657bd0408 Mon Sep 17 00:00:00 2001
From: Andrew Myers <atmyers2@gmail.com>
Date: Wed, 26 Apr 2023 12:17:17 -0700
Subject: Don't rely on managed memory in SmartUtils (#3874)

* Don't rely on managed memory in SmartUtils
* use different copy
* Policies Vector: Explicit Device Copy

Co-authored-by: Axel Huebl <axel.huebl@plasma.ninja>
---
 Source/Particles/ParticleCreation/SmartUtils.cpp | 32 ++++++++++++++++++++----
 1 file changed, 27 insertions(+), 5 deletions(-)

(limited to 'Source/Particles/ParticleCreation/SmartUtils.cpp')

diff --git a/Source/Particles/ParticleCreation/SmartUtils.cpp b/Source/Particles/ParticleCreation/SmartUtils.cpp
index 2ed06b9a2..7e79f58c5 100644
--- a/Source/Particles/ParticleCreation/SmartUtils.cpp
+++ b/Source/Particles/ParticleCreation/SmartUtils.cpp
@@ -7,17 +7,27 @@
  */
 #include "SmartUtils.H"
 
+#include <AMReX_GpuContainers.H>
+
 #include <algorithm>
 #include <utility>
+#include <vector>
 
 PolicyVec getPolicies (const NameMap& names) noexcept
 {
-    PolicyVec policies;
-    policies.resize(names.size());
+    std::vector<InitializationPolicy> h_policies;
+    h_policies.resize(names.size());
     for (const auto& kv : names)
     {
-        policies[kv.second] = initialization_policies[kv.first];
+        h_policies[kv.second] = initialization_policies[kv.first];
     }
+
+    // host-to-device copy
+    PolicyVec policies;
+    policies.resize(names.size());
+    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, h_policies.begin(), h_policies.end(), policies.begin());
+    amrex::Gpu::Device::streamSynchronize();
+
     return policies;
 }
 
@@ -25,6 +35,9 @@ SmartCopyTag getSmartCopyTag (const NameMap& src, const NameMap& dst) noexcept
 {
     SmartCopyTag tag;
 
+    std::vector<int> h_src_comps;
+    std::vector<int> h_dst_comps;
+
     // we use the fact that maps are sorted
     auto i_src = src.begin();
     auto i_dst = dst.begin();
@@ -44,12 +57,21 @@ SmartCopyTag getSmartCopyTag (const NameMap& src, const NameMap& dst) noexcept
         {
             // name is in both...
             tag.common_names.push_back(i_src->first);
-            tag.src_comps.push_back(i_src->second);
-            tag.dst_comps.push_back(i_dst->second);
+            h_src_comps.push_back(i_src->second);
+            h_dst_comps.push_back(i_dst->second);
             ++i_src;
             ++i_dst;
         }
     }
 
+    // host-to-device copies
+    tag.src_comps.resize(h_src_comps.size());
+    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, h_src_comps.begin(), h_src_comps.end(), tag.src_comps.begin());
+
+    tag.dst_comps.resize(h_dst_comps.size());
+    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, h_dst_comps.begin(), h_dst_comps.end(), tag.dst_comps.begin());
+
+    amrex::Gpu::Device::streamSynchronize();
+
     return tag;
 }
-- 
cgit v1.2.3