more changes due to AMReX changes

7 files changed, 60 insertions, 35 deletions

diff --git a/Source/LaserParticleContainer.cpp b/Source/LaserParticleContainer.cpp
index 6ee79e9aa..cf669d32d 100644
--- a/Source/LaserParticleContainer.cpp
+++ b/Source/LaserParticleContainer.cpp
@@ -432,7 +432,11 @@ LaserParticleContainer::Evolve (int lev,
             if (cost) {
                 const Box& tbx = pti.tilebox();
                 wt = (amrex::second() - wt) / tbx.d_numPts();
-                (*cost)[pti].plus(wt, tbx);
+                FArrayBox* costfab = cost->fabPtr(pti);
+                AMREX_LAUNCH_HOST_DEVICE_LAMBDA ( tbx, work_box,
+                {
+                    costfab->plus(wt, work_box);
+                });
             }
 	}
     }
diff --git a/Source/PhysicalParticleContainer.H b/Source/PhysicalParticleContainer.H
index d2648421d..bd6015437 100644
--- a/Source/PhysicalParticleContainer.H
+++ b/Source/PhysicalParticleContainer.H
@@ -80,6 +80,10 @@ public:
     // Inject particles in Box 'part_box'
     virtual void AddParticles (int lev);
     void AddPlasma(int lev, amrex::RealBox part_realbox = amrex::RealBox());
+    void AddPlasmaCPU (int lev, amrex::RealBox part_realbox);
+#ifdef AMREX_USE_GPU
+    void AddPlasmaGPU (int lev, amrex::RealBox part_realbox);
+#endif
 
     void MapParticletoBoostedFrame(amrex::Real& x, amrex::Real& y, amrex::Real& z, std::array<amrex::Real, 3>& u);
 
@@ -110,10 +114,6 @@ protected:
 			    const amrex::RealBox& tile_real_box,
 			    const amrex::RealBox& particle_real_box);
   
-    void AddPlasmaCPU (int lev, amrex::RealBox part_realbox);
-#ifdef AMREX_USE_GPU
-    void AddPlasmaGPU (int lev, amrex::RealBox part_realbox);
-#endif
     int GetRefineFac(const amrex::Real x, const amrex::Real y, const amrex::Real z);
     std::unique_ptr<amrex::IArrayBox> m_refined_injection_mask = nullptr;
 
diff --git a/Source/PhysicalParticleContainer.cpp b/Source/PhysicalParticleContainer.cpp
index f76c74579..c4fdd4464 100644
--- a/Source/PhysicalParticleContainer.cpp
+++ b/Source/PhysicalParticleContainer.cpp
@@ -437,7 +437,11 @@ PhysicalParticleContainer::AddPlasmaCPU (int lev, RealBox part_realbox)
 
             if (cost) {
 	        wt = (amrex::second() - wt) / tile_box.d_numPts();
-		(*cost)[mfi].plus(wt, tile_box);
+                FArrayBox* costfab = cost->fabPtr(mfi);
+                AMREX_LAUNCH_HOST_DEVICE_LAMBDA ( tile_box, work_box,
+                {
+                    costfab->plus(wt, work_box);
+                });
             }
         }
     }
@@ -674,7 +678,11 @@ PhysicalParticleContainer::AddPlasmaGPU (int lev, RealBox part_realbox)
 	    			 
             if (cost) {
 	        wt = (amrex::second() - wt) / tile_box.d_numPts();
-		(*cost)[mfi].plus(wt, tile_box);
+                FArrayBox* costfab = cost->fabPtr(mfi);
+                AMREX_LAUNCH_HOST_DEVICE_LAMBDA ( tile_box, work_box,
+                {
+                    costfab->plus(wt, work_box);
+                });
             }
         }		
     }
@@ -981,7 +989,11 @@ PhysicalParticleContainer::FieldGather (int lev,
             if (cost) {
                 const Box& tbx = pti.tilebox();
                 wt = (amrex::second() - wt) / tbx.d_numPts();
-                (*cost)[pti].plus(wt, tbx);
+                FArrayBox* costfab = cost->fabPtr(pti);
+                AMREX_LAUNCH_HOST_DEVICE_LAMBDA ( tbx, work_box,
+                {
+                    costfab->plus(wt, work_box);
+                });
             }
         }
     }
@@ -1406,7 +1418,11 @@ PhysicalParticleContainer::Evolve (int lev,
             if (cost) {
                 const Box& tbx = pti.tilebox();
                 wt = (amrex::second() - wt) / tbx.d_numPts();
-                (*cost)[pti].plus(wt, tbx);
+                FArrayBox* costfab = cost->fabPtr(pti);
+                AMREX_LAUNCH_HOST_DEVICE_LAMBDA ( tbx, work_box,
+                {
+                    costfab->plus(wt, work_box);
+                });
             }
         }
     }
diff --git a/Source/WarpX.H b/Source/WarpX.H
index 4d74d165f..5678ac79f 100644
--- a/Source/WarpX.H
+++ b/Source/WarpX.H
@@ -144,9 +144,13 @@ public:
     void EvolveB (int lev, amrex::Real dt);
     void EvolveF (         amrex::Real dt, DtType dt_type);
     void EvolveF (int lev, amrex::Real dt, DtType dt_type);
+    void EvolveB (int lev, PatchType patch_type, amrex::Real dt);
+    void EvolveE (int lev, PatchType patch_type, amrex::Real dt);
+    void EvolveF (int lev, PatchType patch_type, amrex::Real dt, DtType dt_type);
 
     void DampPML ();
     void DampPML (int lev);
+    void DampPML (int lev, PatchType patch_type);
 
     void PushParticlesandDepose (int lev, amrex::Real cur_time);
     void PushParticlesandDepose (         amrex::Real cur_time);
@@ -253,11 +257,6 @@ private:
     void FillBoundaryE (int lev, PatchType patch_type);
     void FillBoundaryF (int lev, PatchType patch_type);
 
-    void EvolveB (int lev, PatchType patch_type, amrex::Real dt);
-    void EvolveE (int lev, PatchType patch_type, amrex::Real dt);
-    void EvolveF (int lev, PatchType patch_type, amrex::Real dt, DtType dt_type);
-    void DampPML (int lev, PatchType patch_type);
-
     void OneStep_nosub (amrex::Real t);
     void OneStep_sub1 (amrex::Real t);
 
diff --git a/Source/WarpXEvolve.cpp b/Source/WarpXEvolve.cpp
index 38a630880..6d917e36d 100644
--- a/Source/WarpXEvolve.cpp
+++ b/Source/WarpXEvolve.cpp
@@ -525,8 +525,12 @@ WarpX::EvolveB (int lev, PatchType patch_type, amrex::Real dt)
         if (cost) {
             Box cbx = mfi.tilebox(IntVect{AMREX_D_DECL(0,0,0)});
             if (patch_type == PatchType::coarse) cbx.refine(rr);
-            wt = (amrex::second() - wt) / cbx.d_numPts();
-            (*cost)[mfi].plus(wt, cbx);
+            wt = (amrex::second() - wt) / cbx.d_numPts();\
+            FArrayBox* costfab = cost->fabPtr(mfi);
+            AMREX_LAUNCH_HOST_DEVICE_LAMBDA ( cbx, work_box,
+            {
+                costfab->plus(wt, work_box);
+            });
         }
     }
 
@@ -668,7 +672,11 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real dt)
             Box cbx = mfi.tilebox(IntVect{AMREX_D_DECL(0,0,0)});
             if (patch_type == PatchType::coarse) cbx.refine(rr);
             wt = (amrex::second() - wt) / cbx.d_numPts();
-            (*cost)[mfi].plus(wt, cbx);
+            FArrayBox* costfab = cost->fabPtr(mfi);
+            AMREX_LAUNCH_HOST_DEVICE_LAMBDA ( cbx, work_box,
+            {
+                costfab->plus(wt, work_box);
+            });
         }
     }
 
diff --git a/Source/WarpXMove.cpp b/Source/WarpXMove.cpp
index 899f021f8..908c70573 100644
--- a/Source/WarpXMove.cpp
+++ b/Source/WarpXMove.cpp
@@ -222,15 +222,15 @@ WarpX::shiftMF (MultiFab& mf, const Geometry& geom, int num_shift, int dir)
 #endif
     for (MFIter mfi(tmpmf); mfi.isValid(); ++mfi )
     {
-      FArrayBox* dstfab = &(mf[mfi]);
-      FArrayBox* srcfab = &(tmpmf[mfi]);
+      FArrayBox* dstfab = mf.fabPtr(mfi);
+      FArrayBox* srcfab = tmpmf.fabPtr(mfi);
       Box outbox = mfi.fabbox();
       outbox &= adjBox;
       AMREX_LAUNCH_HOST_DEVICE_LAMBDA(outbox, toutbox,
       {
 	srcfab->setVal(0.0, toutbox, 0, nc);
       });
-      Box dstBox = dstfab->box();
+      Box dstBox = mf[mfi].box();
       if (num_shift > 0) {
 	dstBox.growHi(dir, -num_shift);
       } else {
diff --git a/Source/WarpXParticleContainer.cpp b/Source/WarpXParticleContainer.cpp
index 7c15f9d7d..54199eeb7 100644
--- a/Source/WarpXParticleContainer.cpp
+++ b/Source/WarpXParticleContainer.cpp
@@ -277,21 +277,21 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti,
       BL_PROFILE_VAR_START(blp_accumulate);
 
       FArrayBox const* local_jx_const_ptr = local_jx[thread_num].get();
-      FArrayBox* global_jx_ptr = &jx[pti];
+      FArrayBox* global_jx_ptr = jx.fabPtr(pti);
       AMREX_LAUNCH_HOST_DEVICE_LAMBDA(tbx, thread_bx,
       {
         global_jx_ptr->atomicAdd(*local_jx_const_ptr, thread_bx, thread_bx, 0, 0, 1);
       });
 
       FArrayBox const* local_jy_const_ptr = local_jy[thread_num].get();
-      FArrayBox* global_jy_ptr = &jy[pti];
+      FArrayBox* global_jy_ptr = jy.fabPtr(pti);
       AMREX_LAUNCH_HOST_DEVICE_LAMBDA(tby, thread_bx,
       {
         global_jy_ptr->atomicAdd(*local_jy_const_ptr, thread_bx, thread_bx, 0, 0, 1);
       });
 
       FArrayBox const* local_jz_const_ptr = local_jz[thread_num].get();
-      FArrayBox* global_jz_ptr = &jz[pti];
+      FArrayBox* global_jz_ptr = jz.fabPtr(pti);
       AMREX_LAUNCH_HOST_DEVICE_LAMBDA(tbz, thread_bx,
       {
         global_jz_ptr->atomicAdd(*local_jz_const_ptr, thread_bx, thread_bx, 0, 0, 1);
@@ -364,28 +364,24 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti,
                                &lvect,&WarpX::current_deposition_algo);
       BL_PROFILE_VAR_STOP(blp_pxr_cd);
 
-      FArrayBox& cjxfab = (*cjx)[pti];
-      FArrayBox& cjyfab = (*cjy)[pti];
-      FArrayBox& cjzfab = (*cjz)[pti];
-
       BL_PROFILE_VAR_START(blp_accumulate);
 
       FArrayBox const* local_jx_const_ptr = local_jx[thread_num].get();
-      FArrayBox* global_jx_ptr = &cjxfab;
+      FArrayBox* global_jx_ptr = cjx->fabPtr(pti);
       AMREX_LAUNCH_HOST_DEVICE_LAMBDA(tbx, thread_bx,
       {
         global_jx_ptr->atomicAdd(*local_jx_const_ptr, thread_bx, thread_bx, 0, 0, 1);
       });
 
       FArrayBox const* local_jy_const_ptr = local_jy[thread_num].get();
-      FArrayBox* global_jy_ptr = &cjyfab;
+      FArrayBox* global_jy_ptr = cjx->fabPtr(pti);
       AMREX_LAUNCH_HOST_DEVICE_LAMBDA(tby, thread_bx,
       {
         global_jy_ptr->atomicAdd(*local_jy_const_ptr, thread_bx, thread_bx, 0, 0, 1);
       });
 
       FArrayBox const* local_jz_const_ptr = local_jz[thread_num].get();
-      FArrayBox* global_jz_ptr = &cjzfab;
+      FArrayBox* global_jz_ptr = cjx->fabPtr(pti);
       AMREX_LAUNCH_HOST_DEVICE_LAMBDA(tbz, thread_bx,
       {
         global_jz_ptr->atomicAdd(*local_jz_const_ptr, thread_bx, thread_bx, 0, 0, 1);
@@ -420,7 +416,6 @@ WarpXParticleContainer::DepositCharge ( WarpXParIter& pti, RealVector& wp,
   // Deposit charge for particles that are not in the current buffers
   if (np_current > 0)
     {
-      FArrayBox& rhofab = (*rhomf)[pti];
       const std::array<Real, 3>& xyzmin = xyzmin_tile;
       tile_box.grow(ngRho);
       local_rho[thread_num]->resize(tile_box);
@@ -457,7 +452,7 @@ WarpXParticleContainer::DepositCharge ( WarpXParIter& pti, RealVector& wp,
 
       const int ncomp = 1;
       FArrayBox const* local_fab = local_rho[thread_num].get();
-      FArrayBox*       global_fab = &rhofab;
+      FArrayBox*       global_fab = rhomf->fabPtr(pti);
       BL_PROFILE_VAR_START(blp_accumulate);
       AMREX_LAUNCH_HOST_DEVICE_LAMBDA(tile_box, tbx,
       {
@@ -508,11 +503,10 @@ WarpXParticleContainer::DepositCharge ( WarpXParIter& pti, RealVector& wp,
                               &WarpX::nox,&WarpX::noy,&WarpX::noz,
                               &lvect, &WarpX::charge_deposition_algo);
       BL_PROFILE_VAR_STOP(blp_pxr_chd);
-      FArrayBox& crhofab = (*crhomf)[pti];
 
       const int ncomp = 1;
       FArrayBox const* local_fab = local_rho[thread_num].get();
-      FArrayBox*       global_fab = &crhofab;
+      FArrayBox*       global_fab = crhomf->fabPtr(pti);
       BL_PROFILE_VAR_START(blp_accumulate);
       AMREX_LAUNCH_HOST_DEVICE_LAMBDA(tile_box, tbx,
       {
@@ -878,7 +872,11 @@ WarpXParticleContainer::PushX (int lev, Real dt)
             if (cost) {
                 const Box& tbx = pti.tilebox();
                 wt = (amrex::second() - wt) / tbx.d_numPts();
-                (*cost)[pti].plus(wt, tbx);
+                FArrayBox* costfab = cost->fabPtr(pti);
+                AMREX_LAUNCH_HOST_DEVICE_LAMBDA ( tbx, work_box,
+                {
+                    costfab->plus(wt, work_box);
+                });
             }
         }
     }