Merge pull request #107 from ECP-WarpX/runtime_boosted_frame_particles

Runtime boosted frame particles

1 files changed, 56 insertions, 44 deletions

diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp
index e31d43204..17e6d98d9 100644
--- a/Source/Particles/PhysicalParticleContainer.cpp
+++ b/Source/Particles/PhysicalParticleContainer.cpp
@@ -184,6 +184,18 @@ PhysicalParticleContainer::AddGaussianBeam(Real x_m, Real y_m, Real z_m,
             attribs[PIdx::uz] = u[2];
             attribs[PIdx::w ] = weight;
 
+            if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
+            {
+                auto& particle_tile = DefineAndReturnParticleTile(0, 0, 0);
+                particle_tile.push_back_real(particle_comps["xold"], x);
+                particle_tile.push_back_real(particle_comps["yold"], y);
+                particle_tile.push_back_real(particle_comps["zold"], z);
+                
+                particle_tile.push_back_real(particle_comps["uxold"], u[0]);
+                particle_tile.push_back_real(particle_comps["uyold"], u[1]);
+                particle_tile.push_back_real(particle_comps["uzold"], u[2]);
+            }
+            
             AddOneParticle(0, 0, 0, x, y, z, attribs);
             }
         }
@@ -455,16 +467,18 @@ PhysicalParticleContainer::AddPlasmaCPU (int lev, RealBox part_realbox)
                     attribs[PIdx::ux] = u[0];
                     attribs[PIdx::uy] = u[1];
                     attribs[PIdx::uz] = u[2];
-
-#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS
-                    attribs[PIdx::xold] = x;
-                    attribs[PIdx::yold] = y;
-                    attribs[PIdx::zold] = z;
-
-                    attribs[PIdx::uxold] = u[0];
-                    attribs[PIdx::uyold] = u[1];
-                    attribs[PIdx::uzold] = u[2];
-#endif
+                    
+                    if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
+                    {
+                        auto& particle_tile = DefineAndReturnParticleTile(lev, grid_id, tile_id);
+                        particle_tile.push_back_real(particle_comps["xold"], x);
+                        particle_tile.push_back_real(particle_comps["yold"], y);
+                        particle_tile.push_back_real(particle_comps["zold"], z);
+
+                        particle_tile.push_back_real(particle_comps["uxold"], u[0]);
+                        particle_tile.push_back_real(particle_comps["uyold"], u[1]);
+                        particle_tile.push_back_real(particle_comps["uzold"], u[2]);
+                    }
 
 		    AddOneParticle(lev, grid_id, tile_id, x, y, z, attribs);
                 }
@@ -695,15 +709,18 @@ PhysicalParticleContainer::AddPlasmaGPU (int lev, RealBox part_realbox)
                     attribs[PIdx::uy] = u[1];
                     attribs[PIdx::uz] = u[2];
 
-#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS
-                    attribs[PIdx::xold] = x;
-                    attribs[PIdx::yold] = y;
-                    attribs[PIdx::zold] = z;
-
-                    attribs[PIdx::uxold] = u[0];
-                    attribs[PIdx::uyold] = u[1];
-                    attribs[PIdx::uzold] = u[2];
-#endif
+                    // note - this will be slow on the GPU, need to revisit
+                    if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
+                    {
+                        auto& particle_tile = DefineAndReturnParticleTile(lev, grid_id, tile_id);
+                        particle_tile.push_back_real(particle_comps["xold"], x);
+                        particle_tile.push_back_real(particle_comps["yold"], y);
+                        particle_tile.push_back_real(particle_comps["zold"], z);
+
+                        particle_tile.push_back_real(particle_comps["uxold"], u[0]);
+                        particle_tile.push_back_real(particle_comps["uyold"], u[1]);
+                        particle_tile.push_back_real(particle_comps["uzold"], u[2]);
+                    }
 
 		    ParticleType p;
 		    p.id()  = ParticleType::NextID();
@@ -1666,20 +1683,20 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti,
     auto& Bzp = attribs[PIdx::Bz];
     const long np  = pti.numParticles();
 
-#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS
-    auto& xpold  = attribs[PIdx::xold];
-    auto& ypold  = attribs[PIdx::yold];
-    auto& zpold  = attribs[PIdx::zold];
-    auto& uxpold = attribs[PIdx::uxold];
-    auto& uypold = attribs[PIdx::uyold];
-    auto& uzpold = attribs[PIdx::uzold];
-
-    warpx_copy_attribs(&np, xp.dataPtr(), yp.dataPtr(), zp.dataPtr(),
-                       uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(),
-                       xpold.dataPtr(), ypold.dataPtr(), zpold.dataPtr(),
-                       uxpold.dataPtr(), uypold.dataPtr(), uzpold.dataPtr());
-
-#endif
+    if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
+    {
+        auto& xpold    = pti.GetAttribs(particle_comps["xold"]);
+        auto& ypold    = pti.GetAttribs(particle_comps["yold"]);
+        auto& zpold    = pti.GetAttribs(particle_comps["zold"]);
+        auto& uxpold   = pti.GetAttribs(particle_comps["uxold"]);
+        auto& uypold   = pti.GetAttribs(particle_comps["uyold"]);
+        auto& uzpold   = pti.GetAttribs(particle_comps["uzold"]);
+
+        warpx_copy_attribs(&np, xp.dataPtr(), yp.dataPtr(), zp.dataPtr(),
+                           uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(),
+                           xpold.dataPtr(), ypold.dataPtr(), zpold.dataPtr(),
+                           uxpold.dataPtr(), uypold.dataPtr(), uzpold.dataPtr());
+    }
 
     warpx_particle_pusher(&np,
                           xp.dataPtr(),
@@ -1801,7 +1818,6 @@ void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real
 {
     BL_PROFILE("PhysicalParticleContainer::GetParticleSlice");
 
-#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS
     // Assume that the boost in the positive z direction.
 #if (AMREX_SPACEDIM == 2)
     AMREX_ALWAYS_ASSERT(direction == 1);
@@ -1864,12 +1880,12 @@ void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real
                 auto& uyp_new = attribs[PIdx::uy   ];
                 auto& uzp_new = attribs[PIdx::uz   ];
 
-                auto&  xp_old = attribs[PIdx::xold ];
-                auto&  yp_old = attribs[PIdx::yold ];
-                auto&  zp_old = attribs[PIdx::zold ];
-                auto& uxp_old = attribs[PIdx::uxold];
-                auto& uyp_old = attribs[PIdx::uyold];
-                auto& uzp_old = attribs[PIdx::uzold];
+                auto&  xp_old = pti.GetAttribs(particle_comps["xold"]);
+                auto&  yp_old = pti.GetAttribs(particle_comps["yold"]);
+                auto&  zp_old = pti.GetAttribs(particle_comps["zold"]);
+                auto& uxp_old = pti.GetAttribs(particle_comps["uxold"]);
+                auto& uyp_old = pti.GetAttribs(particle_comps["uyold"]);
+                auto& uzp_old = pti.GetAttribs(particle_comps["uzold"]);
 
                 const long np = pti.numParticles();
 
@@ -1919,10 +1935,6 @@ void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real
             }
         }
     }
-#else
-    AMREX_ALWAYS_ASSERT_WITH_MESSAGE( false ,
-"ERROR: WarpX must be compiled with STORE_OLD_PARTICLE_ATTRIBS=TRUE to use the back-transformed diagnostics");
-#endif
 }
 
 int PhysicalParticleContainer::GetRefineFac(const Real x, const Real y, const Real z)