fix conflicts

1 files changed, 137 insertions, 73 deletions

diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp
index bf0d9b255..793b96db7 100644
--- a/Source/WarpX.cpp
+++ b/Source/WarpX.cpp
@@ -46,11 +46,12 @@ long WarpX::noz = 1;
 bool WarpX::use_fdtd_nci_corr = false;
 int  WarpX::l_lower_order_in_v = true;
 
-bool WarpX::use_laser         = false;
 bool WarpX::use_filter        = false;
 bool WarpX::serialize_ics     = false;
 bool WarpX::refine_plasma     = false;
 
+int WarpX::num_mirrors = 0;
+
 int  WarpX::sort_int = -1;
 
 bool WarpX::do_boosted_frame_diagnostic = false;
@@ -91,6 +92,8 @@ IntVect WarpX::jy_nodal_flag(1,1);  // y is the missing dimension to 2D AMReX
 IntVect WarpX::jz_nodal_flag(1,0);  // z is the second dimension to 2D AMReX
 #endif
 
+IntVect WarpX::filter_npass_each_dir(1);
+
 int WarpX::n_field_gather_buffer = 0;
 int WarpX::n_current_deposition_buffer = -1;
 
@@ -98,8 +101,6 @@ int WarpX::do_nodal = false;
 
 WarpX* WarpX::m_instance = nullptr;
 
-
-
 WarpX&
 WarpX::GetInstance ()
 {
@@ -206,6 +207,9 @@ WarpX::WarpX ()
     dataptr_fp_fft.resize(nlevs_max);
     dataptr_cp_fft.resize(nlevs_max);
 
+    spectral_solver_fp.resize(nlevs_max);
+    spectral_solver_cp.resize(nlevs_max);
+
     ba_valid_fp_fft.resize(nlevs_max);
     ba_valid_cp_fft.resize(nlevs_max);
 
@@ -260,14 +264,14 @@ WarpX::ReadParameters ()
 	pp.query("cfl", cfl);
 	pp.query("verbose", verbose);
 	pp.query("regrid_int", regrid_int);
-        pp.query("do_subcycling", do_subcycling);
+    pp.query("do_subcycling", do_subcycling);
 
-        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(do_subcycling != 1 || max_level <= 1,
-                                         "Subcycling method 1 only works for 2 levels.");
+    AMREX_ALWAYS_ASSERT_WITH_MESSAGE(do_subcycling != 1 || max_level <= 1,
+                                     "Subcycling method 1 only works for 2 levels.");
 
-        ReadBoostedFrameParameters(gamma_boost, beta_boost, boost_direction);
+    ReadBoostedFrameParameters(gamma_boost, beta_boost, boost_direction);
 
-        pp.queryarr("B_external", B_external);
+    pp.queryarr("B_external", B_external);
 
 	pp.query("do_moving_window", do_moving_window);
 	if (do_moving_window)
@@ -298,52 +302,71 @@ WarpX::ReadParameters ()
 
 	pp.query("do_plasma_injection", do_plasma_injection);
 	if (do_plasma_injection) {
-            pp.get("num_injected_species", num_injected_species);
-            injected_plasma_species.resize(num_injected_species);
-            pp.getarr("injected_plasma_species", injected_plasma_species,
-                      0, num_injected_species);
-            if (moving_window_v >= 0){
-                // Inject particles continuously from the right end of the box
-                current_injection_position = geom[0].ProbHi(moving_window_dir);
-            } else {
-                // Inject particles continuously from the left end of the box
-                current_injection_position = geom[0].ProbLo(moving_window_dir);
-            }
+        pp.get("num_injected_species", num_injected_species);
+        injected_plasma_species.resize(num_injected_species);
+        pp.getarr("injected_plasma_species", injected_plasma_species,
+                  0, num_injected_species);
+        if (moving_window_v >= 0){
+            // Inject particles continuously from the right end of the box
+            current_injection_position = geom[0].ProbHi(moving_window_dir);
+        } else {
+            // Inject particles continuously from the left end of the box
+            current_injection_position = geom[0].ProbLo(moving_window_dir);
+        }
 	}
 
-        pp.query("do_boosted_frame_diagnostic", do_boosted_frame_diagnostic);
-        if (do_boosted_frame_diagnostic) {
+    pp.query("do_boosted_frame_diagnostic", do_boosted_frame_diagnostic);
+    if (do_boosted_frame_diagnostic) {
 
-            AMREX_ALWAYS_ASSERT_WITH_MESSAGE(gamma_boost > 1.0,
-                 "gamma_boost must be > 1 to use the boosted frame diagnostic.");
+        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(gamma_boost > 1.0,
+               "gamma_boost must be > 1 to use the boosted frame diagnostic.");
 
-            std::string s;
-    	    pp.get("boost_direction", s);
-            AMREX_ALWAYS_ASSERT_WITH_MESSAGE( (s == "z" || s == "Z"),
-                "The boosted frame diagnostic currently only works if the boost is in the z direction.");
+        std::string s;
+        pp.get("boost_direction", s);
+        AMREX_ALWAYS_ASSERT_WITH_MESSAGE( (s == "z" || s == "Z"),
+               "The boosted frame diagnostic currently only works if the boost is in the z direction.");
 
-            pp.get("num_snapshots_lab", num_snapshots_lab);
-            pp.get("dt_snapshots_lab", dt_snapshots_lab);
-            pp.get("gamma_boost", gamma_boost);
+        pp.get("num_snapshots_lab", num_snapshots_lab);
+        pp.get("dt_snapshots_lab", dt_snapshots_lab);
+        pp.get("gamma_boost", gamma_boost);
 
-            pp.query("do_boosted_frame_fields", do_boosted_frame_fields);
-            pp.query("do_boosted_frame_particles", do_boosted_frame_particles);
+        pp.query("do_boosted_frame_fields", do_boosted_frame_fields);
+        pp.query("do_boosted_frame_particles", do_boosted_frame_particles);
 
 
-            AMREX_ALWAYS_ASSERT_WITH_MESSAGE(do_moving_window,
-                "The moving window should be on if using the boosted frame diagnostic.");
+        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(do_moving_window,
+               "The moving window should be on if using the boosted frame diagnostic.");
 
-	        pp.get("moving_window_dir", s);
-            AMREX_ALWAYS_ASSERT_WITH_MESSAGE( (s == "z" || s == "Z"),
-                "The boosted frame diagnostic currently only works if the moving window is in the z direction.");
-        }
+        pp.get("moving_window_dir", s);
+        AMREX_ALWAYS_ASSERT_WITH_MESSAGE( (s == "z" || s == "Z"),
+               "The boosted frame diagnostic currently only works if the moving window is in the z direction.");
+    }
 
-        pp.query("do_electrostatic", do_electrostatic);
-        pp.query("n_buffer", n_buffer);
-        pp.query("const_dt", const_dt);
+    pp.query("do_electrostatic", do_electrostatic);
+    pp.query("n_buffer", n_buffer);
+    pp.query("const_dt", const_dt);
 
-	pp.query("use_laser", use_laser);
+    // Read filter and fill IntVect filter_npass_each_dir with
+    // proper size for AMREX_SPACEDIM
 	pp.query("use_filter", use_filter);
+    Vector<int> parse_filter_npass_each_dir(AMREX_SPACEDIM,1);
+    pp.queryarr("filter_npass_each_dir", parse_filter_npass_each_dir);
+    filter_npass_each_dir[0] = parse_filter_npass_each_dir[0];
+    filter_npass_each_dir[1] = parse_filter_npass_each_dir[1];
+#if (AMREX_SPACEDIM == 3)
+    filter_npass_each_dir[2] = parse_filter_npass_each_dir[2];
+#endif
+
+    pp.query("num_mirrors", num_mirrors);
+    if (num_mirrors>0){
+        mirror_z.resize(num_mirrors);
+        pp.getarr("mirror_z", mirror_z, 0, num_mirrors);
+        mirror_z_width.resize(num_mirrors);
+        pp.getarr("mirror_z_width", mirror_z_width, 0, num_mirrors);
+        mirror_z_npoints.resize(num_mirrors);
+        pp.getarr("mirror_z_npoints", mirror_z_npoints, 0, num_mirrors);
+    }
+
 	pp.query("serialize_ics", serialize_ics);
 	pp.query("refine_plasma", refine_plasma);
         pp.query("do_dive_cleaning", do_dive_cleaning);
@@ -355,6 +378,8 @@ WarpX::ReadParameters ()
         pp.query("pml_ncell", pml_ncell);
         pp.query("pml_delta", pml_delta);
 
+        pp.query("dump_openpmd", dump_openpmd);
+        pp.query("dump_plotfiles", dump_plotfiles);
         pp.query("plot_raw_fields", plot_raw_fields);
         pp.query("plot_raw_fields_guards", plot_raw_fields_guards);
         if (ParallelDescriptor::NProcs() == 1) {
@@ -368,12 +393,22 @@ WarpX::ReadParameters ()
         pp.query("plot_divb"         , plot_divb);
         pp.query("plot_rho"          , plot_rho);
         pp.query("plot_F"            , plot_F);
+        pp.query("plot_coarsening_ratio", plot_coarsening_ratio);
+        // Check that the coarsening_ratio can divide the blocking factor
+        for (int lev=0; lev<maxLevel(); lev++){
+          for (int comp=0; comp<AMREX_SPACEDIM; comp++){
+            if ( blockingFactor(lev)[comp] % plot_coarsening_ratio != 0 ){
+              amrex::Abort("plot_coarsening_ratio should be an integer divisor of the blocking factor.");
+            }
+          }
+        }
+
         if (plot_F){
         AMREX_ALWAYS_ASSERT_WITH_MESSAGE(do_dive_cleaning,
                 "plot_F only works if warpx.do_dive_cleaning = 1");
         }
+        pp.query("plot_finepatch", plot_finepatch);
         if (maxLevel() > 0) {
-            pp.query("plot_finepatch", plot_finepatch);
             pp.query("plot_crsepatch", plot_crsepatch);
         }
 
@@ -407,6 +442,39 @@ WarpX::ReadParameters ()
             fine_tag_hi = RealVect{hi};
         }
 
+        // select which particle comps to write
+        {
+            pp.queryarr("particle_plot_vars", particle_plot_vars);
+
+            if (particle_plot_vars.size() == 0)
+            {
+                if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
+                {
+                    particle_plot_flags.resize(PIdx::nattribs + 6, 1);
+                }
+                else
+                {
+                    particle_plot_flags.resize(PIdx::nattribs, 1);
+                }                
+            }
+            else
+            {
+                if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles)
+                {
+                    particle_plot_flags.resize(PIdx::nattribs + 6, 0);
+                }
+                else
+                {
+                    particle_plot_flags.resize(PIdx::nattribs, 0);
+                }                
+
+                for (const auto& var : particle_plot_vars)
+                {
+                    particle_plot_flags[ParticleStringNames::to_index.at(var)] = 1;
+                }
+            }
+        }
+
         pp.query("load_balance_int", load_balance_int);
         pp.query("load_balance_with_sfc", load_balance_with_sfc);
         pp.query("load_balance_knapsack_factor", load_balance_knapsack_factor);
@@ -468,11 +536,14 @@ WarpX::ReadParameters ()
 #ifdef WARPX_USE_PSATD
     {
         ParmParse pp("psatd");
+        pp.query("hybrid_mpi_decomposition", fft_hybrid_mpi_decomposition);
         pp.query("ngroups_fft", ngroups_fft);
         pp.query("fftw_plan_measure", fftw_plan_measure);
         pp.query("nox", nox_fft);
         pp.query("noy", noy_fft);
         pp.query("noz", noz_fft);
+        // Override value
+        if (fft_hybrid_mpi_decomposition==false) ngroups_fft=ParallelDescriptor::NProcs();
     }
 #endif
 
@@ -665,10 +736,10 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm
 
     if (do_dive_cleaning || plot_rho)
     {
-        rho_fp[lev].reset(new MultiFab(amrex::convert(ba,IntVect::TheUnitVector()),dm,2,ngRho));    
+        rho_fp[lev].reset(new MultiFab(amrex::convert(ba,IntVect::TheUnitVector()),dm,2,ngRho));
         rho_fp_owner_masks[lev] = std::move(rho_fp[lev]->OwnerMask(period));
     }
-    
+
     if (do_subcycling == 1 && lev == 0)
     {
         current_store[lev][0].reset( new MultiFab(amrex::convert(ba,jx_nodal_flag),dm,1,ngJ));
@@ -782,7 +853,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm
             current_buf[lev][0].reset( new MultiFab(amrex::convert(cba,jx_nodal_flag),dm,1,ngJ));
             current_buf[lev][1].reset( new MultiFab(amrex::convert(cba,jy_nodal_flag),dm,1,ngJ));
             current_buf[lev][2].reset( new MultiFab(amrex::convert(cba,jz_nodal_flag),dm,1,ngJ));
-            if (do_dive_cleaning) {
+            if (do_dive_cleaning || plot_rho) {
                 charge_buf[lev].reset( new MultiFab(amrex::convert(cba,IntVect::TheUnitVector()),dm,2,ngRho));
             }
             current_buffer_masks[lev].reset( new iMultiFab(ba, dm, 1, 1) );
@@ -946,33 +1017,6 @@ WarpX::ComputeDivE (MultiFab& divE, int dcomp,
 }
 
 void
-WarpX::applyFilter (MultiFab& dstmf, const MultiFab& srcmf, int scomp, int dcomp, int ncomp)
-{
-    ncomp = std::min(ncomp, srcmf.nComp());
-#ifdef _OPENMP
-#pragma omp parallel
-#endif
-    {
-        FArrayBox tmpfab;
-        for (MFIter mfi(dstmf,true); mfi.isValid(); ++mfi)
-        {
-            const auto& srcfab = srcmf[mfi];
-            auto& dstfab = dstmf[mfi];
-            const Box& tbx = mfi.growntilebox();
-            const Box& gbx = amrex::grow(tbx,1);
-            tmpfab.resize(gbx,ncomp);
-            tmpfab.setVal(0.0, gbx, 0, ncomp);
-            const Box& ibx = gbx & srcfab.box();
-            tmpfab.copy(srcfab, ibx, scomp, ibx, 0, ncomp);
-            WRPX_FILTER(BL_TO_FORTRAN_BOX(tbx),
-                        BL_TO_FORTRAN_ANYD(tmpfab),
-                        BL_TO_FORTRAN_N_ANYD(dstfab,dcomp),
-                        ncomp);
-        }
-    }
-}
-
-void
 WarpX::BuildBufferMasks ()
 {
     int ngbuffer = std::max(n_field_gather_buffer, n_current_deposition_buffer);
@@ -1041,3 +1085,23 @@ WarpX::RestoreCurrent (int lev)
         }
     }
 }
+
+std::string
+WarpX::Version ()
+{
+#ifdef WARPX_GIT_VERSION
+    return std::string(WARPX_GIT_VERSION);
+#else
+    return std::string("Unknown");
+#endif
+}
+
+std::string
+WarpX::PicsarVersion ()
+{
+#ifdef WARPX_GIT_VERSION
+    return std::string(PICSAR_GIT_VERSION);
+#else
+    return std::string("Unknown");
+#endif
+}