diff options
-rw-r--r-- | Docs/source/running_cpp/parameters.rst | 25 | ||||
-rwxr-xr-x | Examples/Modules/resampling/analysis_leveling_thinning.py | 138 | ||||
-rw-r--r-- | Examples/Modules/resampling/inputs_leveling_thinning | 65 | ||||
-rw-r--r-- | Regression/Checksum/benchmarks_json/leveling_thinning.json | 33 | ||||
-rw-r--r-- | Regression/WarpX-tests.ini | 16 | ||||
-rw-r--r-- | Source/Particles/MultiParticleContainer.H | 2 | ||||
-rw-r--r-- | Source/Particles/MultiParticleContainer.cpp | 2 | ||||
-rw-r--r-- | Source/Particles/PhysicalParticleContainer.H | 4 | ||||
-rw-r--r-- | Source/Particles/PhysicalParticleContainer.cpp | 7 | ||||
-rw-r--r-- | Source/Particles/Resampling/LevelingThinning.H | 10 | ||||
-rw-r--r-- | Source/Particles/Resampling/LevelingThinning.cpp | 6 | ||||
-rw-r--r-- | Source/Particles/Resampling/Resampling.H | 9 | ||||
-rw-r--r-- | Source/Particles/Resampling/Resampling.cpp | 10 | ||||
-rw-r--r-- | Source/Particles/Resampling/ResamplingTrigger.H | 7 | ||||
-rw-r--r-- | Source/Particles/Resampling/ResamplingTrigger.cpp | 8 | ||||
-rw-r--r-- | Source/Particles/WarpXParticleContainer.H | 2 |
16 files changed, 322 insertions, 22 deletions
diff --git a/Docs/source/running_cpp/parameters.rst b/Docs/source/running_cpp/parameters.rst index 05e9808eb..e646d2b6b 100644 --- a/Docs/source/running_cpp/parameters.rst +++ b/Docs/source/running_cpp/parameters.rst @@ -594,6 +594,31 @@ Particle initialization (the name of an existing positron species must be provided). **This feature requires to compile with QED=TRUE** +* ``<species>.do_resampling`` (`0` or `1`) optional (default `0`) + If `1` resampling is performed for this species. This means that the number of macroparticles + will be reduced at specific timesteps while preserving the distribution function as much as + possible (in particular the weight of the remaining particles will be increased on average). + This can be useful in situations with continuous creation of particles (e.g. with ionization + or with QED effects). At least one resampling trigger (see below) must be specified to actually + perform resampling. + +* ``<species>.resampling_algorithm`` (`string`) optional (default `leveling_thinning`) + The algorithm used for resampling. Currently there is only one option, which is already set by + default: + + * ``leveling_thinning`` This algorithm is defined in `Muraviev et al., arXiv:2006.08593 (2020) <https://arxiv.org/abs/2006.08593>`_. + The main parameter for this algorithm can be set with ``<species>.resampling_algorithm_target_ratio``. + It **roughly** corresponds to the ratio between the number of particles before and after + resampling. The default value for this parameter is 1.5. + +* ``<species>.resampling_trigger_intervals`` (`string`) optional (default `0`) + Using the `Intervals parser`_ syntax, this string defines timesteps at which resampling is + performed. + +* ``<species>.resampling_trigger_max_avg_ppc`` (`float`) optional (default `infinity`) + Resampling is performed everytime the number of macroparticles per cell of the species + averaged over the whole simulation domain exceeds this parameter. + .. _running-cpp-parameters-laser: Laser initialization diff --git a/Examples/Modules/resampling/analysis_leveling_thinning.py b/Examples/Modules/resampling/analysis_leveling_thinning.py new file mode 100755 index 000000000..dc96b0138 --- /dev/null +++ b/Examples/Modules/resampling/analysis_leveling_thinning.py @@ -0,0 +1,138 @@ +#! /usr/bin/env python + +# Copyright 2020 Neil Zaim +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +## In this test, we check that leveling thinning works as expected on two simple cases. Each case +## corresponds to a different particle species. + +import yt +import numpy as np +import sys +from scipy.special import erf +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +fn_final = sys.argv[1] +fn0 = fn_final[:-4] + '0000' + +ds0 = yt.load(fn0) +ds = yt.load(fn_final) + +ad0 = ds0.all_data() +ad = ds.all_data() + +numcells = 16*16 +t_r = 1.3 # target ratio +relative_tol = 1.e-13 # tolerance for machine precision errors + + +#### Tests for first species #### +# Particles are present in all simulation cells and all have the same weight + +ppc = 400. +numparts_init = numcells*ppc + +w0 = ad0['resampled_part1','particle_weight'].to_ndarray() # weights before resampling +w = ad['resampled_part1','particle_weight'].to_ndarray() # weights after resampling + +# Renormalize weights for easier calculations +w0 = w0*ppc +w = w*ppc + +# Check that initial number of particles is indeed as expected +assert(w0.shape[0] == numparts_init) +# Check that all initial particles have the same weight +assert(np.unique(w0).shape[0] == 1) +# Check that this weight is 1 (to machine precision) +assert(abs(w0[0] - 1) < relative_tol) + +# Check that the number of particles after resampling is as expected +numparts_final = w.shape[0] +expected_numparts_final = numparts_init/t_r**2 +error = np.abs(numparts_final - expected_numparts_final) +std_numparts_final = np.sqrt(numparts_init/t_r**2*(1.-1./t_r**2)) +# 5 sigma test that has an intrisic probability to fail of 1 over ~2 millions +print("difference between expected and actual final number of particles (1st species): " + str(error)) +print("tolerance: " + str(5*std_numparts_final)) +assert(error<5*std_numparts_final) + +# Check that the final weight is the same for all particles (to machine precision) and is as +# expected +final_weight = t_r**2 +assert(np.amax(np.abs(w-final_weight)) < relative_tol*final_weight ) + + +#### Tests for second species #### +# Particles are only present in a single cell and have a gaussian weight distribution +# Using a single cell makes the analysis easier because leveling thinning is done separately in +# each cell + +ppc = 100000. +numparts_init = ppc + +w0 = ad0['resampled_part2','particle_weight'].to_ndarray() # weights before resampling +w = ad['resampled_part2','particle_weight'].to_ndarray() # weights after resampling + +# Renormalize and sort weights for easier calculations +w0 = np.sort(w0)*ppc +w = np.sort(w)*ppc + +## First we verify that the initial distribution is as expected + +# Check that the mean initial weight is as expected +mean_initial_weight = np.average(w0) +expected_mean_initial_weight = 2.*np.sqrt(2.) +error = np.abs(mean_initial_weight - expected_mean_initial_weight) +expected_std_initial_weight = 1./np.sqrt(2.) +std_mean_initial_weight = expected_std_initial_weight/np.sqrt(numparts_init) +# 5 sigma test that has an intrisic probability to fail of 1 over ~2 millions +print("difference between expected and actual mean initial weight (2nd species): " + str(error)) +print("tolerance: " + str(5*std_mean_initial_weight)) +assert(error<5*std_mean_initial_weight) + +# Check that the initial weight variance is as expected +variance_initial_weight = np.var(w0) +expected_variance_initial_weight = 0.5 +error = np.abs(variance_initial_weight - expected_variance_initial_weight) +std_variance_initial_weight = expected_variance_initial_weight*np.sqrt(2./numparts_init) +# 5 sigma test that has an intrisic probability to fail of 1 over ~2 millions +print("difference between expected and actual variance of initial weight (2nd species): " + str(error)) +print("tolerance: " + str(5*std_variance_initial_weight)) + +## Next we verify that the resampling worked as expected + +# Check that the level weight value is as expected from the initial distribution +level_weight = w[0] +assert(np.abs(level_weight - mean_initial_weight*t_r) < level_weight*relative_tol) + +# Check that the number of particles at the level weight is the same as predicted from analytic +# calculations +numparts_leveled = np.argmax(w > level_weight) # This returns the first index for which +# w > level_weight, which thus corresponds to the number of particles at the level weight +expected_numparts_leveled = numparts_init/(2.*t_r)*(1+erf(expected_mean_initial_weight*(t_r-1.)) \ + -1./(np.sqrt(np.pi)*expected_mean_initial_weight)* \ + np.exp(-(expected_mean_initial_weight*(t_r-1.))**2)) +error = np.abs(numparts_leveled - expected_numparts_leveled) +std_numparts_leveled = np.sqrt(expected_numparts_leveled - numparts_init/np.sqrt(np.pi)/(t_r* \ + expected_mean_initial_weight)**2*(np.sqrt(np.pi)/4.* \ + (2.*expected_mean_initial_weight**2+1.)*(1.-erf(expected_mean_initial_weight* \ + (t_r-1.)))-0.5*np.exp(-(expected_mean_initial_weight*(t_r-1.))**2* \ + (expected_mean_initial_weight*(t_r+1.))))) +# 5 sigma test that has an intrisic probability to fail of 1 over ~2 millions +print("difference between expected and actual number of leveled particles (2nd species): " + str(error)) +print("tolerance: " + str(5*std_numparts_leveled)) + +numparts_unaffected = w.shape[0] - numparts_leveled +numparts_unaffected_anticipated = w0.shape[0] - np.argmax(w0 > level_weight) +# Check that number of particles with weight higher than level weight is the same before and after +# resampling +assert(numparts_unaffected == numparts_unaffected_anticipated) +# Check that particles with weight higher than level weight are unaffected by resampling. +assert(np.all(w[-numparts_unaffected:] == w0[-numparts_unaffected:])) + +test_name = fn_final[:-9] # Could also be os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn_final) diff --git a/Examples/Modules/resampling/inputs_leveling_thinning b/Examples/Modules/resampling/inputs_leveling_thinning new file mode 100644 index 000000000..c761a75dd --- /dev/null +++ b/Examples/Modules/resampling/inputs_leveling_thinning @@ -0,0 +1,65 @@ +max_step = 8 +amr.n_cell = 16 16 +amr.blocking_factor = 8 +amr.max_grid_size = 8 +geometry.is_periodic = 1 1 +geometry.prob_lo = 0. 0. +geometry.prob_hi = 16. 16. +amr.max_level = 0 + +warpx.do_pml = 0 + +particles.species_names = resampled_part1 resampled_part2 + +my_constants.pi = 3.141592653589793 + +# First particle species. The particles are distributed throughout the simulation box and all have +# the same weight. +resampled_part1.species_type = electron +resampled_part1.injection_style = NRandomPerCell +resampled_part1.num_particles_per_cell = 400 +resampled_part1.profile = constant +resampled_part1.density = 1. +resampled_part1.momentum_distribution_type = constant +resampled_part1.do_not_deposit = 1 +resampled_part1.do_not_gather = 1 +resampled_part1.do_not_push = 1 +resampled_part1.do_resampling = 1 +resampled_part1.resampling_algorithm = leveling_thinning +resampled_part1.resampling_algorithm_target_ratio = 1.3 +# This should trigger resampling at timestep 5 only in this case +resampled_part1.resampling_trigger_intervals = 5 +# This should trigger resampling at first timestep +resampled_part1.resampling_trigger_max_avg_ppc = 395 + +# Second particle species. The particles are only present in one cell and have a Gaussian weight +# distribution. +resampled_part2.species_type = electron +resampled_part2.injection_style = NRandomPerCell +resampled_part2.num_particles_per_cell = 100000 +resampled_part2.zmin = 0. +resampled_part2.zmax = 1. +resampled_part2.xmin = 0. +resampled_part2.xmax = 1. +resampled_part2.profile = parse_density_function +# Trick to get a Gaussian weight distribution is to do a Box-Muller transform using the position +# within the cell as the two random variables. Here, we have a distribution with standard deviation +# sigma = 1/sqrt(2) and mean 4*sigma. +resampled_part2.density_function(x,y,z) = 2.*sqrt(2.)+sqrt(-log(x))*cos(2*pi*z) +resampled_part2.momentum_distribution_type = constant +resampled_part2.do_not_deposit = 1 +resampled_part2.do_not_gather = 1 +resampled_part2.do_not_push = 1 +resampled_part2.do_resampling = 1 +resampled_part2.resampling_algorithm = leveling_thinning +resampled_part2.resampling_algorithm_target_ratio = 1.3 +# This should trigger resampling at timestep 7 only in this case. The rest is here to test the +# intervals parser syntax. +resampled_part2.resampling_trigger_intervals = 100 :120:3, 7:7:7 +# This should not trigger resampling: this species only has ~391 ppc on average. +resampled_part2.resampling_trigger_max_avg_ppc = 395 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.period = 8 +diag1.diag_type = Full diff --git a/Regression/Checksum/benchmarks_json/leveling_thinning.json b/Regression/Checksum/benchmarks_json/leveling_thinning.json new file mode 100644 index 000000000..30cb28b20 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/leveling_thinning.json @@ -0,0 +1,33 @@ +{ + "resampled_part1": { + "particle_cpu": 91119.0, + "particle_id": 773368403.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 483987.107681320863775908946990966796875, + "particle_position_y": 484901.85673470445908606052398681640625, + "particle_weight": 255.3040749999979084350343327969312667846680 + }, + "resampled_part2": { + "particle_cpu": 0.0, + "particle_id": 5754560102.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 38483.4033272069573285989463329315185546875, + "particle_position_y": 38200.1600630077518871985375881195068359375, + "particle_weight": 2.8420376578928117083933102549053728580475 + }, + "lev=0": { + "Bx": 0.0, + "By": 0.0, + "Bz": 0.0, + "Ex": 0.0, + "Ey": 0.0, + "Ez": 0.0, + "jx": 0.0, + "jy": 0.0, + "jz": 0.0 + } +} diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 5161589b2..2d3c697fd 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -1736,3 +1736,19 @@ doVis = 0 compareParticles = 0 analysisRoutine = Examples/Tests/initial_distribution/analysis_distribution.py aux1File = Tools/PostProcessing/read_raw_data.py + +[leveling_thinning] +buildDir = . +inputFile = Examples/Modules/resampling/inputs_leveling_thinning +runtime_params = +dim = 2 +addToCompileString = +restartTest = 0 +useMPI = 1 +numprocs = 4 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Modules/resampling/analysis_leveling_thinning.py diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 6d11d53f9..73a8207bd 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -277,8 +277,6 @@ protected: std::vector<PCTypes> species_types; - Resampling m_resampler; - /** Whether to absorb particles exiting the domain */ ParticleBC m_boundary_conditions = ParticleBC::none; diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index a740330d7..9bcb19b2e 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -722,7 +722,7 @@ void MultiParticleContainer::doResampling (const int timestep) // do_resampling can only be true for PhysicalParticleContainers if (!pc->do_resampling){ continue; } - pc->resample(m_resampler, timestep); + pc->resample(timestep); } } diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index 30105ddcb..1e1307072 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -287,7 +287,7 @@ public: * @param[in] resampler the Resampling object used for resampling. * @param[in] timestep the current timestep. */ - void resample (const Resampling& resampler, const int timestep) override final; + void resample (const int timestep) override final; #ifdef WARPX_QED //Functions decleared in WarpXParticleContainer.H @@ -338,6 +338,8 @@ protected: bool boost_adjust_transverse_positions = false; bool do_backward_propagation = false; + Resampling m_resampler; + // Inject particles during the whole simulation void ContinuousInjection (const amrex::RealBox& injection_box) override; diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index b71ff2920..0f0ac96f5 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -123,6 +123,7 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp pp.query("do_field_ionization", do_field_ionization); pp.query("do_resampling", do_resampling); + if (do_resampling) m_resampler = Resampling(species_name); //check if Radiation Reaction is enabled and do consistency checks pp.query("do_classical_radiation_reaction", do_classical_radiation_reaction); @@ -1996,18 +1997,18 @@ PhysicalParticleContainer::getIonizationFunc (const WarpXParIter& pti, ion_atomic_number); } -void PhysicalParticleContainer::resample (const Resampling& resampler, const int timestep) +void PhysicalParticleContainer::resample (const int timestep) { const amrex::Real global_numparts = TotalNumberOfParticles(); - if (resampler.triggered(timestep, global_numparts)) + if (m_resampler.triggered(timestep, global_numparts)) { amrex::Print() << "Resampling " << species_name << ".\n"; for (int lev = 0; lev <= maxLevel(); lev++) { for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { - resampler(pti, lev, this); + m_resampler(pti, lev, this); } } } diff --git a/Source/Particles/Resampling/LevelingThinning.H b/Source/Particles/Resampling/LevelingThinning.H index 920fa07c9..14d0f8e6a 100644 --- a/Source/Particles/Resampling/LevelingThinning.H +++ b/Source/Particles/Resampling/LevelingThinning.H @@ -19,10 +19,18 @@ */ class LevelingThinning: public ResamplingAlgorithm { public: + + /** + * \brief Default constructor of the LevelingThinning class. + */ + LevelingThinning () = default; + /** * \brief Constructor of the LevelingThinning class + * + * @param[in] species_name the name of the resampled species */ - LevelingThinning (); + LevelingThinning (const std::string species_name); /** * \brief A method that performs leveling thinning for the considered species. diff --git a/Source/Particles/Resampling/LevelingThinning.cpp b/Source/Particles/Resampling/LevelingThinning.cpp index 0fc347241..7ac28932e 100644 --- a/Source/Particles/Resampling/LevelingThinning.cpp +++ b/Source/Particles/Resampling/LevelingThinning.cpp @@ -9,12 +9,12 @@ #include <AMReX_Particles.H> -LevelingThinning::LevelingThinning () +LevelingThinning::LevelingThinning (const std::string species_name) { using namespace amrex::literals; - amrex::ParmParse pp("resampling_algorithm"); - pp.query("target_ratio", m_target_ratio); + amrex::ParmParse pp(species_name); + pp.query("resampling_algorithm_target_ratio", m_target_ratio); AMREX_ALWAYS_ASSERT_WITH_MESSAGE( m_target_ratio > 0._rt, "Resampling target ratio should be strictly greater than 0"); if (m_target_ratio <= 1._rt) diff --git a/Source/Particles/Resampling/Resampling.H b/Source/Particles/Resampling/Resampling.H index de919bfcb..b381771f8 100644 --- a/Source/Particles/Resampling/Resampling.H +++ b/Source/Particles/Resampling/Resampling.H @@ -39,10 +39,17 @@ class Resampling public: /** + * \brief Default constructor of the Resampling class. + */ + Resampling () = default; + + /** * \brief Constructor of the Resampling class. Reads the chosen resampling algorithm from the * input file. + * + * @param[in] species_name the name of the resampled species */ - Resampling (); + Resampling (const std::string species_name); /** * \brief A method that returns true if resampling should be done for the considered species diff --git a/Source/Particles/Resampling/Resampling.cpp b/Source/Particles/Resampling/Resampling.cpp index 65332d191..b5c2d8b4f 100644 --- a/Source/Particles/Resampling/Resampling.cpp +++ b/Source/Particles/Resampling/Resampling.cpp @@ -7,18 +7,20 @@ #include "Resampling.H" #include "LevelingThinning.H" -Resampling::Resampling () +Resampling::Resampling (const std::string species_name) { - amrex::ParmParse pp("resampling_algorithm"); + amrex::ParmParse pp(species_name); std::string resampling_algorithm_string = "leveling_thinning"; // default resampling algorithm - pp.query("type", resampling_algorithm_string); + pp.query("resampling_algorithm", resampling_algorithm_string); if (resampling_algorithm_string.compare("leveling_thinning") == 0) { - m_resampling_algorithm = std::make_unique<LevelingThinning>(); + m_resampling_algorithm = std::make_unique<LevelingThinning>(species_name); } else { amrex::Abort("Unknown resampling algorithm."); } + + m_resampling_trigger = ResamplingTrigger(species_name); } bool Resampling::triggered (const int timestep, const amrex::Real global_numparts) const diff --git a/Source/Particles/Resampling/ResamplingTrigger.H b/Source/Particles/Resampling/ResamplingTrigger.H index 7e373289a..830b511f2 100644 --- a/Source/Particles/Resampling/ResamplingTrigger.H +++ b/Source/Particles/Resampling/ResamplingTrigger.H @@ -22,10 +22,15 @@ class ResamplingTrigger public: /** + * \brief Default constructor of the ResamplingTrigger class. + */ + ResamplingTrigger () = default; + + /** * \brief Constructor of the ResamplingTrigger class. Reads the resampling trigger parameters * from the input file. */ - ResamplingTrigger (); + ResamplingTrigger (const std::string species_name); /** * \brief A method that returns true if resampling should be done for the considered species diff --git a/Source/Particles/Resampling/ResamplingTrigger.cpp b/Source/Particles/Resampling/ResamplingTrigger.cpp index c89f01848..97407d51a 100644 --- a/Source/Particles/Resampling/ResamplingTrigger.cpp +++ b/Source/Particles/Resampling/ResamplingTrigger.cpp @@ -7,15 +7,15 @@ #include "ResamplingTrigger.H" #include "WarpX.H" -ResamplingTrigger::ResamplingTrigger () +ResamplingTrigger::ResamplingTrigger (const std::string species_name) { - amrex::ParmParse pprt("resampling_trigger"); + amrex::ParmParse pprt(species_name); std::vector<std::string> resampling_trigger_int_string_vec = {"0"}; - pprt.queryarr("intervals", resampling_trigger_int_string_vec); + pprt.queryarr("resampling_trigger_intervals", resampling_trigger_int_string_vec); m_resampling_intervals = IntervalsParser(resampling_trigger_int_string_vec); - pprt.query("max_avg_ppc", m_max_avg_ppc); + pprt.query("resampling_trigger_max_avg_ppc", m_max_avg_ppc); } bool ResamplingTrigger::triggered (const int timestep, const amrex::Real global_numparts) const diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 8f1cab717..12dbd1289 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -343,7 +343,7 @@ public: * override the method for every derived class. Note that in practice this function is never * called because resample() is only called for PhysicalParticleContainers. */ - virtual void resample (const Resampling& /*resampler*/, const int /*timestep*/) {} + virtual void resample (const int /*timestep*/) {} protected: amrex::Array<amrex::Real,3> m_v_galilean = {{0}}; |