diff options
77 files changed, 3390 insertions, 1884 deletions
diff --git a/.travis.yml b/.travis.yml index 5473bf409..ccabd3852 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ sudo: true env: matrix: - WARPX_TEST_DIM=3 HAS_QED=FALSE + - WARPX_TEST_DIM=RZ HAS_QED=FALSE - WARPX_TEST_DIM=2 HAS_QED=FALSE - HAS_QED=TRUE diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..0eff97030 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at warpx-coc@lbl.gov. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 19a1727c4..ce9f26deb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -98,8 +98,8 @@ it can be convenient to run the tests on your local machine with ./run_tests.sh ``` from WarpX root folder. The tests can be influenced by environment variables: -- `export WARPX_TEST_DIM=3` or `export WARPX_TEST_DIM=2` in order to select -only the tests that correspond to this dimension +- `export WARPX_TEST_DIM=3`, `export WARPX_TEST_DIM=2` or `export WARPX_TEST_DIM=RZ` +in order to select only the tests that correspond to this dimensionality - `export WARPX_TEST_ARCH=CPU` or `export WARPX_TEST_ARCH=GPU` in order to run the tests on CPU or GPU respectively. - `export WARPX_TEST_COMMIT=...` in order to test a specific commit. diff --git a/Docs/source/running_cpp/parameters.rst b/Docs/source/running_cpp/parameters.rst index e5727e376..2c1d31990 100644 --- a/Docs/source/running_cpp/parameters.rst +++ b/Docs/source/running_cpp/parameters.rst @@ -224,6 +224,18 @@ Particle initialization initialization. This can be required whith a moving window and/or when running in a boosted frame. +* ``<species_name>.initialize_self_fields`` (`0` or `1`) + Whether to calculate the space-charge fields associated with this species + at the beginning of the simulation. + +* ``<species_name>.self_fields_required_precision`` (`float`, default: 1.e-11) + The relative precision with which the initial space-charge fields should + be calculated. More specifically, the initial space-charge fields are + computed with an iterative Multi-Level Multi-Grid (MLMG) solver. + For highly-relativistic beams, this solver can fail to reach the default + precision within a reasonable time ; in that case, users can set a + relaxed precision requirement through ``self_fields_required_precision``. + * ``<species_name>.profile`` (`string`) Density profile for this species. The options are: @@ -266,6 +278,37 @@ Particle initialization well as standard deviations along each direction ``<species_name>.ux_th``, ``<species_name>.uy_th`` and ``<species_name>.uz_th``. + * ``maxwell_boltzmann``: Maxwell-Boltzmann distribution that takes a dimensionless + temperature parameter ``<species_name>.theta`` as an input, where theta is kb*T/(m*c^2), + kb is the Boltzmann constant, c is the speed of light, and m is the mass of the species. + It also includes the optional parameter ``<species_name>.beta`` where beta is equal to v/c. + The plasma will be initialized to move at drift velocity beta*c in the positive + ``<species_name>.direction = 'x', 'y', 'z'`` direction. The MB distribution is initialized + in the drifting frame by sampling three Gaussian distributions in each dimension using, + the Box Mueller method, and then the distribution is transformed to the simulation frame + using the flipping method. The flipping method can be found in Zenitani 2015 + section III. B. (Phys. Plasmas 22, 042116). + + Note that though the particles may move at relativistic speeds in the simulation frame, + they are not relativistic in the drift frame. This is as opposed to the Maxwell Juttner + setting, which initializes particles with relativistc momentums in their drifting frame. + + * ``maxwell_juttner``: Maxwell-Juttner distribution for high temperature plasma. This mode + requires a dimensionless temperature parameter ``<species_name>.theta``, where theta is equal + to kb*T/(m*c^2), where kb is the Boltzmann constant, and m is the mass of the species. It also + includes the optional parameter ``<species_name>.beta`` where beta is equal to v/c. The plasma + will be initialized to move at velocity beta*c in the positive + ``<species_name>.direction = 'x', 'y', 'z'`` direction. The MJ distribution will be initialized + in the moving frame using the Sobol method, and then the distribution will be transformed to the + simulation frame using the flipping method. Both the Sobol and the flipping method can be found + in Zenitani 2015 (Phys. Plasmas 22, 042116). + + Please take notice that particles initialized with this setting can be relativistic in two ways. + In the simulation frame, they can drift with a relativistic speed beta. Then, in the drifting + frame they are still moving with relativistic speeds due to high temperature. This is as opposed + to the Maxwell Boltzmann setting, which initializes non-relativistic plasma in their relativistic + drifting frame. + * ``radial_expansion``: momentum depends on the radial coordinate linearly. This requires additional parameter ``u_over_r`` which is the slope. @@ -333,6 +376,10 @@ Particle initialization axes (4 particles in 2D, 6 particles in 3D). When `1`, particles are split along the diagonals (4 particles in 2D, 8 particles in 3D). +* ``<species_name>.do_not_deposit`` (`0` or `1` optional; default `0`) + If `1` is given, both charge deposition and current deposition will + not be done, thus that species does not contribute to the fields. + * ``<species>.plot_species`` (`0` or `1` optional; default `1`) Whether to plot particle quantities for this species. @@ -382,15 +429,19 @@ Particle initialization * ``<species>.do_qed`` (`int`) optional (default `0`) If `<species>.do_qed = 0` all the QED effects are disabled for this species. - If `<species>.do_qed = 1` QED effects can be enabled for this species (see below) + If `<species>.do_qed = 1` QED effects can be enabled for this species (see below). **Implementation of this feature is in progress. It requires to compile with QED=TRUE** * ``<species>.do_qed_quantum_sync`` (`int`) optional (default `0`) It only works if `<species>.do_qed = 1`. Enables Quantum synchrotron emission for this species. + Quantum synchrotron lookup table should be either generated or loaded from disk to enable + this process (see "Lookup tables for QED modules" section below). **Implementation of this feature is in progress. It requires to compile with QED=TRUE** * ``<species>.do_qed_breit_wheeler`` (`int`) optional (default `0`) It only works if `<species>.do_qed = 1`. Enables non-linear Breit-Wheeler process for this species. + Breit-Wheeler lookup table should be either generated or loaded from disk to enable + this process (see "Lookup tables for QED modules" section below). **Implementation of this feature is in progress. It requires to compile with QED=TRUE** * ``warpx.E_external_particle`` & ``warpx.B_external_particle`` (list of `float`) optional (default `0. 0. 0.`) @@ -871,6 +922,92 @@ Diagnostics and output The time interval between the back-transformed reduced diagnostics (where this time interval is expressed in the laboratory frame). +* ``slice.particle_slice_width_lab`` (`float`, in meters) + Only used when ``warpx.do_boosted_frame_diagnostic`` is ``1`` and + ``slice.num_slice_snapshots_lab`` is non-zero. Particles are + copied from the full back-transformed diagnostic to the reduced + slice diagnostic if there are within the user-defined width from + the slice region defined by ``slice.dom_lo`` and ``slice.dom_hi``. + +Lookup tables for QED modules (implementation in progress) +---------------------------------------------------------- +Lookup tables store pre-computed values for functions used by the QED modules. +**Implementation of this feature is in progress. It requires to compile with QED=TRUE** + +* ``qed_bw.lookup_table_mode`` (`string`) + There are three options to prepare the lookup table required by the Breit-Wheeler module: + + * ``dummy_builtin``: a built-in table is used (Warning: the quality of the table is very low, + so this option has to be used only for test purposes). + + * ``generate``: a new table is generated. This option requires Boost math library + (version >= 1.67) and to compile with QED_TABLE_GEN=TRUE. All + the following parameters must be specified: + + * ``qed_bw.chi_min`` (`float`): minimum chi parameter to be considered by the engine + + * ``qed_bw.tab_dndt_chi_min`` (`float`): minimum chi parameter for lookup table 1 ( + used for the evolution of the optical depth of the photons) + + * ``qed_bw.tab_dndt_chi_max`` (`float`): maximum chi parameter for lookup table 1 + + * ``qed_bw.tab_dndt_how_many`` (`int`): number of points to be used for lookup table 1 + + * ``qed_bw.tab_pair_chi_min`` (`float`): minimum chi parameter for lookup table 2 ( + used for pair generation) + + * ``qed_bw.tab_pair_chi_max`` (`float`): maximum chi parameter for lookup table 2 + + * ``qed_bw.tab_pair_chi_how_many`` (`int`): number of points to be used for chi axis in lookup table 2 + + * ``qed_bw.tab_pair_frac_how_many`` (`int`): number of points to be used for the second axis in lookup table 2 + (the second axis is the ratio between the energy of the less energetic particle of the pair and the + energy of the photon). + + * ``qed_bw.save_table_in`` (`string`): where to save the lookup table + + * ``load``: a lookup table is loaded from a pre-generated binary file. The following parameter + must be specified: + + * ``qed_bw.load_table_from`` (`string`): name of the lookup table file to read from. + +* ``qed_qs.lookup_table_mode`` (`string`) + There are three options to prepare the lookup table required by the Quantum Synchrotron module: + + * ``dummy_builtin``: a built-in table is used (Warning: the quality of the table is very low, + so this option has to be used only for test purposes). + + * ``generate``: a new table is generated. This option requires Boost math library + (version >= 1.67) and to compile with QED_TABLE_GEN=TRUE. All + the following parameters must be specified: + + * ``qed_qs.chi_min`` (`float`): minimum chi parameter to be considered by the engine + + * ``qed_qs.tab_dndt_chi_min`` (`float`): minimum chi parameter for lookup table 1 ( + used for the evolution of the optical depth of electrons and positrons) + + * ``qed_qs.tab_dndt_chi_max`` (`float`): maximum chi parameter for lookup table 1 + + * ``qed_qs.tab_dndt_how_many`` (`int`): number of points to be used for lookup table 1 + + * ``qed_qs.tab_em_chi_min`` (`float`): minimum chi parameter for lookup table 2 ( + used for photon emission) + + * ``qed_qs.tab_em_chi_max`` (`float`): maximum chi parameter for lookup table 2 + + * ``qed_qs.tab_em_chi_how_many`` (`int`): number of points to be used for chi axis in lookup table 2 + + * ``qed_qs.tab_em_prob_how_many`` (`int`): number of points to be used for the second axis in lookup table 2 + (the second axis is a cumulative probability). + + * ``qed_bw.save_table_in`` (`string`): where to save the lookup table + + * ``load``: a lookup table is loaded from a pre-generated binary file. The following parameter + must be specified: + + * ``qed_qs.load_table_from`` (`string`): name of the lookup table file to read from. + + Checkpoints and restart ----------------------- WarpX supports checkpoints/restart via AMReX. diff --git a/Docs/source/visualization/yt.rst b/Docs/source/visualization/yt.rst index e2f8b377f..7c98e05aa 100644 --- a/Docs/source/visualization/yt.rst +++ b/Docs/source/visualization/yt.rst @@ -168,6 +168,25 @@ If the back-transformed diagnostics are written in the HDF5 format (This can be plt.figure() plt.plot(f2['Ez'][nx2//2,0,:]) +The back-transformed particle data on the full and reduced diagnostic can be visualized as follows + +:: + species='ions' + iteration = 1 + + snapshot = './lab_frame_data/snapshots/' + 'snapshot' + str(iteration).zfill(5) + xbo = get_particle_field(snapshot, species, 'x') # Read particle data + ybo = get_particle_field(snapshot, species, 'y') + zbo = get_particle_field(snapshot, species, 'z') + + snapshot = './lab_frame_data/slices/' + 'slice' + str(iteration).zfill(5) + xbo_slice = get_particle_field(snapshot, species, 'x') # Read particle data + ybo_slice = get_particle_field(snapshot, species, 'y') + zbo_slice = get_particle_field(snapshot, species, 'z') + plt.figure() + plt.plot(xbo, ybo, 'r.', markersize=1.) + plt.plot(xbo_slice, ybo_slice, 'bx', markersize=1.) + Further information ------------------- diff --git a/Examples/Modules/boosted_diags/inputs.3d.slice b/Examples/Modules/boosted_diags/inputs.3d.slice index 08f2310cd..408b18931 100644 --- a/Examples/Modules/boosted_diags/inputs.3d.slice +++ b/Examples/Modules/boosted_diags/inputs.3d.slice @@ -109,3 +109,4 @@ slice.coarsening_ratio = 1 1 1 slice.plot_int = -1 slice.num_slice_snapshots_lab = 4 slice.dt_slice_snapshots_lab = 3.3356409519815207e-12 +slice.particle_slice_width_lab = 2.e-6 diff --git a/Examples/Modules/qed/breit_wheeler/analysis_2d_tau_init.py b/Examples/Modules/qed/breit_wheeler/analysis_2d_tau_init.py index 850ecc0fe..9168c0502 100755 --- a/Examples/Modules/qed/breit_wheeler/analysis_2d_tau_init.py +++ b/Examples/Modules/qed/breit_wheeler/analysis_2d_tau_init.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python3 +#! /usr/bin/env python import yt import numpy as np import scipy.stats as st diff --git a/Examples/Modules/qed/breit_wheeler/analysis_3d_optical_depth_evolution.py b/Examples/Modules/qed/breit_wheeler/analysis_3d_optical_depth_evolution.py new file mode 100755 index 000000000..617afd6f9 --- /dev/null +++ b/Examples/Modules/qed/breit_wheeler/analysis_3d_optical_depth_evolution.py @@ -0,0 +1,119 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +import yt +import numpy as np +import scipy.stats as st +import sys +import math as m +import scipy.special as spe +import scipy.integrate as integ + +# This script checks if the optical depth of photons undergoing the +# Breit Wheeler process behaves as expected. Four populations of photons +# are initialized with different momenta in a background EM field. The decrease +# of the optical depth (averaged for each population) is used to estimate the +# Breit Wheeler cross section, which is then compared with the theoretical +# formula. Relative discrepancy should be smaller than 2% +# +# References: +# 1) R. Duclous et al 2011 Plasma Phys. Control. Fusion 53 015009 +# 2) A. Gonoskov et al. 2015 Phys. Rev. E 92, 023305 +# 3) M. Lobet. PhD thesis "Effets radiatifs et d'électrodynamique +# quantique dans l'interaction laser-matière ultra-relativiste" +# URL: https://tel.archives-ouvertes.fr/tel-01314224 + + +# Tolerance +tol = 2e-2 + +# EM fields +E_f = np.array([-2433321316961438, 973328526784575, 1459992790176863]) +B_f = np.array([2857142.85714286, 4285714.28571428, 8571428.57142857]) + +# Physical constants +electron_mass = 9.10938356e-31 +elementary_charge = 1.6021766208e-19 +speed_of_light = 299792458 +reduced_plank = 1.054571800e-34 +vacuum_permittivity = 8.854187817e-12 +fine_structure_constant = 0.0072973525664 +classical_elec_radius = (1./4./np.pi/vacuum_permittivity)*( elementary_charge**2 / (electron_mass * speed_of_light**2)) +lambda_ref = 1.0e-6 +field_reference = 2.0 * np.pi * electron_mass*speed_of_light * speed_of_light / (elementary_charge*lambda_ref) +schwinger_field_SI = electron_mass**2 * speed_of_light**3/(reduced_plank*elementary_charge) +schwinger_field_norm = electron_mass*speed_of_light*lambda_ref/(2.0*reduced_plank*m.pi) +#______________ + +# Initial parameters +spec_names = ["p1", "p2", "p3", "p4"] + #Assumes average = 1 at t = 0 (ok for a large number of particles) +tau_begin_avg = np.array([1.0, 1.0, 1.0, 1.0]) +mec = electron_mass*speed_of_light +p_begin = [ + np.array([2000.0,0,0])*mec, + np.array([0.0,5000.0,0.0])*mec, + np.array([0.0,0.0,10000.0])*mec, + np.array([57735.02691896, 57735.02691896, 57735.02691896])*mec +] +#______________ + +def calc_chi_gamma(p, E, B): + p = p / electron_mass / speed_of_light + E = E / field_reference + B = B * speed_of_light / field_reference + gamma_phot = np.linalg.norm(p) + c = p/gamma_phot + loc_field = gamma_phot * np.linalg.norm( E - np.dot(c,E)*c + np.cross(c,B)) + return loc_field/schwinger_field_norm + +#Auxiliary functions +def X(chi_phot, chi_ele): + if (chi_phot > chi_ele and chi_ele != 0): + return np.power(chi_phot/(chi_ele*(chi_phot-chi_ele)), 2./3.) + else: + return 1.0e30 + +def T(chi_phot): + coeff = 1./(np.pi * np.sqrt(3.) * chi_phot * chi_phot) + inner = lambda x : integ.quad(lambda s: np.sqrt(s)*spe.kv(1./3., 2./3. * s**(3./2.)), x, np.inf)[0] + return integ.quad(lambda chi_ele: + coeff*(inner(X(chi_phot, chi_ele)) - + (2.0 - chi_phot*np.power(X(chi_phot, chi_ele), 3./2.))*spe.kv(2./3., 2./3. *X(chi_phot, chi_ele)**(3./2.)) ) + , 0, chi_phot)[0] +#__________________ + +# Breit-Wheeler total cross section +def dNBW_dt(chi_phot, energy_phot): + energy_phot = energy_phot/electron_mass/speed_of_light/speed_of_light + return ((electron_mass*(speed_of_light)**2)*fine_structure_constant/reduced_plank)*(chi_phot/energy_phot)*T(chi_phot) +#__________________ + +def check(): + filename_end = sys.argv[1] + data_set_end = yt.load(filename_end) + + sim_time = data_set_end.current_time.to_value() + + all_data_end = data_set_end.all_data() + + tau_end_avg = np.array([ + np.average(all_data_end[name, 'particle_tau']) + for name in spec_names]) + + dNBW_dt_sim = (tau_begin_avg - tau_end_avg)/sim_time + + dNBW_dt_theo = [ + dNBW_dt(calc_chi_gamma(p, E_f, B_f), np.linalg.norm(p*speed_of_light)) + for p in p_begin + ] + + discrepancy = np.abs(dNBW_dt_sim-dNBW_dt_theo)/dNBW_dt_theo + + assert(np.all(discrepancy < tol)) + +def main(): + check() + +if __name__ == "__main__": + main() diff --git a/Examples/Modules/qed/breit_wheeler/inputs.2d_test_tau_init b/Examples/Modules/qed/breit_wheeler/inputs.2d_test_tau_init index 8e3dd29ca..050414185 100644 --- a/Examples/Modules/qed/breit_wheeler/inputs.2d_test_tau_init +++ b/Examples/Modules/qed/breit_wheeler/inputs.2d_test_tau_init @@ -65,16 +65,19 @@ photons.do_qed_breit_wheeler = 1 ################################# #######QED ENGINE PARAMETERS##### -qed_bw.chi_min = 0.01 -qed_bw.ignore_tables_for_test = 1 -#qed_bw.generate_table = 1 -#qed_bw.tab_dndt_chi_min = 0.01 -#qed_bw.tab_dndt_chi_max = 100 -#qed_bw.tab_dndt_how_many = 200 +qed_bw.lookup_table_mode = "dummy_builtin" + +#qed_bw.lookup_table_mode = "generate" +#qed_bw.chi_min = 0.001 +#qed_bw.tab_dndt_chi_min = 0.1 +#qed_bw.tab_dndt_chi_max = 200 +#qed_bw.tab_dndt_how_many = 64 #qed_bw.tab_pair_chi_min = 0.01 -#qed_bw.tab_pair_chi_max = 100 -#qed_bw.tab_pair_chi_how_many = 30 -#qed_bw.tab_pair_frac_how_many = 30 -#qed_bw.save_table_in = "bw_table" -#qed_bw.load_table_from = "bw_table" +#qed_bw.tab_pair_chi_max = 200 +#qed_bw.tab_pair_chi_how_many = 2 +#qed_bw.tab_pair_frac_how_many = 2 +#qed_bw.save_table_in = "bw_micro_table" + +#qed_bw.lookup_table_mode = "load" +#qed_bw.load_table_from = "bw_micro_table" ################################# diff --git a/Examples/Modules/qed/breit_wheeler/inputs.3d_test_optical_depth_evolution b/Examples/Modules/qed/breit_wheeler/inputs.3d_test_optical_depth_evolution new file mode 100644 index 000000000..0d4cad2b1 --- /dev/null +++ b/Examples/Modules/qed/breit_wheeler/inputs.3d_test_optical_depth_evolution @@ -0,0 +1,166 @@ +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 10 +amr.n_cell = 64 64 64 +amr.max_grid_size = 32 # maximum size of each AMReX box, used to decompose the domain +amr.blocking_factor = 8 # minimum size of each AMReX box, used to decompose the domain +amr.plot_int = 10 +geometry.coord_sys = 0 # 0: Cartesian +geometry.is_periodic = 1 1 1 # Is periodic? +geometry.prob_lo = -1.e-6 -1.e-6 -1e-6 # physical domain +geometry.prob_hi = 1.e-6 1.e-6 1e-6 +amr.max_level = 0 # Maximum level in hierarchy (1 might be unstable, >1 is not supported) + +################################# +############ NUMERICS ########### +################################# +algo.current_deposition = esirkepov +algo.charge_deposition = standard +algo.field_gathering = energy-conserving +algo.particle_pusher = boris +interpolation.nox = 3 # Particle interpolation order. Must be the same in x, y, and z +interpolation.noy = 3 +interpolation.noz = 3 +warpx.verbose = 1 +warpx.do_dive_cleaning = 0 +warpx.plot_raw_fields = 0 +warpx.plot_raw_fields_guards = 0 +warpx.use_filter = 1 +warpx.cfl = 1. # if 1., the time step is set to its CFL limit +warpx.do_pml = 1 # use Perfectly Matched Layer as boundary condition +warpx.serialize_ics = 1 + +################################# +############ PLASMA ############# +################################# +particles.nspecies = 4 # number of species +particles.species_names = p1 p2 p3 p4 +particles.photon_species = p1 p2 p3 p4 +################################# + +p1.charge = -q_e +p1.mass = m_e +p1.injection_style = "NUniformPerCell" +p1.profile = "constant" +p1.xmin = -0.5e-6 +p1.ymin = -0.5e-6 +p1.zmin = -0.5e-6 +p1.xmax = 0.5e6 +p1.ymax = 0.5e6 +p1.zmax = 0.5e6 +p1.num_particles_per_cell_each_dim = 2 2 2 +p1.density = 1e19 +p1.profile = "constant" +p1.momentum_distribution_type = "gaussian" +p1.ux_m = 2000.0 +p1.uy_m = 0.0 +p1.uz_m = 0.0 +p1.ux_th = 0. +p1.uy_th = 0. +p1.uz_th = 0. +##########QED#################### +p1.do_qed = 1 +p1.do_qed_breit_wheeler = 1 +################################# + +p2.charge = -q_e +p2.mass = m_e +p2.injection_style = "NUniformPerCell" +p2.profile = "constant" +p2.xmin = -0.5e-6 +p2.ymin = -0.5e-6 +p2.zmin = -0.5e-6 +p2.xmax = 0.5e6 +p2.ymax = 0.5e6 +p2.zmax = 0.5e6 +p2.num_particles_per_cell_each_dim = 2 2 2 +p2.density = 1e19 +p2.profile = "constant" +p2.momentum_distribution_type = "gaussian" +p2.ux_m = 0.0 +p2.uy_m = 5000.0 +p2.uz_m = 0.0 +p2.ux_th = 0. +p2.uy_th = 0. +p2.uz_th = 0. +##########QED#################### +p2.do_qed = 1 +p2.do_qed_breit_wheeler = 1 +################################# + +p3.charge = -q_e +p3.mass = m_e +p3.injection_style = "NUniformPerCell" +p3.profile = "constant" +p3.xmin = -0.5e-6 +p3.ymin = -0.5e-6 +p3.zmin = -0.5e-6 +p3.xmax = 0.5e6 +p3.ymax = 0.5e6 +p3.zmax = 0.5e6 +p3.num_particles_per_cell_each_dim = 2 2 2 +p3.density = 1e19 +p3.profile = "constant" +p3.momentum_distribution_type = "gaussian" +p3.ux_m = 0.0 +p3.uy_m = 0.0 +p3.uz_m = 10000.0 +p3.ux_th = 0. +p3.uy_th = 0. +p3.uz_th = 0. +##########QED#################### +p3.do_qed = 1 +p3.do_qed_breit_wheeler = 1 +################################# + +p4.charge = -q_e +p4.mass = m_e +p4.injection_style = "NUniformPerCell" +p4.profile = "constant" +p4.xmin = -0.5e-6 +p4.ymin = -0.5e-6 +p4.zmin = -0.5e-6 +p4.xmax = 0.5e6 +p4.ymax = 0.5e6 +p4.zmax = 0.5e6 +p4.num_particles_per_cell_each_dim = 2 2 2 +p4.density = 1e19 +p4.profile = "constant" +p4.momentum_distribution_type = "gaussian" +p4.ux_m = 57735.02691896 +p4.uy_m = 57735.02691896 +p4.uz_m = 57735.02691896 +p4.ux_th = 0. +p4.uy_th = 0. +p4.uz_th = 0. +##########QED#################### +p4.do_qed = 1 +p4.do_qed_breit_wheeler = 1 +################################# + +##########QED TABLES#################### +qed_bw.lookup_table_mode = "dummy_builtin" + +#qed_bw.lookup_table_mode = "generate" +#qed_bw.chi_min = 0.001 +#qed_bw.tab_dndt_chi_min = 0.1 +#qed_bw.tab_dndt_chi_max = 200 +#qed_bw.tab_dndt_how_many = 64 +#qed_bw.tab_pair_chi_min = 0.01 +#qed_bw.tab_pair_chi_max = 200 +#qed_bw.tab_pair_chi_how_many = 2 +#qed_bw.tab_pair_frac_how_many = 2 +#qed_bw.save_table_in = "bw_micro_table" + +#qed_bw.lookup_table_mode = "load" +#qed_bw.load_table_from = "bw_micro_table" +################################# + +### EXTERNAL E FIELD ### (3e15 * [-0.811107105653813 0.324442842261525 0.486664263392288] ) +warpx.E_external_particle = -2433321316961438 973328526784575 1459992790176863 +#### + +### EXTERNAL B FIELD ### (1e7 * [0.28571429 0.42857143 0.85714286] ) +warpx.B_external_particle = 2857142.85714286 4285714.28571428 8571428.57142857 +#### diff --git a/Examples/Modules/qed/quantum_synchrotron/analysis_2d_tau_init.py b/Examples/Modules/qed/quantum_synchrotron/analysis_2d_tau_init.py index 05b313ee6..98c95d5ea 100755 --- a/Examples/Modules/qed/quantum_synchrotron/analysis_2d_tau_init.py +++ b/Examples/Modules/qed/quantum_synchrotron/analysis_2d_tau_init.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python3 +#! /usr/bin/env python import yt import numpy as np import scipy.stats as st diff --git a/Examples/Modules/qed/quantum_synchrotron/inputs.2d_test_tau_init b/Examples/Modules/qed/quantum_synchrotron/inputs.2d_test_tau_init index 03a2a5798..3fa361a2f 100644 --- a/Examples/Modules/qed/quantum_synchrotron/inputs.2d_test_tau_init +++ b/Examples/Modules/qed/quantum_synchrotron/inputs.2d_test_tau_init @@ -90,16 +90,19 @@ positrons.do_qed_quantum_sync = 1 ################################# #######QED ENGINE PARAMETERS##### -qed_qs.chi_min = 0.001 -qed_qs.ignore_tables_for_test = 1 -#qed_qs.generate_table = 1 +qed_qs.lookup_table_mode = "dummy_builtin" + +#qed_qs.lookup_table_mode = "generate" +#qed_qs.chi_min = 0.001 #qed_qs.tab_dndt_chi_min = 0.001 -#qed_qs.tab_dndt_chi_max = 100 -#qed_qs.tab_dndt_how_many = 200 -#qed_qs.tab_em_chi_min = 0.01 -#qed_qs.tab_em_chi_max = 100 +#qed_qs.tab_dndt_chi_max = 200 +#qed_qs.tab_dndt_how_many = 32 +#qed_qs.tab_em_chi_min = 0.001 +#qed_qs.tab_em_chi_max = 200 #qed_qs.tab_em_chi_how_many = 2 #qed_qs.tab_em_prob_how_many = 2 -#qed_qs.save_table_in = "qs_table" -#qed_qs.load_table_from = "qs_table" +#qed_qs.save_table_in = "qs_micro_table" + +#qed_qs.lookup_table_mode = "load" +#qed_qs.load_table_from = "qs_micro_table" ################################# diff --git a/Examples/Modules/relativistic_space_charge_initialization/analysis.py b/Examples/Modules/relativistic_space_charge_initialization/analysis.py new file mode 100755 index 000000000..aad4534f5 --- /dev/null +++ b/Examples/Modules/relativistic_space_charge_initialization/analysis.py @@ -0,0 +1,75 @@ +#! /usr/bin/env python +""" +This script checks the space-charge initialization routine, by +verifying that the space-charge field of a Gaussian beam corresponds to +the expected theoretical field. +""" +import sys +import yt +import numpy as np +import matplotlib.pyplot as plt +import scipy.constants as scc +from scipy.special import gammainc +yt.funcs.mylog.setLevel(0) + +# Parameters from the Simulation +Qtot = -1.e-20 +r0 = 2.e-6 + +# Open data file +filename = sys.argv[1] +ds = yt.load( filename ) +# Extract data +ad0 = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) +Ex_array = ad0['Ex'].to_ndarray().squeeze() +By_array = ad0['By'].to_ndarray() + +# Extract grid coordinates +Nx, Ny, Nz = ds.domain_dimensions +xmin, ymin, zmin = ds.domain_left_edge.v +Lx, Ly, Lz = ds.domain_width.v +x = xmin + Lx/Nx*(0.5+np.arange(Nx)) +y = ymin + Ly/Ny*(0.5+np.arange(Ny)) +z = zmin + Lz/Nz*(0.5+np.arange(Nz)) + +# Compute theoretical field +x_2d, y_2d, z_2d = np.meshgrid(x, y, z, indexing='ij') +r2 = x_2d**2 + y_2d**2 +factor = Qtot/scc.epsilon_0/(2*np.pi*r2) * (1-np.exp(-r2/(2*r0**2))) +factor_z = 1./(2*np.pi)**.5/r0 * np.exp(-z_2d**2/(2*r0**2)) +Ex_th = factor*factor_z*x_2d +Ey_th = factor*factor_z*y_2d + +# Plot theory and data +def make_2d(arr): + if arr.ndim == 3: + return arr[:,Ny//2,:] + else: + return arr +plt.figure(figsize=(10,10)) +plt.subplot(221) +plt.title('Ex: Theory') +plt.imshow(make_2d(Ex_th)) +plt.colorbar() +plt.subplot(222) +plt.title('Ex: Simulation') +plt.imshow(make_2d(Ex_array)) +plt.colorbar() +plt.subplot(223) +plt.title('By: Theory') +plt.imshow(make_2d(Ex_th/scc.c)) +plt.colorbar() +plt.subplot(224) +plt.title('Bz: Simulation') +plt.imshow(make_2d(By_array)) +plt.colorbar() + +plt.savefig('Comparison.png') + +# Automatically check the results +def check(E, E_th, label): + print( 'Relative error in %s: %.3f'%( + label, abs(E-E_th).max()/E_th.max())) + assert np.allclose( E, E_th, atol=0.1*E_th.max() ) + +check( Ex_array, Ex_th, 'Ex' ) diff --git a/Examples/Modules/relativistic_space_charge_initialization/inputs b/Examples/Modules/relativistic_space_charge_initialization/inputs new file mode 100644 index 000000000..eca38e074 --- /dev/null +++ b/Examples/Modules/relativistic_space_charge_initialization/inputs @@ -0,0 +1,30 @@ +max_step = 0 +amr.n_cell = 64 64 64 +amr.max_grid_size = 32 +amr.max_level = 0 + +amr.plot_int = 1 +geometry.coord_sys = 0 # 0: Cartesian +geometry.is_periodic = 0 0 0 # Is periodic? +geometry.prob_lo = -50.e-6 -50.e-6 -50.e-6 +geometry.prob_hi = 50.e-6 50.e-6 50.e-6 + +particles.nspecies = 1 +particles.species_names = beam +beam.charge = -q_e +beam.mass = m_e +beam.injection_style = "gaussian_beam" +beam.initialize_self_fields = 1 +beam.self_fields_required_precision = 1.e-4 +beam.x_rms = 2.e-6 +beam.y_rms = 2.e-6 +beam.z_rms = 2.e-6 +beam.x_m = 0. +beam.y_m = 0. +beam.z_m = 0.e-6 +beam.npart = 20000 +beam.q_tot = -1.e-20 +beam.momentum_distribution_type = "gaussian" +beam.ux_m = 0.0 +beam.uy_m = 0.0 +beam.uz_m = 100.0 diff --git a/Examples/Modules/space_charge_initialization/analysis.py b/Examples/Modules/space_charge_initialization/analysis.py new file mode 100755 index 000000000..67039bad1 --- /dev/null +++ b/Examples/Modules/space_charge_initialization/analysis.py @@ -0,0 +1,89 @@ +#! /usr/bin/env python +""" +This script checks the space-charge initialization routine, by +verifying that the space-charge field of a Gaussian beam corresponds to +the expected theoretical field. +""" +import sys +import yt +import numpy as np +import matplotlib.pyplot as plt +import scipy.constants as scc +from scipy.special import gammainc +yt.funcs.mylog.setLevel(0) + +# Parameters from the Simulation +Qtot = -1.e-20 +r0 = 2.e-6 + +# Open data file +filename = sys.argv[1] +ds = yt.load( filename ) +# Extract data +ad0 = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) +Ex_array = ad0['Ex'].to_ndarray().squeeze() +if ds.dimensionality == 2: + # Rename the z dimension as y, so as to make this script work for 2d and 3d + Ey_array = ad0['Ez'].to_ndarray().squeeze() +elif ds.dimensionality == 3: + Ey_array = ad0['Ey'].to_ndarray() + Ez_array = ad0['Ez'].to_ndarray() + +# Extract grid coordinates +Nx, Ny, Nz = ds.domain_dimensions +xmin, ymin, zmin = ds.domain_left_edge.v +Lx, Ly, Lz = ds.domain_width.v +x = xmin + Lx/Nx*(0.5+np.arange(Nx)) +y = ymin + Ly/Ny*(0.5+np.arange(Ny)) +z = zmin + Lz/Nz*(0.5+np.arange(Nz)) + +# Compute theoretical field +if ds.dimensionality == 2: + x_2d, y_2d = np.meshgrid(x, y, indexing='ij') + r2 = x_2d**2 + y_2d**2 + factor = (Qtot/r0)/(2*np.pi*scc.epsilon_0*r2) * (1-np.exp(-r2/(2*r0**2))) + Ex_th = x_2d * factor + Ey_th = y_2d * factor +elif ds.dimensionality == 3: + x_2d, y_2d, z_2d = np.meshgrid(x, y, z, indexing='ij') + r2 = x_2d**2 + y_2d**2 + z_2d**2 + factor = Qtot/(4*np.pi*scc.epsilon_0*r2**1.5) * gammainc(3./2, r2/(2.*r0**2)) + Ex_th = factor*x_2d + Ey_th = factor*y_2d + Ez_th = factor*z_2d + +# Plot theory and data +def make_2d(arr): + if arr.ndim == 3: + return arr[:,:,Nz//2] + else: + return arr +plt.figure(figsize=(10,10)) +plt.subplot(221) +plt.title('Ex: Theory') +plt.imshow(make_2d(Ex_th)) +plt.colorbar() +plt.subplot(222) +plt.title('Ex: Simulation') +plt.imshow(make_2d(Ex_array)) +plt.colorbar() +plt.subplot(223) +plt.title('Ey: Theory') +plt.imshow(make_2d(Ey_th)) +plt.colorbar() +plt.subplot(224) +plt.title('Ey: Simulation') +plt.imshow(make_2d(Ey_array)) +plt.colorbar() +plt.savefig('Comparison.png') + +# Automatically check the results +def check(E, E_th, label): + print( 'Relative error in %s: %.3f'%( + label, abs(E-E_th).max()/E_th.max())) + assert np.allclose( E, E_th, atol=0.1*E_th.max() ) + +check( Ex_array, Ex_th, 'Ex' ) +check( Ey_array, Ey_th, 'Ey' ) +if ds.dimensionality == 3: + check( Ez_array, Ez_th, 'Ez' ) diff --git a/Examples/Modules/space_charge_initialization/inputs b/Examples/Modules/space_charge_initialization/inputs new file mode 100644 index 000000000..79309e585 --- /dev/null +++ b/Examples/Modules/space_charge_initialization/inputs @@ -0,0 +1,29 @@ +max_step = 0 +amr.n_cell = 64 64 64 +amr.max_grid_size = 32 +amr.max_level = 0 + +amr.plot_int = 1 +geometry.coord_sys = 0 # 0: Cartesian +geometry.is_periodic = 0 0 0 # Is periodic? +geometry.prob_lo = -50.e-6 -50.e-6 -50.e-6 +geometry.prob_hi = 50.e-6 50.e-6 50.e-6 + +particles.nspecies = 1 +particles.species_names = beam +beam.charge = -q_e +beam.mass = m_e +beam.injection_style = "gaussian_beam" +beam.initialize_self_fields = 1 +beam.x_rms = 2.e-6 +beam.y_rms = 2.e-6 +beam.z_rms = 2.e-6 +beam.x_m = 0. +beam.y_m = 0. +beam.z_m = 0.e-6 +beam.npart = 20000 +beam.q_tot = -1.e-20 +beam.momentum_distribution_type = "gaussian" +beam.ux_m = 0.0 +beam.uy_m = 0.0 +beam.uz_m = 0.0 diff --git a/Examples/Tests/Langmuir/PICMI_inputs_langmuir_rz.py b/Examples/Tests/Langmuir/PICMI_inputs_langmuir_rz.py deleted file mode 100644 index 699b7e3d9..000000000 --- a/Examples/Tests/Langmuir/PICMI_inputs_langmuir_rz.py +++ /dev/null @@ -1,44 +0,0 @@ -import numpy as np -from pywarpx import picmi - -constants = picmi.constants - -nr = 64 -nz = 64 - -rmin = 0.e0 -zmin = -20.e-6 -rmax = +20.e-6 -zmax = +20.e-6 - -uniform_plasma = picmi.UniformDistribution(density = 1.e25, - upper_bound = [None, None, 0.], - directed_velocity = [0., 0., 0.1*constants.c]) - -electrons = picmi.Species(particle_type='electron', name='electrons', initial_distribution=uniform_plasma) - -grid = picmi.CylindricalGrid(number_of_cells = [nr, nz], - lower_bound = [rmin, zmin], - upper_bound = [rmax, zmax], - lower_boundary_conditions = ['dirichlet', 'periodic'], - upper_boundary_conditions = ['dirichlet', 'periodic'], - moving_window_velocity = [0., 0.], - warpx_max_grid_size=32) - -solver = picmi.ElectromagneticSolver(grid=grid, cfl=1.) - -sim = picmi.Simulation(solver = solver, - max_steps = 40, - verbose = 1, - warpx_plot_int = 1, - warpx_current_deposition_algo = 'direct') - -sim.add_species(electrons, layout=picmi.GriddedLayout(n_macroparticle_per_cell=[2,1,2], grid=grid)) - -# write_inputs will create an inputs file that can be used to run -# with the compiled version. -sim.write_input_file(file_name='inputsrz_from_PICMI') - -# Alternatively, sim.step will run WarpX, controlling it from Python -#sim.step() - diff --git a/Examples/Tests/radiation_reaction/test_const_B_analytical/analysis_classicalRR.py b/Examples/Tests/radiation_reaction/test_const_B_analytical/analysis_classicalRR.py index b4f17cf7c..e385ebfb7 100755 --- a/Examples/Tests/radiation_reaction/test_const_B_analytical/analysis_classicalRR.py +++ b/Examples/Tests/radiation_reaction/test_const_B_analytical/analysis_classicalRR.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python3 +#! /usr/bin/env python # This script contains few simple tests for the radiation reaction pusher # It initializes an electron or a positron with normalized momentum in different diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index c4e6803d5..6e9c9153b 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -56,7 +56,7 @@ class Species(picmistandard.PICMI_Species): if self.mass is None: self.mass = element.mass*periodictable.constants.atomic_mass_constant - def initialize_inputs(self, layout): + def initialize_inputs(self, layout, initialize_self_fields=False): self.species_number = pywarpx.particles.nspecies pywarpx.particles.nspecies += 1 @@ -68,7 +68,8 @@ class Species(picmistandard.PICMI_Species): else: pywarpx.particles.species_names += ' ' + self.name - self.species = pywarpx.Bucket.Bucket(self.name, mass=self.mass, charge=self.charge, injection_style = 'python') + self.species = pywarpx.Bucket.Bucket(self.name, mass=self.mass, charge=self.charge, + injection_style = 'python', initialize_self_fields=int(initialize_self_fields)) pywarpx.Particles.particles_list.append(self.species) if self.initial_distribution is not None: @@ -77,9 +78,9 @@ class Species(picmistandard.PICMI_Species): picmistandard.PICMI_MultiSpecies.Species_class = Species class MultiSpecies(picmistandard.PICMI_MultiSpecies): - def initialize_inputs(self, layout): + def initialize_inputs(self, layout, initialize_self_fields=False): for species in self.species_instances_list: - species.initialize_inputs(layout) + species.initialize_inputs(layout, initialize_self_fields) class GaussianBunchDistribution(picmistandard.PICMI_GaussianBunchDistribution): @@ -544,8 +545,7 @@ class Simulation(picmistandard.PICMI_Simulation): self.solver.initialize_inputs() for i in range(len(self.species)): - assert not self.initialize_self_fields[i], Exception('WarpX does not support initializing self fields') - self.species[i].initialize_inputs(self.layouts[i]) + self.species[i].initialize_inputs(self.layouts[i], self.initialize_self_fields[i]) for i in range(len(self.lasers)): self.lasers[i].initialize_inputs() diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 19da4ba04..5f1ee0954 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -60,7 +60,7 @@ branch = master [pml_x_yee] buildDir = . inputFile = Examples/Tests/PML/inputs.2d -runtime_params = warpx.do_dynamic_scheduling=0 algo.maxwell_fdtd_solver=yee amrex.abort_on_unused_inputs=1 +runtime_params = warpx.do_dynamic_scheduling=0 algo.maxwell_fdtd_solver=yee dim = 2 addToCompileString = restartTest = 0 @@ -75,7 +75,7 @@ analysisRoutine = Examples/Tests/PML/analysis_pml_yee.py [pml_x_ckc] buildDir = . inputFile = Examples/Tests/PML/inputs.2d -runtime_params = warpx.do_dynamic_scheduling=0 algo.maxwell_fdtd_solver=ckc amrex.abort_on_unused_inputs=1 +runtime_params = warpx.do_dynamic_scheduling=0 algo.maxwell_fdtd_solver=ckc dim = 2 addToCompileString = restartTest = 0 @@ -90,7 +90,7 @@ analysisRoutine = Examples/Tests/PML/analysis_pml_ckc.py [pml_x_psatd] buildDir = . inputFile = Examples/Tests/PML/inputs.2d -runtime_params = warpx.do_dynamic_scheduling=0 amrex.abort_on_unused_inputs=1 +runtime_params = warpx.do_dynamic_scheduling=0 dim = 2 addToCompileString = USE_PSATD=TRUE restartTest = 0 @@ -105,7 +105,7 @@ analysisRoutine = Examples/Tests/PML/analysis_pml_psatd.py [RigidInjection_lab] buildDir = . inputFile = Examples/Modules/RigidInjection/inputs.LabFrame -runtime_params = amrex.abort_on_unused_inputs=1 +runtime_params = dim = 2 addToCompileString = restartTest = 0 @@ -121,7 +121,7 @@ analysisRoutine = Examples/Modules/RigidInjection/analysis_rigid_injection_LabFr [RigidInjection_BTD] buildDir = . inputFile = Examples/Modules/RigidInjection/inputs.BoostedFrame -runtime_params = amrex.abort_on_unused_inputs=1 +runtime_params = dim = 2 addToCompileString = restartTest = 0 @@ -156,7 +156,7 @@ analysisRoutine = Examples/Modules/boosted_diags/analysis_3Dbacktransformed_diag [nci_corrector] buildDir = . inputFile = Examples/Modules/nci_corrector/inputs.2d -runtime_params = amr.max_level=0 particles.use_fdtd_nci_corr=1 amrex.abort_on_unused_inputs=1 +runtime_params = amr.max_level=0 particles.use_fdtd_nci_corr=1 dim = 2 addToCompileString = restartTest = 0 @@ -172,7 +172,7 @@ analysisRoutine = Examples/Modules/nci_corrector/analysis_ncicorr.py [nci_correctorMR] buildDir = . inputFile = Examples/Modules/nci_corrector/inputs.2d -runtime_params = amr.max_level=1 particles.use_fdtd_nci_corr=1 amr.n_cell=64 64 amrex.abort_on_unused_inputs=1 warpx.fine_tag_lo=-20.e-6 -20.e-6 warpx.fine_tag_hi=20.e-6 20.e-6 +runtime_params = amr.max_level=1 particles.use_fdtd_nci_corr=1 amr.n_cell=64 64 warpx.fine_tag_lo=-20.e-6 -20.e-6 warpx.fine_tag_hi=20.e-6 20.e-6 dim = 2 addToCompileString = restartTest = 0 @@ -188,7 +188,7 @@ analysisRoutine = Examples/Modules/nci_corrector/analysis_ncicorr.py [ionization_lab] buildDir = . inputFile = Examples/Modules/ionization/inputs.rt -runtime_params = amrex.abort_on_unused_inputs=1 +runtime_params = dim = 2 addToCompileString = restartTest = 0 @@ -203,7 +203,7 @@ analysisRoutine = Examples/Modules/ionization/analysis_ionization.py [ionization_boost] buildDir = . inputFile = Examples/Modules/ionization/inputs.bf.rt -runtime_params = amrex.abort_on_unused_inputs=1 +runtime_params = dim = 2 addToCompileString = restartTest = 0 @@ -218,7 +218,7 @@ analysisRoutine = Examples/Modules/ionization/analysis_ionization.py [bilinear_filter] buildDir = . inputFile = Examples/Tests/SingleParticle/inputs -runtime_params = warpx.use_filter=1 warpx.filter_npass_each_dir=1 5 amrex.abort_on_unused_inputs=1 +runtime_params = warpx.use_filter=1 warpx.filter_npass_each_dir=1 5 dim = 2 addToCompileString = restartTest = 0 @@ -233,7 +233,7 @@ analysisRoutine = Examples/Tests/SingleParticle/analysis_bilinear_filter.py [Langmuir_2d] buildDir = . inputFile = Examples/Tests/Langmuir/inputs.rt -runtime_params = electrons.ux=0.01 electrons.xmax=0.e-6 warpx.fields_to_plot=Ex jx electrons.plot_vars=w ux Ex amrex.abort_on_unused_inputs=1 +runtime_params = electrons.ux=0.01 electrons.xmax=0.e-6 warpx.fields_to_plot=Ex jx electrons.plot_vars=w ux Ex dim = 2 addToCompileString = restartTest = 0 @@ -251,7 +251,7 @@ analysisOutputImage = langmuir2d_analysis.png [Langmuir_2d_nompi] buildDir = . inputFile = Examples/Tests/Langmuir/inputs.rt -runtime_params = electrons.ux=0.01 electrons.xmax=0.e-6 warpx.fields_to_plot=Ex jx electrons.plot_vars=w ux Ex amrex.abort_on_unused_inputs=1 +runtime_params = electrons.ux=0.01 electrons.xmax=0.e-6 warpx.fields_to_plot=Ex jx electrons.plot_vars=w ux Ex dim = 2 addToCompileString = restartTest = 0 @@ -269,7 +269,7 @@ analysisOutputImage = langmuir2d_analysis.png [Langmuir_x] buildDir = . inputFile = Examples/Tests/Langmuir/inputs.rt -runtime_params = electrons.ux=0.01 electrons.xmax=0.e-6 warpx.do_dynamic_scheduling=0 warpx.fields_to_plot = Ex jx electrons.plot_vars=w ux Ex amrex.abort_on_unused_inputs=1 +runtime_params = electrons.ux=0.01 electrons.xmax=0.e-6 warpx.do_dynamic_scheduling=0 warpx.fields_to_plot = Ex jx electrons.plot_vars=w ux Ex dim = 3 addToCompileString = restartTest = 0 @@ -287,7 +287,7 @@ analysisOutputImage = langmuir_x_analysis.png [Langmuir_y] buildDir = . inputFile = Examples/Tests/Langmuir/inputs.rt -runtime_params = electrons.uy=0.01 electrons.ymax=0.e-6 warpx.do_dynamic_scheduling=0 warpx.fields_to_plot = Ey jy electrons.plot_vars=w uy Ey amrex.abort_on_unused_inputs=1 +runtime_params = electrons.uy=0.01 electrons.ymax=0.e-6 warpx.do_dynamic_scheduling=0 warpx.fields_to_plot = Ey jy electrons.plot_vars=w uy Ey dim = 3 addToCompileString = restartTest = 0 @@ -305,7 +305,7 @@ analysisOutputImage = langmuir_y_analysis.png [Langmuir_z] buildDir = . inputFile = Examples/Tests/Langmuir/inputs.rt -runtime_params = electrons.uz=0.01 electrons.zmax=0.e-6 warpx.do_dynamic_scheduling=0 warpx.fields_to_plot = Ez jz electrons.plot_vars=w uz Ez amrex.abort_on_unused_inputs=1 +runtime_params = electrons.uz=0.01 electrons.zmax=0.e-6 warpx.do_dynamic_scheduling=0 warpx.fields_to_plot = Ez jz electrons.plot_vars=w uz Ez dim = 3 addToCompileString = restartTest = 0 @@ -323,7 +323,7 @@ analysisOutputImage = langmuir_z_analysis.png [Langmuir_multi] buildDir = . inputFile = Examples/Tests/Langmuir/inputs.multi.rt -runtime_params = warpx.do_dynamic_scheduling=0 amrex.abort_on_unused_inputs=1 +runtime_params = warpx.do_dynamic_scheduling=0 dim = 3 addToCompileString = restartTest = 0 @@ -341,7 +341,7 @@ analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_nodal] buildDir = . inputFile = Examples/Tests/Langmuir/inputs.multi.rt -runtime_params = warpx.do_dynamic_scheduling=0 warpx.do_nodal=1 amrex.abort_on_unused_inputs=1 +runtime_params = warpx.do_dynamic_scheduling=0 warpx.do_nodal=1 dim = 3 addToCompileString = restartTest = 0 @@ -359,7 +359,7 @@ analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd] buildDir = . inputFile = Examples/Tests/Langmuir/inputs.multi.rt -runtime_params = psatd.fftw_plan_measure=0 amrex.abort_on_unused_inputs=1 +runtime_params = psatd.fftw_plan_measure=0 dim = 3 addToCompileString = USE_PSATD=TRUE restartTest = 0 @@ -378,7 +378,7 @@ analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_nodal] buildDir = . inputFile = Examples/Tests/Langmuir/inputs.multi.rt -runtime_params = psatd.fftw_plan_measure=0 warpx.do_dynamic_scheduling=0 warpx.do_nodal=1 amrex.abort_on_unused_inputs=1 +runtime_params = psatd.fftw_plan_measure=0 warpx.do_dynamic_scheduling=0 warpx.do_nodal=1 dim = 3 addToCompileString = USE_PSATD=TRUE restartTest = 0 @@ -397,7 +397,7 @@ analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_2d_nodal] buildDir = . inputFile = Examples/Tests/Langmuir/inputs.multi.2d.rt -runtime_params = warpx.do_nodal=1 algo.current_deposition=direct electrons.plot_vars=w ux uy uz Ex Ey Ez positrons.plot_vars=w ux uy uz Ex Ey Ez amrex.abort_on_unused_inputs=1 +runtime_params = warpx.do_nodal=1 algo.current_deposition=direct electrons.plot_vars=w ux uy uz Ex Ey Ez positrons.plot_vars=w ux uy uz Ex Ey Ez dim = 2 addToCompileString = restartTest = 0 @@ -415,7 +415,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd] buildDir = . inputFile = Examples/Tests/Langmuir/inputs.multi.2d.rt -runtime_params = psatd.fftw_plan_measure=0 electrons.plot_vars=w ux uy uz Ex Ey Ez positrons.plot_vars=w ux uy uz Ex Ey Ez warpx.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell amrex.abort_on_unused_inputs=1 +runtime_params = psatd.fftw_plan_measure=0 electrons.plot_vars=w ux uy uz Ex Ey Ez positrons.plot_vars=w ux uy uz Ex Ey Ez warpx.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell dim = 2 addToCompileString = USE_PSATD=TRUE restartTest = 0 @@ -433,7 +433,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_nodal] buildDir = . inputFile = Examples/Tests/Langmuir/inputs.multi.2d.rt -runtime_params = psatd.fftw_plan_measure=0 warpx.do_nodal=1 algo.current_deposition=direct electrons.plot_vars=w ux uy uz Ex Ey Ez positrons.plot_vars=w ux uy uz Ex Ey Ez warpx.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell amrex.abort_on_unused_inputs=1 +runtime_params = psatd.fftw_plan_measure=0 warpx.do_nodal=1 algo.current_deposition=direct electrons.plot_vars=w ux uy uz Ex Ey Ez positrons.plot_vars=w ux uy uz Ex Ey Ez warpx.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell dim = 2 addToCompileString = USE_PSATD=TRUE restartTest = 0 @@ -451,7 +451,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_rz] buildDir = . inputFile = Examples/Tests/Langmuir/inputs.multi.rz.rt -runtime_params = electrons.plot_vars=w ux uy uz Ex Ey Ez Bx By ions.plot_vars=w ux uy uz Ex Ey Ez Bx By amrex.abort_on_unused_inputs=1 +runtime_params = electrons.plot_vars=w ux uy uz Ex Ey Ez Bx By ions.plot_vars=w ux uy uz Ex Ey Ez Bx By dim = 2 addToCompileString = USE_RZ=TRUE restartTest = 0 @@ -469,7 +469,7 @@ analysisOutputImage = langmuir_multi_rz_analysis.png [Python_Langmuir_rz_multimode] buildDir = . inputFile = Examples/Tests/Langmuir/PICMI_inputs_langmuir_rz_multimode_analyze.py -runtime_params = amrex.abort_on_unused_inputs=1 +runtime_params = customRunCmd = python PICMI_inputs_langmuir_rz_multimode_analyze.py dim = 2 addToCompileString = USE_PYTHON_MAIN=TRUE USE_RZ=TRUE @@ -487,7 +487,7 @@ outputFile = diags/plotfiles/plt00040 [LaserInjection] buildDir = . inputFile = Examples/Modules/laser_injection/inputs.rt -runtime_params = amrex.abort_on_unused_inputs=1 +runtime_params = dim = 3 addToCompileString = restartTest = 0 @@ -504,7 +504,7 @@ analysisOutputImage = laser_analysis.png [LaserInjection_2d] buildDir = . inputFile = Examples/Modules/laser_injection/inputs.2d.rt -runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_ics=1 amrex.abort_on_unused_inputs=1 +runtime_params = warpx.do_dynamic_scheduling=0 warpx.serialize_ics=1 dim = 2 addToCompileString = restartTest = 0 @@ -519,7 +519,7 @@ compareParticles = 0 [LaserAcceleration] buildDir = . inputFile = Examples/Physics_applications/laser_acceleration/inputs.3d -runtime_params = warpx.do_dynamic_scheduling=0 amr.n_cell=32 32 256 max_step=100 electrons.zmin=0.e-6 warpx.serialize_ics=1 amrex.abort_on_unused_inputs=1 +runtime_params = warpx.do_dynamic_scheduling=0 amr.n_cell=32 32 256 max_step=100 electrons.zmin=0.e-6 warpx.serialize_ics=1 dim = 3 addToCompileString = restartTest = 0 @@ -535,7 +535,7 @@ particleTypes = electrons [subcyclingMR] buildDir = . inputFile = Examples/Tests/subcycling/inputs.2d -runtime_params = warpx.serialize_ics=1 warpx.do_dynamic_scheduling=0 amrex.abort_on_unused_inputs=1 +runtime_params = warpx.serialize_ics=1 warpx.do_dynamic_scheduling=0 dim = 2 addToCompileString = restartTest = 0 @@ -551,7 +551,7 @@ compareParticles = 0 [LaserAccelerationMR] buildDir = . inputFile = Examples/Physics_applications/laser_acceleration/inputs.2d -runtime_params = amr.max_level=1 max_step=200 warpx.serialize_ics=1 amrex.abort_on_unused_inputs=1 warpx.fine_tag_lo=-5.e-6 -35.e-6 warpx.fine_tag_hi=5.e-6 -25.e-6 +runtime_params = amr.max_level=1 max_step=200 warpx.serialize_ics=1 warpx.fine_tag_lo=-5.e-6 -35.e-6 warpx.fine_tag_hi=5.e-6 -25.e-6 dim = 2 addToCompileString = restartTest = 0 @@ -567,7 +567,7 @@ particleTypes = electrons beam [PlasmaAccelerationMR] buildDir = . inputFile = Examples/Physics_applications/plasma_acceleration/inputs.2d -runtime_params = amr.max_level=1 amr.n_cell=32 512 max_step=400 warpx.serialize_ics=1 warpx.do_dynamic_scheduling=0 amrex.abort_on_unused_inputs=1 +runtime_params = amr.max_level=1 amr.n_cell=32 512 max_step=400 warpx.serialize_ics=1 warpx.do_dynamic_scheduling=0 dim = 2 addToCompileString = restartTest = 0 @@ -583,7 +583,7 @@ particleTypes = beam driver plasma_e [Python_Langmuir] buildDir = . inputFile = Examples/Tests/Langmuir/PICMI_inputs_langmuir_rt.py -runtime_params = amrex.abort_on_unused_inputs=1 +runtime_params = customRunCmd = python PICMI_inputs_langmuir_rt.py dim = 3 addToCompileString = USE_PYTHON_MAIN=TRUE @@ -601,7 +601,7 @@ outputFile = diags/plotfiles/plt00040 [uniform_plasma_restart] buildDir = . inputFile = Examples/Physics_applications/uniform_plasma/inputs.3d -runtime_params = amrex.abort_on_unused_inputs=1 +runtime_params = dim = 3 addToCompileString = restartTest = 1 @@ -616,10 +616,64 @@ compareParticles = 0 particleTypes = electrons tolerance = 1.e-14 +[space_charge_initialization_2d] +buildDir = . +inputFile = Examples/Modules/space_charge_initialization/inputs +dim = 2 +addToCompileString = +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 2 +compileTest = 0 +doVis = 0 +compareParticles = 0 +runtime_params = warpx.do_dynamic_scheduling=0 +analysisRoutine = Examples/Modules/space_charge_initialization/analysis.py +analysisOutputImage = Comparison.png +outputFile=space_charge_initialization_2d_plt00000 + +[space_charge_initialization] +buildDir = . +inputFile = Examples/Modules/space_charge_initialization/inputs +dim = 3 +addToCompileString = +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 2 +compileTest = 0 +doVis = 0 +compareParticles = 0 +runtime_params = warpx.do_dynamic_scheduling=0 +analysisRoutine = Examples/Modules/space_charge_initialization/analysis.py +analysisOutputImage = Comparison.png +outputFile=space_charge_initialization_plt00000 + +[relativistic_space_charge_initialization] +buildDir = . +inputFile = Examples/Modules/relativistic_space_charge_initialization/inputs +dim = 3 +addToCompileString = +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 2 +compileTest = 0 +doVis = 0 +compareParticles = 0 +runtime_params = warpx.do_dynamic_scheduling=0 +analysisRoutine = Examples/Modules/relativistic_space_charge_initialization/analysis.py +analysisOutputImage = Comparison.png +outputFile=relativistic_space_charge_initialization_plt00000 + [particles_in_pml_2d] buildDir = . inputFile = Examples/Tests/particles_in_PML/inputs.2d -runtime_params = amrex.abort_on_unused_inputs=1 +runtime_params = dim = 2 restartTest = 0 useMPI = 1 @@ -634,7 +688,7 @@ analysisRoutine = Examples/Tests/particles_in_PML/analysis_particles_in_pml.py [particles_in_pml] buildDir = . inputFile = Examples/Tests/particles_in_PML/inputs -runtime_params = amrex.abort_on_unused_inputs=1 +runtime_params = dim = 3 restartTest = 0 useMPI = 1 @@ -649,7 +703,7 @@ analysisRoutine = Examples/Tests/particles_in_PML/analysis_particles_in_pml.py [photon_pusher] buildDir = . inputFile = Examples/Tests/photon_pusher/inputs -runtime_params = amrex.abort_on_unused_inputs=1 +runtime_params = dim = 3 restartTest = 0 useMPI = 1 @@ -679,7 +733,7 @@ analysisRoutine = Examples/Tests/radiation_reaction/test_const_B_analytical/ana [qed_breit_wheeler_tau_init] buildDir = . inputFile = Examples/Modules/qed/breit_wheeler/inputs.2d_test_tau_init -runtime_params = amrex.abort_on_unused_inputs=1 +runtime_params = dim = 2 addToCompileString = QED=TRUE restartTest = 0 @@ -692,10 +746,26 @@ doVis = 0 compareParticles = 0 analysisRoutine = Examples/Modules/qed/breit_wheeler/analysis_2d_tau_init.py +[qed_breit_wheeler_opt_depth_evolution] +buildDir = . +inputFile = Examples/Modules/qed/breit_wheeler/inputs.3d_test_optical_depth_evolution +runtime_params = +dim = 3 +addToCompileString = QED=TRUE +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 2 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Modules/qed/breit_wheeler/analysis_3d_optical_depth_evolution.py + [qed_quantum_sync_tau_init] buildDir = . inputFile = Examples/Modules/qed/quantum_synchrotron/inputs.2d_test_tau_init -runtime_params = amrex.abort_on_unused_inputs=1 +runtime_params = dim = 2 addToCompileString = QED=TRUE restartTest = 0 @@ -737,6 +807,7 @@ compileTest = 0 doVis = 0 compareParticles = 1 particleTypes = electrons +outputFile = diags/plotfiles/plt00010 [PlasmaAccelerationBoost2d] buildDir = . @@ -767,6 +838,7 @@ compileTest = 0 doVis = 0 compareParticles = 1 particleTypes = beam +outputFile = diags/plotfiles/plt00010 [Python_PlasmaAccelerationMR] buildDir = . @@ -783,6 +855,7 @@ compileTest = 0 doVis = 0 compareParticles = 1 particleTypes = beam +outputFile = diags/plotfiles/plt00010 [PlasmaAccelerationBoost3d] buildDir = . @@ -860,6 +933,7 @@ compileTest = 0 doVis = 0 compareParticles = 1 particleTypes = electrons +outputFile = diags/plotfiles/plt00010 [Python_Langmuir_2d] buildDir = . @@ -876,20 +950,7 @@ compileTest = 0 doVis = 0 compareParticles = 1 particleTypes = electrons - -[Python_Langmuir_rz_multimode] -buildDir = . -inputFile = Examples/Tests/Langmuir/PICMI_inputs_langmuir_rz.py -customRunCmd = python PICMI_inputs_langmuir_rz.py -dim = 2 -addToCompileString = USE_PYTHON_MAIN=TRUE USE_RZ=TRUE -restartTest = 0 -useMPI = 1 -numprocs = 4 -useOMP = 1 -numthreads = 1 -compileTest = 0 -doVis = 0 +outputFile = diags/plotfiles/plt00040 [GPU_inputs] buildDir = . @@ -959,4 +1020,3 @@ useOMP = 1 numthreads = 2 compileTest = 0 doVis = 0 - diff --git a/Regression/prepare_file_travis.py b/Regression/prepare_file_travis.py index 1fc7e35b1..d461f6f81 100644 --- a/Regression/prepare_file_travis.py +++ b/Regression/prepare_file_travis.py @@ -29,16 +29,22 @@ text = re.sub( '\[(?P<name>.*)\]\nbuildDir = ', # Change compile options when running on GPU if arch == 'GPU': text = re.sub( 'addToCompileString =', - 'addToCompileString = USE_GPU=TRUE USE_OMP=FALSE USE_ACC=TRUE', text) + 'addToCompileString = USE_GPU=TRUE USE_OMP=FALSE USE_ACC=TRUE ', text) text = re.sub( 'COMP\s*=.*', 'COMP = pgi', text ) print('Compiling for %s' %arch) +# Add runtime option: crash for unused variables +text = re.sub('runtime_params =', + 'runtime_params = amrex.abort_on_unused_inputs=1 ', + text) + # Use only 2 cores for compiling text = re.sub( 'numMakeJobs = \d+', 'numMakeJobs = 2', text ) - # Use only 1 MPI and 1 thread proc for tests text = re.sub( 'numprocs = \d+', 'numprocs = 2', text) text = re.sub( 'numthreads = \d+', 'numthreads = 1', text) +# Prevent emails from being sent +text = re.sub( 'sendEmailWhenFail = 1', 'sendEmailWhenFail = 0', text ) # Remove Python test (does not compile) text = re.sub( '\[Python_Langmuir\]\n(.+\n)*', '', text) @@ -46,22 +52,37 @@ text = re.sub( '\[Python_Langmuir\]\n(.+\n)*', '', text) # Remove Langmuir_x/y/z test (too long; not that useful) text = re.sub( '\[Langmuir_[xyz]\]\n(.+\n)*', '', text) -# Remove tests that do not have the right dimension +# Select the tests to be run +# -------------------------- + +# - Extract test blocks (they are identified by the fact that they contain "inputFile") +select_test_regex = r'(\[(.+\n)*inputFile(.+\n)*)' +test_blocks = [ match[0] for match in re.findall(select_test_regex, text) ] +# - Remove the test blocks from `text` (only the selected ones will be added back) +text = re.sub( select_test_regex, '', text ) + +# Keep tests that have the right dimension if dim is not None: print('Selecting tests with dim = %s' %dim) - text = re.sub('\[.+\n(.+\n)*dim = [^%s]\n(.+\n)*' %dim, '', text) + # Cartesian tests + if dim in ['2', '3']: + test_blocks = [ block for block in test_blocks \ + if ('dim = %s'%dim in block) and not ('USE_RZ' in block) ] + elif dim == 'RZ': + test_blocks = [ block for block in test_blocks if 'USE_RZ' in block ] + else: + raise ValueError('Unkown dimension: %s' %dim) # Remove or keep QED tests according to 'qed' variable if qed is not None: print('Selecting tests with QED = %s' %qed) if (qed == "FALSE"): - text = re.sub('\[qed.+\n(.+\n)*\n*', '', text) + test_blocks = [ block for block in test_blocks if not 'QED=TRUE' in block ] else: - text = re.sub('^\[(?!qed).*$\n(.+\n)*(dim = .+\n)(.+\n)*\n*', '', text, flags=re.MULTILINE) - + test_blocks = [ block for block in test_blocks if 'QED=TRUE' in block ] -# Prevent emails from being sent -text = re.sub( 'sendEmailWhenFail = 1', 'sendEmailWhenFail = 0', text ) +# - Add the selected test blocks to the text +text = text + '\n' + '\n'.join(test_blocks) with open('travis-tests.ini', 'w') as f: f.write(text) diff --git a/Source/Diagnostics/BackTransformedDiagnostic.H b/Source/Diagnostics/BackTransformedDiagnostic.H index 9e24caa1b..5621d48c6 100644 --- a/Source/Diagnostics/BackTransformedDiagnostic.H +++ b/Source/Diagnostics/BackTransformedDiagnostic.H @@ -8,12 +8,10 @@ #include <AMReX_PlotFileUtil.H> #include <AMReX_ParallelDescriptor.H> #include <AMReX_Geometry.H> -#include <AMReX_CudaContainers.H> #include "MultiParticleContainer.H" #include "WarpXConst.H" - /** \brief * The capability for back-transformed lab-frame data is implemented to generate * the full diagnostic snapshot for the entire domain and reduced diagnostic @@ -37,39 +35,38 @@ class LabFrameDiag { public: - std::string file_name; - amrex::Real t_lab; - amrex::RealBox prob_domain_lab_; - amrex::IntVect prob_ncells_lab_; - amrex::RealBox diag_domain_lab_; - amrex::Box buff_box_; - - amrex::Real current_z_lab; - amrex::Real current_z_boost; - amrex::Real inv_gamma_boost_; - amrex::Real inv_beta_boost_; - amrex::Real dz_lab_; - amrex::Real dx_; - amrex::Real dy_; - - int ncomp_to_dump_; - std::vector<std::string> mesh_field_names_; - - int file_num; - int initial_i; + std::string m_file_name; + amrex::Real m_t_lab; + amrex::RealBox m_prob_domain_lab_; + amrex::IntVect m_prob_ncells_lab_; + amrex::RealBox m_diag_domain_lab_; + amrex::Box m_buff_box_; + + amrex::Real m_current_z_lab; + amrex::Real m_current_z_boost; + amrex::Real m_inv_gamma_boost_; + amrex::Real m_inv_beta_boost_; + amrex::Real m_dz_lab_; + amrex::Real m_particle_slice_dx_lab_; + + int m_ncomp_to_dump_; + std::vector<std::string> m_mesh_field_names_; + + int m_file_num; + int m_initial_i; // For back-transformed diagnostics of grid fields, data_buffer_ // stores a buffer of the fields in the lab frame (in a MultiFab, i.e. // with all box data etc.). When the buffer if full, dump to file. - std::unique_ptr<amrex::MultiFab> data_buffer_; + std::unique_ptr<amrex::MultiFab> m_data_buffer_; // particles_buffer_ is currently blind to refinement level. // particles_buffer_[j] is a WarpXParticleContainer::DiagnosticParticleData // where - j is the species number for the current diag - amrex::Vector<WarpXParticleContainer::DiagnosticParticleData> particles_buffer_; + amrex::Vector<WarpXParticleContainer::DiagnosticParticleData> m_particles_buffer_; // buff_counter_ is the number of z slices in data_buffer_ - int buff_counter_; - int num_buffer_ = 256; - int max_box_size = 256; + int m_buff_counter_; + int m_num_buffer_ = 256; + int m_max_box_size = 256; void updateCurrentZPositions(amrex::Real t_boost, amrex::Real inv_gamma, amrex::Real inv_beta); @@ -141,7 +138,7 @@ class LabFrameSlice : public LabFrameDiag { std::vector<std::string> mesh_field_names, amrex::RealBox diag_domain_lab, amrex::Box diag_box, int file_num_in, - amrex::Real cell_dx, amrex::Real cell_dy); + amrex::Real particle_slice_dx_lab); void AddDataToBuffer( amrex::MultiFab& tmp_slice_ptr, int i_lab, amrex::Gpu::ManagedDeviceVector<int> map_actual_fields_to_dump) override; void AddPartDataToParticleBuffer( @@ -175,7 +172,8 @@ public: int N_slice_snapshots, amrex::Real gamma_boost, amrex::Real t_boost, amrex::Real dt_boost, int boost_direction, const amrex::Geometry& geom, - amrex::RealBox& slice_realbox); + amrex::RealBox& slice_realbox, + amrex::Real particle_slice_width_lab); /// Flush() is called at the end of the simulation when the buffers that contain /// back-transformed lab-frame data even if they are not full. @@ -211,23 +209,24 @@ public: void writeMetaData(); private: - amrex::Real gamma_boost_; - amrex::Real inv_gamma_boost_; - amrex::Real beta_boost_; - amrex::Real inv_beta_boost_; - amrex::Real dz_lab_; - amrex::Real inv_dz_lab_; - amrex::Real dt_snapshots_lab_; - amrex::Real dt_boost_; - int N_snapshots_; - int boost_direction_; - int N_slice_snapshots_; - amrex::Real dt_slice_snapshots_lab_; - - int num_buffer_ = 256; - int max_box_size_ = 256; - - std::vector<std::unique_ptr<LabFrameDiag> > LabFrameDiags_; + amrex::Real m_gamma_boost_; + amrex::Real m_inv_gamma_boost_; + amrex::Real m_beta_boost_; + amrex::Real m_inv_beta_boost_; + amrex::Real m_dz_lab_; + amrex::Real m_inv_dz_lab_; + amrex::Real m_dt_snapshots_lab_; + amrex::Real m_dt_boost_; + int m_N_snapshots_; + int m_boost_direction_; + int m_N_slice_snapshots_; + amrex::Real m_dt_slice_snapshots_lab_; + amrex::Real m_particle_slice_width_lab_; + + int m_num_buffer_ = 256; + int m_max_box_size_ = 256; + + std::vector<std::unique_ptr<LabFrameDiag> > m_LabFrameDiags_; void writeParticleData( const WarpXParticleContainer::DiagnosticParticleData& pdata, @@ -239,7 +238,7 @@ private: const std::string& name, const std::string& species_name); #endif // Map field names and component number in cell_centered_data - std::map<std::string, int> possible_fields_to_dump = { + std::map<std::string, int> m_possible_fields_to_dump = { {"Ex" , 0}, {"Ey" , 1}, {"Ez" , 2}, @@ -257,10 +256,10 @@ private: amrex::Gpu::ManagedDeviceVector<int> map_actual_fields_to_dump; // Name of fields to dump. By default, all fields in cell_centered_data. // Needed for file headers only. - std::vector<std::string> mesh_field_names = {"Ex", "Ey", "Ez", + std::vector<std::string> m_mesh_field_names = {"Ex", "Ey", "Ez", "Bx", "By", "Bz", "jx", "jy", "jz", "rho"}; - int ncomp_to_dump = 10; + int m_ncomp_to_dump = 10; }; diff --git a/Source/Diagnostics/BackTransformedDiagnostic.cpp b/Source/Diagnostics/BackTransformedDiagnostic.cpp index 2880b37b1..452828f02 100644 --- a/Source/Diagnostics/BackTransformedDiagnostic.cpp +++ b/Source/Diagnostics/BackTransformedDiagnostic.cpp @@ -459,7 +459,7 @@ namespace bool compare_tlab_uptr(const std::unique_ptr<LabFrameDiag>&a, const std::unique_ptr<LabFrameDiag>&b) { - return a->t_lab < b->t_lab; + return a->m_t_lab < b->m_t_lab; } namespace @@ -468,7 +468,7 @@ void LorentzTransformZ(MultiFab& data, Real gamma_boost, Real beta_boost, int ncomp) { // Loop over tiles/boxes and in-place convert each slice from boosted - // frame to lab frame. + // frame to back-transformed lab frame. #ifdef _OPENMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif @@ -520,14 +520,16 @@ BackTransformedDiagnostic(Real zmin_lab, Real zmax_lab, Real v_window_lab, Real dt_slice_snapshots_lab, int N_slice_snapshots, Real gamma_boost, Real t_boost, Real dt_boost, int boost_direction, const Geometry& geom, - amrex::RealBox& slice_realbox) - : gamma_boost_(gamma_boost), - dt_snapshots_lab_(dt_snapshots_lab), - dt_boost_(dt_boost), - N_snapshots_(N_snapshots), - boost_direction_(boost_direction), - N_slice_snapshots_(N_slice_snapshots), - dt_slice_snapshots_lab_(dt_slice_snapshots_lab) + amrex::RealBox& slice_realbox, + amrex::Real particle_slice_width_lab) + : m_gamma_boost_(gamma_boost), + m_dt_snapshots_lab_(dt_snapshots_lab), + m_dt_boost_(dt_boost), + m_N_snapshots_(N_snapshots), + m_boost_direction_(boost_direction), + m_N_slice_snapshots_(N_slice_snapshots), + m_dt_slice_snapshots_lab_(dt_slice_snapshots_lab), + m_particle_slice_width_lab_(particle_slice_width_lab) { @@ -536,13 +538,13 @@ BackTransformedDiagnostic(Real zmin_lab, Real zmax_lab, Real v_window_lab, AMREX_ALWAYS_ASSERT(WarpX::do_back_transformed_fields or WarpX::do_back_transformed_particles); - inv_gamma_boost_ = 1.0 / gamma_boost_; - beta_boost_ = std::sqrt(1.0 - inv_gamma_boost_*inv_gamma_boost_); - inv_beta_boost_ = 1.0 / beta_boost_; + m_inv_gamma_boost_ = 1.0 / m_gamma_boost_; + m_beta_boost_ = std::sqrt(1.0 - m_inv_gamma_boost_*m_inv_gamma_boost_); + m_inv_beta_boost_ = 1.0 / m_beta_boost_; - dz_lab_ = PhysConst::c * dt_boost_ * inv_beta_boost_ * inv_gamma_boost_; - inv_dz_lab_ = 1.0 / dz_lab_; - int Nz_lab = static_cast<unsigned>((zmax_lab - zmin_lab) * inv_dz_lab_); + m_dz_lab_ = PhysConst::c * m_dt_boost_ * m_inv_beta_boost_ * m_inv_gamma_boost_; + m_inv_dz_lab_ = 1.0 / m_dz_lab_; + int Nz_lab = static_cast<unsigned>((zmax_lab - zmin_lab) * m_inv_dz_lab_); int Nx_lab = geom.Domain().length(0); #if (AMREX_SPACEDIM == 3) int Ny_lab = geom.Domain().length(1); @@ -565,40 +567,38 @@ BackTransformedDiagnostic(Real zmin_lab, Real zmax_lab, Real v_window_lab, map_actual_fields_to_dump.push_back(i); if (do_user_fields){ - ncomp_to_dump = user_fields_to_dump.size(); - map_actual_fields_to_dump.resize(ncomp_to_dump); - mesh_field_names.resize(ncomp_to_dump); - for (int i=0; i<ncomp_to_dump; i++){ + m_ncomp_to_dump = user_fields_to_dump.size(); + map_actual_fields_to_dump.resize(m_ncomp_to_dump); + m_mesh_field_names.resize(m_ncomp_to_dump); + for (int i=0; i<m_ncomp_to_dump; i++){ std::string fieldstr = user_fields_to_dump[i]; - mesh_field_names[i] = fieldstr; - map_actual_fields_to_dump[i] = possible_fields_to_dump[fieldstr]; + m_mesh_field_names[i] = fieldstr; + map_actual_fields_to_dump[i] = m_possible_fields_to_dump[fieldstr]; } } // allocating array with total number of lab frame diags (snapshots+slices) - LabFrameDiags_.resize(N_snapshots+N_slice_snapshots); + m_LabFrameDiags_.resize(N_snapshots+N_slice_snapshots); for (int i = 0; i < N_snapshots; ++i) { - Real t_lab = i * dt_snapshots_lab_; + Real t_lab = i * m_dt_snapshots_lab_; // Get simulation domain physical coordinates (in boosted frame). RealBox prob_domain_lab = geom.ProbDomain(); // Replace z bounds by lab-frame coordinates - // x and y bounds are the same for lab frame and boosted frame + // x and y bounds are the same for back-transformed lab frame and boosted frame prob_domain_lab.setLo(AMREX_SPACEDIM-1, zmin_lab + v_window_lab * t_lab); prob_domain_lab.setHi(AMREX_SPACEDIM-1, zmax_lab + v_window_lab * t_lab); Box diag_box = geom.Domain(); - LabFrameDiags_[i].reset(new LabFrameSnapShot(t_lab, t_boost, - inv_gamma_boost_, inv_beta_boost_, dz_lab_, + m_LabFrameDiags_[i].reset(new LabFrameSnapShot(t_lab, t_boost, + m_inv_gamma_boost_, m_inv_beta_boost_, m_dz_lab_, prob_domain_lab, prob_ncells_lab, - ncomp_to_dump, mesh_field_names, prob_domain_lab, + m_ncomp_to_dump, m_mesh_field_names, prob_domain_lab, diag_box, i)); } for (int i = 0; i < N_slice_snapshots; ++i) { - amrex::Real cell_dx = 0; - amrex::Real cell_dy = 0; IntVect slice_ncells_lab ; // To construct LabFrameSlice(), the location of lo() and hi() of the @@ -611,18 +611,17 @@ BackTransformedDiagnostic(Real zmin_lab, Real zmax_lab, Real v_window_lab, const amrex::Real* current_slice_hi = slice_realbox.hi(); const amrex::Real zmin_slice_lab = current_slice_lo[AMREX_SPACEDIM-1] / - ( (1.+beta_boost_)*gamma_boost_); + ( (1.+m_beta_boost_)*m_gamma_boost_); const amrex::Real zmax_slice_lab = current_slice_hi[AMREX_SPACEDIM-1] / - ( (1.+beta_boost_)*gamma_boost_); + ( (1.+m_beta_boost_)*m_gamma_boost_); int Nz_slice_lab = static_cast<unsigned>(( - zmax_slice_lab - zmin_slice_lab) * inv_dz_lab_); + zmax_slice_lab - zmin_slice_lab) * m_inv_dz_lab_); int Nx_slice_lab = ( current_slice_hi[0] - current_slice_lo[0] ) / geom.CellSize(0); if (Nx_slice_lab == 0 ) Nx_slice_lab = 1; // if the x-dimension is reduced, increase total_cells by 1 // to be consistent with the number of cells created for the output. if (Nx_lab != Nx_slice_lab) Nx_slice_lab++; - cell_dx = geom.CellSize(0); #if (AMREX_SPACEDIM == 3) int Ny_slice_lab = ( current_slice_hi[1] - current_slice_lo[1]) / geom.CellSize(1); @@ -631,7 +630,6 @@ BackTransformedDiagnostic(Real zmin_lab, Real zmax_lab, Real v_window_lab, // to be consistent with the number of cells created for the output. if (Ny_lab != Ny_slice_lab) Ny_slice_lab++; slice_ncells_lab = {Nx_slice_lab, Ny_slice_lab, Nz_slice_lab}; - cell_dy = geom.CellSize(1); #else slice_ncells_lab = {Nx_slice_lab, Nz_slice_lab}; #endif @@ -653,7 +651,7 @@ BackTransformedDiagnostic(Real zmin_lab, Real zmax_lab, Real v_window_lab, Box stmp(slice_lo,slice_hi); Box slicediag_box = stmp; - Real t_slice_lab = i * dt_slice_snapshots_lab_ ; + Real t_slice_lab = i * m_dt_slice_snapshots_lab_ ; RealBox prob_domain_lab = geom.ProbDomain(); // replace z bounds by lab-frame coordinates prob_domain_lab.setLo(AMREX_SPACEDIM-1, zmin_lab + v_window_lab * t_slice_lab); @@ -667,16 +665,16 @@ BackTransformedDiagnostic(Real zmin_lab, Real zmax_lab, Real v_window_lab, v_window_lab * t_slice_lab ); // construct labframeslice - LabFrameDiags_[i+N_snapshots].reset(new LabFrameSlice(t_slice_lab, t_boost, - inv_gamma_boost_, inv_beta_boost_, dz_lab_, + m_LabFrameDiags_[i+N_snapshots].reset(new LabFrameSlice(t_slice_lab, t_boost, + m_inv_gamma_boost_, m_inv_beta_boost_, m_dz_lab_, prob_domain_lab, slice_ncells_lab, - ncomp_to_dump, mesh_field_names, slice_dom_lab, - slicediag_box, i, cell_dx, cell_dy)); + m_ncomp_to_dump, m_mesh_field_names, slice_dom_lab, + slicediag_box, i, m_particle_slice_width_lab_)); } // sort diags based on their respective t_lab - std::stable_sort(LabFrameDiags_.begin(), LabFrameDiags_.end(), compare_tlab_uptr); + std::stable_sort(m_LabFrameDiags_.begin(), m_LabFrameDiags_.end(), compare_tlab_uptr); - AMREX_ALWAYS_ASSERT(max_box_size_ >= num_buffer_); + AMREX_ALWAYS_ASSERT(m_max_box_size_ >= m_num_buffer_); } void BackTransformedDiagnostic::Flush(const Geometry& geom) @@ -690,42 +688,42 @@ void BackTransformedDiagnostic::Flush(const Geometry& geom) const std::vector<std::string> species_names = mypc.GetSpeciesNames(); // Loop over BFD snapshots - for (int i = 0; i < LabFrameDiags_.size(); ++i) { + for (int i = 0; i < m_LabFrameDiags_.size(); ++i) { - Real zmin_lab = LabFrameDiags_[i]->prob_domain_lab_.lo(AMREX_SPACEDIM-1); - int i_lab = (LabFrameDiags_[i]->current_z_lab - zmin_lab) / dz_lab_; + Real zmin_lab = m_LabFrameDiags_[i]->m_prob_domain_lab_.lo(AMREX_SPACEDIM-1); + int i_lab = (m_LabFrameDiags_[i]->m_current_z_lab - zmin_lab) / m_dz_lab_; - if (LabFrameDiags_[i]->buff_counter_ != 0) { + if (m_LabFrameDiags_[i]->m_buff_counter_ != 0) { if (WarpX::do_back_transformed_fields) { - const BoxArray& ba = LabFrameDiags_[i]->data_buffer_->boxArray(); - const int hi = ba[0].bigEnd(boost_direction_); - const int lo = hi - LabFrameDiags_[i]->buff_counter_ + 1; + const BoxArray& ba = m_LabFrameDiags_[i]->m_data_buffer_->boxArray(); + const int hi = ba[0].bigEnd(m_boost_direction_); + const int lo = hi - m_LabFrameDiags_[i]->m_buff_counter_ + 1; //Box buff_box = geom.Domain(); - Box buff_box = LabFrameDiags_[i]->buff_box_; - buff_box.setSmall(boost_direction_, lo); - buff_box.setBig(boost_direction_, hi); + Box buff_box = m_LabFrameDiags_[i]->m_buff_box_; + buff_box.setSmall(m_boost_direction_, lo); + buff_box.setBig(m_boost_direction_, hi); BoxArray buff_ba(buff_box); - buff_ba.maxSize(max_box_size_); + buff_ba.maxSize(m_max_box_size_); DistributionMapping buff_dm(buff_ba); - const int ncomp = LabFrameDiags_[i]->data_buffer_->nComp(); + const int ncomp = m_LabFrameDiags_[i]->m_data_buffer_->nComp(); MultiFab tmp(buff_ba, buff_dm, ncomp, 0); - tmp.copy(*LabFrameDiags_[i]->data_buffer_, 0, 0, ncomp); + tmp.copy(*m_LabFrameDiags_[i]->m_data_buffer_, 0, 0, ncomp); #ifdef WARPX_USE_HDF5 for (int comp = 0; comp < ncomp; ++comp) { - output_write_field(LabFrameDiags_[i]->file_name, - mesh_field_names[comp], tmp, comp, + output_write_field(m_LabFrameDiags_[i]->m_file_name, + m_mesh_field_names[comp], tmp, comp, lbound(buff_box).x, lbound(buff_box).y, lbound(buff_box).z); } #else std::stringstream ss; - ss << LabFrameDiags_[i]->file_name << "/Level_0/" + ss << m_LabFrameDiags_[i]->m_file_name << "/Level_0/" << Concatenate("buffer", i_lab, 5); VisMF::Write(tmp, ss.str()); #endif @@ -739,21 +737,21 @@ void BackTransformedDiagnostic::Flush(const Geometry& geom) species_names[mypc.mapSpeciesBackTransformedDiagnostics(j)]; #ifdef WARPX_USE_HDF5 // Dump species data - writeParticleDataHDF5(LabFrameDiags_[i]->particles_buffer_[j], - LabFrameDiags_[i]->file_name, + writeParticleDataHDF5(m_LabFrameDiags_[i]->m_particles_buffer_[j], + m_LabFrameDiags_[i]->m_file_name, species_name); #else std::stringstream part_ss; - part_ss << LabFrameDiags_[i]->file_name + "/" + + part_ss << m_LabFrameDiags_[i]->m_file_name + "/" + species_name + "/"; // Dump species data - writeParticleData(LabFrameDiags_[i]->particles_buffer_[j], + writeParticleData(m_LabFrameDiags_[i]->m_particles_buffer_[j], part_ss.str(), i_lab); #endif } - LabFrameDiags_[i]->particles_buffer_.clear(); + m_LabFrameDiags_[i]->m_particles_buffer_.clear(); } - LabFrameDiags_[i]->buff_counter_ = 0; + m_LabFrameDiags_[i]->m_buff_counter_ = 0; } } @@ -775,8 +773,8 @@ writeLabFrameData(const MultiFab* cell_centered_data, VisMF::SetHeaderVersion(amrex::VisMF::Header::NoFabHeader_v1); const RealBox& domain_z_boost = geom.ProbDomain(); - const Real zlo_boost = domain_z_boost.lo(boost_direction_); - const Real zhi_boost = domain_z_boost.hi(boost_direction_); + const Real zlo_boost = domain_z_boost.lo(m_boost_direction_); + const Real zhi_boost = domain_z_boost.hi(m_boost_direction_); const std::vector<std::string> species_names = mypc.GetSpeciesNames(); Real prev_t_lab = -dt; @@ -785,43 +783,43 @@ writeLabFrameData(const MultiFab* cell_centered_data, amrex::Vector<WarpXParticleContainer::DiagnosticParticleData> tmp_particle_buffer; // Loop over snapshots - for (int i = 0; i < LabFrameDiags_.size(); ++i) { + for (int i = 0; i < m_LabFrameDiags_.size(); ++i) { // Get updated z position of snapshot - const Real old_z_boost = LabFrameDiags_[i]->current_z_boost; - LabFrameDiags_[i]->updateCurrentZPositions(t_boost, - inv_gamma_boost_, - inv_beta_boost_); + const Real old_z_boost = m_LabFrameDiags_[i]->m_current_z_boost; + m_LabFrameDiags_[i]->updateCurrentZPositions(t_boost, + m_inv_gamma_boost_, + m_inv_beta_boost_); - Real diag_zmin_lab = LabFrameDiags_[i]->diag_domain_lab_.lo(AMREX_SPACEDIM-1); - Real diag_zmax_lab = LabFrameDiags_[i]->diag_domain_lab_.hi(AMREX_SPACEDIM-1); + Real diag_zmin_lab = m_LabFrameDiags_[i]->m_diag_domain_lab_.lo(AMREX_SPACEDIM-1); + Real diag_zmax_lab = m_LabFrameDiags_[i]->m_diag_domain_lab_.hi(AMREX_SPACEDIM-1); - if ( ( LabFrameDiags_[i]->current_z_boost < zlo_boost) or - ( LabFrameDiags_[i]->current_z_boost > zhi_boost) or - ( LabFrameDiags_[i]->current_z_lab < diag_zmin_lab) or - ( LabFrameDiags_[i]->current_z_lab > diag_zmax_lab) ) continue; + if ( ( m_LabFrameDiags_[i]->m_current_z_boost < zlo_boost) or + ( m_LabFrameDiags_[i]->m_current_z_boost > zhi_boost) or + ( m_LabFrameDiags_[i]->m_current_z_lab < diag_zmin_lab) or + ( m_LabFrameDiags_[i]->m_current_z_lab > diag_zmax_lab) ) continue; // Get z index of data_buffer_ (i.e. in the lab frame) where // simulation domain (t', [zmin',zmax']), back-transformed to lab // frame, intersects with snapshot. - Real dom_zmin_lab = LabFrameDiags_[i]->prob_domain_lab_.lo(AMREX_SPACEDIM-1); - int i_lab = ( LabFrameDiags_[i]->current_z_lab - dom_zmin_lab) / dz_lab_; + Real dom_zmin_lab = m_LabFrameDiags_[i]->m_prob_domain_lab_.lo(AMREX_SPACEDIM-1); + int i_lab = ( m_LabFrameDiags_[i]->m_current_z_lab - dom_zmin_lab) / m_dz_lab_; // If buffer of snapshot i is empty... - if ( LabFrameDiags_[i]->buff_counter_ == 0) { + if ( m_LabFrameDiags_[i]->m_buff_counter_ == 0) { // ... reset fields buffer data_buffer_ if (WarpX::do_back_transformed_fields) { - LabFrameDiags_[i]->buff_box_.setSmall(boost_direction_, - i_lab - num_buffer_ + 1); - LabFrameDiags_[i]->buff_box_.setBig(boost_direction_, i_lab); + m_LabFrameDiags_[i]->m_buff_box_.setSmall(m_boost_direction_, + i_lab - m_num_buffer_ + 1); + m_LabFrameDiags_[i]->m_buff_box_.setBig(m_boost_direction_, i_lab); - BoxArray buff_ba(LabFrameDiags_[i]->buff_box_); - buff_ba.maxSize(max_box_size_); + BoxArray buff_ba(m_LabFrameDiags_[i]->m_buff_box_); + buff_ba.maxSize(m_max_box_size_); DistributionMapping buff_dm(buff_ba); - LabFrameDiags_[i]->data_buffer_.reset( new MultiFab(buff_ba, - buff_dm, ncomp_to_dump, 0) ); + m_LabFrameDiags_[i]->m_data_buffer_.reset( new MultiFab(buff_ba, + buff_dm, m_ncomp_to_dump, 0) ); } // ... reset particle buffer particles_buffer_[i] if (WarpX::do_back_transformed_particles) - LabFrameDiags_[i]->particles_buffer_.resize( + m_LabFrameDiags_[i]->m_particles_buffer_.resize( mypc.nSpeciesBackTransformedDiagnostics()); } @@ -830,34 +828,34 @@ writeLabFrameData(const MultiFab* cell_centered_data, const int start_comp = 0; const bool interpolate = true; // slice containing back-transformed data is generated only if t_lab != prev_t_lab and is re-used if multiple diags have the same z_lab,t_lab. - if (LabFrameDiags_[i]->t_lab != prev_t_lab ) { + if (m_LabFrameDiags_[i]->m_t_lab != prev_t_lab ) { if (slice) { slice.reset(new MultiFab); slice.reset(nullptr); } - slice = amrex::get_slice_data(boost_direction_, - LabFrameDiags_[i]->current_z_boost, + slice = amrex::get_slice_data(m_boost_direction_, + m_LabFrameDiags_[i]->m_current_z_boost, *cell_centered_data, geom, start_comp, ncomp, interpolate); // Back-transform data to the lab-frame - LorentzTransformZ(*slice, gamma_boost_, beta_boost_, ncomp); + LorentzTransformZ(*slice, m_gamma_boost_, m_beta_boost_, ncomp); } // Create a 2D box for the slice in the boosted frame - Real dx = geom.CellSize(boost_direction_); - int i_boost = ( LabFrameDiags_[i]->current_z_boost - - geom.ProbLo(boost_direction_))/dx; + Real dx = geom.CellSize(m_boost_direction_); + int i_boost = ( m_LabFrameDiags_[i]->m_current_z_boost - + geom.ProbLo(m_boost_direction_))/dx; //Box slice_box = geom.Domain(); - Box slice_box = LabFrameDiags_[i]->buff_box_; - slice_box.setSmall(boost_direction_, i_boost); - slice_box.setBig(boost_direction_, i_boost); + Box slice_box = m_LabFrameDiags_[i]->m_buff_box_; + slice_box.setSmall(m_boost_direction_, i_boost); + slice_box.setBig(m_boost_direction_, i_boost); // Make it a BoxArray slice_ba BoxArray slice_ba(slice_box); - slice_ba.maxSize(max_box_size_); + slice_ba.maxSize(m_max_box_size_); tmp_slice_ptr = std::unique_ptr<MultiFab>(new MultiFab(slice_ba, - LabFrameDiags_[i]->data_buffer_->DistributionMap(), + m_LabFrameDiags_[i]->m_data_buffer_->DistributionMap(), ncomp, 0)); // slice is re-used if the t_lab of a diag is equal to @@ -867,7 +865,7 @@ writeLabFrameData(const MultiFab* cell_centered_data, // tmp_slice_ptr which has the dmap of the // data_buffer that stores the back-transformed data. tmp_slice_ptr->copy(*slice, 0, 0, ncomp); - LabFrameDiags_[i]->AddDataToBuffer(*tmp_slice_ptr, i_lab, + m_LabFrameDiags_[i]->AddDataToBuffer(*tmp_slice_ptr, i_lab, map_actual_fields_to_dump); tmp_slice_ptr.reset(new MultiFab); tmp_slice_ptr.reset(nullptr); @@ -875,44 +873,43 @@ writeLabFrameData(const MultiFab* cell_centered_data, if (WarpX::do_back_transformed_particles) { - if (LabFrameDiags_[i]->t_lab != prev_t_lab ) { + if (m_LabFrameDiags_[i]->m_t_lab != prev_t_lab ) { if (tmp_particle_buffer.size()>0) { tmp_particle_buffer.clear(); tmp_particle_buffer.shrink_to_fit(); } tmp_particle_buffer.resize(mypc.nSpeciesBackTransformedDiagnostics()); - mypc.GetLabFrameData( LabFrameDiags_[i]->file_name, i_lab, - boost_direction_, old_z_boost, - LabFrameDiags_[i]->current_z_boost, - t_boost, LabFrameDiags_[i]->t_lab, dt, - tmp_particle_buffer); + mypc.GetLabFrameData(m_LabFrameDiags_[i]->m_file_name, i_lab, + m_boost_direction_, old_z_boost, + m_LabFrameDiags_[i]->m_current_z_boost, + t_boost, m_LabFrameDiags_[i]->m_t_lab, dt, + tmp_particle_buffer); } - LabFrameDiags_[i]->AddPartDataToParticleBuffer(tmp_particle_buffer, + m_LabFrameDiags_[i]->AddPartDataToParticleBuffer(tmp_particle_buffer, mypc.nSpeciesBackTransformedDiagnostics()); } - ++LabFrameDiags_[i]->buff_counter_; - prev_t_lab = LabFrameDiags_[i]->t_lab; - + ++m_LabFrameDiags_[i]->m_buff_counter_; + prev_t_lab = m_LabFrameDiags_[i]->m_t_lab; // If buffer full, write to disk. - if ( LabFrameDiags_[i]->buff_counter_ == num_buffer_) { + if (m_LabFrameDiags_[i]->m_buff_counter_ == m_num_buffer_) { if (WarpX::do_back_transformed_fields) { #ifdef WARPX_USE_HDF5 - Box buff_box = LabFrameDiags_[i]->buff_box_; - for (int comp = 0; comp < LabFrameDiags_[i]->data_buffer_->nComp(); ++comp) - output_write_field( LabFrameDiags_[i]->file_name, - mesh_field_names[comp], - *LabFrameDiags_[i]->data_buffer_, comp, - lbound(buff_box).x, lbound(buff_box).y, - lbound(buff_box).z); + Box buff_box = m_LabFrameDiags_[i]->m_buff_box_; + for (int comp = 0; comp < m_LabFrameDiags_[i]->m_data_buffer_->nComp(); ++comp) + output_write_field(m_LabFrameDiags_[i]->m_file_name, + m_mesh_field_names[comp], + *m_LabFrameDiags_[i]->m_data_buffer_, comp, + lbound(buff_box).x, lbound(buff_box).y, + lbound(buff_box).z); #else std::stringstream mesh_ss; - mesh_ss << LabFrameDiags_[i]->file_name << "/Level_0/" << + mesh_ss << m_LabFrameDiags_[i]->m_file_name << "/Level_0/" << Concatenate("buffer", i_lab, 5); - VisMF::Write( (*LabFrameDiags_[i]->data_buffer_), mesh_ss.str()); + VisMF::Write( (*m_LabFrameDiags_[i]->m_data_buffer_), mesh_ss.str()); #endif } @@ -924,23 +921,23 @@ writeLabFrameData(const MultiFab* cell_centered_data, mypc.mapSpeciesBackTransformedDiagnostics(j)]; #ifdef WARPX_USE_HDF5 // Write data to disk (HDF5) - writeParticleDataHDF5(LabFrameDiags_[i]->particles_buffer_[j], - LabFrameDiags_[i]->file_name, + writeParticleDataHDF5(m_LabFrameDiags_[i]->m_particles_buffer_[j], + m_LabFrameDiags_[i]->m_file_name, species_name); #else std::stringstream part_ss; - part_ss << LabFrameDiags_[i]->file_name + "/" + + part_ss << m_LabFrameDiags_[i]->m_file_name + "/" + species_name + "/"; // Write data to disk (custom) - writeParticleData(LabFrameDiags_[i]->particles_buffer_[j], + writeParticleData(m_LabFrameDiags_[i]->m_particles_buffer_[j], part_ss.str(), i_lab); #endif } - LabFrameDiags_[i]->particles_buffer_.clear(); + m_LabFrameDiags_[i]->m_particles_buffer_.clear(); } - LabFrameDiags_[i]->buff_counter_ = 0; + m_LabFrameDiags_[i]->m_buff_counter_ = 0; } } @@ -1069,12 +1066,12 @@ writeMetaData () HeaderFile.precision(17); - HeaderFile << N_snapshots_ << "\n"; - HeaderFile << dt_snapshots_lab_ << "\n"; - HeaderFile << gamma_boost_ << "\n"; - HeaderFile << beta_boost_ << "\n"; + HeaderFile << m_N_snapshots_ << "\n"; + HeaderFile << m_dt_snapshots_lab_ << "\n"; + HeaderFile << m_gamma_boost_ << "\n"; + HeaderFile << m_beta_boost_ << "\n"; - if (N_slice_snapshots_ > 0) { + if (m_N_slice_snapshots_ > 0) { const std::string fullpath_slice = WarpX::lab_data_directory + "/slices"; if (!UtilCreateDirectory(fullpath_slice, 0755)) CreateDirectoryFailed(fullpath_slice); @@ -1095,10 +1092,10 @@ writeMetaData () HeaderFile_slice.precision(17); - HeaderFile_slice << N_slice_snapshots_ << "\n"; - HeaderFile_slice << dt_slice_snapshots_lab_ << "\n"; - HeaderFile_slice << gamma_boost_ << "\n"; - HeaderFile_slice << beta_boost_ << "\n"; + HeaderFile_slice << m_N_slice_snapshots_ << "\n"; + HeaderFile_slice << m_dt_slice_snapshots_lab_ << "\n"; + HeaderFile_slice << m_gamma_boost_ << "\n"; + HeaderFile_slice << m_beta_boost_ << "\n"; } @@ -1114,35 +1111,35 @@ LabFrameSnapShot(Real t_lab_in, Real t_boost, Real inv_gamma_boost_in, std::vector<std::string> mesh_field_names, amrex::RealBox diag_domain_lab, Box diag_box, int file_num_in) { - t_lab = t_lab_in; - dz_lab_ = dz_lab_in; - inv_gamma_boost_ = inv_gamma_boost_in; - inv_beta_boost_ = inv_beta_boost_in; - prob_domain_lab_ = prob_domain_lab; - prob_ncells_lab_ = prob_ncells_lab; - diag_domain_lab_ = diag_domain_lab; - buff_box_ = diag_box; - ncomp_to_dump_ = ncomp_to_dump; - mesh_field_names_ = mesh_field_names; - file_num = file_num_in; - current_z_lab = 0.0; - current_z_boost = 0.0; - updateCurrentZPositions(t_boost, inv_gamma_boost_, inv_beta_boost_); - Real zmin_lab = prob_domain_lab_.lo(AMREX_SPACEDIM-1); - initial_i = (current_z_lab - zmin_lab) / dz_lab_ ; - file_name = Concatenate(WarpX::lab_data_directory + "/snapshots/snapshot", - file_num, 5); + m_t_lab = t_lab_in; + m_dz_lab_ = dz_lab_in; + m_inv_gamma_boost_ = inv_gamma_boost_in; + m_inv_beta_boost_ = inv_beta_boost_in; + m_prob_domain_lab_ = prob_domain_lab; + m_prob_ncells_lab_ = prob_ncells_lab; + m_diag_domain_lab_ = diag_domain_lab; + m_buff_box_ = diag_box; + m_ncomp_to_dump_ = ncomp_to_dump; + m_mesh_field_names_ = mesh_field_names; + m_file_num = file_num_in; + m_current_z_lab = 0.0; + m_current_z_boost = 0.0; + updateCurrentZPositions(t_boost, m_inv_gamma_boost_, m_inv_beta_boost_); + Real zmin_lab = m_prob_domain_lab_.lo(AMREX_SPACEDIM-1); + m_initial_i = (m_current_z_lab - zmin_lab) / m_dz_lab_ ; + m_file_name = Concatenate(WarpX::lab_data_directory + "/snapshots/snapshot", + m_file_num, 5); createLabFrameDirectories(); - buff_counter_ = 0; - if (WarpX::do_back_transformed_fields) data_buffer_.reset(nullptr); + m_buff_counter_ = 0; + if (WarpX::do_back_transformed_fields) m_data_buffer_.reset(nullptr); } void LabFrameDiag:: updateCurrentZPositions(Real t_boost, Real inv_gamma, Real inv_beta) { - current_z_boost = (t_lab*inv_gamma - t_boost)*PhysConst::c*inv_beta; - current_z_lab = (t_lab - t_boost*inv_gamma)*PhysConst::c*inv_beta; + m_current_z_boost = (m_t_lab*inv_gamma - t_boost)*PhysConst::c*inv_beta; + m_current_z_lab = (m_t_lab - t_boost*inv_gamma)*PhysConst::c*inv_beta; } void @@ -1151,7 +1148,7 @@ createLabFrameDirectories() { #ifdef WARPX_USE_HDF5 if (ParallelDescriptor::IOProcessor()) { - output_create(file_name); + output_create(m_file_name); } ParallelDescriptor::Barrier(); @@ -1160,16 +1157,16 @@ createLabFrameDirectories() { { if (WarpX::do_back_transformed_fields) { - const auto lo = lbound(buff_box_); - for (int comp = 0; comp < ncomp_to_dump_; ++comp) { - output_create_field(file_name, mesh_field_names_[comp], - prob_ncells_lab_[0], + const auto lo = lbound(m_buff_box_); + for (int comp = 0; comp < m_ncomp_to_dump_; ++comp) { + output_create_field(m_file_name, m_mesh_field_names_[comp], + m_prob_ncells_lab_[0], #if ( AMREX_SPACEDIM == 3 ) - prob_ncells_lab_[1], + m_prob_ncells_lab_[1], #else 1, #endif - prob_ncells_lab_[AMREX_SPACEDIM-1]+1); + m_prob_ncells_lab_[AMREX_SPACEDIM-1]+1); } } } @@ -1185,23 +1182,23 @@ createLabFrameDirectories() { // Loop over species to be dumped to BFD std::string species_name = species_names[mypc.mapSpeciesBackTransformedDiagnostics(j)]; - output_create_species_group(file_name, species_name); + output_create_species_group(m_file_name, species_name); for (int k = 0; k < static_cast<int>(particle_field_names.size()); ++k) { std::string field_path = species_name + "/" + particle_field_names[k]; - output_create_particle_field(file_name, field_path); + output_create_particle_field(m_file_name, field_path); } } } #else if (ParallelDescriptor::IOProcessor()) { - if (!UtilCreateDirectory(file_name, 0755)) - CreateDirectoryFailed(file_name); + if (!UtilCreateDirectory(m_file_name, 0755)) + CreateDirectoryFailed(m_file_name); const int nlevels = 1; for(int i = 0; i < nlevels; ++i) { - const std::string &fullpath = LevelFullPath(i, file_name); + const std::string &fullpath = LevelFullPath(i, m_file_name); if (!UtilCreateDirectory(fullpath, 0755)) CreateDirectoryFailed(fullpath); } @@ -1215,7 +1212,7 @@ createLabFrameDirectories() { // Get species name std::string species_name = species_names[mypc.mapSpeciesBackTransformedDiagnostics(i)]; - const std::string fullpath = file_name + "/" + species_name; + const std::string fullpath = m_file_name + "/" + species_name; if (!UtilCreateDirectory(fullpath, 0755)) CreateDirectoryFailed(fullpath); } @@ -1234,7 +1231,7 @@ writeLabFrameHeader() { VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size); std::ofstream HeaderFile; HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size()); - std::string HeaderFileName(file_name + "/Header"); + std::string HeaderFileName(m_file_name + "/Header"); HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out | std::ofstream::trunc | std::ofstream::binary); @@ -1243,30 +1240,30 @@ writeLabFrameHeader() { HeaderFile.precision(17); - HeaderFile << t_lab << "\n"; + HeaderFile << m_t_lab << "\n"; // Write domain number of cells - HeaderFile << prob_ncells_lab_[0] << ' ' + HeaderFile << m_prob_ncells_lab_[0] << ' ' #if ( AMREX_SPACEDIM==3 ) - << prob_ncells_lab_[1] << ' ' + << m_prob_ncells_lab_[1] << ' ' #endif - << prob_ncells_lab_[AMREX_SPACEDIM-1] <<'\n'; + << m_prob_ncells_lab_[AMREX_SPACEDIM-1] <<'\n'; // Write domain physical boundaries // domain lower bound - HeaderFile << diag_domain_lab_.lo(0) << ' ' + HeaderFile << m_diag_domain_lab_.lo(0) << ' ' #if ( AMREX_SPACEDIM==3 ) - << diag_domain_lab_.lo(1) << ' ' + << m_diag_domain_lab_.lo(1) << ' ' #endif - << diag_domain_lab_.lo(AMREX_SPACEDIM-1) <<'\n'; + << m_diag_domain_lab_.lo(AMREX_SPACEDIM-1) <<'\n'; // domain higher bound - HeaderFile << diag_domain_lab_.hi(0) << ' ' + HeaderFile << m_diag_domain_lab_.hi(0) << ' ' #if ( AMREX_SPACEDIM==3 ) - << diag_domain_lab_.hi(1) << ' ' + << m_diag_domain_lab_.hi(1) << ' ' #endif - << diag_domain_lab_.hi(AMREX_SPACEDIM-1) <<'\n'; + << m_diag_domain_lab_.hi(AMREX_SPACEDIM-1) <<'\n'; // List of fields dumped to file - for (int i=0; i<ncomp_to_dump_; i++) + for (int i=0; i<m_ncomp_to_dump_; i++) { - HeaderFile << mesh_field_names_[i] << ' '; + HeaderFile << m_mesh_field_names_[i] << ' '; } HeaderFile << "\n"; } @@ -1281,28 +1278,30 @@ LabFrameSlice(Real t_lab_in, Real t_boost, Real inv_gamma_boost_in, IntVect prob_ncells_lab, int ncomp_to_dump, std::vector<std::string> mesh_field_names, RealBox diag_domain_lab, Box diag_box, int file_num_in, - amrex::Real cell_dx, amrex::Real cell_dy) + amrex::Real particle_slice_dx_lab) { - t_lab = t_lab_in; - prob_domain_lab_ = prob_domain_lab; - prob_ncells_lab_ = prob_ncells_lab; - diag_domain_lab_ = diag_domain_lab; - buff_box_ = diag_box; - ncomp_to_dump_ = ncomp_to_dump; - mesh_field_names_ = mesh_field_names; - file_num = file_num_in; - current_z_lab = 0.0; - current_z_boost = 0.0; - updateCurrentZPositions(t_boost, inv_gamma_boost_, inv_beta_boost_); - Real zmin_lab = prob_domain_lab_.lo(AMREX_SPACEDIM-1); - initial_i = (current_z_lab - zmin_lab)/dz_lab_; - file_name = Concatenate(WarpX::lab_data_directory+"/slices/slice",file_num,5); + m_t_lab = t_lab_in; + m_dz_lab_ = dz_lab_in; + m_inv_gamma_boost_ = inv_gamma_boost_in; + m_inv_beta_boost_ = inv_beta_boost_in; + m_prob_domain_lab_ = prob_domain_lab; + m_prob_ncells_lab_ = prob_ncells_lab; + m_diag_domain_lab_ = diag_domain_lab; + m_buff_box_ = diag_box; + m_ncomp_to_dump_ = ncomp_to_dump; + m_mesh_field_names_ = mesh_field_names; + m_file_num = file_num_in; + m_current_z_lab = 0.0; + m_current_z_boost = 0.0; + updateCurrentZPositions(t_boost, m_inv_gamma_boost_, m_inv_beta_boost_); + Real zmin_lab = m_prob_domain_lab_.lo(AMREX_SPACEDIM-1); + m_initial_i = (m_current_z_lab - zmin_lab)/m_dz_lab_; + m_file_name = Concatenate(WarpX::lab_data_directory+"/slices/slice",m_file_num,5); createLabFrameDirectories(); - buff_counter_ = 0; - dx_ = cell_dx; - dy_ = cell_dy; + m_buff_counter_ = 0; + m_particle_slice_dx_lab_ = particle_slice_dx_lab; - if (WarpX::do_back_transformed_fields) data_buffer_.reset(nullptr); + if (WarpX::do_back_transformed_fields) m_data_buffer_.reset(nullptr); } void @@ -1311,7 +1310,7 @@ AddDataToBuffer( MultiFab& tmp, int k_lab, amrex::Gpu::ManagedDeviceVector<int> map_actual_fields_to_dump) { const int ncomp_to_dump = map_actual_fields_to_dump.size(); - MultiFab& buf = *data_buffer_; + MultiFab& buf = *m_data_buffer_; for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) { Array4<Real> tmp_arr = tmp[mfi].array(); Array4<Real> buf_arr = buf[mfi].array(); @@ -1340,10 +1339,10 @@ AddDataToBuffer( MultiFab& tmp, int k_lab, amrex::Gpu::ManagedDeviceVector<int> map_actual_fields_to_dump) { const int ncomp_to_dump = map_actual_fields_to_dump.size(); - MultiFab& buf = *data_buffer_; + MultiFab& buf = *m_data_buffer_; for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) { - Box& bx = buff_box_; + Box& bx = m_buff_box_; const Box& bx_bf = mfi.tilebox(); bx.setSmall(AMREX_SPACEDIM-1,bx_bf.smallEnd(AMREX_SPACEDIM-1)); bx.setBig(AMREX_SPACEDIM-1,bx_bf.bigEnd(AMREX_SPACEDIM-1)); @@ -1374,40 +1373,56 @@ AddPartDataToParticleBuffer( auto np = tmp_particle_buffer[isp].GetRealData(DiagIdx::w).size(); if (np == 0) return; - particles_buffer_[isp].GetRealData(DiagIdx::w).insert( - particles_buffer_[isp].GetRealData(DiagIdx::w).end(), - tmp_particle_buffer[isp].GetRealData(DiagIdx::w).begin(), - tmp_particle_buffer[isp].GetRealData(DiagIdx::w).end()); - - particles_buffer_[isp].GetRealData(DiagIdx::x).insert( - particles_buffer_[isp].GetRealData(DiagIdx::x).end(), - tmp_particle_buffer[isp].GetRealData(DiagIdx::x).begin(), - tmp_particle_buffer[isp].GetRealData(DiagIdx::x).end()); - - particles_buffer_[isp].GetRealData(DiagIdx::y).insert( - particles_buffer_[isp].GetRealData(DiagIdx::y).end(), - tmp_particle_buffer[isp].GetRealData(DiagIdx::y).begin(), - tmp_particle_buffer[isp].GetRealData(DiagIdx::y).end()); - - particles_buffer_[isp].GetRealData(DiagIdx::z).insert( - particles_buffer_[isp].GetRealData(DiagIdx::z).end(), - tmp_particle_buffer[isp].GetRealData(DiagIdx::z).begin(), - tmp_particle_buffer[isp].GetRealData(DiagIdx::z).end()); - - particles_buffer_[isp].GetRealData(DiagIdx::ux).insert( - particles_buffer_[isp].GetRealData(DiagIdx::ux).end(), - tmp_particle_buffer[isp].GetRealData(DiagIdx::ux).begin(), - tmp_particle_buffer[isp].GetRealData(DiagIdx::ux).end()); - - particles_buffer_[isp].GetRealData(DiagIdx::uy).insert( - particles_buffer_[isp].GetRealData(DiagIdx::uy).end(), - tmp_particle_buffer[isp].GetRealData(DiagIdx::uy).begin(), - tmp_particle_buffer[isp].GetRealData(DiagIdx::uy).end()); - - particles_buffer_[isp].GetRealData(DiagIdx::uz).insert( - particles_buffer_[isp].GetRealData(DiagIdx::uz).end(), - tmp_particle_buffer[isp].GetRealData(DiagIdx::uz).begin(), - tmp_particle_buffer[isp].GetRealData(DiagIdx::uz).end()); + // allocate size of particle buffer array to np + // This is a growing array. Each time we add np elements + // to the existing array which has size = init_size + const int init_size = m_particles_buffer_[isp].GetRealData(DiagIdx::w).size(); + const int total_size = init_size + np; + m_particles_buffer_[isp].resize(total_size); + + // Data pointers to particle attributes // + Real* const AMREX_RESTRICT wp_buff = + m_particles_buffer_[isp].GetRealData(DiagIdx::w).data(); + Real* const AMREX_RESTRICT x_buff = + m_particles_buffer_[isp].GetRealData(DiagIdx::x).data(); + Real* const AMREX_RESTRICT y_buff = + m_particles_buffer_[isp].GetRealData(DiagIdx::y).data(); + Real* const AMREX_RESTRICT z_buff = + m_particles_buffer_[isp].GetRealData(DiagIdx::z).data(); + Real* const AMREX_RESTRICT ux_buff = + m_particles_buffer_[isp].GetRealData(DiagIdx::ux).data(); + Real* const AMREX_RESTRICT uy_buff = + m_particles_buffer_[isp].GetRealData(DiagIdx::uy).data(); + Real* const AMREX_RESTRICT uz_buff = + m_particles_buffer_[isp].GetRealData(DiagIdx::uz).data(); + + Real const* const AMREX_RESTRICT wp_temp = + tmp_particle_buffer[isp].GetRealData(DiagIdx::w).data(); + Real const* const AMREX_RESTRICT x_temp = + tmp_particle_buffer[isp].GetRealData(DiagIdx::x).data(); + Real const* const AMREX_RESTRICT y_temp = + tmp_particle_buffer[isp].GetRealData(DiagIdx::y).data(); + Real const* const AMREX_RESTRICT z_temp = + tmp_particle_buffer[isp].GetRealData(DiagIdx::z).data(); + Real const* const AMREX_RESTRICT ux_temp = + tmp_particle_buffer[isp].GetRealData(DiagIdx::ux).data(); + Real const* const AMREX_RESTRICT uy_temp = + tmp_particle_buffer[isp].GetRealData(DiagIdx::uy).data(); + Real const* const AMREX_RESTRICT uz_temp = + tmp_particle_buffer[isp].GetRealData(DiagIdx::uz).data(); + + // copy all the particles from tmp to buffer + amrex::ParallelFor(np, + [=] AMREX_GPU_DEVICE(int i) + { + wp_buff[init_size + i] = wp_temp[i]; + x_buff[init_size + i] = x_temp[i]; + y_buff[init_size + i] = y_temp[i]; + z_buff[init_size + i] = z_temp[i]; + ux_buff[init_size + i] = ux_temp[i]; + uy_buff[init_size + i] = uy_temp[i]; + uz_buff[init_size + i] = uz_temp[i]; + }); } } @@ -1415,53 +1430,107 @@ void LabFrameSlice:: AddPartDataToParticleBuffer( Vector<WarpXParticleContainer::DiagnosticParticleData> tmp_particle_buffer, - int nSpeciesBoostedFrame) { - for (int isp = 0; isp < nSpeciesBoostedFrame; ++isp) { + int nSpeciesBackTransformedDiagnostics) { + + + for (int isp = 0; isp < nSpeciesBackTransformedDiagnostics; ++isp) { auto np = tmp_particle_buffer[isp].GetRealData(DiagIdx::w).size(); if (np == 0) return; - auto const& wpc = tmp_particle_buffer[isp].GetRealData(DiagIdx::w); - auto const& xpc = tmp_particle_buffer[isp].GetRealData(DiagIdx::x); - auto const& ypc = tmp_particle_buffer[isp].GetRealData(DiagIdx::y); - auto const& zpc = tmp_particle_buffer[isp].GetRealData(DiagIdx::z); - auto const& uxpc = tmp_particle_buffer[isp].GetRealData(DiagIdx::ux); - auto const& uypc = tmp_particle_buffer[isp].GetRealData(DiagIdx::uy); - auto const& uzpc = tmp_particle_buffer[isp].GetRealData(DiagIdx::uz); - - particles_buffer_[isp].resize(np); - auto wpc_buff = particles_buffer_[isp].GetRealData(DiagIdx::w); - auto xpc_buff = particles_buffer_[isp].GetRealData(DiagIdx::x); - auto ypc_buff = particles_buffer_[isp].GetRealData(DiagIdx::y); - auto zpc_buff = particles_buffer_[isp].GetRealData(DiagIdx::z); - auto uxpc_buff = particles_buffer_[isp].GetRealData(DiagIdx::ux); - auto uypc_buff = particles_buffer_[isp].GetRealData(DiagIdx::uy); - auto uzpc_buff = particles_buffer_[isp].GetRealData(DiagIdx::uz); - - - int partcounter = 0; - for (int i = 0; i < np; ++i) - { - if( xpc[i] >= (diag_domain_lab_.lo(0)-dx_) && - xpc[i] <= (diag_domain_lab_.hi(0)+dx_) ) { - #if (AMREX_SPACEDIM == 3) - if( ypc[i] >= (diag_domain_lab_.lo(1)-dy_) && - ypc[i] <= (diag_domain_lab_.hi(1) + dy_)) - #endif - { - wpc_buff[partcounter] = wpc[i]; - xpc_buff[partcounter] = xpc[i]; - ypc_buff[partcounter] = ypc[i]; - zpc_buff[partcounter] = zpc[i]; - uxpc_buff[partcounter] = uxpc[i]; - uypc_buff[partcounter] = uypc[i]; - uzpc_buff[partcounter] = uzpc[i]; - ++partcounter; - } - } - } + Real const* const AMREX_RESTRICT wp_temp = + tmp_particle_buffer[isp].GetRealData(DiagIdx::w).data(); + Real const* const AMREX_RESTRICT x_temp = + tmp_particle_buffer[isp].GetRealData(DiagIdx::x).data(); + Real const* const AMREX_RESTRICT y_temp = + tmp_particle_buffer[isp].GetRealData(DiagIdx::y).data(); + Real const* const AMREX_RESTRICT z_temp = + tmp_particle_buffer[isp].GetRealData(DiagIdx::z).data(); + Real const* const AMREX_RESTRICT ux_temp = + tmp_particle_buffer[isp].GetRealData(DiagIdx::ux).data(); + Real const* const AMREX_RESTRICT uy_temp = + tmp_particle_buffer[isp].GetRealData(DiagIdx::uy).data(); + Real const* const AMREX_RESTRICT uz_temp = + tmp_particle_buffer[isp].GetRealData(DiagIdx::uz).data(); + + // temporary arrays to store copy_flag and copy_index + // for particles that cross the reduced domain for diagnostics. + amrex::Gpu::ManagedDeviceVector<int> FlagForPartCopy(np); + amrex::Gpu::ManagedDeviceVector<int> IndexForPartCopy(np); + + int* const AMREX_RESTRICT Flag = FlagForPartCopy.dataPtr(); + int* const AMREX_RESTRICT IndexLocation = IndexForPartCopy.dataPtr(); + + // Compute extent of the reduced domain +/- user-defined physical width + Real const xmin = m_diag_domain_lab_.lo(0)-m_particle_slice_dx_lab_; + Real const xmax = m_diag_domain_lab_.hi(0)+m_particle_slice_dx_lab_; +#if (AMREX_SPACEDIM == 3) + Real const ymin = m_diag_domain_lab_.lo(1)-m_particle_slice_dx_lab_; + Real const ymax = m_diag_domain_lab_.hi(1)+m_particle_slice_dx_lab_; +#endif - particles_buffer_[isp].resize(partcounter); + //Flag particles that need to be copied if they are + // within the reduced slice +/- user-defined physical width + amrex::ParallelFor(np, + [=] AMREX_GPU_DEVICE(int i) + { + Flag[i] = 0; + if ( x_temp[i] >= (xmin) && + x_temp[i] <= (xmax) ) { +#if (AMREX_SPACEDIM == 3) + if (y_temp[i] >= (ymin) && + y_temp[i] <= (ymax) ) +#endif + { + Flag[i] = 1; + } + } + }); + + // Call exclusive scan to obtain location indices using + // flag values. These location indices are used to copy data + // from src to dst when the copy-flag is set to 1. + amrex::Gpu::exclusive_scan(Flag,Flag+np,IndexLocation); + const int copy_size = IndexLocation[np-1] + Flag[np-1]; + const int init_size = m_particles_buffer_[isp].GetRealData(DiagIdx::w).size(); + const int total_reducedDiag_size = copy_size + init_size; + + // allocate array size for reduced diagnostic buffer array + m_particles_buffer_[isp].resize(total_reducedDiag_size); + + // Data pointers to particle attributes // + Real* const AMREX_RESTRICT wp_buff = + m_particles_buffer_[isp].GetRealData(DiagIdx::w).data(); + Real* const AMREX_RESTRICT x_buff = + m_particles_buffer_[isp].GetRealData(DiagIdx::x).data(); + Real* const AMREX_RESTRICT y_buff = + m_particles_buffer_[isp].GetRealData(DiagIdx::y).data(); + Real* const AMREX_RESTRICT z_buff = + m_particles_buffer_[isp].GetRealData(DiagIdx::z).data(); + Real* const AMREX_RESTRICT ux_buff = + m_particles_buffer_[isp].GetRealData(DiagIdx::ux).data(); + Real* const AMREX_RESTRICT uy_buff = + m_particles_buffer_[isp].GetRealData(DiagIdx::uy).data(); + Real* const AMREX_RESTRICT uz_buff = + m_particles_buffer_[isp].GetRealData(DiagIdx::uz).data(); + + // Selective copy of particle data from tmp array to reduced buffer + // array on the GPU using the flag value and index location. + amrex::ParallelFor(np, + [=] AMREX_GPU_DEVICE(int i) + { + if (Flag[i] == 1) + { + const int loc = IndexLocation[i] + init_size; + wp_buff[loc] = wp_temp[i]; + x_buff[loc] = x_temp[i]; + y_buff[loc] = y_temp[i]; + z_buff[loc] = z_temp[i]; + ux_buff[loc] = ux_temp[i]; + uy_buff[loc] = uy_temp[i]; + uz_buff[loc] = uz_temp[i]; + } + }); } } diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index 3b7481b8c..c08d58d36 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -1,4 +1,3 @@ - #include <MultiParticleContainer.H> #include <WarpX.H> @@ -172,7 +171,8 @@ PhysicalParticleContainer::ConvertUnits(ConvertDirection convert_direction) factor = 1./mass; } - for (int lev=0; lev<=finestLevel(); lev++){ + const int nLevels = finestLevel(); + for (int lev=0; lev<=nLevels; lev++){ #ifdef _OPENMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif diff --git a/Source/Evolve/WarpXEvolveES.cpp b/Source/Evolve/WarpXEvolveES.cpp index effd6ec96..61a8a5f32 100644 --- a/Source/Evolve/WarpXEvolveES.cpp +++ b/Source/Evolve/WarpXEvolveES.cpp @@ -179,30 +179,6 @@ void WarpX::zeroOutBoundary(amrex::MultiFab& input_data, bndry_data.FillBoundary(); } -void WarpX::sumFineToCrseNodal(const amrex::MultiFab& fine, - amrex::MultiFab& crse, - const amrex::Geometry& cgeom, - const amrex::IntVect& ratio) { - const BoxArray& fine_BA = fine.boxArray(); - const DistributionMapping& fine_dm = fine.DistributionMap(); - BoxArray coarsened_fine_BA = fine_BA; - coarsened_fine_BA.coarsen(ratio); - - MultiFab coarsened_fine_data(coarsened_fine_BA, fine_dm, 1, 0); - coarsened_fine_data.setVal(0.0); - - for (MFIter mfi(coarsened_fine_data); mfi.isValid(); ++mfi) { - const Box& bx = mfi.validbox(); - const Box& crse_box = coarsened_fine_data[mfi].box(); - const Box& fine_box = fine[mfi].box(); - WRPX_SUM_FINE_TO_CRSE_NODAL(bx.loVect(), bx.hiVect(), ratio.getVect(), - coarsened_fine_data[mfi].dataPtr(), crse_box.loVect(), crse_box.hiVect(), - fine[mfi].dataPtr(), fine_box.loVect(), fine_box.hiVect()); - } - - crse.copy(coarsened_fine_data, cgeom.periodicity(), FabArrayBase::ADD); -} - void WarpX::fixRHSForSolve(Vector<std::unique_ptr<MultiFab> >& rhs, const Vector<std::unique_ptr<FabArray<BaseFab<int> > > >& masks) const { @@ -242,104 +218,3 @@ void WarpX::getLevelMasks(Vector<std::unique_ptr<FabArray<BaseFab<int> > > >& ma } } } - - -void WarpX::computePhi(const Vector<std::unique_ptr<MultiFab> >& rho, - Vector<std::unique_ptr<MultiFab> >& phi) const { - - - int num_levels = rho.size(); - Vector<std::unique_ptr<MultiFab> > rhs(num_levels); - for (int lev = 0; lev < num_levels; ++lev) { - phi[lev]->setVal(0.0, 2); - rhs[lev].reset(new MultiFab(rho[lev]->boxArray(), dmap[lev], 1, 0)); - MultiFab::Copy(*rhs[lev], *rho[lev], 0, 0, 1, 0); - rhs[lev]->mult(-1.0/PhysConst::ep0, 0); - } - - fixRHSForSolve(rhs, masks); - - bool nodal = true; - bool have_rhcc = false; - int nc = 0; - int Ncomp = 1; - int stencil = ND_CROSS_STENCIL; - int verbose = 0; - Vector<int> mg_bc(2*AMREX_SPACEDIM, 1); // this means Dirichlet - Real rel_tol = 1.0e-14; - Real abs_tol = 1.0e-14; - - Vector<Geometry> level_geom(1); - Vector<BoxArray> level_grids(1); - Vector<DistributionMapping> level_dm(1); - Vector<MultiFab*> level_phi(1); - Vector<MultiFab*> level_rhs(1); - - for (int lev = 0; lev < num_levels; ++lev) { - level_phi[0] = phi[lev].get(); - level_rhs[0] = rhs[lev].get(); - level_geom[0] = geom[lev]; - level_grids[0] = grids[lev]; - level_dm[0] = dmap[lev]; - - MGT_Solver solver(level_geom, mg_bc.dataPtr(), level_grids, - level_dm, nodal, - stencil, have_rhcc, nc, Ncomp, verbose); - - solver.set_nodal_const_coefficients(1.0); - - solver.solve_nodal(level_phi, level_rhs, rel_tol, abs_tol); - - if (lev < num_levels-1) { - - NoOpPhysBC cphysbc, fphysbc; -#if AMREX_SPACEDIM == 3 - int lo_bc[] = {BCType::int_dir, BCType::int_dir, BCType::int_dir}; - int hi_bc[] = {BCType::int_dir, BCType::int_dir, BCType::int_dir}; -#else - int lo_bc[] = {BCType::int_dir, BCType::int_dir}; - int hi_bc[] = {BCType::int_dir, BCType::int_dir}; -#endif - Vector<BCRec> bcs(1, BCRec(lo_bc, hi_bc)); - NodeBilinear mapper; - - amrex::InterpFromCoarseLevel(*phi[lev+1], 0.0, *phi[lev], - 0, 0, 1, geom[lev], geom[lev+1], - cphysbc, fphysbc, - IntVect(AMREX_D_DECL(2, 2, 2)), &mapper, bcs); - } - } - - for (int lev = 0; lev < num_levels; ++lev) { - const Geometry& gm = geom[lev]; - phi[lev]->FillBoundary(gm.periodicity()); - } -} - -void WarpX::computeE(Vector<std::array<std::unique_ptr<MultiFab>, 3> >& E, - const Vector<std::unique_ptr<MultiFab> >& phi) const { - - const int num_levels = E.size(); - for (int lev = 0; lev < num_levels; ++lev) { - const auto& gm = GetInstance().Geom(lev); - const Real* dx = gm.CellSize(); - for (MFIter mfi(*phi[lev]); mfi.isValid(); ++mfi) { - const Box& bx = mfi.validbox(); - - WRPX_COMPUTE_E_NODAL(bx.loVect(), bx.hiVect(), - (*phi[lev] )[mfi].dataPtr(), - (*E[lev][0])[mfi].dataPtr(), - (*E[lev][1])[mfi].dataPtr(), -#if AMREX_SPACEDIM == 3 - (*E[lev][2])[mfi].dataPtr(), -#endif - dx); - } - - E[lev][0]->FillBoundary(gm.periodicity()); - E[lev][1]->FillBoundary(gm.periodicity()); -#if AMREX_SPACEDIM == 3 - E[lev][2]->FillBoundary(gm.periodicity()); -#endif - } -} diff --git a/Source/FieldSolver/Make.package b/Source/FieldSolver/Make.package index 076be41bb..018cfbfba 100644 --- a/Source/FieldSolver/Make.package +++ b/Source/FieldSolver/Make.package @@ -7,12 +7,6 @@ ifeq ($(USE_PSATD),TRUE) include $(WARPX_HOME)/Source/FieldSolver/PicsarHybridSpectralSolver/Make.package endif endif -ifeq ($(USE_OPENBC_POISSON),TRUE) - F90EXE_sources += openbc_poisson_solver.F90 -endif -ifeq ($DO_ELECTROSTATIC,TRUE) - F90EXE_sources += solve_E_nodal.F90 -endif INCLUDE_LOCATIONS += $(WARPX_HOME)/Source/FieldSolver VPATH_LOCATIONS += $(WARPX_HOME)/Source/FieldSolver diff --git a/Source/FieldSolver/openbc_poisson_solver.F90 b/Source/FieldSolver/openbc_poisson_solver.F90 deleted file mode 100644 index 8e962a2ef..000000000 --- a/Source/FieldSolver/openbc_poisson_solver.F90 +++ /dev/null @@ -1,62 +0,0 @@ - -module warpx_openbc_module - - implicit none - - integer, parameter :: idecomp = 0 ! 0=xyz, 1=xy, 2=yz, 3=xz, 4=x, 5=y, 6=z. - integer, parameter :: igfflag = 1 ! =0 for ordinary 1/r Green function; - ! =1 for integrated Green function - - integer, save :: gb_lo(3), gb_hi(3) - integer, save :: lc_lo(3), lc_hi(3) - -contains - - subroutine warpx_openbc_decompose(glo, ghi, lo, hi) bind(c,name='warpx_openbc_decompose') - - use mpi - - integer, intent(in) :: glo(3), ghi(3) - integer, intent(out) :: lo(3), hi(3) - integer :: myrank, mprocs, ierr, npx, npy, npz - - call MPI_Comm_size(MPI_COMM_WORLD, mprocs, ierr); - call MPI_COMM_RANK(MPI_COMM_WORLD,myrank,ierr) - - gb_lo = glo + 1 ! +1 because AMReX's domain index starts with 0 - gb_hi = ghi + 2 ! +2 to nodalize it - - call procgriddecomp(mprocs, gb_lo(1),gb_hi(1), & - & gb_lo(2),gb_hi(2), & - & gb_lo(3),gb_hi(3), & - & idecomp, npx, npy, npz) - - call decompose(myrank,mprocs, gb_lo(1),gb_hi(1), & - & gb_lo(2),gb_hi(2), & - & gb_lo(3),gb_hi(3), & - idecomp, lc_lo(1), lc_hi(1), & - & lc_lo(2), lc_hi(2), & - & lc_lo(3), lc_hi(3)) - - lo = lc_lo - 1 ! AMReX's domain index starts with zero. - hi = lc_hi - 1 - - end subroutine warpx_openbc_decompose - - subroutine warpx_openbc_potential(rho, phi, dx) & - bind(C, name="warpx_openbc_potential") - - double precision, intent(in ) :: dx(3) - double precision, intent(in ) :: rho(lc_lo(1):lc_hi(1),lc_lo(2):lc_hi(2),lc_lo(3):lc_hi(3)) - double precision, intent(out) :: phi(lc_lo(1):lc_hi(1),lc_lo(2):lc_hi(2),lc_lo(3):lc_hi(3)) - - integer :: ierr - - call openbcpotential(rho, phi, dx(1), dx(2), dx(3), & - lc_lo(1), lc_hi(1), lc_lo(2), lc_hi(2), lc_lo(3), lc_hi(3), & - gb_lo(1), gb_hi(1), gb_lo(2), gb_hi(2), gb_lo(3), gb_hi(3), & - idecomp, igfflag, ierr) - - end subroutine warpx_openbc_potential - -end module warpx_openbc_module diff --git a/Source/FieldSolver/solve_E_nodal.F90 b/Source/FieldSolver/solve_E_nodal.F90 deleted file mode 100644 index 4ed9bb845..000000000 --- a/Source/FieldSolver/solve_E_nodal.F90 +++ /dev/null @@ -1,73 +0,0 @@ -module warpx_ES_solve_E_nodal - - use iso_c_binding - use amrex_fort_module, only : amrex_real - - implicit none - -contains - -! This routine computes the node-centered electric field given a node-centered phi. -! The gradient is computed using 2nd-order centered differences. It assumes the -! Boundary conditions have already been set and that you have two rows of ghost cells -! for phi and one row of ghost cells for Ex, Ey, and Ez. -! Note that this routine includes the minus sign in E = - grad phi. -! -! Arguments: -! lo, hi: The corners of the valid box over which the gradient is taken -! Ex, Ey, Ez: The electric field in the x, y, and z directions. -! dx: The cell spacing -! - subroutine warpx_compute_E_nodal_3d (lo, hi, phi, Ex, Ey, Ez, dx) & - bind(c,name='warpx_compute_E_nodal_3d') - integer(c_int), intent(in) :: lo(3), hi(3) - real(amrex_real), intent(in) :: dx(3) - real(amrex_real), intent(in ) :: phi(lo(1)-2:hi(1)+2,lo(2)-2:hi(2)+2,lo(3)-2:hi(3)+2) - real(amrex_real), intent(inout) :: Ex (lo(1)-1:hi(1)+1,lo(2)-1:hi(2)+1,lo(3)-1:hi(3)+1) - real(amrex_real), intent(inout) :: Ey (lo(1)-1:hi(1)+1,lo(2)-1:hi(2)+1,lo(3)-1:hi(3)+1) - real(amrex_real), intent(inout) :: Ez (lo(1)-1:hi(1)+1,lo(2)-1:hi(2)+1,lo(3)-1:hi(3)+1) - - integer :: i, j, k - real(amrex_real) :: fac(3) - - fac = 0.5d0 / dx - - do k = lo(3)-1, hi(3)+1 - do j = lo(2)-1, hi(2)+1 - do i = lo(1)-1, hi(1)+1 - - Ex(i,j,k) = fac(1) * (phi(i-1,j,k) - phi(i+1,j,k)) - Ey(i,j,k) = fac(2) * (phi(i,j-1,k) - phi(i,j+1,k)) - Ez(i,j,k) = fac(3) * (phi(i,j,k-1) - phi(i,j,k+1)) - - end do - end do - end do - - end subroutine warpx_compute_E_nodal_3d - - subroutine warpx_compute_E_nodal_2d (lo, hi, phi, Ex, Ey, dx) & - bind(c,name='warpx_compute_E_nodal_2d') - integer(c_int), intent(in) :: lo(2), hi(2) - real(amrex_real), intent(in) :: dx(2) - real(amrex_real), intent(in ) :: phi(lo(1)-2:hi(1)+2,lo(2)-2:hi(2)+2) - real(amrex_real), intent(inout) :: Ex (lo(1)-1:hi(1)+1,lo(2)-1:hi(2)+1) - real(amrex_real), intent(inout) :: Ey (lo(1)-1:hi(1)+1,lo(2)-1:hi(2)+1) - - integer :: i, j - real(amrex_real) :: fac(2) - - fac = 0.5d0 / dx - - do j = lo(2)-1, hi(2)+1 - do i = lo(1)-1, hi(1)+1 - - Ex(i,j) = fac(1) * (phi(i-1,j) - phi(i+1,j)) - Ey(i,j) = fac(2) * (phi(i,j-1) - phi(i,j+1)) - - end do - end do - - end subroutine warpx_compute_E_nodal_2d - -end module warpx_ES_solve_E_nodal diff --git a/Source/Filter/Filter.H b/Source/Filter/Filter.H index 5eb84b96a..1ecb1bff4 100644 --- a/Source/Filter/Filter.H +++ b/Source/Filter/Filter.H @@ -1,5 +1,4 @@ #include <AMReX_MultiFab.H> -#include <AMReX_CudaContainers.H> #ifndef WARPX_FILTER_H_ #define WARPX_FILTER_H_ diff --git a/Source/FortranInterface/WarpX_f.H b/Source/FortranInterface/WarpX_f.H index b51a83387..e32719adf 100644 --- a/Source/FortranInterface/WarpX_f.H +++ b/Source/FortranInterface/WarpX_f.H @@ -16,11 +16,8 @@ #if (AMREX_SPACEDIM == 3) -#define WRPX_SUM_FINE_TO_CRSE_NODAL warpx_sum_fine_to_crse_nodal_3d #define WRPX_ZERO_OUT_BNDRY warpx_zero_out_bndry_3d #define WRPX_BUILD_MASK warpx_build_mask_3d -#define WRPX_COMPUTE_E_NODAL warpx_compute_E_nodal_3d -#define WRPX_DEPOSIT_CIC warpx_deposit_cic_3d #define WRPX_INTERPOLATE_CIC warpx_interpolate_cic_3d #define WRPX_INTERPOLATE_CIC_TWO_LEVELS warpx_interpolate_cic_two_levels_3d #define WRPX_PUSH_LEAPFROG warpx_push_leapfrog_3d @@ -28,11 +25,8 @@ #elif (AMREX_SPACEDIM == 2) -#define WRPX_SUM_FINE_TO_CRSE_NODAL warpx_sum_fine_to_crse_nodal_2d #define WRPX_ZERO_OUT_BNDRY warpx_zero_out_bndry_2d #define WRPX_BUILD_MASK warpx_build_mask_2d -#define WRPX_COMPUTE_E_NODAL warpx_compute_E_nodal_2d -#define WRPX_DEPOSIT_CIC warpx_deposit_cic_2d #define WRPX_INTERPOLATE_CIC warpx_interpolate_cic_2d #define WRPX_INTERPOLATE_CIC_TWO_LEVELS warpx_interpolate_cic_two_levels_2d #define WRPX_PUSH_LEAPFROG warpx_push_leapfrog_2d @@ -45,26 +39,9 @@ extern "C" { #endif -#ifdef USE_OPENBC_POISSON - void warpx_openbc_potential (amrex::Real* rho, amrex::Real* phi, const amrex::Real* dx); - void warpx_openbc_decompose (const int*, const int*, int*, int*); -#endif - - void warpx_compute_E (const int* lo, const int* hi, - const BL_FORT_FAB_ARG_3D(phi), - BL_FORT_FAB_ARG_3D(Ex), - BL_FORT_FAB_ARG_3D(Ey), - BL_FORT_FAB_ARG_3D(Ez), - const amrex::Real* dx); - /// /// These functions are used in electrostatic mode. /// - - void WRPX_SUM_FINE_TO_CRSE_NODAL(const int* lo, const int* hi, const int* lrat, - amrex::Real* crse, const int* clo, const int* chi, - const amrex::Real* fine, const int* flo, const int* fhi); - void WRPX_ZERO_OUT_BNDRY(const int* lo, const int* hi, amrex::Real* input_data, amrex::Real* bndry_data, const int* mask); @@ -72,21 +49,6 @@ extern "C" void WRPX_BUILD_MASK(const int* lo, const int* hi, const int* tmp_mask, int* mask, const int* ncells); - void WRPX_COMPUTE_E_NODAL(const int* lo, const int* hi, - const amrex::Real* phi, - amrex::Real* Ex, amrex::Real* Ey, -#if (AMREX_SPACEDIM == 3) - amrex::Real* Ez, -#endif - const amrex::Real* dx); - - void WRPX_DEPOSIT_CIC(const amrex::ParticleReal* particles, int ns, int np, - const amrex::ParticleReal* weights, - const amrex::Real* charge, - amrex::Real* rho, const int* lo, const int* hi, - const amrex::Real* plo, const amrex::Real* dx, - const int* ng); - void WRPX_INTERPOLATE_CIC_TWO_LEVELS(const amrex::ParticleReal* particles, int ns, int np, amrex::Real* Ex_p, amrex::Real* Ey_p, #if (AMREX_SPACEDIM == 3) @@ -138,93 +100,6 @@ extern "C" const amrex::Real* dt, const amrex::Real* prob_lo, const amrex::Real* prob_hi); -// These functions are used to evolve E and B in the PML - - void WRPX_PUSH_PML_BVEC(const int* xlo, const int* xhi, - const int* ylo, const int* yhi, - const int* zlo, const int* zhi, - const BL_FORT_FAB_ARG_3D(ex), - const BL_FORT_FAB_ARG_3D(ey), - const BL_FORT_FAB_ARG_3D(ez), - BL_FORT_FAB_ARG_3D(bx), - BL_FORT_FAB_ARG_3D(by), - BL_FORT_FAB_ARG_3D(bz), - const amrex::Real* dtsdx, - const amrex::Real* dtsdy, - const amrex::Real* dtsdz, - const int* maxwell_fdtd_solver_id); - - - void WRPX_PUSH_PML_EVEC(const int* xlo, const int* xhi, - const int* ylo, const int* yhi, - const int* zlo, const int* zhi, - BL_FORT_FAB_ARG_3D(ex), - BL_FORT_FAB_ARG_3D(ey), - BL_FORT_FAB_ARG_3D(ez), - const BL_FORT_FAB_ARG_3D(bx), - const BL_FORT_FAB_ARG_3D(by), - const BL_FORT_FAB_ARG_3D(bz), - const amrex::Real* dtsdx, - const amrex::Real* dtsdy, - const amrex::Real* dtsdz); - - void WRPX_PUSH_PML_EVEC_F(const int* xlo, const int* xhi, - const int* ylo, const int* yhi, - const int* zlo, const int* zhi, - BL_FORT_FAB_ARG_3D(ex), - BL_FORT_FAB_ARG_3D(ey), - BL_FORT_FAB_ARG_3D(ez), - const BL_FORT_FAB_ARG_3D(f), - const amrex::Real* dtsdx, - const amrex::Real* dtsdy, - const amrex::Real* dtsdz, - const int* maxwell_fdtd_solver_id); - - void WRPX_PUSH_PML_F(const int* lo, const int* hi, - BL_FORT_FAB_ARG_3D(f), - const BL_FORT_FAB_ARG_3D(ex), - const BL_FORT_FAB_ARG_3D(ey), - const BL_FORT_FAB_ARG_3D(ez), - const amrex::Real* dtsdx, - const amrex::Real* dtsdy, - const amrex::Real* dtsdz); - - void WRPX_DAMP_PML (const int* texlo, const int* texhi, - const int* teylo, const int* teyhi, - const int* tezlo, const int* tezhi, - const int* tbxlo, const int* tbxhi, - const int* tbylo, const int* tbyhi, - const int* tbzlo, const int* tbzhi, - amrex::Real* ex, const int* exlo, const int* exhi, - amrex::Real* ey, const int* eylo, const int* eyhi, - amrex::Real* ez, const int* ezlo, const int* ezhi, - amrex::Real* bx, const int* bxlo, const int* bxhi, - amrex::Real* by, const int* bylo, const int* byhi, - amrex::Real* bz, const int* bzlo, const int* bzhi, - const amrex::Real* sigex, int sigex_lo, int sigex_hi, -#if (AMREX_SPACEDIM == 3) - const amrex::Real* sigey, int sigey_lo, int sigey_hi, -#endif - const amrex::Real* sigez, int sigez_lo, int sigez_hi, - const amrex::Real* sigbx, int sigbx_lo, int sigbx_hi, -#if (AMREX_SPACEDIM == 3) - const amrex::Real* sigby, int sigby_lo, int sigby_hi, -#endif - const amrex::Real* sigbz, int sigbz_lo, int sigbz_hi); - - void WRPX_DAMP_PML_F (const int* tndlo, const int* tndhi, - amrex::Real* F, const int* flo, const int* fhi, - const amrex::Real* sigex, int sigex_lo, int sigex_hi, -#if (AMREX_SPACEDIM == 3) - const amrex::Real* sigey, int sigey_lo, int sigey_hi, -#endif - const amrex::Real* sigez, int sigez_lo, int sigez_hi, - const amrex::Real* sigbx, int sigbx_lo, int sigbx_hi, -#if (AMREX_SPACEDIM == 3) - const amrex::Real* sigby, int sigby_lo, int sigby_hi, -#endif - const amrex::Real* sigbz, int sigbz_lo, int sigbz_hi); - #ifdef WARPX_USE_PSATD void warpx_fft_mpi_init (int fcomm); void warpx_fft_domain_decomp (int* warpx_local_nz, int* warpx_local_z0, diff --git a/Source/Initialization/InitSpaceChargeField.cpp b/Source/Initialization/InitSpaceChargeField.cpp new file mode 100644 index 000000000..0a873b742 --- /dev/null +++ b/Source/Initialization/InitSpaceChargeField.cpp @@ -0,0 +1,303 @@ + +#include <AMReX_ParallelDescriptor.H> +#include <AMReX_MLMG.H> +#include <AMReX_MLNodeTensorLaplacian.H> + +#include <WarpX.H> + +using namespace amrex; + +void +WarpX::InitSpaceChargeField (WarpXParticleContainer& pc) +{ + +#ifdef WARPX_DIM_RZ + amrex::Abort("The initialization of space-charge field has not yet been implemented in RZ geometry."); +#endif + + // Allocate fields for charge and potential + const int num_levels = max_level + 1; + Vector<std::unique_ptr<MultiFab> > rho(num_levels); + Vector<std::unique_ptr<MultiFab> > phi(num_levels); + const int ng = WarpX::nox; + for (int lev = 0; lev <= max_level; lev++) { + BoxArray nba = boxArray(lev); + nba.surroundingNodes(); + rho[lev].reset(new MultiFab(nba, dmap[lev], 1, ng)); // Make ng big enough/use rho from sim + phi[lev].reset(new MultiFab(nba, dmap[lev], 1, 1)); + phi[lev]->setVal(0.); + } + + // Deposit particle charge density (source of Poisson solver) + bool const local = false; + bool const reset = true; + bool const do_rz_volume_scaling = true; + pc.DepositCharge(rho, local, reset, do_rz_volume_scaling); + + // Get the particle beta vector + bool const local_average = false; // Average across all MPI ranks + std::array<Real, 3> beta = pc.meanParticleVelocity(local_average); + for (Real& beta_comp : beta) beta_comp /= PhysConst::c; // Normalize + + // Compute the potential phi, by solving the Poisson equation + computePhi( rho, phi, beta, pc.self_fields_required_precision ); + + // Compute the corresponding electric and magnetic field, from the potential phi + computeE( Efield_fp, phi, beta ); + computeB( Bfield_fp, phi, beta ); + +} + +/* Compute the potential `phi` by solving the Poisson equation with `rho` as + a source, assuming that the source moves at a constant speed \f$\vec{\beta}\f$. + This uses the amrex solver. + + More specifically, this solves the equation + \f[ + \vec{\nabla}^2\phi - (\vec{\beta}\cdot\vec{\nabla})^2\phi = -\frac{\rho}{\epsilon_0} + \f] + + \param[in] rho The charge density a given species + \param[out] phi The potential to be computed by this function + \param[in] beta Represents the velocity of the source of `phi` +*/ +void +WarpX::computePhi (const amrex::Vector<std::unique_ptr<amrex::MultiFab> >& rho, + amrex::Vector<std::unique_ptr<amrex::MultiFab> >& phi, + std::array<Real, 3> const beta, + Real const required_precision) const +{ + // Define the boundary conditions + Array<LinOpBCType,AMREX_SPACEDIM> lobc, hibc; + for (int idim=0; idim<AMREX_SPACEDIM; idim++){ + if ( Geom(0).isPeriodic(idim) ) { + lobc[idim] = LinOpBCType::Periodic; + hibc[idim] = LinOpBCType::Periodic; + } else { + // Use Dirichlet boundary condition by default. + // Ideally, we would often want open boundary conditions here. + lobc[idim] = LinOpBCType::Dirichlet; + hibc[idim] = LinOpBCType::Dirichlet; + } + } + + // Define the linear operator (Poisson operator) + MLNodeTensorLaplacian linop( Geom(), boxArray(), DistributionMap() ); + linop.setDomainBC( lobc, hibc ); + // Set the value of beta + amrex::Array<amrex::Real,AMREX_SPACEDIM> beta_solver = +#if (AMREX_SPACEDIM==2) + {{ beta[0], beta[2] }}; // beta_x and beta_z +#else + {{ beta[0], beta[1], beta[2] }}; +#endif + linop.setBeta( beta_solver ); + + // Solve the Poisson equation + MLMG mlmg(linop); + mlmg.setVerbose(2); + mlmg.solve( GetVecOfPtrs(phi), GetVecOfConstPtrs(rho), required_precision, 0.0); + + // Normalize by the correct physical constant + for (int lev=0; lev < rho.size(); lev++){ + phi[lev]->mult(-1./PhysConst::ep0); + } +} + +/* \bried Compute the electric field that corresponds to `phi`, and + add it to the set of MultiFab `E`. + + The electric field is calculated by assuming that the source that + produces the `phi` potential is moving with a constant speed \f$\vec{\beta}\f$: + \f[ + \vec{E} = -\vec{\nabla}\phi + (\vec{\beta}\cdot\vec{\beta})\phi \vec{\beta} + \f] + (where the second term represent the term \f$\partial_t \vec{A}\f$, in + the case of a moving source) + + \param[inout] E Electric field on the grid + \param[in] phi The potential from which to compute the electric field + \param[in] beta Represents the velocity of the source of `phi` +*/ +void +WarpX::computeE (amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>, 3> >& E, + const amrex::Vector<std::unique_ptr<amrex::MultiFab> >& phi, + std::array<amrex::Real, 3> const beta ) const +{ + for (int lev = 0; lev <= max_level; lev++) { + + const Real* dx = Geom(lev).CellSize(); + +#ifdef _OPENMP + #pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for ( MFIter mfi(*phi[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi ) + { + const Real inv_dx = 1./dx[0]; +#if (AMREX_SPACEDIM == 3) + const Real inv_dy = 1./dx[1]; + const Real inv_dz = 1./dx[2]; +#else + const Real inv_dz = 1./dx[1]; +#endif + const Box& tbx = mfi.tilebox(Ex_nodal_flag); + const Box& tby = mfi.tilebox(Ey_nodal_flag); + const Box& tbz = mfi.tilebox(Ez_nodal_flag); + + const auto& phi_arr = phi[lev]->array(mfi); + const auto& Ex_arr = (*E[lev][0])[mfi].array(); + const auto& Ey_arr = (*E[lev][1])[mfi].array(); + const auto& Ez_arr = (*E[lev][2])[mfi].array(); + + Real beta_x = beta[0]; + Real beta_y = beta[1]; + Real beta_z = beta[2]; + + // Calculate the electric field + // Use discretized derivative that matches the staggering of the grid. +#if (AMREX_SPACEDIM == 3) + amrex::ParallelFor( tbx, tby, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Ex_arr(i,j,k) += + +(beta_x*beta_x-1)*inv_dx*( phi_arr(i+1,j,k)-phi_arr(i,j,k) ) + +beta_x*beta_y*0.25*inv_dy*(phi_arr(i ,j+1,k)-phi_arr(i ,j-1,k) + + phi_arr(i+1,j+1,k)-phi_arr(i+1,j-1,k)) + +beta_x*beta_z*0.25*inv_dz*(phi_arr(i ,j,k+1)-phi_arr(i ,j,k-1) + + phi_arr(i+1,j,k+1)-phi_arr(i+1,j,k-1)); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Ey_arr(i,j,k) += + +beta_y*beta_x*0.25*inv_dx*(phi_arr(i+1,j ,k)-phi_arr(i-1,j ,k) + + phi_arr(i+1,j+1,k)-phi_arr(i-1,j+1,k)) + +(beta_y*beta_y-1)*inv_dy*( phi_arr(i,j+1,k)-phi_arr(i,j,k) ) + +beta_y*beta_z*0.25*inv_dz*(phi_arr(i,j ,k+1)-phi_arr(i,j ,k-1) + + phi_arr(i,j+1,k+1)-phi_arr(i,j+1,k-1)); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Ez_arr(i,j,k) += + +beta_z*beta_x*0.25*inv_dx*(phi_arr(i+1,j,k )-phi_arr(i-1,j,k ) + + phi_arr(i+1,j,k+1)-phi_arr(i-1,j,k+1)) + +beta_z*beta_y*0.25*inv_dy*(phi_arr(i,j+1,k )-phi_arr(i,j-1,k ) + + phi_arr(i,j+1,k+1)-phi_arr(i,j-1,k+1)) + +(beta_y*beta_z-1)*inv_dz*( phi_arr(i,j,k+1)-phi_arr(i,j,k) ); + } + ); +#else + amrex::ParallelFor( tbx, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Ex_arr(i,j,k) += + +(beta_x*beta_x-1)*inv_dx*( phi_arr(i+1,j,k)-phi_arr(i,j,k) ) + +beta_x*beta_z*0.25*inv_dz*(phi_arr(i ,j+1,k)-phi_arr(i ,j-1,k) + + phi_arr(i+1,j+1,k)-phi_arr(i+1,j-1,k)); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Ez_arr(i,j,k) += + +beta_z*beta_x*0.25*inv_dx*(phi_arr(i+1,j ,k)-phi_arr(i-1,j ,k) + + phi_arr(i+1,j+1,k)-phi_arr(i-1,j+1,k)) + +(beta_y*beta_z-1)*inv_dz*( phi_arr(i,j+1,k)-phi_arr(i,j,k) ); + } + ); +#endif + } + } +} + + +/* \bried Compute the magnetic field that corresponds to `phi`, and + add it to the set of MultiFab `B`. + + The magnetic field is calculated by assuming that the source that + produces the `phi` potential is moving with a constant speed \f$\vec{\beta}\f$: + \f[ + \vec{B} = -\frac{1}{c}\vec{\beta}\times\vec{\nabla}\phi + \f] + (this represents the term \f$\vec{\nabla} \times \vec{A}\f$, in the case of a moving source) + + \param[inout] E Electric field on the grid + \param[in] phi The potential from which to compute the electric field + \param[in] beta Represents the velocity of the source of `phi` +*/ +void +WarpX::computeB (amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>, 3> >& B, + const amrex::Vector<std::unique_ptr<amrex::MultiFab> >& phi, + std::array<amrex::Real, 3> const beta ) const +{ + for (int lev = 0; lev <= max_level; lev++) { + + const Real* dx = Geom(lev).CellSize(); + +#ifdef _OPENMP + #pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for ( MFIter mfi(*phi[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi ) + { + const Real inv_dx = 1./dx[0]; +#if (AMREX_SPACEDIM == 3) + const Real inv_dy = 1./dx[1]; + const Real inv_dz = 1./dx[2]; +#else + const Real inv_dz = 1./dx[1]; +#endif + const Box& tbx = mfi.tilebox(Bx_nodal_flag); + const Box& tby = mfi.tilebox(By_nodal_flag); + const Box& tbz = mfi.tilebox(Bz_nodal_flag); + + const auto& phi_arr = phi[0]->array(mfi); + const auto& Bx_arr = (*B[lev][0])[mfi].array(); + const auto& By_arr = (*B[lev][1])[mfi].array(); + const auto& Bz_arr = (*B[lev][2])[mfi].array(); + + Real beta_x = beta[0]; + Real beta_y = beta[1]; + Real beta_z = beta[2]; + + constexpr Real inv_c = 1./PhysConst::c; + + // Calculate the magnetic field + // Use discretized derivative that matches the staggering of the grid. +#if (AMREX_SPACEDIM == 3) + amrex::ParallelFor( tbx, tby, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Bx_arr(i,j,k) += inv_c * ( + -beta_y*inv_dz*0.5*(phi_arr(i,j ,k+1)-phi_arr(i,j ,k) + + phi_arr(i,j+1,k+1)-phi_arr(i,j+1,k)) + +beta_z*inv_dy*0.5*(phi_arr(i,j+1,k )-phi_arr(i,j,k ) + + phi_arr(i,j+1,k+1)-phi_arr(i,j,k+1))); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + By_arr(i,j,k) += inv_c * ( + -beta_z*inv_dx*0.5*(phi_arr(i+1,j,k )-phi_arr(i,j,k ) + + phi_arr(i+1,j,k+1)-phi_arr(i,j,k+1)) + +beta_x*inv_dz*0.5*(phi_arr(i ,j,k+1)-phi_arr(i ,j,k) + + phi_arr(i+1,j,k+1)-phi_arr(i+1,j,k))); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Bz_arr(i,j,k) += inv_c * ( + -beta_x*inv_dy*0.5*(phi_arr(i ,j+1,k)-phi_arr(i ,j,k) + + phi_arr(i+1,j+1,k)-phi_arr(i+1,j,k)) + +beta_y*inv_dx*0.5*(phi_arr(i+1,j ,k)-phi_arr(i,j ,k) + + phi_arr(i+1,j+1,k)-phi_arr(i,j+1,k))); + } + ); +#else + amrex::ParallelFor( tbx, tby, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Bx_arr(i,j,k) += inv_c * ( + -beta_y*inv_dz*( phi_arr(i,j+1,k)-phi_arr(i,j,k) )); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + By_arr(i,j,k) += inv_c * ( + -beta_z*inv_dx*0.5*(phi_arr(i+1,j ,k)-phi_arr(i,j ,k) + + phi_arr(i+1,j+1,k)-phi_arr(i,j+1,k)) + +beta_x*inv_dz*0.5*(phi_arr(i ,j+1,k)-phi_arr(i ,j,k) + + phi_arr(i+1,j+1,k)-phi_arr(i+1,j,k))); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + Bz_arr(i,j,k) += inv_c * ( + +beta_y*inv_dx*( phi_arr(i+1,j,k)-phi_arr(i,j,k) )); + } + ); +#endif + } + } +} diff --git a/Source/Initialization/InjectorMomentum.H b/Source/Initialization/InjectorMomentum.H index ba0d26fc4..5e0494d92 100644 --- a/Source/Initialization/InjectorMomentum.H +++ b/Source/Initialization/InjectorMomentum.H @@ -47,6 +47,140 @@ private: amrex::Real m_ux_th, m_uy_th, m_uz_th; }; +// struct whose getMomentum returns momentum for 1 particle with relativistic +// drift velocity beta, from the Maxwell-Boltzmann distribution. +struct InjectorMomentumBoltzmann +{ + // Constructor whose inputs are: + // the temperature parameter theta, + // boost velocity/c beta, + // and boost direction dir respectively. + InjectorMomentumBoltzmann(amrex::Real t, amrex::Real b, int d) noexcept + : vave(std::sqrt(2*t)), beta(b), dir(d) + {} + + AMREX_GPU_HOST_DEVICE + amrex::XDim3 + getMomentum (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept + { + amrex::Real x1, x2, gamma; + amrex::Real u[3]; + x1 = amrex::Random(); + x2 = amrex::Random(); + // Each value of sqrt(-log(x1))*sin(2*pi*x2) is a sample from a Gaussian + // distribution with sigma = average velocity / c + // using the Box-Mueller Method. + u[(dir+1)%3] = vave*std::sqrt(-std::log(x1)) *std::sin(2*M_PI*x2); + u[(dir+2)%3] = vave*std::sqrt(-std::log(x1)) *std::cos(2*M_PI*x2); + u[dir] = vave*std::sqrt(-std::log(amrex::Random()))* + std::sin(2*M_PI*amrex::Random()); + gamma = std::pow(u[0],2)+std::pow(u[1],2)+std::pow(u[2],2); + gamma = std::sqrt(1+gamma); + // The following condition is equtaion 32 in Zenitani 2015 + // (Phys. Plasmas 22, 042116) , called the flipping method. It + // transforms the intergral: d3x' -> d3x where d3x' is the volume + // element for positions in the boosted frame. The particle positions + // and densities can be initialized in the simulation frame. + // The flipping method can transform any symmetric distribution from one + // reference frame to another moving at a relative velocity of beta. + // An equivalent alternative to this method native to WarpX would be to + // initialize the particle positions and densities in the frame moving + // at speed beta, and then perform a Lorentz transform on the positions + // and MB sampled velocities to the simulation frame. + x1 = amrex::Random(); + if(-beta*u[dir]/gamma > x1) + { + u[dir] = -u[dir]; + } + // This Lorentz transform is equation 17 in Zenitani. + // It transforms the integral d3u' -> d3u + // where d3u' is the volume element for momentum in the boosted frame. + u[dir] = 1/std::sqrt(1-pow(beta,2))*(u[dir]+gamma*beta); + // Note that if beta = 0 then the flipping method and Lorentz transform + // have no effect on the u[dir] direction. + return amrex::XDim3 {u[0],u[1],u[2]}; + } + +private: + int dir; + amrex::Real beta, vave; +}; + +// struct whose getMomentum returns momentum for 1 particle with relativistc +// drift velocity beta, from the Maxwell-Juttner distribution. Method is from +// Zenitani 2015 (Phys. Plasmas 22, 042116). +struct InjectorMomentumJuttner +{ + // Constructor whose inputs are: + // the temperature parameter theta, + // boost velocity/c beta, + // and boost direction dir respectively. + InjectorMomentumJuttner(amrex::Real t, amrex::Real b, int d) noexcept + : theta(t), beta(b), dir(d) + {} + + AMREX_GPU_HOST_DEVICE + amrex::XDim3 + getMomentum (amrex::Real x, amrex::Real y, amrex::Real z) const noexcept + { + // Sobol method for sampling MJ Speeds, + // from Zenitani 2015 (Phys. Plasmas 22, 042116). + amrex::Real x1, x2, gamma; + amrex::Real u [3]; + x1 = 0.; + gamma = 0.; + u[dir] = 0.; + // This condition is equation 10 in Zenitani, + // though x1 is defined differently. + while(u[dir]-gamma <= x1) + { + u[dir] = -theta* + std::log(amrex::Random()*amrex::Random()*amrex::Random()); + gamma = std::sqrt(1+std::pow(u[dir],2)); + x1 = theta*std::log(amrex::Random()); + } + // The following code samples a random unit vector + // and multiplies the result by speed u[dir]. + x1 = amrex::Random(); + x2 = amrex::Random(); + // Direction dir is an input parameter that sets the boost direction: + // 'x' -> d = 0, 'y' -> d = 1, 'z' -> d = 2. + u[(dir+1)%3] = 2*u[dir]*std::sqrt(x1*(1-x1))*std::sin(2*M_PI*x2); + u[(dir+2)%3] = 2*u[dir]*std::sqrt(x1*(1-x1))*std::cos(2*M_PI*x2); + // The value of dir is the boost direction to be transformed. + u[dir] = u[dir]*(2*x1-1); + x1 = amrex::Random(); + // The following condition is equtaion 32 in Zenitani, called + // The flipping method. It transforms the intergral: d3x' -> d3x + // where d3x' is the volume element for positions in the boosted frame. + // The particle positions and densities can be initialized in the + // simulation frame with this method. + // The flipping method can similarly transform any + // symmetric distribution from one reference frame to another moving at + // a relative velocity of beta. + // An equivalent alternative to this method native to WarpX + // would be to initialize the particle positions and densities in the + // frame moving at speed beta, and then perform a Lorentz transform + // on their positions and MJ sampled velocities to the simulation frame. + if(-beta*u[dir]/gamma>x1) + { + u[dir] = -u[dir]; + } + // This Lorentz transform is equation 17 in Zenitani. + // It transforms the integral d3u' -> d3u + // where d3u' is the volume element for momentum in the boosted frame. + u[dir] = 1/std::sqrt(1-pow(beta,2))*(u[dir]+gamma*beta); + // Note that if beta = 0 then the flipping method and Lorentz transform + // have no effect on the u[dir] direction. + return amrex::XDim3 {u[0],u[1],u[2]}; + } + +private: + int dir; + amrex::Real beta, theta; +}; + + // struct whose getMomentum returns momentum for 1 particle, for // radial expansion struct InjectorMomentumRadialExpansion @@ -124,6 +258,19 @@ struct InjectorMomentum object(t,a_ux_m,a_uy_m,a_uz_m,a_ux_th,a_uy_th,a_uz_th) { } + InjectorMomentum (InjectorMomentumBoltzmann* t, + amrex::Real theta, amrex::Real beta, int dir) + : type(Type::boltzmann), + object(t, theta, beta, dir) + { } + + // This constructor stores a InjectorMomentumJuttner in union object. + InjectorMomentum (InjectorMomentumJuttner* t, + amrex::Real theta, amrex::Real beta, int dir) + : type(Type::juttner), + object(t, theta, beta, dir) + { } + // This constructor stores a InjectorMomentumCustom in union object. InjectorMomentum (InjectorMomentumCustom* t, std::string const& a_species_name) @@ -165,6 +312,14 @@ struct InjectorMomentum { return object.gaussian.getMomentum(x,y,z); } + case Type::boltzmann: + { + return object.boltzmann.getMomentum(x,y,z); + } + case Type::juttner: + { + return object.juttner.getMomentum(x,y,z); + } case Type::constant: { return object.constant.getMomentum(x,y,z); @@ -186,7 +341,7 @@ struct InjectorMomentum } private: - enum struct Type { constant, custom, gaussian, radial_expansion, parser }; + enum struct Type { constant, custom, gaussian, boltzmann, juttner, radial_expansion, parser}; Type type; // An instance of union Object constructs and stores any one of @@ -204,6 +359,12 @@ private: amrex::Real a_uz_m, amrex::Real a_ux_th, amrex::Real a_uy_th, amrex::Real a_uz_th) noexcept : gaussian(a_ux_m,a_uy_m,a_uz_m,a_ux_th,a_uy_th,a_uz_th) {} + Object (InjectorMomentumBoltzmann*, + amrex::Real t, amrex::Real b, int dir) noexcept + : boltzmann(t,b,dir) {} + Object (InjectorMomentumJuttner*, + amrex::Real t, amrex::Real b, int dir) noexcept + : juttner(t,b,dir) {} Object (InjectorMomentumRadialExpansion*, amrex::Real u_over_r) noexcept : radial_expansion(u_over_r) {} @@ -215,6 +376,8 @@ private: InjectorMomentumConstant constant; InjectorMomentumCustom custom; InjectorMomentumGaussian gaussian; + InjectorMomentumBoltzmann boltzmann; + InjectorMomentumJuttner juttner; InjectorMomentumRadialExpansion radial_expansion; InjectorMomentumParser parser; }; diff --git a/Source/Initialization/Make.package b/Source/Initialization/Make.package index 2c6458b6d..0b65c1ab1 100644 --- a/Source/Initialization/Make.package +++ b/Source/Initialization/Make.package @@ -14,5 +14,7 @@ CEXE_sources += InjectorMomentum.cpp CEXE_headers += CustomDensityProb.H CEXE_headers += CustomMomentumProb.H +CEXE_sources += InitSpaceChargeField.cpp + INCLUDE_LOCATIONS += $(WARPX_HOME)/Source/Initialization VPATH_LOCATIONS += $(WARPX_HOME)/Source/Initialization diff --git a/Source/Initialization/PlasmaInjector.cpp b/Source/Initialization/PlasmaInjector.cpp index af04c6b31..300c41f7b 100644 --- a/Source/Initialization/PlasmaInjector.cpp +++ b/Source/Initialization/PlasmaInjector.cpp @@ -269,6 +269,50 @@ void PlasmaInjector::parseMomentum (ParmParse& pp) // Construct InjectorMomentum with InjectorMomentumGaussian. inj_mom.reset(new InjectorMomentum((InjectorMomentumGaussian*)nullptr, ux_m, uy_m, uz_m, ux_th, uy_th, uz_th)); + } else if (mom_dist_s == "maxwell_boltzmann"){ + Real beta = 0.; + Real theta = 10.; + int dir = 0; + std::string direction = "x"; + pp.query("beta", beta); + pp.query("theta", theta); + pp.query("direction", direction); + if(direction == "x" || direction == "X"){ + dir = 0; + } else if (direction == "y" || direction == "Y"){ + dir = 1; + } else if (direction == "z" || direction == "Z"){ + dir = 2; + } else{ + std::stringstream stringstream; + stringstream << "Direction " << direction << " is not recognzied. Please enter x, y, or z."; + direction = stringstream.str(); + amrex::Abort(direction.c_str()); + } + // Construct InjectorMomentum with InjectorMomentumBoltzmann. + inj_mom.reset(new InjectorMomentum((InjectorMomentumBoltzmann*)nullptr, theta, beta, dir)); + } else if (mom_dist_s == "maxwell_juttner"){ + Real beta = 0.; + Real theta = 10.; + int dir = 0; + std::string direction = "x"; + pp.query("beta", beta); + pp.query("theta", theta); + pp.query("direction", direction); + if(direction == "x" || direction == "X"){ + dir = 0; + } else if (direction == "y" || direction == "Y"){ + dir = 1; + } else if (direction == "z" || direction == "Z"){ + dir = 2; + } else{ + std::stringstream stringstream; + stringstream << "Direction " << direction << " is not recognzied. Please enter x, y, or z."; + direction = stringstream.str(); + amrex::Abort(direction.c_str()); + } + // Construct InjectorMomentum with InjectorMomentumJuttner. + inj_mom.reset(new InjectorMomentum((InjectorMomentumJuttner*)nullptr, theta, beta, dir)); } else if (mom_dist_s == "radial_expansion") { Real u_over_r = 0.; pp.query("u_over_r", u_over_r); diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 0814f369b..29c9a8955 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -97,7 +97,8 @@ WarpX::InitDiagnostics () { num_slice_snapshots_lab, gamma_boost, t_new[0], dt_boost, moving_window_dir, geom[0], - slice_realbox)); + slice_realbox, + particle_slice_width_lab)); } } @@ -111,9 +112,13 @@ WarpX::InitFromScratch () mypc->AllocData(); mypc->InitData(); -#ifdef USE_OPENBC_POISSON - InitOpenbc(); -#endif + // Loop through species and calculate their space-charge field + for (int ispecies=0; ispecies<mypc->nSpecies(); ispecies++){ + WarpXParticleContainer& species = mypc->GetParticleContainer(ispecies); + if (species.initialize_self_fields) { + InitSpaceChargeField(species); + } + } InitPML(); @@ -226,79 +231,6 @@ WarpX::PostRestart () mypc->PostRestart(); } -#ifdef USE_OPENBC_POISSON -void -WarpX::InitOpenbc () -{ -#ifndef BL_USE_MPI - static_assert(false, "must use MPI"); -#endif - - static_assert(AMREX_SPACEDIM == 3, "Openbc is 3D only"); - BL_ASSERT(finestLevel() == 0); - - const int lev = 0; - - const Geometry& gm = Geom(lev); - const Box& gbox = gm.Domain(); - int lohi[6]; - warpx_openbc_decompose(gbox.loVect(), gbox.hiVect(), lohi, lohi+3); - - int nprocs = ParallelDescriptor::NProcs(); - int myproc = ParallelDescriptor::MyProc(); - Vector<int> alllohi(6*nprocs,100000); - - MPI_Allgather(lohi, 6, MPI_INT, alllohi.data(), 6, MPI_INT, ParallelDescriptor::Communicator()); - - BoxList bl{IndexType::TheNodeType()}; - for (int i = 0; i < nprocs; ++i) - { - bl.push_back(Box(IntVect(alllohi[6*i ],alllohi[6*i+1],alllohi[6*i+2]), - IntVect(alllohi[6*i+3],alllohi[6*i+4],alllohi[6*i+5]), - IndexType::TheNodeType())); - } - BoxArray ba{bl}; - - Vector<int> iprocmap(nprocs+1); - std::iota(iprocmap.begin(), iprocmap.end(), 0); - iprocmap.back() = myproc; - - DistributionMapping dm{iprocmap}; - - MultiFab rho_openbc(ba, dm, 1, 0); - MultiFab phi_openbc(ba, dm, 1, 0); - - bool local = true; - const std::unique_ptr<MultiFab>& rho = mypc->GetChargeDensity(lev, local); - - rho_openbc.setVal(0.0); - rho_openbc.copy(*rho, 0, 0, 1, rho->nGrow(), 0, gm.periodicity(), FabArrayBase::ADD); - - const Real* dx = gm.CellSize(); - - warpx_openbc_potential(rho_openbc[myproc].dataPtr(), phi_openbc[myproc].dataPtr(), dx); - - BoxArray nba = boxArray(lev); - nba.surroundingNodes(); - MultiFab phi(nba, DistributionMap(lev), 1, 0); - phi.copy(phi_openbc, gm.periodicity()); - -#ifdef _OPENMP -#pragma omp parallel -#endif - for (MFIter mfi(phi); mfi.isValid(); ++mfi) - { - const Box& bx = mfi.validbox(); - warpx_compute_E(bx.loVect(), bx.hiVect(), - BL_TO_FORTRAN_3D(phi[mfi]), - BL_TO_FORTRAN_3D((*Efield[lev][0])[mfi]), - BL_TO_FORTRAN_3D((*Efield[lev][1])[mfi]), - BL_TO_FORTRAN_3D((*Efield[lev][2])[mfi]), - dx); - } -} -#endif - void WarpX::InitLevelData (int lev, Real time) { diff --git a/Source/Laser/LaserParticleContainer.H b/Source/Laser/LaserParticleContainer.H index e2a0743bc..63ace31fb 100644 --- a/Source/Laser/LaserParticleContainer.H +++ b/Source/Laser/LaserParticleContainer.H @@ -1,13 +1,14 @@ #ifndef WARPX_LaserParticleContainer_H_ #define WARPX_LaserParticleContainer_H_ -#include <limits> +#include <LaserProfiles.H> #include <WarpXParticleContainer.H> #include <WarpXConst.H> #include <WarpXParser.H> -enum class laser_t { Null, Gaussian, Harris, parse_field_function }; +#include <memory> +#include <limits> class LaserParticleContainer : public WarpXParticleContainer @@ -44,14 +45,6 @@ public: virtual void PostRestart () final; - void gaussian_laser_profile (const int np, amrex::Real const * AMREX_RESTRICT const Xp, - amrex::Real const * AMREX_RESTRICT const Yp, amrex::Real t, - amrex::Real * AMREX_RESTRICT const amplitude); - - void harris_laser_profile (const int np, amrex::Real const * AMREX_RESTRICT const Xp, - amrex::Real const * AMREX_RESTRICT const Yp, amrex::Real t, - amrex::Real * AMREX_RESTRICT const amplitude); - void calculate_laser_plane_coordinates (const int np, const int thread_num, amrex::Real * AMREX_RESTRICT const pplane_Xp, amrex::Real * AMREX_RESTRICT const pplane_Yp); @@ -68,17 +61,15 @@ protected: std::string laser_name; private: - // runtime paramters - laser_t profile = laser_t::Null; - amrex::Vector<amrex::Real> position; - amrex::Vector<amrex::Real> nvec; - amrex::Vector<amrex::Real> p_X; - amrex::Vector<amrex::Real> stc_direction; + amrex::Vector<amrex::Real> position; //! Coordinates of one of the point of the antenna + amrex::Vector<amrex::Real> nvec; //! Normal of the plane of the antenna + amrex::Vector<amrex::Real> p_X;// ! Polarization long pusher_algo = -1; amrex::Real e_max = std::numeric_limits<amrex::Real>::quiet_NaN(); amrex::Real wavelength = std::numeric_limits<amrex::Real>::quiet_NaN(); + amrex::Real Z0_lab = 0; // Position of the antenna in the lab frame long min_particles_per_mode = 4; @@ -90,19 +81,6 @@ private: amrex::Real weight = std::numeric_limits<amrex::Real>::quiet_NaN(); amrex::Real mobility = std::numeric_limits<amrex::Real>::quiet_NaN(); - // Gaussian profile - amrex::Real profile_waist = std::numeric_limits<amrex::Real>::quiet_NaN(); - amrex::Real profile_duration = std::numeric_limits<amrex::Real>::quiet_NaN(); - amrex::Real profile_t_peak = std::numeric_limits<amrex::Real>::quiet_NaN(); - amrex::Real profile_focal_distance = std::numeric_limits<amrex::Real>::quiet_NaN(); - amrex::Real zeta = 0.; - amrex::Real beta = 0.; - amrex::Real phi2 = 0.; - amrex::Real theta_stc = 0.; - - // parse_field_function profile - WarpXParser parser; - std::string field_function; // laser particle domain amrex::RealBox laser_injection_box; @@ -118,6 +96,10 @@ private: void ContinuousInjection(const amrex::RealBox& injection_box) override; // Update position of the antenna void UpdateContinuousInjectionPosition(amrex::Real dt) override; + + // Unique (smart) pointer to the laser profile + std::unique_ptr<WarpXLaserProfiles::ILaserProfile> m_up_laser_profile; + }; #endif diff --git a/Source/Laser/LaserParticleContainer.cpp b/Source/Laser/LaserParticleContainer.cpp index 9493672e0..a330200cc 100644 --- a/Source/Laser/LaserParticleContainer.cpp +++ b/Source/Laser/LaserParticleContainer.cpp @@ -1,4 +1,3 @@ - #include <limits> #include <cmath> #include <algorithm> @@ -11,6 +10,7 @@ #include <MultiParticleContainer.H> using namespace amrex; +using namespace WarpXLaserProfiles; namespace { @@ -34,70 +34,24 @@ LaserParticleContainer::LaserParticleContainer (AmrCore* amr_core, int ispecies, std::string laser_type_s; pp.get("profile", laser_type_s); std::transform(laser_type_s.begin(), laser_type_s.end(), laser_type_s.begin(), ::tolower); - if (laser_type_s == "gaussian") { - profile = laser_t::Gaussian; - } else if(laser_type_s == "harris") { - profile = laser_t::Harris; - } else if(laser_type_s == "parse_field_function") { - profile = laser_t::parse_field_function; - } else { - amrex::Abort("Unknown laser type"); - } // Parse the properties of the antenna pp.getarr("position", position); pp.getarr("direction", nvec); pp.getarr("polarization", p_X); + pp.query("pusher_algo", pusher_algo); pp.get("wavelength", wavelength); pp.get("e_max", e_max); pp.query("do_continuous_injection", do_continuous_injection); pp.query("min_particles_per_mode", min_particles_per_mode); - if ( profile == laser_t::Gaussian ) { - // Parse the properties of the Gaussian profile - pp.get("profile_waist", profile_waist); - pp.get("profile_duration", profile_duration); - pp.get("profile_t_peak", profile_t_peak); - pp.get("profile_focal_distance", profile_focal_distance); - stc_direction = p_X; - pp.queryarr("stc_direction", stc_direction); - pp.query("zeta", zeta); - pp.query("beta", beta); - pp.query("phi2", phi2); - } - - if ( profile == laser_t::Harris ) { - // Parse the properties of the Harris profile - pp.get("profile_waist", profile_waist); - pp.get("profile_duration", profile_duration); - pp.get("profile_focal_distance", profile_focal_distance); - } - - if ( profile == laser_t::parse_field_function ) { - // Parse the properties of the parse_field_function profile - pp.get("field_function(X,Y,t)", field_function); - parser.define(field_function); - parser.registerVariables({"X","Y","t"}); - - ParmParse ppc("my_constants"); - std::set<std::string> symbols = parser.symbols(); - symbols.erase("X"); - symbols.erase("Y"); - symbols.erase("t"); // after removing variables, we are left with constants - for (auto it = symbols.begin(); it != symbols.end(); ) { - Real v; - if (ppc.query(it->c_str(), v)) { - parser.setConstant(*it, v); - it = symbols.erase(it); - } else { - ++it; - } - } - for (auto const& s : symbols) { // make sure there no unknown symbols - amrex::Abort("Laser Profile: Unknown symbol "+s); - } + //Select laser profile + if(laser_profiles_dictionary.count(laser_type_s) == 0){ + amrex::Abort(std::string("Unknown laser type: ").append(laser_type_s)); } + m_up_laser_profile = laser_profiles_dictionary.at(laser_type_s)(); + //__________ // Plane normal Real s = 1.0_rt / std::sqrt(nvec[0]*nvec[0] + nvec[1]*nvec[1] + nvec[2]*nvec[2]); @@ -128,22 +82,6 @@ LaserParticleContainer::LaserParticleContainer (AmrCore* amr_core, int ispecies, p_Y = CrossProduct(nvec, p_X); // The second polarization vector - s = 1.0_rt / std::sqrt(stc_direction[0]*stc_direction[0] + stc_direction[1]*stc_direction[1] + stc_direction[2]*stc_direction[2]); - stc_direction = { stc_direction[0]*s, stc_direction[1]*s, stc_direction[2]*s }; - Real const dp2 = std::inner_product(nvec.begin(), nvec.end(), stc_direction.begin(), 0.0); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(std::abs(dp2) < 1.0e-14, - "stc_direction is not perpendicular to the laser plane vector"); - - // Get angle between p_X and stc_direction - // in 2d, stcs are in the simulation plane -#if AMREX_SPACEDIM == 3 - theta_stc = acos(stc_direction[0]*p_X[0] + - stc_direction[1]*p_X[1] + - stc_direction[2]*p_X[2]); -#else - theta_stc = 0.; -#endif - #if (defined WARPX_DIM_3D) || (defined WARPX_DIM_RZ) u_X = p_X; u_Y = p_Y; @@ -189,6 +127,14 @@ LaserParticleContainer::LaserParticleContainer (AmrCore* amr_core, int ispecies, "warpx.boost_direction = z. TODO: all directions."); } } + + //Init laser profile + CommonLaserParameters common_params; + common_params.wavelength = wavelength; + common_params.e_max = e_max; + common_params.p_X = p_X; + common_params.nvec = nvec; + m_up_laser_profile->init(pp, ParmParse{"my_constants"}, common_params); } /* \brief Check if laser particles enter the box, and inject if necessary. @@ -454,7 +400,7 @@ LaserParticleContainer::Evolve (int lev, int const thread_num = 0; #endif - Cuda::ManagedDeviceVector<Real> plane_Xp, plane_Yp, amplitude_E; + Gpu::ManagedDeviceVector<Real> plane_Xp, plane_Yp, amplitude_E; for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { @@ -503,21 +449,9 @@ LaserParticleContainer::Evolve (int lev, // Calculate the laser amplitude to be emitted, // at the position of the emission plane - if (profile == laser_t::Gaussian) { - gaussian_laser_profile(np, plane_Xp.dataPtr(), plane_Yp.dataPtr(), - t_lab, amplitude_E.dataPtr()); - } - - if (profile == laser_t::Harris) { - harris_laser_profile(np, plane_Xp.dataPtr(), plane_Yp.dataPtr(), - t_lab, amplitude_E.dataPtr()); - } - - if (profile == laser_t::parse_field_function) { - for (int i = 0; i < np; ++i) { - amplitude_E[i] = parser.eval(plane_Xp[i], plane_Yp[i], t); - } - } + m_up_laser_profile->fill_amplitude( + np, plane_Xp.dataPtr(), plane_Yp.dataPtr(), + t_lab, amplitude_E.dataPtr()); // Calculate the corresponding momentum and position for the particles update_laser_particle(np, uxp.dataPtr(), uyp.dataPtr(), diff --git a/Source/Laser/LaserProfiles.H b/Source/Laser/LaserProfiles.H new file mode 100644 index 000000000..528309492 --- /dev/null +++ b/Source/Laser/LaserProfiles.H @@ -0,0 +1,202 @@ +#ifndef WARPX_LaserProfiles_H_ +#define WARPX_LaserProfiles_H_ + +#include <AMReX_REAL.H> +#include <WarpXParser.H> +#include <AMReX_ParmParse.H> +#include <AMReX_Vector.H> + +#include <WarpXParser.H> + +#include <map> +#include <string> +#include <memory> +#include <functional> +#include <limits> + +namespace WarpXLaserProfiles { + +/** Common laser profile parameters + * + * Parameters for each laser profile as shared among all laser profile classes. + */ +struct CommonLaserParameters +{ + amrex::Real wavelength; //! central wavelength + amrex::Real e_max; //! maximum electric field at peak + amrex::Vector<amrex::Real> p_X;// ! Polarization + amrex::Vector<amrex::Real> nvec; //! Normal of the plane of the antenna +}; + + +/** Abstract interface for laser profile classes + * + * Each new laser profile should inherit this interface and implement two + * methods: init and fill_amplitude (described below). + * + * The declaration of a LaserProfile class should be placed in this file, + * while the implementation of the methods should be in a dedicated file in + * LaserProfilesImpl folder. LaserProfile classes should appear in + * laser_profiles_dictionary to be used by LaserParticleContainer. + */ +class ILaserProfile +{ +public: + /** Initialize Laser Profile + * + * Reads the section of the inputfile relative to the laser beam + * (e.g. <laser_name>.profile_t_peak, <laser_name>.profile_duration...) + * and the "my_constants" section. It also receives some common + * laser profile parameters. It uses these data to initialize the + * member variables of the laser profile class. + * + * @param[in] ppl should be amrex::ParmParse(laser_name) + * @param[in] ppc should be amrex::ParmParse("my_constants") + * @param[in] params common laser profile parameters + */ + virtual void + init ( + const amrex::ParmParse& ppl, + const amrex::ParmParse& ppc, + CommonLaserParameters params) = 0; + + /** Fill Electric Field Amplitude for each particle of the antenna. + * + * Xp, Yp and amplitude must be arrays with the same length + * + * @param[in] Xp X coordinate of the particles of the antenna + * @param[in] Yp Y coordinate of the particles of the antenna + * @param[in] t time (seconds) + * @param[out] amplitude of the electric field (V/m) + */ + virtual void + fill_amplitude ( + const int np, + amrex::Real const * AMREX_RESTRICT const Xp, + amrex::Real const * AMREX_RESTRICT const Yp, + amrex::Real t, + amrex::Real * AMREX_RESTRICT const amplitude) = 0; + + virtual ~ILaserProfile(){}; +}; + +/** + * Gaussian laser profile + */ +class GaussianLaserProfile : public ILaserProfile +{ + +public: + void + init ( + const amrex::ParmParse& ppl, + const amrex::ParmParse& ppc, + CommonLaserParameters params) override final; + + void + fill_amplitude ( + const int np, + amrex::Real const * AMREX_RESTRICT const Xp, + amrex::Real const * AMREX_RESTRICT const Yp, + amrex::Real t, + amrex::Real * AMREX_RESTRICT const amplitude) override final; + +private: + struct { + amrex::Real waist = std::numeric_limits<amrex::Real>::quiet_NaN(); + amrex::Real duration = std::numeric_limits<amrex::Real>::quiet_NaN(); + amrex::Real t_peak = std::numeric_limits<amrex::Real>::quiet_NaN(); + amrex::Real focal_distance = std::numeric_limits<amrex::Real>::quiet_NaN(); + amrex::Real zeta = 0; + amrex::Real beta = 0; + amrex::Real phi2 = 0; + + amrex::Vector<amrex::Real> stc_direction; //! Direction of the spatio-temporal couplings + amrex::Real theta_stc; //! Angle between polarization (p_X) and direction of spatiotemporal coupling (stc_direction) + } m_params; + + CommonLaserParameters m_common_params; +}; + +/** + * Harris laser profile + */ +class HarrisLaserProfile : public ILaserProfile +{ + +public: + void + init ( + const amrex::ParmParse& ppl, + const amrex::ParmParse& ppc, + CommonLaserParameters params) override final; + + void + fill_amplitude ( + const int np, + amrex::Real const * AMREX_RESTRICT const Xp, + amrex::Real const * AMREX_RESTRICT const Yp, + amrex::Real t, + amrex::Real * AMREX_RESTRICT const amplitude) override final; + +private: + struct { + amrex::Real waist = std::numeric_limits<amrex::Real>::quiet_NaN(); + amrex::Real duration = std::numeric_limits<amrex::Real>::quiet_NaN(); + amrex::Real focal_distance = std::numeric_limits<amrex::Real>::quiet_NaN(); + } m_params; + + CommonLaserParameters m_common_params; +}; + +/** + * Laser profile defined by the used with an analytical expression + */ +class FieldFunctionLaserProfile : public ILaserProfile +{ + +public: + void + init ( + const amrex::ParmParse& ppl, + const amrex::ParmParse& ppc, + CommonLaserParameters params) override final; + + void + fill_amplitude ( + const int np, + amrex::Real const * AMREX_RESTRICT const Xp, + amrex::Real const * AMREX_RESTRICT const Yp, + amrex::Real t, + amrex::Real * AMREX_RESTRICT const amplitude) override final; + +private: + struct{ + std::string field_function; + } m_params; + + WarpXParser m_parser; +}; + +/** + * Maps laser profile names to lambdas returing unique pointers + * to the corresponding laser profile objects. + */ +const +std::map< +std::string, +std::function<std::unique_ptr<ILaserProfile>()> +> +laser_profiles_dictionary = +{ + {"gaussian", + [] () {return std::make_unique<GaussianLaserProfile>();} }, + {"harris", + [] () {return std::make_unique<HarrisLaserProfile>();} }, + {"parse_field_function", + [] () {return std::make_unique<FieldFunctionLaserProfile>();} } +}; + +} //WarpXLaserProfiles + +#endif //WARPX_LaserProfiles_H_ diff --git a/Source/Laser/LaserProfiles.cpp b/Source/Laser/LaserProfiles.cpp deleted file mode 100644 index 44411cedf..000000000 --- a/Source/Laser/LaserProfiles.cpp +++ /dev/null @@ -1,133 +0,0 @@ - -#include <WarpX_Complex.H> -#include <LaserParticleContainer.H> - -using namespace amrex; - -/* \brief compute field amplitude for a Gaussian laser, at particles' position - * - * Both Xp and Yp are given in laser plane coordinate. - * For each particle with position Xp and Yp, this routine computes the - * amplitude of the laser electric field, stored in array amplitude. - * - * \param np: number of laser particles - * \param Xp: pointer to first component of positions of laser particles - * \param Yp: pointer to second component of positions of laser particles - * \param t: Current physical time - * \param amplitude: pointer to array of field amplitude. - */ -void -LaserParticleContainer::gaussian_laser_profile ( - const int np, Real const * AMREX_RESTRICT const Xp, Real const * AMREX_RESTRICT const Yp, - Real t, Real * AMREX_RESTRICT const amplitude) -{ - Complex I(0,1); - // Calculate a few factors which are independent of the macroparticle - const Real k0 = 2.*MathConst::pi/wavelength; - const Real inv_tau2 = 1. / (profile_duration * profile_duration); - const Real oscillation_phase = k0 * PhysConst::c * ( t - profile_t_peak ); - // The coefficients below contain info about Gouy phase, - // laser diffraction, and phase front curvature - const Complex diffract_factor = 1._rt + I * profile_focal_distance - * 2._rt/( k0 * profile_waist * profile_waist ); - const Complex inv_complex_waist_2 = 1._rt / ( profile_waist*profile_waist * diffract_factor ); - - // Time stretching due to STCs and phi2 complex envelope - // (1 if zeta=0, beta=0, phi2=0) - const Complex stretch_factor = 1._rt + 4._rt * - (zeta+beta*profile_focal_distance) * (zeta+beta*profile_focal_distance) - * (inv_tau2*inv_complex_waist_2) + - 2._rt *I*(phi2 - beta*beta*k0*profile_focal_distance) * inv_tau2; - - // Amplitude and monochromatic oscillations - Complex prefactor = e_max * MathFunc::exp( I * oscillation_phase ); - - // Because diffract_factor is a complex, the code below takes into - // account the impact of the dimensionality on both the Gouy phase - // and the amplitude of the laser -#if (AMREX_SPACEDIM == 3) - prefactor = prefactor / diffract_factor; -#elif (AMREX_SPACEDIM == 2) - prefactor = prefactor / MathFunc::sqrt(diffract_factor); -#endif - - // Copy member variables to tmp copies for GPU runs. - Real tmp_profile_t_peak = profile_t_peak; - Real tmp_beta = beta; - Real tmp_zeta = zeta; - Real tmp_theta_stc = theta_stc; - Real tmp_profile_focal_distance = profile_focal_distance; - // Loop through the macroparticle to calculate the proper amplitude - amrex::ParallelFor( - np, - [=] AMREX_GPU_DEVICE (int i) { - const Complex stc_exponent = 1._rt / stretch_factor * inv_tau2 * - MathFunc::pow((t - tmp_profile_t_peak - - tmp_beta*k0*(Xp[i]*std::cos(tmp_theta_stc) + Yp[i]*std::sin(tmp_theta_stc)) - - 2._rt *I*(Xp[i]*std::cos(tmp_theta_stc) + Yp[i]*std::sin(tmp_theta_stc)) - *( tmp_zeta - tmp_beta*tmp_profile_focal_distance ) * inv_complex_waist_2),2); - // stcfactor = everything but complex transverse envelope - const Complex stcfactor = prefactor * MathFunc::exp( - stc_exponent ); - // Exp argument for transverse envelope - const Complex exp_argument = - ( Xp[i]*Xp[i] + Yp[i]*Yp[i] ) * inv_complex_waist_2; - // stcfactor + transverse envelope - amplitude[i] = ( stcfactor * MathFunc::exp( exp_argument ) ).real(); - } - ); -} - -/* \brief compute field amplitude for a Harris laser function, at particles' position - * - * Both Xp and Yp are given in laser plane coordinate. - * For each particle with position Xp and Yp, this routine computes the - * amplitude of the laser electric field, stored in array amplitude. - * - * \param np: number of laser particles - * \param Xp: pointer to first component of positions of laser particles - * \param Yp: pointer to second component of positions of laser particles - * \param t: Current physical time - * \param amplitude: pointer to array of field amplitude. - */ -void -LaserParticleContainer::harris_laser_profile ( - const int np, Real const * AMREX_RESTRICT const Xp, Real const * AMREX_RESTRICT const Yp, - Real t, Real * AMREX_RESTRICT const amplitude) -{ - // This function uses the Harris function as the temporal profile of the pulse - const Real omega0 = 2.*MathConst::pi*PhysConst::c/wavelength; - const Real zR = MathConst::pi * profile_waist*profile_waist / wavelength; - const Real wz = profile_waist * - std::sqrt(1. + profile_focal_distance*profile_focal_distance/zR*zR); - const Real inv_wz_2 = 1./(wz*wz); - Real inv_Rz; - if (profile_focal_distance == 0.){ - inv_Rz = 0.; - } else { - inv_Rz = -profile_focal_distance / - ( profile_focal_distance*profile_focal_distance + zR*zR ); - } - const Real arg_env = 2.*MathConst::pi*t/profile_duration; - - // time envelope is given by the Harris function - Real time_envelope = 0.; - if (t < profile_duration) - time_envelope = 1./32. * (10. - 15.*std::cos(arg_env) + - 6.*std::cos(2.*arg_env) - - std::cos(3.*arg_env)); - - // Copy member variables to tmp copies for GPU runs. - Real tmp_e_max = e_max; - // Loop through the macroparticle to calculate the proper amplitude - amrex::ParallelFor( - np, - [=] AMREX_GPU_DEVICE (int i) { - const Real space_envelope = - std::exp(- ( Xp[i]*Xp[i] + Yp[i]*Yp[i] ) * inv_wz_2); - const Real arg_osc = omega0*t - omega0/PhysConst::c* - (Xp[i]*Xp[i] + Yp[i]*Yp[i]) * inv_Rz / 2.; - const Real oscillations = std::cos(arg_osc); - amplitude[i] = tmp_e_max * time_envelope * - space_envelope * oscillations; - } - ); -} diff --git a/Source/Laser/LaserProfilesImpl/LaserProfileFieldFunction.cpp b/Source/Laser/LaserProfilesImpl/LaserProfileFieldFunction.cpp new file mode 100644 index 000000000..3c9d7379a --- /dev/null +++ b/Source/Laser/LaserProfilesImpl/LaserProfileFieldFunction.cpp @@ -0,0 +1,45 @@ +#include <LaserProfiles.H> + +#include <WarpX_Complex.H> + +using namespace amrex; +using namespace WarpXLaserProfiles; + +void +FieldFunctionLaserProfile::init ( + const amrex::ParmParse& ppl, + const amrex::ParmParse& ppc, + CommonLaserParameters params) +{ + // Parse the properties of the parse_field_function profile + ppl.get("field_function(X,Y,t)", m_params.field_function); + m_parser.define(m_params.field_function); + m_parser.registerVariables({"X","Y","t"}); + + std::set<std::string> symbols = m_parser.symbols(); + symbols.erase("X"); + symbols.erase("Y"); + symbols.erase("t"); // after removing variables, we are left with constants + for (auto it = symbols.begin(); it != symbols.end(); ) { + Real v; + if (ppc.query(it->c_str(), v)) { + m_parser.setConstant(*it, v); + it = symbols.erase(it); + } else { + ++it; + } + } + for (auto const& s : symbols) { // make sure there no unknown symbols + amrex::Abort("Laser Profile: Unknown symbol "+s); + } +} + +void +FieldFunctionLaserProfile::fill_amplitude ( + const int np, Real const * AMREX_RESTRICT const Xp, Real const * AMREX_RESTRICT const Yp, + Real t, Real * AMREX_RESTRICT const amplitude) +{ + for (int i = 0; i < np; ++i) { + amplitude[i] = m_parser.eval(Xp[i], Yp[i], t); + } +} diff --git a/Source/Laser/LaserProfilesImpl/LaserProfileGaussian.cpp b/Source/Laser/LaserProfilesImpl/LaserProfileGaussian.cpp new file mode 100644 index 000000000..a0b5dd855 --- /dev/null +++ b/Source/Laser/LaserProfilesImpl/LaserProfileGaussian.cpp @@ -0,0 +1,134 @@ +#include <LaserProfiles.H> + +#include <WarpX_Complex.H> +#include <WarpXConst.H> + +#include <cmath> + +using namespace amrex; +using namespace WarpXLaserProfiles; + +void +GaussianLaserProfile::init ( + const amrex::ParmParse& ppl, + const amrex::ParmParse& /* ppc */, + CommonLaserParameters params) +{ + //Copy common params + m_common_params = params; + + // Parse the properties of the Gaussian profile + ppl.get("profile_waist", m_params.waist); + ppl.get("profile_duration", m_params.duration); + ppl.get("profile_t_peak", m_params.t_peak); + ppl.get("profile_focal_distance", m_params.focal_distance); + ppl.query("zeta", m_params.zeta); + ppl.query("beta", m_params.beta); + ppl.query("phi2", m_params.phi2); + + m_params.stc_direction = m_common_params.p_X; + ppl.queryarr("stc_direction", m_params.stc_direction); + auto const s = 1.0_rt / std::sqrt( + m_params.stc_direction[0]*m_params.stc_direction[0] + + m_params.stc_direction[1]*m_params.stc_direction[1] + + m_params.stc_direction[2]*m_params.stc_direction[2]); + m_params.stc_direction = { + m_params.stc_direction[0]*s, + m_params.stc_direction[1]*s, + m_params.stc_direction[2]*s }; + auto const dp2 = + std::inner_product( + m_common_params.nvec.begin(), + m_common_params.nvec.end(), + m_params.stc_direction.begin(), 0.0); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(std::abs(dp2) < 1.0e-14, + "stc_direction is not perpendicular to the laser plane vector"); + + // Get angle between p_X and stc_direction + // in 2d, stcs are in the simulation plane +#if AMREX_SPACEDIM == 3 + m_params.theta_stc = acos( + m_params.stc_direction[0]*m_common_params.p_X[0] + + m_params.stc_direction[1]*m_common_params.p_X[1] + + m_params.stc_direction[2]*m_common_params.p_X[2]); +#else + m_params.theta_stc = 0.; +#endif + +} + +/* \brief compute field amplitude for a Gaussian laser, at particles' position + * + * Both Xp and Yp are given in laser plane coordinate. + * For each particle with position Xp and Yp, this routine computes the + * amplitude of the laser electric field, stored in array amplitude. + * + * \param np: number of laser particles + * \param Xp: pointer to first component of positions of laser particles + * \param Yp: pointer to second component of positions of laser particles + * \param t: Current physical time + * \param amplitude: pointer to array of field amplitude. + */ +void +GaussianLaserProfile::fill_amplitude ( + const int np, Real const * AMREX_RESTRICT const Xp, Real const * AMREX_RESTRICT const Yp, + Real t, Real * AMREX_RESTRICT const amplitude) +{ + Complex I(0,1); + // Calculate a few factors which are independent of the macroparticle + const Real k0 = 2.*MathConst::pi/m_common_params.wavelength; + const Real inv_tau2 = 1._rt /(m_params.duration * m_params.duration); + const Real oscillation_phase = k0 * PhysConst::c * ( t - m_params.t_peak ); + // The coefficients below contain info about Gouy phase, + // laser diffraction, and phase front curvature + const Complex diffract_factor = + 1._rt + I * m_params.focal_distance * 2._rt/ + ( k0 * m_params.waist * m_params.waist ); + const Complex inv_complex_waist_2 = + 1._rt /(m_params.waist*m_params.waist * diffract_factor ); + + // Time stretching due to STCs and phi2 complex envelope + // (1 if zeta=0, beta=0, phi2=0) + const Complex stretch_factor = 1._rt + 4._rt * + (m_params.zeta+m_params.beta*m_params.focal_distance) + * (m_params.zeta+m_params.beta*m_params.focal_distance) + * (inv_tau2*inv_complex_waist_2) + 2._rt *I * (m_params.phi2 + - m_params.beta*m_params.beta*k0*m_params.focal_distance) * inv_tau2; + + // Amplitude and monochromatic oscillations + Complex prefactor = + m_common_params.e_max * MathFunc::exp( I * oscillation_phase ); + + // Because diffract_factor is a complex, the code below takes into + // account the impact of the dimensionality on both the Gouy phase + // and the amplitude of the laser +#if (AMREX_SPACEDIM == 3) + prefactor = prefactor / diffract_factor; +#elif (AMREX_SPACEDIM == 2) + prefactor = prefactor / MathFunc::sqrt(diffract_factor); +#endif + + // Copy member variables to tmp copies for GPU runs. + auto const tmp_profile_t_peak = m_params.t_peak; + auto const tmp_beta = m_params.beta; + auto const tmp_zeta = m_params.zeta; + auto const tmp_theta_stc = m_params.theta_stc; + auto const tmp_profile_focal_distance = m_params.focal_distance; + // Loop through the macroparticle to calculate the proper amplitude + amrex::ParallelFor( + np, + [=] AMREX_GPU_DEVICE (int i) { + const Complex stc_exponent = 1._rt / stretch_factor * inv_tau2 * + MathFunc::pow((t - tmp_profile_t_peak - + tmp_beta*k0*(Xp[i]*std::cos(tmp_theta_stc) + Yp[i]*std::sin(tmp_theta_stc)) - + 2._rt *I*(Xp[i]*std::cos(tmp_theta_stc) + Yp[i]*std::sin(tmp_theta_stc)) + *( tmp_zeta - tmp_beta*tmp_profile_focal_distance ) * inv_complex_waist_2),2); + // stcfactor = everything but complex transverse envelope + const Complex stcfactor = prefactor * MathFunc::exp( - stc_exponent ); + // Exp argument for transverse envelope + const Complex exp_argument = - ( Xp[i]*Xp[i] + Yp[i]*Yp[i] ) * inv_complex_waist_2; + // stcfactor + transverse envelope + amplitude[i] = ( stcfactor * MathFunc::exp( exp_argument ) ).real(); + } + ); +} diff --git a/Source/Laser/LaserProfilesImpl/LaserProfileHarris.cpp b/Source/Laser/LaserProfilesImpl/LaserProfileHarris.cpp new file mode 100644 index 000000000..55374c5ea --- /dev/null +++ b/Source/Laser/LaserProfilesImpl/LaserProfileHarris.cpp @@ -0,0 +1,79 @@ +#include <LaserProfiles.H> + +#include <WarpX_Complex.H> +#include <WarpXConst.H> + +using namespace amrex; +using namespace WarpXLaserProfiles; + +void +HarrisLaserProfile::init ( + const amrex::ParmParse& ppl, + const amrex::ParmParse& /* ppc */, + CommonLaserParameters params) +{ + // Parse the properties of the Harris profile + ppl.get("profile_waist", m_params.waist); + ppl.get("profile_duration", m_params.duration); + ppl.get("profile_focal_distance", m_params.focal_distance); + //Copy common params + m_common_params = params; +} + +/* \brief compute field amplitude for a Harris laser function, at particles' position + * + * Both Xp and Yp are given in laser plane coordinate. + * For each particle with position Xp and Yp, this routine computes the + * amplitude of the laser electric field, stored in array amplitude. + * + * \param np: number of laser particles + * \param Xp: pointer to first component of positions of laser particles + * \param Yp: pointer to second component of positions of laser particles + * \param t: Current physical time + * \param amplitude: pointer to array of field amplitude. + */ +void +HarrisLaserProfile::fill_amplitude ( + const int np, Real const * AMREX_RESTRICT const Xp, Real const * AMREX_RESTRICT const Yp, + Real t, Real * AMREX_RESTRICT const amplitude) +{ + // This function uses the Harris function as the temporal profile of the pulse + const Real omega0 = + 2._rt*MathConst::pi*PhysConst::c/m_common_params.wavelength; + const Real zR = MathConst::pi * m_params.waist*m_params.waist + / m_common_params.wavelength; + const Real wz = m_params.waist * + std::sqrt(1._rt + m_params.focal_distance*m_params.focal_distance/(zR*zR)); + const Real inv_wz_2 = 1._rt/(wz*wz); + Real inv_Rz; + if (m_params.focal_distance == 0.){ + inv_Rz = 0.; + } else { + inv_Rz = -m_params.focal_distance / + ( m_params.focal_distance*m_params.focal_distance + zR*zR ); + } + const Real arg_env = 2._rt*MathConst::pi*t/m_params.duration; + + // time envelope is given by the Harris function + Real time_envelope = 0.; + if (t < m_params.duration) + time_envelope = 1._rt/32._rt * (10._rt - 15._rt*std::cos(arg_env) + + 6._rt*std::cos(2._rt*arg_env) - + std::cos(3._rt*arg_env)); + + // Copy member variables to tmp copies for GPU runs. + const auto tmp_e_max = m_common_params.e_max; + // Loop through the macroparticle to calculate the proper amplitude + amrex::ParallelFor( + np, + [=] AMREX_GPU_DEVICE (int i) { + const Real space_envelope = + std::exp(- ( Xp[i]*Xp[i] + Yp[i]*Yp[i] ) * inv_wz_2); + const Real arg_osc = omega0*t - omega0/PhysConst::c* + (Xp[i]*Xp[i] + Yp[i]*Yp[i]) * inv_Rz / 2._rt; + const Real oscillations = std::cos(arg_osc); + amplitude[i] = tmp_e_max * time_envelope * + space_envelope * oscillations; + } + ); +} diff --git a/Source/Laser/LaserProfilesImpl/Make.package b/Source/Laser/LaserProfilesImpl/Make.package new file mode 100644 index 000000000..32284c4e4 --- /dev/null +++ b/Source/Laser/LaserProfilesImpl/Make.package @@ -0,0 +1,6 @@ +CEXE_sources += LaserProfileGaussian.cpp +CEXE_sources += LaserProfileHarris.cpp +CEXE_sources += LaserProfileFieldFunction.cpp + +INCLUDE_LOCATIONS += $(WARPX_HOME)/Source/Laser/LaserProfilesImpl +VPATH_LOCATIONS += $(WARPX_HOME)/Source/Laser/LaserProfilesImpl diff --git a/Source/Laser/Make.package b/Source/Laser/Make.package index 0babbb8e4..f2de8797d 100644 --- a/Source/Laser/Make.package +++ b/Source/Laser/Make.package @@ -1,6 +1,9 @@ -CEXE_sources += LaserParticleContainer.cpp -CEXE_sources += LaserProfiles.cpp CEXE_headers += LaserParticleContainer.H +CEXE_sources += LaserParticleContainer.cpp +CEXE_headers += LaserProfiles.H +CEXE_headers += LaserAntennaParams.H + +include $(WARPX_HOME)/Source/Laser/LaserProfilesImpl/Make.package INCLUDE_LOCATIONS += $(WARPX_HOME)/Source/Laser VPATH_LOCATIONS += $(WARPX_HOME)/Source/Laser diff --git a/Source/Make.WarpX b/Source/Make.WarpX index b81fed147..ec004f8fd 100644 --- a/Source/Make.WarpX +++ b/Source/Make.WarpX @@ -76,6 +76,7 @@ include $(AMREX_HOME)/Src/Base/Make.package include $(AMREX_HOME)/Src/Particle/Make.package include $(AMREX_HOME)/Src/Boundary/Make.package include $(AMREX_HOME)/Src/AmrCore/Make.package +include $(AMREX_HOME)/Src/LinearSolvers/MLMG/Make.package ifeq ($(USE_SENSEI_INSITU),TRUE) include $(AMREX_HOME)/Src/Amr/Make.package diff --git a/Source/Parallelization/Make.package b/Source/Parallelization/Make.package index c74583522..7c3c38627 100644 --- a/Source/Parallelization/Make.package +++ b/Source/Parallelization/Make.package @@ -2,6 +2,7 @@ CEXE_sources += WarpXComm.cpp CEXE_sources += WarpXRegrid.cpp CEXE_headers += WarpXSumGuardCells.H CEXE_headers += WarpXComm_K.H +CEXE_headers += WarpXComm.H INCLUDE_LOCATIONS += $(WARPX_HOME)/Source/Parallelization VPATH_LOCATIONS += $(WARPX_HOME)/Source/Parallelization diff --git a/Source/Parallelization/WarpXComm.H b/Source/Parallelization/WarpXComm.H new file mode 100644 index 000000000..81f00088e --- /dev/null +++ b/Source/Parallelization/WarpXComm.H @@ -0,0 +1,33 @@ +#ifndef WARPX_PARALLELIZATION_COMM_H_ +#define WARPX_PARALLELIZATION_COMM_H_ + +#include <AMReX_MultiFab.H> + +/** \brief Fills the values of the current on the coarse patch by + * averaging the values of the current of the fine patch (on the same level). + * Also fills the guards of the coarse patch. + * + * \param[in] fine fine patches to interpolate from + * \param[out] coarse coarse patches to interpolate to + * \param[in] refinement_ratio integer ratio between the two + */ +void +interpolateCurrentFineToCoarse ( + std::array< amrex::MultiFab const *, 3 > const & fine, + std::array< amrex::MultiFab *, 3 > const & coarse, + int const refinement_ratio); + +/** \brief Fills the values of the charge density on the coarse patch by + * averaging the values of the charge density of the fine patch (on the same level). + * + * \param[in] fine fine patches to interpolate from + * \param[out] coarse coarse patches to interpolate to + * \param[in] refinement_ratio integer ratio between the two + */ +void +interpolateDensityFineToCoarse ( + const amrex::MultiFab& fine, + amrex::MultiFab& coarse, + int const refinement_ratio); + +#endif // WARPX_PARALLELIZATION_COMM_H_ diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 52df3dc25..b61ae4fc7 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -1,14 +1,14 @@ +#include <WarpXComm.H> #include <WarpXComm_K.H> #include <WarpX.H> #include <WarpX_f.H> #include <WarpXSumGuardCells.H> -#include <Parallelization/InterpolateCurrentFineToCoarse.H> -#include <Parallelization/InterpolateDensityFineToCoarse.H> +#include <InterpolateCurrentFineToCoarse.H> +#include <InterpolateDensityFineToCoarse.H> #include <algorithm> #include <cstdlib> - using namespace amrex; void @@ -503,9 +503,9 @@ WarpX::SyncCurrent () } void -WarpX::interpolateCurrentFineToCoarse ( std::array< amrex::MultiFab const *, 3 > const & fine, - std::array< amrex::MultiFab *, 3 > const & coarse, - int const refinement_ratio) +interpolateCurrentFineToCoarse ( std::array< amrex::MultiFab const *, 3 > const & fine, + std::array< amrex::MultiFab *, 3 > const & coarse, + int const refinement_ratio) { BL_PROFILE("interpolateCurrentFineToCoarse()"); BL_ASSERT(refinement_ratio == 2); @@ -563,7 +563,7 @@ WarpX::SyncRho () } void -WarpX::interpolateDensityFineToCoarse (const MultiFab& fine, MultiFab& coarse, int const refinement_ratio) +interpolateDensityFineToCoarse (const MultiFab& fine, MultiFab& coarse, int const refinement_ratio) { BL_PROFILE("interpolateDensityFineToCoarse()"); BL_ASSERT(refinement_ratio == 2); diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index 2ae167283..29ccc8f4d 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -1,4 +1,3 @@ - #include <WarpX.H> #include <AMReX_BLProfiler.H> @@ -12,7 +11,8 @@ WarpX::LoadBalance () AMREX_ALWAYS_ASSERT(costs[0] != nullptr); - for (int lev = 0; lev <= finestLevel(); ++lev) + const int nLevels = finestLevel(); + for (int lev = 0; lev <= nLevels; ++lev) { const Real nboxes = costs[lev]->size(); const Real nprocs = ParallelDescriptor::NProcs(); diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 2737eb008..c1502e311 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -15,15 +15,14 @@ required to have the charge of each macroparticle since q is a scalar. For non-ionizable species, ion_lev is a null pointer. - * \param jx_arr : Array4 of current density, either full array or tile. - * \param jy_arr : Array4 of current density, either full array or tile. - * \param jz_arr : Array4 of current density, either full array or tile. + * \param jx_fab : FArrayBox of current density, either full array or tile. + * \param jy_fab : FArrayBox of current density, either full array or tile. + * \param jz_fab : FArrayBox of current density, either full array or tile. * \param np_to_depose : Number of particles for which current is deposited. * \param dt : Time step for particle level * \param dx : 3D cell size * \param xyzmin : Physical lower bounds of domain. * \param lo : Index lower bounds of domain. - * \param stagger_shift: 0 if nodal, 0.5 if staggered. * /param q : species charge. */ template <int depos_order> @@ -35,14 +34,13 @@ void doDepositionShapeN(const amrex::ParticleReal * const xp, const amrex::ParticleReal * const uyp, const amrex::ParticleReal * const uzp, const int * const ion_lev, - const amrex::Array4<amrex::Real>& jx_arr, - const amrex::Array4<amrex::Real>& jy_arr, - const amrex::Array4<amrex::Real>& jz_arr, + amrex::FArrayBox& jx_fab, + amrex::FArrayBox& jy_fab, + amrex::FArrayBox& jz_fab, const long np_to_depose, const amrex::Real dt, const std::array<amrex::Real,3>& dx, - const std::array<amrex::Real, 3> xyzmin, + const std::array<amrex::Real,3>& xyzmin, const amrex::Dim3 lo, - const amrex::Real stagger_shift, const amrex::Real q) { // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer @@ -63,16 +61,28 @@ void doDepositionShapeN(const amrex::ParticleReal * const xp, const amrex::Real xmin = xyzmin[0]; const amrex::Real ymin = xyzmin[1]; const amrex::Real zmin = xyzmin[2]; + const amrex::Real clightsq = 1.0/PhysConst::c/PhysConst::c; - // Loop over particles and deposit into jx_arr, jy_arr and jz_arr + amrex::Array4<amrex::Real> const& jx_arr = jx_fab.array(); + amrex::Array4<amrex::Real> const& jy_arr = jy_fab.array(); + amrex::Array4<amrex::Real> const& jz_arr = jz_fab.array(); + amrex::IntVect const jx_type = jx_fab.box().type(); + amrex::IntVect const jy_type = jy_fab.box().type(); + amrex::IntVect const jz_type = jz_fab.box().type(); + + constexpr int zdir = (AMREX_SPACEDIM - 1); + constexpr int NODE = amrex::IndexType::NODE; + constexpr int CELL = amrex::IndexType::CELL; + + // Loop over particles and deposit into jx_fab, jy_fab and jz_fab amrex::ParallelFor( np_to_depose, [=] AMREX_GPU_DEVICE (long ip) { // --- Get particle quantities const amrex::Real gaminv = 1.0/std::sqrt(1.0 + uxp[ip]*uxp[ip]*clightsq - + uyp[ip]*uyp[ip]*clightsq - + uzp[ip]*uzp[ip]*clightsq); + + uyp[ip]*uyp[ip]*clightsq + + uzp[ip]*uzp[ip]*clightsq); amrex::Real wq = q*wp[ip]; if (do_ionization){ wq *= ion_lev[ip]; @@ -108,47 +118,84 @@ void doDepositionShapeN(const amrex::ParticleReal * const xp, // x direction // Get particle position after 1/2 push back in position #if (defined WARPX_DIM_RZ) - const amrex::Real xmid = (rpmid-xmin)*dxi; + const amrex::Real xmid = (rpmid - xmin)*dxi; #else - const amrex::Real xmid = (xp[ip]-xmin)*dxi-dts2dx*vx; + const amrex::Real xmid = (xp[ip] - xmin)*dxi - dts2dx*vx; #endif - // Compute shape factors for node-centered quantities - amrex::Real sx [depos_order + 1]; - // j: leftmost grid point (node-centered) that the particle touches - const int j = compute_shape_factor<depos_order>(sx, xmid); - // Compute shape factors for cell-centered quantities - amrex::Real sx0[depos_order + 1]; - // j0: leftmost grid point (cell-centered) that the particle touches - const int j0 = compute_shape_factor<depos_order>(sx0, xmid-stagger_shift); + // j_j[xyz] leftmost grid point in x that the particle touches for the centering of each current + // sx_j[xyz] shape factor along x for the centering of each current + // There are only two possible centerings, node or cell centered, so at most only two shape factor + // arrays will be needed. + amrex::Real sx_node[depos_order + 1]; + amrex::Real sx_cell[depos_order + 1]; + int j_node; + int j_cell; + if (jx_type[0] == NODE || jy_type[0] == NODE || jz_type[0] == NODE) { + j_node = compute_shape_factor<depos_order>(sx_node, xmid); + } + if (jx_type[0] == CELL || jy_type[0] == CELL || jz_type[0] == CELL) { + j_cell = compute_shape_factor<depos_order>(sx_cell, xmid - 0.5); + } + const amrex::Real (&sx_jx)[depos_order + 1] = ((jx_type[0] == NODE) ? sx_node : sx_cell); + const amrex::Real (&sx_jy)[depos_order + 1] = ((jy_type[0] == NODE) ? sx_node : sx_cell); + const amrex::Real (&sx_jz)[depos_order + 1] = ((jz_type[0] == NODE) ? sx_node : sx_cell); + int const j_jx = ((jx_type[0] == NODE) ? j_node : j_cell); + int const j_jy = ((jy_type[0] == NODE) ? j_node : j_cell); + int const j_jz = ((jz_type[0] == NODE) ? j_node : j_cell); #if (defined WARPX_DIM_3D) // y direction - const amrex::Real ymid= (yp[ip]-ymin)*dyi-dts2dy*vy; - amrex::Real sy [depos_order + 1]; - const int k = compute_shape_factor<depos_order>(sy, ymid); - amrex::Real sy0[depos_order + 1]; - const int k0 = compute_shape_factor<depos_order>(sy0, ymid-stagger_shift); + const amrex::Real ymid = (yp[ip] - ymin)*dyi - dts2dy*vy; + amrex::Real sy_node[depos_order + 1]; + amrex::Real sy_cell[depos_order + 1]; + int k_node; + int k_cell; + if (jx_type[1] == NODE || jy_type[1] == NODE || jz_type[1] == NODE) { + k_node = compute_shape_factor<depos_order>(sy_node, ymid); + } + if (jx_type[1] == CELL || jy_type[1] == CELL || jz_type[1] == CELL) { + k_cell = compute_shape_factor<depos_order>(sy_cell, ymid - 0.5); + } + const amrex::Real (&sy_jx)[depos_order + 1] = ((jx_type[1] == NODE) ? sy_node : sy_cell); + const amrex::Real (&sy_jy)[depos_order + 1] = ((jy_type[1] == NODE) ? sy_node : sy_cell); + const amrex::Real (&sy_jz)[depos_order + 1] = ((jz_type[1] == NODE) ? sy_node : sy_cell); + int const k_jx = ((jx_type[1] == NODE) ? k_node : k_cell); + int const k_jy = ((jy_type[1] == NODE) ? k_node : k_cell); + int const k_jz = ((jz_type[1] == NODE) ? k_node : k_cell); #endif + // z direction - const amrex::Real zmid= (zp[ip]-zmin)*dzi-dts2dz*vz; - amrex::Real sz [depos_order + 1]; - const int l = compute_shape_factor<depos_order>(sz, zmid); - amrex::Real sz0[depos_order + 1]; - const int l0 = compute_shape_factor<depos_order>(sz0, zmid-stagger_shift); + const amrex::Real zmid = (zp[ip] - zmin)*dzi - dts2dz*vz; + amrex::Real sz_node[depos_order + 1]; + amrex::Real sz_cell[depos_order + 1]; + int l_node; + int l_cell; + if (jx_type[zdir] == NODE || jy_type[zdir] == NODE || jz_type[zdir] == NODE) { + l_node = compute_shape_factor<depos_order>(sz_node, zmid); + } + if (jx_type[zdir] == CELL || jy_type[zdir] == CELL || jz_type[zdir] == CELL) { + l_cell = compute_shape_factor<depos_order>(sz_cell, zmid - 0.5); + } + const amrex::Real (&sz_jx)[depos_order + 1] = ((jx_type[zdir] == NODE) ? sz_node : sz_cell); + const amrex::Real (&sz_jy)[depos_order + 1] = ((jy_type[zdir] == NODE) ? sz_node : sz_cell); + const amrex::Real (&sz_jz)[depos_order + 1] = ((jz_type[zdir] == NODE) ? sz_node : sz_cell); + int const l_jx = ((jx_type[zdir] == NODE) ? l_node : l_cell); + int const l_jy = ((jy_type[zdir] == NODE) ? l_node : l_cell); + int const l_jz = ((jz_type[zdir] == NODE) ? l_node : l_cell); // Deposit current into jx_arr, jy_arr and jz_arr #if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) for (int iz=0; iz<=depos_order; iz++){ for (int ix=0; ix<=depos_order; ix++){ amrex::Gpu::Atomic::Add( - &jx_arr(lo.x+j0+ix, lo.y+l +iz, 0), - sx0[ix]*sz [iz]*wqx); + &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 0), + sx_jx[ix]*sz_jx[iz]*wqx); amrex::Gpu::Atomic::Add( - &jy_arr(lo.x+j +ix, lo.y+l +iz, 0), - sx [ix]*sz [iz]*wqy); + &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 0), + sx_jy[ix]*sz_jy[iz]*wqy); amrex::Gpu::Atomic::Add( - &jz_arr(lo.x+j +ix, lo.y+l0+iz, 0), - sx [ix]*sz0[iz]*wqz); + &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 0), + sx_jz[ix]*sz_jz[iz]*wqz); } } #elif (defined WARPX_DIM_3D) @@ -156,14 +203,14 @@ void doDepositionShapeN(const amrex::ParticleReal * const xp, for (int iy=0; iy<=depos_order; iy++){ for (int ix=0; ix<=depos_order; ix++){ amrex::Gpu::Atomic::Add( - &jx_arr(lo.x+j0+ix, lo.y+k +iy, lo.z+l +iz), - sx0[ix]*sy [iy]*sz [iz]*wqx); + &jx_arr(lo.x+j_jx+ix, lo.y+k_jx+iy, lo.z+l_jx+iz), + sx_jx[ix]*sy_jx[iy]*sz_jx[iz]*wqx); amrex::Gpu::Atomic::Add( - &jy_arr(lo.x+j +ix, lo.y+k0+iy, lo.z+l +iz), - sx [ix]*sy0[iy]*sz [iz]*wqy); + &jy_arr(lo.x+j_jy+ix, lo.y+k_jy+iy, lo.z+l_jy+iz), + sx_jy[ix]*sy_jy[iy]*sz_jy[iz]*wqy); amrex::Gpu::Atomic::Add( - &jz_arr(lo.x+j +ix, lo.y+k +iy, lo.z+l0+iz), - sx [ix]*sy [iy]*sz0[iz]*wqz); + &jz_arr(lo.x+j_jz+ix, lo.y+k_jz+iy, lo.z+l_jz+iz), + sx_jz[ix]*sy_jz[iy]*sz_jz[iz]*wqz); } } } diff --git a/Source/Particles/Make.package b/Source/Particles/Make.package index a6c124ddd..6d34fa0ef 100644 --- a/Source/Particles/Make.package +++ b/Source/Particles/Make.package @@ -1,4 +1,3 @@ -F90EXE_sources += deposit_cic.F90 F90EXE_sources += interpolate_cic.F90 F90EXE_sources += push_particles_ES.F90 diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index f9a0e51d7..f6d6acafb 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -11,6 +11,7 @@ #include <AMReX_Particles.H> #ifdef WARPX_QED + #include <QedChiFunctions.H> #include <BreitWheelerEngineWrapper.H> #include <QuantumSyncEngineWrapper.H> #endif @@ -19,8 +20,6 @@ #include <map> #include <string> #include <algorithm> -#include <utility> -#include <tuple> // // MultiParticleContainer holds multiple (nspecies or npsecies+1 when @@ -256,22 +255,17 @@ protected: void InitBreitWheeler (); /** - * Parses inputfile parameters for Quantum Synchrotron engine - * @return {a tuple containing a flag which is true if tables - * have to be generate, a filename (where tables should be stored - * or read from) and control parameters.} + * Called by InitQuantumSync if a new table has + * to be generated. */ - std::tuple<bool, std::string, PicsarQuantumSynchrotronCtrl> - ParseQuantumSyncParams (); + void QuantumSyncGenerateTable(); /** - * Parses inputfile parameters for Breit Wheeler engine - * @return {a tuple containing a flag which is true if tables - * have to be generate, a filename (where tables should be stored - * or read from) and control parameters.} + * Called by InitBreitWheeler if a new table has + * to be generated. */ - std::tuple<bool, std::string, PicsarBreitWheelerCtrl> - ParseBreitWheelerParams (); + void BreitWheelerGenerateTable(); + #endif private: diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index fca22daa2..bf1ffb74e 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -321,11 +321,7 @@ void MultiParticleContainer::Redistribute () { for (auto& pc : allcontainers) { - if ( (pc->NumRuntimeRealComps()>0) || (pc->NumRuntimeIntComps()>0) ) { - pc->RedistributeCPU(); - } else { - pc->Redistribute(); - } + pc->Redistribute(); } } @@ -333,11 +329,7 @@ void MultiParticleContainer::RedistributeLocal (const int num_ghost) { for (auto& pc : allcontainers) { - if ( (pc->NumRuntimeRealComps()>0) || (pc->NumRuntimeIntComps()>0) ) { - pc->RedistributeCPU(0, 0, 0, num_ghost); - } else { - pc->Redistribute(0, 0, 0, num_ghost); - } + pc->Redistribute(0, 0, 0, num_ghost); } } @@ -649,100 +641,106 @@ void MultiParticleContainer::InitQED () void MultiParticleContainer::InitQuantumSync () { - bool generate_table; - PicsarQuantumSynchrotronCtrl ctrl; - std::string filename; - std::tie(generate_table, filename, ctrl) = ParseQuantumSyncParams(); - - //Only temporary for test purposes, will be removed + std::string lookup_table_mode; ParmParse pp("qed_qs"); - bool ignore_tables = false; - pp.query("ignore_tables_for_test", ignore_tables); - if(ignore_tables) return; - //_________________________________________________ - - - if(generate_table && ParallelDescriptor::IOProcessor()){ - m_shr_p_qs_engine->compute_lookup_tables(ctrl); - Vector<char> all_data = m_shr_p_qs_engine->export_lookup_tables_data(); - WarpXUtilIO::WriteBinaryDataOnFile(filename, all_data); - } - ParallelDescriptor::Barrier(); - - Vector<char> table_data; - ParallelDescriptor::ReadAndBcastFile(filename, table_data); - ParallelDescriptor::Barrier(); + pp.query("lookup_table_mode", lookup_table_mode); + if(lookup_table_mode.empty()){ + amrex::Abort("Quantum Synchrotron table mode should be provided"); + } - //No need to initialize from raw data for the processor that - //has just generated the table - if(!generate_table || !ParallelDescriptor::IOProcessor()){ + if(lookup_table_mode == "generate"){ + amrex::Print() << "Quantum Synchrotron table will be generated. \n" ; +#ifndef WARPX_QED_TABLE_GEN + amrex::Error("Error: Compile with QED_TABLE_GEN=TRUE to enable table generation!\n"); +#else + QuantumSyncGenerateTable(); +#endif + } + else if(lookup_table_mode == "load"){ + amrex::Print() << "Quantum Synchrotron table will be read from file. \n" ; + std::string load_table_name; + pp.query("load_table_from", load_table_name); + if(load_table_name.empty()){ + amrex::Abort("Quantum Synchrotron table name should be provided"); + } + Vector<char> table_data; + ParallelDescriptor::ReadAndBcastFile(load_table_name, table_data); + ParallelDescriptor::Barrier(); m_shr_p_qs_engine->init_lookup_tables_from_raw_data(table_data); } + else if(lookup_table_mode == "dummy_builtin"){ + amrex::Print() << "Built-in Quantum Synchrotron dummy table will be used. \n" ; + m_shr_p_qs_engine->init_dummy_tables(); + } + else{ + amrex::Abort("Unknown Quantum Synchrotron table mode"); + } - if(!m_shr_p_qs_engine->are_lookup_tables_initialized()) - amrex::Error("Table initialization has failed!\n"); + if(!m_shr_p_qs_engine->are_lookup_tables_initialized()){ + amrex::Abort("Table initialization has failed!"); + } } void MultiParticleContainer::InitBreitWheeler () { - bool generate_table; - PicsarBreitWheelerCtrl ctrl; - std::string filename; - std::tie(generate_table, filename, ctrl) = ParseBreitWheelerParams(); - - //Only temporary for test purposes, will be removed + std::string lookup_table_mode; ParmParse pp("qed_bw"); - bool ignore_tables = false; - pp.query("ignore_tables_for_test", ignore_tables); - if(ignore_tables) return; - //_________________________________________________ - - if(generate_table && ParallelDescriptor::IOProcessor()){ - m_shr_p_bw_engine->compute_lookup_tables(ctrl); - Vector<char> all_data = m_shr_p_bw_engine->export_lookup_tables_data(); - WarpXUtilIO::WriteBinaryDataOnFile(filename, all_data); + pp.query("lookup_table_mode", lookup_table_mode); + if(lookup_table_mode.empty()){ + amrex::Abort("Breit Wheeler table mode should be provided"); } - ParallelDescriptor::Barrier(); - Vector<char> table_data; - ParallelDescriptor::ReadAndBcastFile(filename, table_data); - ParallelDescriptor::Barrier(); - - //No need to initialize from raw data for the processor that - //has just generated the table - if(!generate_table || !ParallelDescriptor::IOProcessor()){ + if(lookup_table_mode == "generate"){ + amrex::Print() << "Breit Wheeler table will be generated. \n" ; +#ifndef WARPX_QED_TABLE_GEN + amrex::Error("Error: Compile with QED_TABLE_GEN=TRUE to enable table generation!\n"); +#else + BreitWheelerGenerateTable(); +#endif + } + else if(lookup_table_mode == "load"){ + amrex::Print() << "Breit Wheeler table will be read from file. \n" ; + std::string load_table_name; + pp.query("load_table_from", load_table_name); + if(load_table_name.empty()){ + amrex::Abort("Breit Wheeler table name should be provided"); + } + Vector<char> table_data; + ParallelDescriptor::ReadAndBcastFile(load_table_name, table_data); + ParallelDescriptor::Barrier(); m_shr_p_bw_engine->init_lookup_tables_from_raw_data(table_data); } + else if(lookup_table_mode == "dummy_builtin"){ + amrex::Print() << "Built-in Breit Wheeler dummy table will be used. \n" ; + m_shr_p_bw_engine->init_dummy_tables(); + } + else{ + amrex::Abort("Unknown Breit Wheeler table mode"); + } - if(!m_shr_p_bw_engine->are_lookup_tables_initialized()) - amrex::Error("Table initialization has failed!\n"); + if(!m_shr_p_bw_engine->are_lookup_tables_initialized()){ + amrex::Abort("Table initialization has failed!"); + } } -std::tuple<bool,std::string,PicsarQuantumSynchrotronCtrl> -MultiParticleContainer::ParseQuantumSyncParams () +void +MultiParticleContainer::QuantumSyncGenerateTable () { - PicsarQuantumSynchrotronCtrl ctrl = - m_shr_p_qs_engine->get_default_ctrl(); - bool generate_table{false}; - std::string table_name; - ParmParse pp("qed_qs"); + std::string table_name; + pp.query("save_table_in", table_name); + if(table_name.empty()) + amrex::Abort("qed_qs.save_table_in should be provided!"); - // Engine paramenter: chi_part_min is the minium chi parameter to be - // considered by the engine. If a lepton has chi < chi_part_min, - // the optical depth is not evolved and photon generation is ignored - pp.query("chi_min", ctrl.chi_part_min); - - //Only temporary for test purposes, will be removed - bool ignore_tables = false; - pp.query("ignore_tables_for_test", ignore_tables); - if(ignore_tables) - return std::make_tuple(false, "__DUMMY__", ctrl); - //_________________________________________________ + if(ParallelDescriptor::IOProcessor()){ + PicsarQuantumSynchrotronCtrl ctrl; + int t_int; - pp.query("generate_table", generate_table); - if(generate_table){ - int t_int = 0; + // Engine paramenter: chi_part_min is the minium chi parameter to be + // considered by the engine. If a lepton has chi < chi_part_min, + // the optical depth is not evolved and photon generation is ignored + if(!pp.query("chi_min", ctrl.chi_part_min)) + amrex::Abort("qed_qs.chi_min should be provided!"); //==Table parameters== @@ -751,16 +749,19 @@ MultiParticleContainer::ParseQuantumSyncParams () //which appears in the evolution of the optical depth //Minimun chi for the table. If a lepton has chi < chi_part_tdndt_min, - ///chi is considered as it were equal to chi_part_tdndt_min - pp.query("tab_dndt_chi_min", ctrl.chi_part_tdndt_min); + //chi is considered as it were equal to chi_part_tdndt_min + if(!pp.query("tab_dndt_chi_min", ctrl.chi_part_tdndt_min)) + amrex::Abort("qed_qs.tab_dndt_chi_min should be provided!"); //Maximum chi for the table. If a lepton has chi > chi_part_tdndt_max, - ///chi is considered as it were equal to chi_part_tdndt_max - pp.query("tab_dndt_chi_max", ctrl.chi_part_tdndt_max); + //chi is considered as it were equal to chi_part_tdndt_max + if(!pp.query("tab_dndt_chi_max", ctrl.chi_part_tdndt_max)) + amrex::Abort("qed_qs.tab_dndt_chi_max should be provided!"); //How many points should be used for chi in the table - pp.query("tab_dndt_how_many", t_int); - ctrl.chi_part_tdndt_how_many= t_int; + if(!pp.query("tab_dndt_how_many", t_int)) + amrex::Abort("qed_qs.tab_dndt_how_many should be provided!"); + ctrl.chi_part_tdndt_how_many = t_int; //------ //--- sub-table 2 (2D) @@ -769,78 +770,64 @@ MultiParticleContainer::ParseQuantumSyncParams () //photons. //Minimun chi for the table. If a lepton has chi < chi_part_tem_min, - ///chi is considered as it were equal to chi_part_tem_min - pp.query("tab_em_chi_min", ctrl.chi_part_tem_min); + //chi is considered as it were equal to chi_part_tem_min + if(!pp.query("tab_em_chi_min", ctrl.chi_part_tem_min)) + amrex::Abort("qed_qs.tab_em_chi_min should be provided!"); //Maximum chi for the table. If a lepton has chi > chi_part_tem_max, - ///chi is considered as it were equal to chi_part_tem_max - pp.query("tab_em_chi_max", ctrl.chi_part_tem_max); + //chi is considered as it were equal to chi_part_tem_max + if(!pp.query("tab_em_chi_max", ctrl.chi_part_tem_max)) + amrex::Abort("qed_qs.tab_em_chi_max should be provided!"); //How many points should be used for chi in the table - pp.query("tab_em_chi_how_many", t_int); + if(!pp.query("tab_em_chi_how_many", t_int)) + amrex::Abort("qed_qs.tab_em_chi_how_many should be provided!"); ctrl.chi_part_tem_how_many = t_int; //The other axis of the table is a cumulative probability distribution //(corresponding to different energies of the generated particles) //This parameter is the number of different points to consider - pp.query("tab_em_prob_how_many", t_int); + if(!pp.query("tab_em_prob_how_many", t_int)) + amrex::Abort("qed_qs.tab_em_prob_how_many should be provided!"); ctrl.prob_tem_how_many = t_int; //==================== - pp.query("save_table_in", table_name); - - } - - std::string load_table_name; - pp.query("load_table_from", load_table_name); - if(!load_table_name.empty()){ - if(generate_table && ParallelDescriptor::IOProcessor()){ - amrex::Print() << "Warning, Quantum Synchrotron table will be loaded, not generated. \n"; - } - table_name = load_table_name; - generate_table = false; + m_shr_p_qs_engine->compute_lookup_tables(ctrl); + WarpXUtilIO::WriteBinaryDataOnFile(table_name, + m_shr_p_qs_engine->export_lookup_tables_data()); } -#ifndef WARPX_QED_TABLE_GEN - if(generate_table){ - amrex::Error("Error: Compile with QED_TABLE_GEN=TRUE to enable table generation!\n"); - } -#endif + ParallelDescriptor::Barrier(); + Vector<char> table_data; + ParallelDescriptor::ReadAndBcastFile(table_name, table_data); + ParallelDescriptor::Barrier(); - if(table_name.empty()){ - amrex::Error("Error: Quantum Synchrotron table has either to be generated or to be loaded.\n"); + //No need to initialize from raw data for the processor that + //has just generated the table + if(!ParallelDescriptor::IOProcessor()){ + m_shr_p_qs_engine->init_lookup_tables_from_raw_data(table_data); } - - return std::make_tuple(generate_table, table_name, ctrl); } -std::tuple<bool,std::string,PicsarBreitWheelerCtrl> -MultiParticleContainer::ParseBreitWheelerParams () +void +MultiParticleContainer::BreitWheelerGenerateTable () { - PicsarBreitWheelerCtrl ctrl = - m_shr_p_bw_engine->get_default_ctrl(); - bool generate_table{false}; - std::string table_name; - ParmParse pp("qed_bw"); + std::string table_name; + pp.query("save_table_in", table_name); + if(table_name.empty()) + amrex::Abort("qed_bw.save_table_in should be provided!"); - // Engine paramenter: chi_phot_min is the minium chi parameter to be - // considered by the engine. If a photon has chi < chi_phot_min, - // the optical depth is not evolved and pair generation is ignored - pp.query("chi_min", ctrl.chi_phot_min); - - //Only temporary for test purposes, will be removed - bool ignore_tables = false; - pp.query("ignore_tables_for_test", ignore_tables); - if(ignore_tables) - return std::make_tuple(false, "__DUMMY__", ctrl); - //_________________________________________________ - - pp.query("generate_table", generate_table); - if(generate_table){ - + if(ParallelDescriptor::IOProcessor()){ + PicsarBreitWheelerCtrl ctrl; int t_int; + // Engine paramenter: chi_phot_min is the minium chi parameter to be + // considered by the engine. If a photon has chi < chi_phot_min, + // the optical depth is not evolved and pair generation is ignored + if(!pp.query("chi_min", ctrl.chi_phot_min)) + amrex::Abort("qed_bw.chi_min should be provided!"); + //==Table parameters== //--- sub-table 1 (1D) @@ -849,14 +836,17 @@ MultiParticleContainer::ParseBreitWheelerParams () //Minimun chi for the table. If a photon has chi < chi_phot_tdndt_min, //an analytical approximation is used. - pp.query("tab_dndt_chi_min", ctrl.chi_phot_tdndt_min); + if(!pp.query("tab_dndt_chi_min", ctrl.chi_phot_tdndt_min)) + amrex::Abort("qed_bw.tab_dndt_chi_min should be provided!"); //Maximum chi for the table. If a photon has chi > chi_phot_tdndt_min, //an analytical approximation is used. - pp.query("tab_dndt_chi_max", ctrl.chi_phot_tdndt_max); + if(!pp.query("tab_dndt_chi_max", ctrl.chi_phot_tdndt_max)) + amrex::Abort("qed_bw.tab_dndt_chi_max should be provided!"); //How many points should be used for chi in the table - pp.query("tab_dndt_how_many", t_int); + if(!pp.query("tab_dndt_how_many", t_int)) + amrex::Abort("qed_bw.tab_dndt_how_many should be provided!"); ctrl.chi_phot_tdndt_how_many = t_int; //------ @@ -867,48 +857,41 @@ MultiParticleContainer::ParseBreitWheelerParams () //Minimun chi for the table. If a photon has chi < chi_phot_tpair_min //chi is considered as it were equal to chi_phot_tpair_min - pp.query("tab_pair_chi_min", ctrl.chi_phot_tpair_min); + if(!pp.query("tab_pair_chi_min", ctrl.chi_phot_tpair_min)) + amrex::Abort("qed_bw.tab_pair_chi_min should be provided!"); //Maximum chi for the table. If a photon has chi > chi_phot_tpair_max //chi is considered as it were equal to chi_phot_tpair_max - pp.query("tab_pair_chi_max", ctrl.chi_phot_tpair_max); + if(!pp.query("tab_pair_chi_max", ctrl.chi_phot_tpair_max)) + amrex::Abort("qed_bw.tab_pair_chi_max should be provided!"); //How many points should be used for chi in the table - pp.query("tab_pair_chi_how_many", t_int); + if(!pp.query("tab_pair_chi_how_many", t_int)) + amrex::Abort("qed_bw.tab_pair_chi_how_many should be provided!"); ctrl.chi_phot_tpair_how_many = t_int; //The other axis of the table is the fraction of the initial energy //'taken away' by the most energetic particle of the pair. //This parameter is the number of different fractions to consider - pp.query("tab_pair_frac_how_many", t_int); + if(!pp.query("tab_pair_frac_how_many", t_int)) + amrex::Abort("qed_bw.tab_pair_frac_how_many should be provided!"); ctrl.chi_frac_tpair_how_many = t_int; //==================== - pp.query("save_table_in", table_name); - } - - std::string load_table_name; - pp.query("load_table_from", load_table_name); - if(!load_table_name.empty()){ - if(generate_table && ParallelDescriptor::IOProcessor()){ - amrex::Print() << "Warning, Breit Wheeler table will be loaded, not generated. \n"; - } - table_name = load_table_name; - generate_table = false; + m_shr_p_bw_engine->compute_lookup_tables(ctrl); + WarpXUtilIO::WriteBinaryDataOnFile(table_name, + m_shr_p_bw_engine->export_lookup_tables_data()); } -#ifndef WARPX_QED_TABLE_GEN - if(generate_table){ - if(ParallelDescriptor::IOProcessor()){ - amrex::Error("Error: Compile with QED_TABLE_GEN=TRUE to enable table generation!\n"); - } - } -#endif + ParallelDescriptor::Barrier(); + Vector<char> table_data; + ParallelDescriptor::ReadAndBcastFile(table_name, table_data); + ParallelDescriptor::Barrier(); - if(table_name.empty()){ - amrex::Error("Error: Breit Wheeler table has either to be generated or to be loaded.\n"); + //No need to initialize from raw data for the processor that + //has just generated the table + if(!ParallelDescriptor::IOProcessor()){ + m_shr_p_bw_engine->init_lookup_tables_from_raw_data(table_data); } - - return std::make_tuple(generate_table, table_name, ctrl); } #endif diff --git a/Source/Particles/ParticleCreation/CopyParticle.H b/Source/Particles/ParticleCreation/CopyParticle.H index bd2d530fb..5e51c5283 100644 --- a/Source/Particles/ParticleCreation/CopyParticle.H +++ b/Source/Particles/ParticleCreation/CopyParticle.H @@ -13,7 +13,7 @@ * Pointers to SoA data are saved when constructor is called. * AoS data is passed as argument of operator (). */ -class copyParticle +class CopyParticle { public: @@ -33,7 +33,7 @@ public: // Simple constructor AMREX_GPU_HOST_DEVICE - copyParticle( + CopyParticle( const int cpuid, const int do_back_transformed_product, const amrex::GpuArray<amrex::ParticleReal*,3> runtime_uold_source, const amrex::GpuArray<amrex::ParticleReal*,PIdx::nattribs> attribs_source, @@ -54,7 +54,7 @@ public: * \param ip index of product particle * \param pid_product ID of first product particle * \param p_source Struct with data for 1 source particle (position etc.) - * \param p_source Struct with data for 1 product particle (position etc.) + * \param p_product Struct with data for 1 product particle (position etc.) */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void operator () (int is, int ip, int pid_product, diff --git a/Source/Particles/ParticleCreation/ElementaryProcess.H b/Source/Particles/ParticleCreation/ElementaryProcess.H index fa791c244..75443cb38 100644 --- a/Source/Particles/ParticleCreation/ElementaryProcess.H +++ b/Source/Particles/ParticleCreation/ElementaryProcess.H @@ -21,8 +21,8 @@ * The class is templated on the process type. This gives flexibility * on source and product particle transformations. */ -template<elementaryProcessType ProcessT> -class elementaryProcess +template<ElementaryProcessType ProcessT> +class ElementaryProcess { public: @@ -36,16 +36,17 @@ public: * \param attribs_product attribs of product particles * \param runtime_attribs_product runtime attribs for product, to store old attribs */ - copyParticle initialize_copy( + CopyParticle initialize_copy( const int cpuid, const int do_back_transformed_product, const amrex::GpuArray<amrex::ParticleReal*,3> runtime_uold_source, const amrex::GpuArray<amrex::ParticleReal*,PIdx::nattribs> attribs_source, const amrex::GpuArray<amrex::ParticleReal*,PIdx::nattribs> attribs_product, const amrex::GpuArray<amrex::ParticleReal*,6> runtime_attribs_product) const noexcept { - return copyParticle ( + return CopyParticle( cpuid, do_back_transformed_product, runtime_uold_source, - attribs_source, attribs_product, runtime_attribs_product); + attribs_source, attribs_product, runtime_attribs_product + ); }; /** @@ -123,7 +124,8 @@ public: int np_flagged = i_product[np_source-1]; if (np_flagged == 0) return; - amrex::Vector<copyParticle> v_copy_functor; + amrex::Gpu::ManagedDeviceVector<CopyParticle> v_copy_functor; + v_copy_functor.reserve(nproducts); amrex::Gpu::ManagedDeviceVector<int> v_pid_product(nproducts); amrex::Gpu::ManagedDeviceVector<WarpXParticleContainer::ParticleType*> v_particles_product(nproducts); for (int iproduct=0; iproduct<nproducts; iproduct++){ @@ -172,12 +174,13 @@ public: const int cpuid = amrex::ParallelDescriptor::MyProc(); // Create instance of copy functor, and add it to the vector - v_copy_functor.push_back (initialize_copy( - cpuid, v_do_back_transformed_product[iproduct], - runtime_uold_source, - attribs_source, - attribs_product, - runtime_attribs_product) ); + v_copy_functor.push_back( initialize_copy( + cpuid, v_do_back_transformed_product[iproduct], + runtime_uold_source, + attribs_source, + attribs_product, + runtime_attribs_product + ) ); v_pid_product[iproduct] = pid_product; } @@ -212,12 +215,12 @@ public: amrex::Gpu::ManagedDeviceVector<int> v_pid_product, amrex::Gpu::ManagedDeviceVector<WarpXParticleContainer::ParticleType*> v_particles_product, WarpXParticleContainer::ParticleType* particles_source, - const amrex::Vector<copyParticle>& v_copy_functor, + const amrex::Gpu::ManagedDeviceVector<CopyParticle>& v_copy_functor, amrex::GpuArray<int*,1> runtime_iattribs_source) { int const * const AMREX_RESTRICT p_is_flagged = is_flagged.dataPtr(); int const * const AMREX_RESTRICT p_i_product = i_product.dataPtr(); - copyParticle const * const p_copy_functor = v_copy_functor.data(); + CopyParticle const * const AMREX_RESTRICT p_copy_functor = v_copy_functor.dataPtr(); WarpXParticleContainer::ParticleType * * p_particles_product = v_particles_product.data(); int const * const p_pid_product = v_pid_product.data(); @@ -251,7 +254,7 @@ public: }; // Derived class for ionization process -class IonizationProcess: public elementaryProcess<elementaryProcessType::Ionization> +class IonizationProcess: public ElementaryProcess<ElementaryProcessType::Ionization> {}; #endif // ELEMENTARYPROCESS_H_ diff --git a/Source/Particles/ParticleCreation/TransformParticle.H b/Source/Particles/ParticleCreation/TransformParticle.H index 14e534d0e..1b71df723 100644 --- a/Source/Particles/ParticleCreation/TransformParticle.H +++ b/Source/Particles/ParticleCreation/TransformParticle.H @@ -3,7 +3,7 @@ #include "WarpXParticleContainer.H" -enum struct elementaryProcessType { Ionization }; +enum struct ElementaryProcessType { Ionization }; /** * \brief small modifications on source particle @@ -11,7 +11,7 @@ enum struct elementaryProcessType { Ionization }; * \param particle particle struct * \param runtime_iattribs integer attribs */ -template < elementaryProcessType ProcessT > +template < ElementaryProcessType ProcessT > AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void transformSourceParticle( int i, @@ -23,7 +23,7 @@ void transformSourceParticle( * \param i index of particle * \param particle particle struct */ -template < elementaryProcessType ProcessT > +template < ElementaryProcessType ProcessT > AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void transformProductParticle( int i, @@ -32,7 +32,7 @@ void transformProductParticle( // For ionization, increase ionization level of source particle template <> AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE -void transformSourceParticle < elementaryProcessType::Ionization > ( +void transformSourceParticle < ElementaryProcessType::Ionization > ( int i, WarpXParticleContainer::ParticleType& particle, amrex::GpuArray<int*,1> runtime_iattribs) diff --git a/Source/Particles/PhotonParticleContainer.H b/Source/Particles/PhotonParticleContainer.H index 9132cf4a9..580f13f86 100644 --- a/Source/Particles/PhotonParticleContainer.H +++ b/Source/Particles/PhotonParticleContainer.H @@ -41,12 +41,12 @@ public: DtType a_dt_type=DtType::Full) override; virtual void PushPX(WarpXParIter& pti, - amrex::Cuda::ManagedDeviceVector<amrex::ParticleReal>& xp, - amrex::Cuda::ManagedDeviceVector<amrex::ParticleReal>& yp, - amrex::Cuda::ManagedDeviceVector<amrex::ParticleReal>& zp, + amrex::Gpu::ManagedDeviceVector<amrex::ParticleReal>& xp, + amrex::Gpu::ManagedDeviceVector<amrex::ParticleReal>& yp, + amrex::Gpu::ManagedDeviceVector<amrex::ParticleReal>& zp, amrex::Real dt, DtType a_dt_type=DtType::Full) override; - // Don't push momenta for photons + // Do nothing virtual void PushP (int lev, amrex::Real dt, const amrex::MultiFab& Ex, @@ -79,6 +79,18 @@ public: return false; }; +#ifdef WARPX_QED + /** + * This function evolves the optical depth of the photons if QED effects + * are enabled. + * @param[in,out] pti particle iterator (optical depth will be modified) + * @param[in] dt temporal step + */ + virtual void EvolveOpticalDepth(WarpXParIter& pti, + amrex::Real dt) override; + +#endif + }; #endif // #ifndef WARPX_PhotonParticleContainer_H_ diff --git a/Source/Particles/PhotonParticleContainer.cpp b/Source/Particles/PhotonParticleContainer.cpp index 7e52b52e1..c03ed00c2 100644 --- a/Source/Particles/PhotonParticleContainer.cpp +++ b/Source/Particles/PhotonParticleContainer.cpp @@ -51,9 +51,9 @@ void PhotonParticleContainer::InitData() void PhotonParticleContainer::PushPX(WarpXParIter& pti, - Cuda::ManagedDeviceVector<ParticleReal>& xp, - Cuda::ManagedDeviceVector<ParticleReal>& yp, - Cuda::ManagedDeviceVector<ParticleReal>& zp, + Gpu::ManagedDeviceVector<ParticleReal>& xp, + Gpu::ManagedDeviceVector<ParticleReal>& yp, + Gpu::ManagedDeviceVector<ParticleReal>& zp, Real dt, DtType a_dt_type) { @@ -78,8 +78,6 @@ PhotonParticleContainer::PushPX(WarpXParIter& pti, copy_attribs(pti, x, y, z); } - //No need to update momentum for photons (for now) - amrex::ParallelFor( pti.numParticles(), [=] AMREX_GPU_DEVICE (long i) { @@ -88,7 +86,6 @@ PhotonParticleContainer::PushPX(WarpXParIter& pti, ux[i], uy[i], uz[i], dt ); } ); - } void @@ -116,3 +113,48 @@ PhotonParticleContainer::Evolve (int lev, t, dt); } + +#ifdef WARPX_QED + +void +PhotonParticleContainer::EvolveOpticalDepth( + WarpXParIter& pti,amrex::Real dt) +{ + if(!has_breit_wheeler()) + return; + + auto& attribs = pti.GetAttribs(); + ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); + ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); + ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); + const ParticleReal* const AMREX_RESTRICT Ex = attribs[PIdx::Ex].dataPtr(); + const ParticleReal* const AMREX_RESTRICT Ey = attribs[PIdx::Ey].dataPtr(); + const ParticleReal* const AMREX_RESTRICT Ez = attribs[PIdx::Ez].dataPtr(); + const ParticleReal* const AMREX_RESTRICT Bx = attribs[PIdx::Bx].dataPtr(); + const ParticleReal* const AMREX_RESTRICT By = attribs[PIdx::By].dataPtr(); + const ParticleReal* const AMREX_RESTRICT Bz = attribs[PIdx::Bz].dataPtr(); + + BreitWheelerEvolveOpticalDepth evolve_opt = + m_shr_p_bw_engine->build_evolve_functor(); + + amrex::Real* AMREX_RESTRICT p_tau = + pti.GetAttribs(particle_comps["tau"]).dataPtr(); + + const auto me = PhysConst::m_e; + + amrex::ParallelFor( + pti.numParticles(), + [=] AMREX_GPU_DEVICE (long i) { + const ParticleReal px = me * ux[i]; + const ParticleReal py = me * uy[i]; + const ParticleReal pz = me * uz[i]; + + bool has_event_happened = evolve_opt( + px, py, pz, + Ex[i], Ey[i], Ez[i], + Bx[i], By[i], Bz[i], + dt, p_tau[i]); + } + ); +} +#endif diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index 17a504719..e70b470b8 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -94,9 +94,9 @@ public: DtType a_dt_type=DtType::Full) override; virtual void PushPX(WarpXParIter& pti, - amrex::Cuda::ManagedDeviceVector<amrex::ParticleReal>& xp, - amrex::Cuda::ManagedDeviceVector<amrex::ParticleReal>& yp, - amrex::Cuda::ManagedDeviceVector<amrex::ParticleReal>& zp, + amrex::Gpu::ManagedDeviceVector<amrex::ParticleReal>& xp, + amrex::Gpu::ManagedDeviceVector<amrex::ParticleReal>& yp, + amrex::Gpu::ManagedDeviceVector<amrex::ParticleReal>& zp, amrex::Real dt, DtType a_dt_type=DtType::Full); virtual void PushP (int lev, amrex::Real dt, @@ -187,13 +187,47 @@ public: amrex::FArrayBox const * & byfab, amrex::FArrayBox const * & bzfab); #ifdef WARPX_QED + //Functions decleared in WarpXParticleContainer.H + //containers for which QED processes could be relevant + //are expected to override these functions + + /** + * Tells if this PhysicalParticleContainer has Quantum + * Synchrotron process enabled + * @return true if process is enabled + */ bool has_quantum_sync() override; + + /** + * Tells if this PhysicalParticleContainer has Breit + * Wheeler process enabled + * @return true if process is enabled + */ bool has_breit_wheeler() override; + /** + * Acquires a shared smart pointer to a BreitWheelerEngine + * @param[in] ptr the pointer + */ void set_breit_wheeler_engine_ptr - (std::shared_ptr<BreitWheelerEngine>) override; + (std::shared_ptr<BreitWheelerEngine> ptr) override; + + /** + * Acquires a shared smart pointer to a QuantumSynchrotronEngine + * @param[in] ptr the pointer + */ void set_quantum_sync_engine_ptr - (std::shared_ptr<QuantumSynchrotronEngine>) override; + (std::shared_ptr<QuantumSynchrotronEngine> ptr) override; + //__________ + + /** + * This function evolves the optical depth of the particles if QED effects + * are enabled. + * @param[in,out] pti particle iterator (optical depth will be modified) + * @param[in] dt temporal step + */ + virtual void EvolveOpticalDepth(WarpXParIter& pti, + amrex::Real dt); #endif protected: diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 51690d659..008eb9c89 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -39,8 +39,11 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp // Initialize splitting pp.query("do_splitting", do_splitting); pp.query("split_type", split_type); + pp.query("do_not_deposit", do_not_deposit); pp.query("do_continuous_injection", do_continuous_injection); + pp.query("initialize_self_fields", initialize_self_fields); + pp.query("self_fields_required_precision", self_fields_required_precision); // Whether to plot back-transformed (lab-frame) diagnostics // for this species. pp.query("do_back_transformed_diagnostics", do_back_transformed_diagnostics); @@ -62,16 +65,6 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp "Radiation reaction can be enabled only if Boris pusher is used"); //_____________________________ -#ifdef AMREX_USE_GPU - Print()<<"\n-----------------------------------------------------\n"; - Print()<<"WARNING: field ionization on GPU uses RedistributeCPU\n"; - Print()<<"-----------------------------------------------------\n\n"; - //AMREX_ALWAYS_ASSERT_WITH_MESSAGE( - //do_field_ionization == 0, - //"Field ionization does not work on GPU so far, because the current " - //"version of Redistribute in AMReX does not work with runtime parameters"); -#endif - #ifdef WARPX_QED //Add real component if QED is enabled pp.query("do_qed", m_do_qed); @@ -1062,6 +1055,7 @@ PhysicalParticleContainer::Evolve (int lev, BL_PROFILE("PPC::Evolve()"); BL_PROFILE_VAR_NS("PPC::Evolve::Copy", blp_copy); BL_PROFILE_VAR_NS("PPC::FieldGather", blp_fg); + BL_PROFILE_VAR_NS("PPC::EvolveOpticalDepth", blp_ppc_qed_ev); BL_PROFILE_VAR_NS("PPC::ParticlePush", blp_ppc_pp); const std::array<Real,3>& dx = WarpX::CellSize(lev); @@ -1253,6 +1247,15 @@ PhysicalParticleContainer::Evolve (int lev, BL_PROFILE_VAR_STOP(blp_fg); +#ifdef WARPX_QED + // + //Evolve Optical Depth + // + BL_PROFILE_VAR_START(blp_ppc_qed_ev); + EvolveOpticalDepth(pti, dt); + BL_PROFILE_VAR_STOP(blp_ppc_qed_ev); +#endif + // // Particle Push // @@ -1406,7 +1409,7 @@ PhysicalParticleContainer::SplitParticles(int lev) { auto& mypc = WarpX::GetInstance().GetPartContainer(); auto& pctmp_split = mypc.GetPCtmp(); - Cuda::ManagedDeviceVector<ParticleReal> xp, yp, zp; + Gpu::ManagedDeviceVector<ParticleReal> xp, yp, zp; RealVector psplit_x, psplit_y, psplit_z, psplit_w; RealVector psplit_ux, psplit_uy, psplit_uz; long np_split_to_add = 0; @@ -1564,9 +1567,9 @@ PhysicalParticleContainer::SplitParticles(int lev) void PhysicalParticleContainer::PushPX(WarpXParIter& pti, - Cuda::ManagedDeviceVector<ParticleReal>& xp, - Cuda::ManagedDeviceVector<ParticleReal>& yp, - Cuda::ManagedDeviceVector<ParticleReal>& zp, + Gpu::ManagedDeviceVector<ParticleReal>& xp, + Gpu::ManagedDeviceVector<ParticleReal>& yp, + Gpu::ManagedDeviceVector<ParticleReal>& zp, Real dt, DtType a_dt_type) { @@ -1600,8 +1603,45 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti, const Real q = this->charge; const Real m = this-> mass; +#ifdef WARPX_QED + + auto t_chi_max = m_shr_p_qs_engine->get_ref_ctrl().chi_part_min; - //Assumes that all consistency checks have been done at initialization + if(do_classical_radiation_reaction){ + if(m_do_qed_quantum_sync){ + amrex::ParallelFor( + pti.numParticles(), + [=] AMREX_GPU_DEVICE (long i) { + auto chi = QedUtils::chi_lepton(m*ux[i], m*uy[i], m*uz[i], + Ex[i], Ey[i], Ez[i], + Bx[i], By[i], Bz[i]); + if(chi < t_chi_max){ + UpdateMomentumBorisWithRadiationReaction( ux[i], uy[i], uz[i], + Ex[i], Ey[i], Ez[i], Bx[i], + By[i], Bz[i], q, m, dt); + } + else{ + UpdateMomentumBoris( ux[i], uy[i], uz[i], + Ex[i], Ey[i], Ez[i], Bx[i], + By[i], Bz[i], q, m, dt); + } + UpdatePosition( x[i], y[i], z[i], + ux[i], uy[i], uz[i], dt ); + } + ); + }else{ + amrex::ParallelFor( + pti.numParticles(), + [=] AMREX_GPU_DEVICE (long i) { + UpdateMomentumBorisWithRadiationReaction( ux[i], uy[i], uz[i], + Ex[i], Ey[i], Ez[i], Bx[i], + By[i], Bz[i], q, m, dt); + UpdatePosition( x[i], y[i], z[i], + ux[i], uy[i], uz[i], dt ); + } + ); + } +#else if(do_classical_radiation_reaction){ amrex::ParallelFor( pti.numParticles(), @@ -1615,6 +1655,7 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti, ux[i], uy[i], uz[i], dt ); } ); +#endif } else if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Boris){ amrex::ParallelFor( pti.numParticles(), @@ -1659,6 +1700,49 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti, }; } +#ifdef WARPX_QED +void PhysicalParticleContainer::EvolveOpticalDepth( + WarpXParIter& pti, amrex::Real dt) +{ + if(!has_quantum_sync()) + return; + + QuantumSynchrotronEvolveOpticalDepth evolve_opt = + m_shr_p_qs_engine->build_evolve_functor(); + + auto& attribs = pti.GetAttribs(); + const ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); + const ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); + const ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); + const ParticleReal* const AMREX_RESTRICT Ex = attribs[PIdx::Ex].dataPtr(); + const ParticleReal* const AMREX_RESTRICT Ey = attribs[PIdx::Ey].dataPtr(); + const ParticleReal* const AMREX_RESTRICT Ez = attribs[PIdx::Ez].dataPtr(); + const ParticleReal* const AMREX_RESTRICT Bx = attribs[PIdx::Bx].dataPtr(); + const ParticleReal* const AMREX_RESTRICT By = attribs[PIdx::By].dataPtr(); + const ParticleReal* const AMREX_RESTRICT Bz = attribs[PIdx::Bz].dataPtr(); + + ParticleReal* const AMREX_RESTRICT p_tau = + pti.GetAttribs(particle_comps["tau"]).dataPtr(); + + const ParticleReal m = this->mass; + + amrex::ParallelFor(pti.numParticles(), + [=] AMREX_GPU_DEVICE (long i) { + const ParticleReal px = m * ux[i]; + const ParticleReal py = m * uy[i]; + const ParticleReal pz = m * uz[i]; + + bool has_event_happened = evolve_opt( + px, py, pz, + Ex[i], Ey[i], Ez[i], + Bx[i], By[i], Bz[i], + dt, p_tau[i]); + } + ); + +} +#endif + void PhysicalParticleContainer::PushP (int lev, Real dt, const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, @@ -1742,12 +1826,10 @@ PhysicalParticleContainer::PushP (int lev, Real dt, ion_lev = pti.GetiAttribs(particle_icomps["ionization_level"]).dataPtr(); } - //Assumes that all consistency checks have been done at initialization if(do_classical_radiation_reaction){ - amrex::ParallelFor( - pti.numParticles(), - [=] AMREX_GPU_DEVICE (long i) { + amrex::ParallelFor(pti.numParticles(), + [=] AMREX_GPU_DEVICE (long i){ Real qp = q; if (ion_lev){ qp *= ion_lev[i]; } UpdateMomentumBorisWithRadiationReaction( @@ -1758,7 +1840,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, } ); } else if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Boris){ - amrex::ParallelFor( pti.numParticles(), + amrex::ParallelFor(pti.numParticles(), [=] AMREX_GPU_DEVICE (long i) { Real qp = q; if (ion_lev){ qp *= ion_lev[i]; } @@ -1769,8 +1851,8 @@ PhysicalParticleContainer::PushP (int lev, Real dt, qp, m, dt); } ); - } else if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Vay) { - amrex::ParallelFor( pti.numParticles(), + } else if (WarpX::particle_pusher_algo == ParticlePusherAlgo::Vay){ + amrex::ParallelFor(pti.numParticles(), [=] AMREX_GPU_DEVICE (long i) { Real qp = q; if (ion_lev){ qp *= ion_lev[i]; } @@ -1781,16 +1863,19 @@ PhysicalParticleContainer::PushP (int lev, Real dt, qp, m, dt); } ); - } else if (WarpX::particle_pusher_algo == ParticlePusherAlgo::HigueraCary) { - amrex::ParallelFor( pti.numParticles(), + } else if (WarpX::particle_pusher_algo == ParticlePusherAlgo::HigueraCary){ + amrex::ParallelFor(pti.numParticles(), [=] AMREX_GPU_DEVICE (long i) { UpdateMomentumHigueraCary( ux[i], uy[i], uz[i], - Expp[i], Eypp[i], Ezpp[i], Bxpp[i], Bypp[i], Bzpp[i], q, m, dt); + Expp[i], Eypp[i], Ezpp[i], + Bxpp[i], Bypp[i], Bzpp[i], + q, m, dt); } ); } else { amrex::Abort("Unknown particle pusher"); - }; + } + } } } @@ -1875,80 +1960,156 @@ void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real #pragma omp parallel #endif { - RealVector xp_new, yp_new, zp_new; - +#ifdef _OPENMP + int thread_num = omp_get_thread_num(); +#else + int thread_num = 0; +#endif for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { + int counter_for_ParticleCopy = 0; const Box& box = pti.validbox(); - auto index = std::make_pair(pti.index(), pti.LocalTileIndex()); - const RealBox tile_real_box(box, dx, plo); if ( !slice_box.intersects(tile_real_box) ) continue; - pti.GetPosition(xp_new, yp_new, zp_new); + pti.GetPosition(m_xp[thread_num],m_yp[thread_num],m_zp[thread_num]); + Real *const AMREX_RESTRICT xpnew = m_xp[thread_num].dataPtr(); + Real *const AMREX_RESTRICT ypnew = m_yp[thread_num].dataPtr(); + Real *const AMREX_RESTRICT zpnew = m_zp[thread_num].dataPtr(); auto& attribs = pti.GetAttribs(); - - auto& wp = attribs[PIdx::w ]; - - auto& uxp_new = attribs[PIdx::ux ]; - auto& uyp_new = attribs[PIdx::uy ]; - auto& uzp_new = attribs[PIdx::uz ]; - - auto& xp_old = tmp_particle_data[lev][index][TmpIdx::xold]; - auto& yp_old = tmp_particle_data[lev][index][TmpIdx::yold]; - auto& zp_old = tmp_particle_data[lev][index][TmpIdx::zold]; - auto& uxp_old = tmp_particle_data[lev][index][TmpIdx::uxold]; - auto& uyp_old = tmp_particle_data[lev][index][TmpIdx::uyold]; - auto& uzp_old = tmp_particle_data[lev][index][TmpIdx::uzold]; + Real* const AMREX_RESTRICT wpnew = attribs[PIdx::w].dataPtr(); + Real* const AMREX_RESTRICT uxpnew = attribs[PIdx::ux].dataPtr(); + Real* const AMREX_RESTRICT uypnew = attribs[PIdx::uy].dataPtr(); + Real* const AMREX_RESTRICT uzpnew = attribs[PIdx::uz].dataPtr(); + + Real* const AMREX_RESTRICT + xpold = tmp_particle_data[lev][index][TmpIdx::xold].dataPtr(); + Real* const AMREX_RESTRICT + ypold = tmp_particle_data[lev][index][TmpIdx::yold].dataPtr(); + Real* const AMREX_RESTRICT + zpold = tmp_particle_data[lev][index][TmpIdx::zold].dataPtr(); + Real* const AMREX_RESTRICT + uxpold = tmp_particle_data[lev][index][TmpIdx::uxold].dataPtr(); + Real* const AMREX_RESTRICT + uypold = tmp_particle_data[lev][index][TmpIdx::uyold].dataPtr(); + Real* const AMREX_RESTRICT + uzpold = tmp_particle_data[lev][index][TmpIdx::uzold].dataPtr(); const long np = pti.numParticles(); Real uzfrm = -WarpX::gamma_boost*WarpX::beta_boost*PhysConst::c; Real inv_c2 = 1.0/PhysConst::c/PhysConst::c; - for (long i = 0; i < np; ++i) { - - // if the particle did not cross the plane of z_boost in the last - // timestep, skip it. - if ( not (((zp_new[i] >= z_new) && (zp_old[i] <= z_old)) || - ((zp_new[i] <= z_new) && (zp_old[i] >= z_old))) ) continue; - - // Lorentz transform particles to lab frame - Real gamma_new_p = std::sqrt(1.0 + inv_c2*(uxp_new[i]*uxp_new[i] + uyp_new[i]*uyp_new[i] + uzp_new[i]*uzp_new[i])); - Real t_new_p = WarpX::gamma_boost*t_boost - uzfrm*zp_new[i]*inv_c2; - Real z_new_p = WarpX::gamma_boost*(zp_new[i] + WarpX::beta_boost*PhysConst::c*t_boost); - Real uz_new_p = WarpX::gamma_boost*uzp_new[i] - gamma_new_p*uzfrm; - - Real gamma_old_p = std::sqrt(1.0 + inv_c2*(uxp_old[i]*uxp_old[i] + uyp_old[i]*uyp_old[i] + uzp_old[i]*uzp_old[i])); - Real t_old_p = WarpX::gamma_boost*(t_boost - dt) - uzfrm*zp_old[i]*inv_c2; - Real z_old_p = WarpX::gamma_boost*(zp_old[i] + WarpX::beta_boost*PhysConst::c*(t_boost-dt)); - Real uz_old_p = WarpX::gamma_boost*uzp_old[i] - gamma_old_p*uzfrm; - - // interpolate in time to t_lab - Real weight_old = (t_new_p - t_lab) / (t_new_p - t_old_p); - Real weight_new = (t_lab - t_old_p) / (t_new_p - t_old_p); + // temporary arrays to store copy_flag and copy_index + // for particles that cross the z-slice + amrex::Gpu::ManagedDeviceVector<int> FlagForPartCopy(np); + amrex::Gpu::ManagedDeviceVector<int> IndexForPartCopy(np); - Real xp = xp_old[i]*weight_old + xp_new[i]*weight_new; - Real yp = yp_old[i]*weight_old + yp_new[i]*weight_new; - Real zp = z_old_p *weight_old + z_new_p *weight_new; + int* const AMREX_RESTRICT Flag = FlagForPartCopy.dataPtr(); + int* const AMREX_RESTRICT IndexLocation = IndexForPartCopy.dataPtr(); - Real uxp = uxp_old[i]*weight_old + uxp_new[i]*weight_new; - Real uyp = uyp_old[i]*weight_old + uyp_new[i]*weight_new; - Real uzp = uz_old_p *weight_old + uz_new_p *weight_new; - - diagnostic_particles[lev][index].GetRealData(DiagIdx::w).push_back(wp[i]); - - diagnostic_particles[lev][index].GetRealData(DiagIdx::x).push_back(xp); - diagnostic_particles[lev][index].GetRealData(DiagIdx::y).push_back(yp); - diagnostic_particles[lev][index].GetRealData(DiagIdx::z).push_back(zp); - - diagnostic_particles[lev][index].GetRealData(DiagIdx::ux).push_back(uxp); - diagnostic_particles[lev][index].GetRealData(DiagIdx::uy).push_back(uyp); - diagnostic_particles[lev][index].GetRealData(DiagIdx::uz).push_back(uzp); - } + //Flag particles that need to be copied if they cross the z_slice + amrex::ParallelFor(np, + [=] AMREX_GPU_DEVICE(int i) + { + Flag[i] = 0; + if ( (((zpnew[i] >= z_new) && (zpold[i] <= z_old)) || + ((zpnew[i] <= z_new) && (zpold[i] >= z_old))) ) + { + Flag[i] = 1; + } + }); + + // exclusive scan to obtain location indices using flag values + // These location indices are used to copy data from + // src to dst when the copy-flag is set to 1. + amrex::Gpu::exclusive_scan(Flag,Flag+np,IndexLocation); + + const int total_partdiag_size = IndexLocation[np-1] + Flag[np-1]; + + // allocate array size for diagnostic particle array + diagnostic_particles[lev][index].resize(total_partdiag_size); + + amrex::Real gammaboost = WarpX::gamma_boost; + amrex::Real betaboost = WarpX::beta_boost; + amrex::Real Phys_c = PhysConst::c; + + Real* const AMREX_RESTRICT diag_wp = + diagnostic_particles[lev][index].GetRealData(DiagIdx::w).data(); + Real* const AMREX_RESTRICT diag_xp = + diagnostic_particles[lev][index].GetRealData(DiagIdx::x).data(); + Real* const AMREX_RESTRICT diag_yp = + diagnostic_particles[lev][index].GetRealData(DiagIdx::y).data(); + Real* const AMREX_RESTRICT diag_zp = + diagnostic_particles[lev][index].GetRealData(DiagIdx::z).data(); + Real* const AMREX_RESTRICT diag_uxp = + diagnostic_particles[lev][index].GetRealData(DiagIdx::ux).data(); + Real* const AMREX_RESTRICT diag_uyp = + diagnostic_particles[lev][index].GetRealData(DiagIdx::uy).data(); + Real* const AMREX_RESTRICT diag_uzp = + diagnostic_particles[lev][index].GetRealData(DiagIdx::uz).data(); + + // Copy particle data to diagnostic particle array on the GPU + // using flag and index values + amrex::ParallelFor(np, + [=] AMREX_GPU_DEVICE(int i) + { + if (Flag[i] == 1) + { + // Lorentz Transform particles to lab-frame + const Real gamma_new_p = std::sqrt(1.0 + inv_c2* + (uxpnew[i]*uxpnew[i] + + uypnew[i]*uypnew[i] + + uzpnew[i]*uzpnew[i])); + const Real t_new_p = gammaboost*t_boost + - uzfrm*zpnew[i]*inv_c2; + const Real z_new_p = gammaboost*(zpnew[i] + + betaboost*Phys_c*t_boost); + const Real uz_new_p = gammaboost*uzpnew[i] + - gamma_new_p*uzfrm; + const Real gamma_old_p = std::sqrt(1.0 + inv_c2* + (uxpold[i]*uxpold[i] + + uypold[i]*uypold[i] + + uzpold[i]*uzpold[i])); + const Real t_old_p = gammaboost*(t_boost - dt) + - uzfrm*zpold[i]*inv_c2; + const Real z_old_p = gammaboost*(zpold[i] + + betaboost*Phys_c*(t_boost-dt)); + const Real uz_old_p = gammaboost*uzpold[i] + - gamma_old_p*uzfrm; + + // interpolate in time to t_lab + const Real weight_old = (t_new_p - t_lab) + / (t_new_p - t_old_p); + const Real weight_new = (t_lab - t_old_p) + / (t_new_p - t_old_p); + + const Real xp = xpold[i]*weight_old + + xpnew[i]*weight_new; + const Real yp = ypold[i]*weight_old + + ypnew[i]*weight_new; + const Real zp = z_old_p*weight_old + + z_new_p*weight_new; + const Real uxp = uxpold[i]*weight_old + + uxpnew[i]*weight_new; + const Real uyp = uypold[i]*weight_old + + uypnew[i]*weight_new; + const Real uzp = uz_old_p*weight_old + + uz_new_p *weight_new; + + const int loc = IndexLocation[i]; + diag_wp[loc] = wpnew[i]; + diag_xp[loc] = xp; + diag_yp[loc] = yp; + diag_zp[loc] = zp; + diag_uxp[loc] = uxp; + diag_uyp[loc] = uyp; + diag_uzp[loc] = uzp; + } + }); } } } diff --git a/Source/Particles/RigidInjectedParticleContainer.H b/Source/Particles/RigidInjectedParticleContainer.H index 3abbb4afe..a2473c5ad 100644 --- a/Source/Particles/RigidInjectedParticleContainer.H +++ b/Source/Particles/RigidInjectedParticleContainer.H @@ -44,9 +44,9 @@ public: DtType a_dt_type=DtType::Full) override; virtual void PushPX(WarpXParIter& pti, - amrex::Cuda::ManagedDeviceVector<amrex::ParticleReal>& xp, - amrex::Cuda::ManagedDeviceVector<amrex::ParticleReal>& yp, - amrex::Cuda::ManagedDeviceVector<amrex::ParticleReal>& zp, + amrex::Gpu::ManagedDeviceVector<amrex::ParticleReal>& xp, + amrex::Gpu::ManagedDeviceVector<amrex::ParticleReal>& yp, + amrex::Gpu::ManagedDeviceVector<amrex::ParticleReal>& zp, amrex::Real dt, DtType a_dt_type=DtType::Full) override; virtual void PushP (int lev, amrex::Real dt, diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index 507206041..bee71fba1 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -78,7 +78,7 @@ RigidInjectedParticleContainer::RemapParticles() // Note that the particles are already in the boosted frame. // This value is saved to advance the particles not injected yet - Cuda::ManagedDeviceVector<ParticleReal> xp, yp, zp; + Gpu::ManagedDeviceVector<ParticleReal> xp, yp, zp; for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { @@ -138,7 +138,7 @@ RigidInjectedParticleContainer::BoostandRemapParticles() #pragma omp parallel #endif { - Cuda::ManagedDeviceVector<ParticleReal> xp, yp, zp; + Gpu::ManagedDeviceVector<ParticleReal> xp, yp, zp; for (WarpXParIter pti(*this, 0); pti.isValid(); ++pti) { @@ -209,9 +209,9 @@ RigidInjectedParticleContainer::BoostandRemapParticles() void RigidInjectedParticleContainer::PushPX(WarpXParIter& pti, - Cuda::ManagedDeviceVector<ParticleReal>& xp, - Cuda::ManagedDeviceVector<ParticleReal>& yp, - Cuda::ManagedDeviceVector<ParticleReal>& zp, + Gpu::ManagedDeviceVector<ParticleReal>& xp, + Gpu::ManagedDeviceVector<ParticleReal>& yp, + Gpu::ManagedDeviceVector<ParticleReal>& zp, Real dt, DtType a_dt_type) { @@ -222,7 +222,7 @@ RigidInjectedParticleContainer::PushPX(WarpXParIter& pti, auto& uzp = attribs[PIdx::uz]; // Save the position and momenta, making copies - Cuda::ManagedDeviceVector<ParticleReal> xp_save, yp_save, zp_save; + Gpu::ManagedDeviceVector<ParticleReal> xp_save, yp_save, zp_save; RealVector uxp_save, uyp_save, uzp_save; ParticleReal* const AMREX_RESTRICT x = xp.dataPtr(); diff --git a/Source/Particles/Sorting/SortingUtils.H b/Source/Particles/Sorting/SortingUtils.H index 28a1b73b7..80eaaf9cb 100644 --- a/Source/Particles/Sorting/SortingUtils.H +++ b/Source/Particles/Sorting/SortingUtils.H @@ -2,7 +2,6 @@ #define WARPX_PARTICLES_SORTING_SORTINGUTILS_H_ #include <WarpXParticleContainer.H> -#include <AMReX_CudaContainers.H> #include <AMReX_Gpu.H> #ifdef AMREX_USE_GPU #include <thrust/partition.h> @@ -43,7 +42,7 @@ ForwardIterator stablePartition(ForwardIterator const index_begin, // On GPU: Use thrust int const* AMREX_RESTRICT predicate_ptr = predicate.dataPtr(); ForwardIterator const sep = thrust::stable_partition( - thrust::cuda::par(amrex::Cuda::The_ThrustCachedAllocator()), + thrust::cuda::par(amrex::Gpu::The_ThrustCachedAllocator()), index_begin, index_end, [predicate_ptr] AMREX_GPU_DEVICE (long i) { return predicate_ptr[i]; } ); diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index dbd913c5b..866947565 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -73,12 +73,12 @@ public: WarpXParIter (ContainerType& pc, int level); #if (AMREX_SPACEDIM == 2) - void GetPosition (amrex::Cuda::ManagedDeviceVector<amrex::ParticleReal>& x, - amrex::Cuda::ManagedDeviceVector<amrex::ParticleReal>& y, - amrex::Cuda::ManagedDeviceVector<amrex::ParticleReal>& z) const; - void SetPosition (const amrex::Cuda::ManagedDeviceVector<amrex::ParticleReal>& x, - const amrex::Cuda::ManagedDeviceVector<amrex::ParticleReal>& y, - const amrex::Cuda::ManagedDeviceVector<amrex::ParticleReal>& z); + void GetPosition (amrex::Gpu::ManagedDeviceVector<amrex::ParticleReal>& x, + amrex::Gpu::ManagedDeviceVector<amrex::ParticleReal>& y, + amrex::Gpu::ManagedDeviceVector<amrex::ParticleReal>& z) const; + void SetPosition (const amrex::Gpu::ManagedDeviceVector<amrex::ParticleReal>& x, + const amrex::Gpu::ManagedDeviceVector<amrex::ParticleReal>& y, + const amrex::Gpu::ManagedDeviceVector<amrex::ParticleReal>& z); #endif const std::array<RealVector, PIdx::nattribs>& GetAttribs () const { return GetStructOfArrays().GetRealData(); @@ -185,7 +185,8 @@ public: const amrex::MultiFab& Bz) = 0; void DepositCharge(amrex::Vector<std::unique_ptr<amrex::MultiFab> >& rho, - bool local = false); + bool local = false, bool reset = false, + bool do_rz_volume_scaling = false ); std::unique_ptr<amrex::MultiFab> GetChargeDensity(int lev, bool local = false); virtual void DepositCharge(WarpXParIter& pti, @@ -254,6 +255,8 @@ public: void setNextID(int next_id) { ParticleType::NextID(next_id); } bool do_splitting = false; + bool initialize_self_fields = false; + amrex::Real self_fields_required_precision = 1.e-11; // split along diagonals (0) or axes (1) int split_type = 0; @@ -299,6 +302,7 @@ protected: bool m_gather_from_main_grid = false; static int do_not_push; + static int do_not_deposit; // Whether to allow particles outside of the simulation domain to be // initialized when they enter the domain. @@ -322,9 +326,12 @@ protected: #ifdef WARPX_QED bool m_do_qed = false; + //Species for which QED effects are relevant should override these methods virtual bool has_quantum_sync(){return false;}; virtual bool has_breit_wheeler(){return false;}; + //Species can receive a shared pointer to a QED engine (species for + //which this is relevant should override these functions) virtual void set_breit_wheeler_engine_ptr(std::shared_ptr<BreitWheelerEngine>){}; virtual void diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 7fb57500d..9f02da338 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -1,13 +1,13 @@ - #include <limits> #include <MultiParticleContainer.H> #include <WarpXParticleContainer.H> #include <AMReX_AmrParGDB.H> +#include <WarpXComm.H> #include <WarpX_f.H> #include <WarpX.H> #include <WarpXAlgorithmSelection.H> - +#include <WarpXComm.H> // Import low-level single-particle kernels #include <GetAndSetPosition.H> #include <UpdatePosition.H> @@ -17,6 +17,7 @@ using namespace amrex; int WarpXParticleContainer::do_not_push = 0; +int WarpXParticleContainer::do_not_deposit = 0; WarpXParIter::WarpXParIter (ContainerType& pc, int level) : ParIter(pc, level, MFItInfo().SetDynamic(WarpX::do_dynamic_scheduling)) @@ -270,11 +271,12 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, // If no particles, do not do anything if (np_to_depose == 0) return; + // If user decides not to deposit + if (do_not_deposit) return; + const long ngJ = jx->nGrow(); const std::array<Real,3>& dx = WarpX::CellSize(std::max(depos_lev,0)); - int j_is_nodal = jx->is_nodal() and jy->is_nodal() and jz->is_nodal(); Real q = this->charge; - const Real stagger_shift = j_is_nodal ? 0.0 : 0.5; BL_PROFILE_VAR_NS("PPC::Evolve::Accumulate", blp_accumulate); BL_PROFILE_VAR_NS("PPC::CurrentDeposition", blp_deposit); @@ -300,6 +302,9 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, #ifdef AMREX_USE_GPU // No tiling on GPU: jx_ptr points to the full // jx array (same for jy_ptr and jz_ptr). + auto & jx_fab = jx->get(pti); + auto & jy_fab = jy->get(pti); + auto & jz_fab = jz->get(pti); Array4<Real> const& jx_arr = jx->array(pti); Array4<Real> const& jy_arr = jy->array(pti); Array4<Real> const& jz_arr = jz->array(pti); @@ -319,6 +324,9 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, local_jy[thread_num].setVal(0.0); local_jz[thread_num].setVal(0.0); + auto & jx_fab = local_jx[thread_num]; + auto & jy_fab = local_jy[thread_num]; + auto & jz_fab = local_jz[thread_num]; Array4<Real> const& jx_arr = local_jx[thread_num].array(); Array4<Real> const& jy_arr = local_jy[thread_num].array(); Array4<Real> const& jz_arr = local_jz[thread_num].array(); @@ -333,13 +341,8 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, // Lower corner of tile box physical domain // Note that this includes guard cells since it is after tilebox.ngrow - const std::array<Real, 3>& xyzmin = WarpX::LowerCorner(tilebox, depos_lev); - // xyzmin is built on pti.tilebox(), so it does - // not include staggering, so the stagger_shift has to be done by hand. - // Alternatively, we could define xyzminx from tbx (and the same for 3 - // directions and for jx, jy, jz). This way, sx0 would not be needed. - // Better for memory? worth trying? const Dim3 lo = lbound(tilebox); + const std::array<Real, 3>& xyzmin = WarpX::LowerCorner(tilebox, depos_lev); BL_PROFILE_VAR_START(blp_deposit); if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { @@ -367,20 +370,20 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, doDepositionShapeN<1>( xp, yp, zp, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_depose, dt, dx, xyzmin, lo, - stagger_shift, q); + jx_fab, jy_fab, jz_fab, np_to_depose, dt, dx, + xyzmin, lo, q); } else if (WarpX::nox == 2){ doDepositionShapeN<2>( xp, yp, zp, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_depose, dt, dx, xyzmin, lo, - stagger_shift, q); + jx_fab, jy_fab, jz_fab, np_to_depose, dt, dx, + xyzmin, lo, q); } else if (WarpX::nox == 3){ doDepositionShapeN<3>( xp, yp, zp, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_depose, dt, dx, xyzmin, lo, - stagger_shift, q); + jx_fab, jy_fab, jz_fab, np_to_depose, dt, dx, + xyzmin, lo, q); } } BL_PROFILE_VAR_STOP(blp_deposit); @@ -428,6 +431,9 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector& wp, // If no particles, do not do anything if (np_to_depose == 0) return; + // If user decides not to deposit + if (do_not_deposit) return; + const long ngRho = rho->nGrow(); const std::array<Real,3>& dx = WarpX::CellSize(std::max(depos_lev,0)); const Real q = this->charge; @@ -501,68 +507,68 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector& wp, } void -WarpXParticleContainer::DepositCharge (Vector<std::unique_ptr<MultiFab> >& rho, bool local) +WarpXParticleContainer::DepositCharge (Vector<std::unique_ptr<MultiFab> >& rho, + bool local, bool reset, + bool do_rz_volume_scaling) { + // Loop over the refinement levels + int const finest_level = rho.size() - 1; + for (int lev = 0; lev <= finest_level; ++lev) { - int num_levels = rho.size(); - int finest_level = num_levels - 1; - - // each level deposits it's own particles - const int ng = rho[0]->nGrow(); - for (int lev = 0; lev < num_levels; ++lev) { - - rho[lev]->setVal(0.0, ng); - - const auto& gm = m_gdb->Geom(lev); - const auto& ba = m_gdb->ParticleBoxArray(lev); + // Reset the `rho` array if `reset` is True + if (reset) rho[lev]->setVal(0.0, rho[lev]->nGrow()); - const Real* dx = gm.CellSize(); - const Real* plo = gm.ProbLo(); - BoxArray nba = ba; - nba.surroundingNodes(); + // Loop over particle tiles and deposit charge on each level +#ifdef _OPENMP + #pragma omp parallel + { + int thread_num = omp_get_thread_num(); +#else + int thread_num = 0; +#endif + for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) + { + const long np = pti.numParticles(); + auto& wp = pti.GetAttribs(PIdx::w); - for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { - const Box& box = nba[pti]; + pti.GetPosition(m_xp[thread_num], m_yp[thread_num], m_zp[thread_num]); - auto& wp = pti.GetAttribs(PIdx::w); - const auto& particles = pti.GetArrayOfStructs(); - int nstride = particles.dataShape().first; - const long np = pti.numParticles(); + int* AMREX_RESTRICT ion_lev; + if (do_field_ionization){ + ion_lev = pti.GetiAttribs(particle_icomps["ionization_level"]).dataPtr(); + } else { + ion_lev = nullptr; + } - FArrayBox& rhofab = (*rho[lev])[pti]; + DepositCharge(pti, wp, ion_lev, rho[lev].get(), 0, 0, np, thread_num, lev, lev); + } +#ifdef _OPENMP + } +#endif - WRPX_DEPOSIT_CIC(particles.dataPtr(), nstride, np, - wp.dataPtr(), &this->charge, - rhofab.dataPtr(), box.loVect(), box.hiVect(), - plo, dx, &ng); +#ifdef WARPX_DIM_RZ + if (do_rz_volume_scaling) { + WarpX::GetInstance().ApplyInverseVolumeScalingToChargeDensity(rho[lev].get(), lev); } +#endif - if (!local) rho[lev]->SumBoundary(gm.periodicity()); + // Exchange guard cells + if (!local) rho[lev]->SumBoundary( m_gdb->Geom(lev).periodicity() ); } - // now we average down fine to crse - std::unique_ptr<MultiFab> crse; + // Now that the charge has been deposited at each level, + // we average down from fine to crse for (int lev = finest_level - 1; lev >= 0; --lev) { - const BoxArray& fine_BA = rho[lev+1]->boxArray(); const DistributionMapping& fine_dm = rho[lev+1]->DistributionMap(); - BoxArray coarsened_fine_BA = fine_BA; + BoxArray coarsened_fine_BA = rho[lev+1]->boxArray(); coarsened_fine_BA.coarsen(m_gdb->refRatio(lev)); - MultiFab coarsened_fine_data(coarsened_fine_BA, fine_dm, rho[lev+1]->nComp(), 0); coarsened_fine_data.setVal(0.0); - IntVect ratio(AMREX_D_DECL(2, 2, 2)); // FIXME - - for (MFIter mfi(coarsened_fine_data); mfi.isValid(); ++mfi) { - const Box& bx = mfi.validbox(); - const Box& crse_box = coarsened_fine_data[mfi].box(); - const Box& fine_box = (*rho[lev+1])[mfi].box(); - WRPX_SUM_FINE_TO_CRSE_NODAL(bx.loVect(), bx.hiVect(), ratio.getVect(), - coarsened_fine_data[mfi].dataPtr(), crse_box.loVect(), crse_box.hiVect(), - (*rho[lev+1])[mfi].dataPtr(), fine_box.loVect(), fine_box.hiVect()); - } + int const refinement_ratio = 2; - rho[lev]->copy(coarsened_fine_data, m_gdb->Geom(lev).periodicity(), FabArrayBase::ADD); + interpolateDensityFineToCoarse( *rho[lev+1], coarsened_fine_data, refinement_ratio ); + rho[lev]->ParallelAdd( coarsened_fine_data, m_gdb->Geom(lev).periodicity() ); } } @@ -624,7 +630,8 @@ Real WarpXParticleContainer::sumParticleCharge(bool local) { amrex::Real total_charge = 0.0; - for (int lev = 0; lev < finestLevel(); ++lev) + const int nLevels = finestLevel(); + for (int lev = 0; lev < nLevels; ++lev) { #ifdef _OPENMP @@ -654,7 +661,8 @@ std::array<Real, 3> WarpXParticleContainer::meanParticleVelocity(bool local) { amrex::Real inv_clight_sq = 1.0/PhysConst::c/PhysConst::c; - for (int lev = 0; lev <= finestLevel(); ++lev) { + const int nLevels = finestLevel(); + for (int lev = 0; lev <= nLevels; ++lev) { #ifdef _OPENMP #pragma omp parallel reduction(+:vx_total, vy_total, vz_total, np_total) @@ -698,7 +706,8 @@ Real WarpXParticleContainer::maxParticleVelocity(bool local) { amrex::ParticleReal max_v = 0.0; - for (int lev = 0; lev <= finestLevel(); ++lev) + const int nLevels = finestLevel(); + for (int lev = 0; lev <= nLevels; ++lev) { #ifdef _OPENMP @@ -724,7 +733,7 @@ WarpXParticleContainer::PushXES (Real dt) { BL_PROFILE("WPC::PushXES()"); - int num_levels = finestLevel() + 1; + const int num_levels = finestLevel() + 1; for (int lev = 0; lev < num_levels; ++lev) { const auto& gm = m_gdb->Geom(lev); @@ -753,7 +762,8 @@ WarpXParticleContainer::PushXES (Real dt) void WarpXParticleContainer::PushX (Real dt) { - for (int lev = 0; lev <= finestLevel(); ++lev) { + const int nLevels = finestLevel(); + for (int lev = 0; lev <= nLevels; ++lev) { PushX(lev, dt); } } @@ -841,5 +851,3 @@ WarpXParticleContainer::particlePostLocate(ParticleType& p, if (pld.m_lev == lev-1){ } } - - diff --git a/Source/Particles/deposit_cic.F90 b/Source/Particles/deposit_cic.F90 deleted file mode 100644 index 2831ce96b..000000000 --- a/Source/Particles/deposit_cic.F90 +++ /dev/null @@ -1,134 +0,0 @@ -module warpx_ES_deposit_cic - - use iso_c_binding - use amrex_fort_module, only : amrex_real, amrex_particle_real - - implicit none - -contains - -! This routine computes the charge density due to the particles using cloud-in-cell -! deposition. The Fab rho is assumed to be node-centered. -! -! Arguments: -! particles : a pointer to the particle array-of-structs -! ns : the stride length of particle struct (the size of the struct in number of reals) -! np : the number of particles -! weights : the particle weights -! charge : the charge of this particle species -! rho : a Fab that will contain the charge density on exit -! lo : a pointer to the lo corner of this valid box for rho, in index space -! hi : a pointer to the hi corner of this valid box for rho, in index space -! plo : the real position of the left-hand corner of the problem domain -! dx : the mesh spacing -! ng : the number of ghost cells in rho -! - subroutine warpx_deposit_cic_3d(particles, ns, np, & - weights, charge, rho, lo, hi, plo, dx, & - ng) & - bind(c,name='warpx_deposit_cic_3d') - integer, value, intent(in) :: ns, np - real(amrex_particle_real), intent(in) :: particles(ns,np) - real(amrex_particle_real), intent(in) :: weights(np) - real(amrex_real), intent(in) :: charge - integer, intent(in) :: lo(3) - integer, intent(in) :: hi(3) - integer, intent(in) :: ng - real(amrex_real), intent(inout) :: rho(lo(1)-ng:hi(1)+ng, lo(2)-ng:hi(2)+ng, lo(3)-ng:hi(3)+ng) - real(amrex_real), intent(in) :: plo(3) - real(amrex_real), intent(in) :: dx(3) - - integer i, j, k, n - real(amrex_real) wx_lo, wy_lo, wz_lo, wx_hi, wy_hi, wz_hi - real(amrex_real) lx, ly, lz - real(amrex_real) inv_dx(3) - real(amrex_real) qp, inv_vol - - inv_dx = 1.0d0/dx - - inv_vol = inv_dx(1) * inv_dx(2) * inv_dx(3) - - do n = 1, np - - qp = weights(n) * charge * inv_vol - - lx = (particles(1, n) - plo(1))*inv_dx(1) - ly = (particles(2, n) - plo(2))*inv_dx(2) - lz = (particles(3, n) - plo(3))*inv_dx(3) - - i = floor(lx) - j = floor(ly) - k = floor(lz) - - wx_hi = lx - i - wy_hi = ly - j - wz_hi = lz - k - - wx_lo = 1.0d0 - wx_hi - wy_lo = 1.0d0 - wy_hi - wz_lo = 1.0d0 - wz_hi - - rho(i, j, k ) = rho(i, j, k ) + wx_lo*wy_lo*wz_lo*qp - rho(i, j, k+1) = rho(i, j, k+1) + wx_lo*wy_lo*wz_hi*qp - rho(i, j+1, k ) = rho(i, j+1, k ) + wx_lo*wy_hi*wz_lo*qp - rho(i, j+1, k+1) = rho(i, j+1, k+1) + wx_lo*wy_hi*wz_hi*qp - rho(i+1, j, k ) = rho(i+1, j, k ) + wx_hi*wy_lo*wz_lo*qp - rho(i+1, j, k+1) = rho(i+1, j, k+1) + wx_hi*wy_lo*wz_hi*qp - rho(i+1, j+1, k ) = rho(i+1, j+1, k ) + wx_hi*wy_hi*wz_lo*qp - rho(i+1, j+1, k+1) = rho(i+1, j+1, k+1) + wx_hi*wy_hi*wz_hi*qp - - end do - - end subroutine warpx_deposit_cic_3d - - subroutine warpx_deposit_cic_2d(particles, ns, np, & - weights, charge, rho, lo, hi, plo, dx, & - ng) & - bind(c,name='warpx_deposit_cic_2d') - integer, value, intent(in) :: ns, np - real(amrex_particle_real), intent(in) :: particles(ns,np) - real(amrex_particle_real), intent(in) :: weights(np) - real(amrex_real), intent(in) :: charge - integer, intent(in) :: lo(2) - integer, intent(in) :: hi(2) - integer, intent(in) :: ng - real(amrex_real), intent(inout) :: rho(lo(1)-ng:hi(1)+ng, lo(2)-ng:hi(2)+ng) - real(amrex_real), intent(in) :: plo(2) - real(amrex_real), intent(in) :: dx(2) - - integer i, j, n - real(amrex_real) wx_lo, wy_lo, wx_hi, wy_hi - real(amrex_real) lx, ly - real(amrex_real) inv_dx(2) - real(amrex_real) qp, inv_vol - - inv_dx = 1.0d0/dx - - inv_vol = inv_dx(1) * inv_dx(2) - - do n = 1, np - - qp = weights(n) * charge * inv_vol - - lx = (particles(1, n) - plo(1))*inv_dx(1) - ly = (particles(2, n) - plo(2))*inv_dx(2) - - i = floor(lx) - j = floor(ly) - - wx_hi = lx - i - wy_hi = ly - j - - wx_lo = 1.0d0 - wx_hi - wy_lo = 1.0d0 - wy_hi - - rho(i, j ) = rho(i, j ) + wx_lo*wy_lo*qp - rho(i, j+1) = rho(i, j+1) + wx_lo*wy_hi*qp - rho(i+1, j ) = rho(i+1, j ) + wx_hi*wy_lo*qp - rho(i+1, j+1) = rho(i+1, j+1) + wx_hi*wy_hi*qp - - end do - - end subroutine warpx_deposit_cic_2d - -end module warpx_ES_deposit_cic diff --git a/Source/QED/BreitWheelerDummyTable.H b/Source/QED/BreitWheelerDummyTable.H new file mode 100644 index 000000000..e03f9d20b --- /dev/null +++ b/Source/QED/BreitWheelerDummyTable.H @@ -0,0 +1,78 @@ +#ifndef WARPX_breit_wheeler_dummy_tables_h_ +#define WARPX_breit_wheeler_dummy_tables_h_ + +#include "BreitWheelerEngineInnards.H" + +#include <AMReX_REAL.H> + +#include <limits> +#include <vector> + +namespace QedUtils{ + +//A default mini-table used for test purposes +const struct //BreitWheelerEngineInnardsDummy +{ + picsar::multi_physics::breit_wheeler_engine_ctrl<amrex::Real> ctrl{ + 0.001, /*chi_phot_min*/ + 0.1, /*chi_phot_tdndt_min*/ + 200, /*chi_phot_tdndt_max*/ + 64, /*chi_phot_tdndt_how_many*/ + 0.01, /*chi_phot_tpair_min*/ + 200, /*chi_phot_tpair_max*/ + 2, /*chi_phot_tpair_how_many*/ + 2 /*chi_frac_tpair_how_many*/ + }; + std::vector<amrex::Real> TTfunc_coords{ + -2.302585093, -2.181935848, -2.061286602, -1.940637357, + -1.819988111, -1.699338866, -1.578689621, -1.458040375, + -1.33739113, -1.216741884, -1.096092639, -0.9754433937, + -0.8547941483, -0.7341449029, -0.6134956575, -0.4928464122, + -0.3721971668, -0.2515479214, -0.130898676, -0.01024943059, + 0.1103998148, 0.2310490602, 0.3516983056, 0.472347551, + 0.5929967964, 0.7136460417, 0.8342952871, 0.9549445325, + 1.075593778, 1.196243023, 1.316892269, 1.437541514, + 1.558190759, 1.678840005, 1.79948925, 1.920138496, + 2.040787741, 2.161436986, 2.282086232, 2.402735477, + 2.523384723, 2.644033968, 2.764683213, 2.885332459, + 3.005981704, 3.12663095, 3.247280195, 3.36792944, + 3.488578686, 3.609227931, 3.729877176, 3.850526422, + 3.971175667, 4.091824913, 4.212474158, 4.333123403, + 4.453772649, 4.574421894, 4.69507114, 4.815720385, + 4.93636963, 5.057018876, 5.177668121, 5.298317367 + }; + std::vector<amrex::Real> TTfunc_data{ + -32.75941722, -29.48929687, -26.5638705, -23.94401054, + -21.59504068, -19.48623016, -17.59034545, -15.88325289, + -14.34356639, -12.95233518, -11.69276696, -10.54998244, + -9.510797773, -8.563531596, -7.697833953, -6.90453462, + -6.175508601, -5.503556869, -4.882300625, -4.306087546, + -3.76990867, -3.269324729, -2.800400851, -2.359648711, + -1.943975268, -1.550637379, -1.177201596, -0.8215085915, + -0.4816416657, -0.1558988919, 0.1572315255, 0.4590930403, + 0.7508800304, 1.033654828, 1.308362775, 1.575845526, + 1.836852816, 2.092052851, 2.342041503, 2.587350441, + 2.828454313, 3.065777115, 3.299697807, 3.530555306, + 3.758652908, 3.984262221, 4.207626639, 4.428964367, + 4.648471027, 4.866321957, 5.082674363, 5.297669484, + 5.511434783, 5.724086013, 5.935728862, 6.146459973, + 6.356367276, 6.565529804, 6.774017335, 6.981890203, + 7.189199547, 7.395988084, 7.602291337, 7.808139168 + }; + std::vector<amrex::Real> cum_distrib_coords_1{ + -4.605170186, 5.298317367 + };//_____________________________ + std::vector<amrex::Real> cum_distrib_coords_2{ + 0, 0.5 + };//_____________________________ + std::vector<amrex::Real> cum_distrib_data{ + -std::numeric_limits<amrex::Real>::infinity(), + -0.6931471806, + -std::numeric_limits<amrex::Real>::infinity(), + -0.6931471806 + };//_____________________________ +} BreitWheelerEngineInnardsDummy; + +}; + +#endif // WARPX_breit_wheeler_dummy_tables_h_ diff --git a/Source/QED/BreitWheelerEngineWrapper.H b/Source/QED/BreitWheelerEngineWrapper.H index 1033ff7c9..369c64375 100644 --- a/Source/QED/BreitWheelerEngineWrapper.H +++ b/Source/QED/BreitWheelerEngineWrapper.H @@ -151,7 +151,7 @@ class BreitWheelerGeneratePairs { public: /** - * Constructor acquires a reference to control parameters and + * Constructor acquires pointers to control parameters and * lookup tables data. * lookup_table uses non-owning vectors under the hood. So no new data * allocations should be triggered on GPU @@ -270,6 +270,12 @@ public: bool init_lookup_tables_from_raw_data (const amrex::Vector<char>& raw_data); /** + * Init lookup tables using built-in dummy tables + * for test purposes. + */ + void init_dummy_tables(); + + /** * Export lookup tables data into a raw binary Vector * @return the data in binary format. The Vector is empty if tables were * not previously initialized. @@ -288,6 +294,12 @@ public: */ PicsarBreitWheelerCtrl get_default_ctrl() const; + /** + * returns a constant reference to the control parameters + * @return const reference to control parameters + */ + const PicsarBreitWheelerCtrl& get_ref_ctrl() const; + private: bool m_lookup_tables_initialized = false; diff --git a/Source/QED/BreitWheelerEngineWrapper.cpp b/Source/QED/BreitWheelerEngineWrapper.cpp index 42953c97f..c963d44d1 100644 --- a/Source/QED/BreitWheelerEngineWrapper.cpp +++ b/Source/QED/BreitWheelerEngineWrapper.cpp @@ -1,6 +1,7 @@ #include "BreitWheelerEngineWrapper.H" #include "QedTableParserHelperFunctions.H" +#include "BreitWheelerDummyTable.H" #include <utility> @@ -140,6 +141,28 @@ BreitWheelerEngine::init_lookup_tables_from_raw_data ( return true; } +void BreitWheelerEngine::init_dummy_tables() +{ + m_innards.ctrl = QedUtils::BreitWheelerEngineInnardsDummy.ctrl; + m_innards.TTfunc_coords.assign( + QedUtils::BreitWheelerEngineInnardsDummy.TTfunc_coords.begin(), + QedUtils::BreitWheelerEngineInnardsDummy.TTfunc_coords.end()); + m_innards.TTfunc_data.assign( + QedUtils::BreitWheelerEngineInnardsDummy.TTfunc_data.begin(), + QedUtils::BreitWheelerEngineInnardsDummy.TTfunc_data.end()); + m_innards.cum_distrib_coords_1.assign( + QedUtils::BreitWheelerEngineInnardsDummy.cum_distrib_coords_1.begin(), + QedUtils::BreitWheelerEngineInnardsDummy.cum_distrib_coords_1.end()); + m_innards.cum_distrib_coords_2.assign( + QedUtils::BreitWheelerEngineInnardsDummy.cum_distrib_coords_2.begin(), + QedUtils::BreitWheelerEngineInnardsDummy.cum_distrib_coords_2.end()); + m_innards.cum_distrib_data.assign( + QedUtils::BreitWheelerEngineInnardsDummy.cum_distrib_data.begin(), + QedUtils::BreitWheelerEngineInnardsDummy.cum_distrib_data.end()); + + m_lookup_tables_initialized = true; +} + Vector<char> BreitWheelerEngine::export_lookup_tables_data () const { Vector<char> res{}; @@ -176,6 +199,12 @@ BreitWheelerEngine::get_default_ctrl() const return PicsarBreitWheelerCtrl(); } +const PicsarBreitWheelerCtrl& +BreitWheelerEngine::get_ref_ctrl() const +{ + return m_innards.ctrl; +} + void BreitWheelerEngine::compute_lookup_tables ( PicsarBreitWheelerCtrl ctrl) { diff --git a/Source/QED/Make.package b/Source/QED/Make.package index c9cd73cc2..bfc6f2369 100644 --- a/Source/QED/Make.package +++ b/Source/QED/Make.package @@ -5,6 +5,8 @@ CEXE_headers += BreitWheelerEngineInnards.H CEXE_headers += QuantumSyncEngineInnards.H CEXE_headers += BreitWheelerEngineWrapper.H CEXE_headers += QuantumSyncEngineWrapper.H +CEXE_headers += BreitWheelerDummyTable.H +CEXE_headers += QuantumSyncDummyTable.H CEXE_sources += BreitWheelerEngineWrapper.cpp CEXE_sources += QuantumSyncEngineWrapper.cpp diff --git a/Source/QED/QuantumSyncDummyTable.H b/Source/QED/QuantumSyncDummyTable.H new file mode 100644 index 000000000..587e8b546 --- /dev/null +++ b/Source/QED/QuantumSyncDummyTable.H @@ -0,0 +1,78 @@ +#ifndef WARPX_quantum_sync_dummy_tables_h_ +#define WARPX_quantum_sync_dummy_tables_h_ + +#include "QuantumSyncEngineInnards.H" + +#include <AMReX_REAL.H> +#include <AMReX_Gpu.H> + +#include <limits> + +namespace QedUtils{ + +//A default mini-table used for test purposes +const struct //QuantumSyncEngineInnardsDummy +{ + picsar::multi_physics::quantum_synchrotron_engine_ctrl<amrex::Real> ctrl{ + 0.001, /*chi_part_min*/ + 0.001, /*chi_part_tdndt_min*/ + 200, /*chi_part_tdndt_max*/ + 64, /*chi_part_tdndt_how_many*/ + 0.001, /*chi_part_tem_min*/ + 200, /*chi_part_tem_max*/ + 2, /*chi_part_tem_how_many*/ + 2 /*prob_tem_how_many*/ + }; + std::vector<amrex::Real> KKfunc_coords{ + -6.907755279, -6.714008094, -6.520260909, -6.326513724, + -6.13276654, -5.939019355, -5.74527217, -5.551524985, + -5.3577778, -5.164030615, -4.97028343, -4.776536246, + -4.582789061, -4.389041876, -4.195294691, -4.001547506, + -3.807800321, -3.614053137, -3.420305952, -3.226558767, + -3.032811582, -2.839064397, -2.645317212, -2.451570027, + -2.257822843, -2.064075658, -1.870328473, -1.676581288, + -1.482834103, -1.289086918, -1.095339733, -0.9015925486, + -0.7078453638, -0.5140981789, -0.3203509941, -0.1266038092, + 0.06714337561, 0.2608905605, 0.4546377453, 0.6483849302, + 0.842132115, 1.0358793, 1.229626485, 1.42337367, + 1.617120854, 1.810868039, 2.004615224, 2.198362409, + 2.392109594, 2.585856779, 2.779603964, 2.973351148, + 3.167098333, 3.360845518, 3.554592703, 3.748339888, + 3.942087073, 4.135834257, 4.329581442, 4.523328627, + 4.717075812, 4.910822997, 5.104570182, 5.298317367 + }; + std::vector<amrex::Real> KKfunc_data{ + -7.968431811, -7.639082211, -7.326295546, -7.02752527, + -6.740710773, -6.464172009, -6.196529608, -5.93664402 + -5.683568899, -5.436515162, -5.194823127, -4.957940775, + -4.725406674, -4.49683649, -4.2719122, -4.050373372, + -3.832009948, -3.616656119, -3.404184903, -3.194503151, + -2.987546751, -2.783275883, -2.581670257, -2.382724345, + -2.1864427, -1.992835514, -1.801914573, -1.613689793, + -1.428166439, -1.2453431, -1.065210351, -0.8877500928, + -0.7129353081, -0.5407301909, -0.3710904422, -0.2039636495, + -0.03928968527, 0.1229988926, 0.2829764221, 0.4407236701, + 0.5963272798, 0.7498791107, 0.9014754584, 1.051216164, + 1.199203636, 1.345541816, 1.490335133, 1.633687478, + 1.775701241, 1.916476434, 2.056109933, 2.194694829, + 2.332319922, 2.469069336, 2.605022252, 2.740252763, + 2.874829832, 3.008817314, 3.142273988, 3.27525366, + 3.407805563, 3.539975021, 3.671803889, 3.803330346 + };//_____________________________ + std::vector<amrex::Real> cum_distrib_coords_1{ + -6.907755279, 5.298317367 + };//_____________________________ + std::vector<amrex::Real> cum_distrib_coords_2{ + 0, 0.5 + };//_____________________________ + std::vector<amrex::Real> cum_distrib_data{ + -std::numeric_limits<amrex::Real>::infinity(), + -0.6931471806, + -std::numeric_limits<amrex::Real>::infinity(), + -0.6931471806 + };//_____________________________ +} QuantumSyncEngineInnardsDummy; + +}; + +#endif //WARPX_quantum_sync_dummy_tables_h_ diff --git a/Source/QED/QuantumSyncEngineWrapper.H b/Source/QED/QuantumSyncEngineWrapper.H index 1a6ffe4f3..df0bdc5f5 100644 --- a/Source/QED/QuantumSyncEngineWrapper.H +++ b/Source/QED/QuantumSyncEngineWrapper.H @@ -79,7 +79,7 @@ class QuantumSynchrotronEvolveOpticalDepth { public: /** - * Constructor acquires a reference to control parameters and + * Constructor acquires pointers to control parameters and * lookup tables data. * lookup_table uses non-owning vectors under the hood. So no new data * allocations should be triggered on GPU @@ -152,7 +152,7 @@ class QuantumSynchrotronGeneratePhotonAndUpdateMomentum { public: /** - * Constructor acquires a reference to control parameters and + * Constructor acquires pointers to control parameters and * lookup tables data. * lookup_table uses non-owning vectors under the hood. So no new data * allocations should be triggered on GPU @@ -267,6 +267,12 @@ public: bool init_lookup_tables_from_raw_data (const amrex::Vector<char>& raw_data); /** + * Init lookup tables using built-in dummy tables + * for test purposes. + */ + void init_dummy_tables(); + + /** * Export lookup tables data into a raw binary Vector * @return the data in binary format. The Vector is empty if tables were * not previously initialized. @@ -285,6 +291,12 @@ public: */ PicsarQuantumSynchrotronCtrl get_default_ctrl() const; + /** + * returns a constant reference to the control parameters + * @return const reference to control parameters + */ + const PicsarQuantumSynchrotronCtrl& get_ref_ctrl() const; + private: bool m_lookup_tables_initialized = false; diff --git a/Source/QED/QuantumSyncEngineWrapper.cpp b/Source/QED/QuantumSyncEngineWrapper.cpp index b2630dc4d..ffafec761 100644 --- a/Source/QED/QuantumSyncEngineWrapper.cpp +++ b/Source/QED/QuantumSyncEngineWrapper.cpp @@ -1,6 +1,7 @@ #include "QuantumSyncEngineWrapper.H" #include "QedTableParserHelperFunctions.H" +#include "QuantumSyncDummyTable.H" #include <utility> @@ -139,6 +140,28 @@ QuantumSynchrotronEngine::init_lookup_tables_from_raw_data ( return true; } +void QuantumSynchrotronEngine::init_dummy_tables() +{ + m_innards.ctrl = QedUtils::QuantumSyncEngineInnardsDummy.ctrl; + m_innards.KKfunc_coords.assign( + QedUtils::QuantumSyncEngineInnardsDummy.KKfunc_coords.begin(), + QedUtils::QuantumSyncEngineInnardsDummy.KKfunc_coords.end()); + m_innards.KKfunc_data.assign( + QedUtils::QuantumSyncEngineInnardsDummy.KKfunc_data.begin(), + QedUtils::QuantumSyncEngineInnardsDummy.KKfunc_data.end()); + m_innards.cum_distrib_coords_1.assign( + QedUtils::QuantumSyncEngineInnardsDummy.cum_distrib_coords_1.begin(), + QedUtils::QuantumSyncEngineInnardsDummy.cum_distrib_coords_1.end()); + m_innards.cum_distrib_coords_2.assign( + QedUtils::QuantumSyncEngineInnardsDummy.cum_distrib_coords_2.begin(), + QedUtils::QuantumSyncEngineInnardsDummy.cum_distrib_coords_2.end()); + m_innards.cum_distrib_data.assign( + QedUtils::QuantumSyncEngineInnardsDummy.cum_distrib_data.begin(), + QedUtils::QuantumSyncEngineInnardsDummy.cum_distrib_data.end()); + + m_lookup_tables_initialized = true; +} + Vector<char> QuantumSynchrotronEngine::export_lookup_tables_data () const { Vector<char> res{}; @@ -175,6 +198,12 @@ QuantumSynchrotronEngine::get_default_ctrl() const return PicsarQuantumSynchrotronCtrl(); } +const PicsarQuantumSynchrotronCtrl& +QuantumSynchrotronEngine::get_ref_ctrl() const +{ + return m_innards.ctrl; +} + void QuantumSynchrotronEngine::compute_lookup_tables ( PicsarQuantumSynchrotronCtrl ctrl) { diff --git a/Source/Utils/utils_ES.F90 b/Source/Utils/utils_ES.F90 index ce143bb94..baadeb7af 100644 --- a/Source/Utils/utils_ES.F90 +++ b/Source/Utils/utils_ES.F90 @@ -7,79 +7,6 @@ module warpx_ES_utils contains - subroutine warpx_sum_fine_to_crse_nodal_3d (lo, hi, lrat, crse, clo, chi, fine, flo, fhi) & - bind(c, name="warpx_sum_fine_to_crse_nodal_3d") - - integer, intent(in) :: lo(3), hi(3) - integer, intent(in) :: clo(3), chi(3) - integer, intent(in) :: flo(3), fhi(3) - integer, intent(in) :: lrat(3) - real(amrex_real), intent(inout) :: crse(clo(1):chi(1),clo(2):chi(2),clo(3):chi(3)) - real(amrex_real), intent(in) :: fine(flo(1):fhi(1),flo(2):fhi(2),flo(3):fhi(3)) - - integer :: i, j, k, ii, jj, kk - - do k = lo(3), hi(3) - kk = k * lrat(3) - do j = lo(2), hi(2) - jj = j * lrat(2) - do i = lo(1), hi(1) - ii = i * lrat(1) - crse(i,j,k) = fine(ii,jj,kk) + & -! These six fine nodes are shared by two coarse nodes... - 0.5d0 * (fine(ii-1,jj,kk) + fine(ii+1,jj,kk) + & - fine(ii,jj-1,kk) + fine(ii,jj+1,kk) + & - fine(ii,jj,kk-1) + fine(ii,jj,kk+1)) + & -! ... these twelve are shared by four... - 0.25d0 * (fine(ii,jj-1,kk-1) + fine(ii,jj+1,kk-1) + & - fine(ii,jj-1,kk+1) + fine(ii,jj+1,kk+1) + & - fine(ii-1,jj,kk-1) + fine(ii+1,jj,kk-1) + & - fine(ii-1,jj,kk+1) + fine(ii+1,jj,kk+1) + & - fine(ii-1,jj-1,kk) + fine(ii+1,jj-1,kk) + & - fine(ii-1,jj+1,kk) + fine(ii+1,jj+1,kk)) + & -! ... and these eight are shared by eight - 0.125d0 * (fine(ii-1,jj-1,kk-1) + fine(ii-1,jj-1,kk+1) + & - fine(ii-1,jj+1,kk-1) + fine(ii-1,jj+1,kk+1) + & - fine(ii+1,jj-1,kk-1) + fine(ii+1,jj-1,kk+1) + & - fine(ii+1,jj+1,kk-1) + fine(ii+1,jj+1,kk+1)) -! ... note that we have 27 nodes in total... - crse(i,j,k) = crse(i,j,k) / 8.d0 - end do - end do - end do - - end subroutine warpx_sum_fine_to_crse_nodal_3d - - subroutine warpx_sum_fine_to_crse_nodal_2d (lo, hi, lrat, crse, clo, chi, fine, flo, fhi) & - bind(c, name="warpx_sum_fine_to_crse_nodal_2d") - - integer, intent(in) :: lo(2), hi(2) - integer, intent(in) :: clo(2), chi(2) - integer, intent(in) :: flo(2), fhi(2) - integer, intent(in) :: lrat(2) - real(amrex_real), intent(inout) :: crse(clo(1):chi(1),clo(2):chi(2)) - real(amrex_real), intent(in) :: fine(flo(1):fhi(1),flo(2):fhi(2)) - - integer :: i, j, ii, jj - - do j = lo(2), hi(2) - jj = j * lrat(2) - do i = lo(1), hi(1) - ii = i * lrat(1) - crse(i,j) = fine(ii,jj) + & -! These four fine nodes are shared by two coarse nodes... - 0.5d0 * (fine(ii-1,jj) + fine(ii+1,jj) + & - fine(ii,jj-1) + fine(ii,jj+1)) + & -! ... and these four are shared by four... - 0.25d0 * (fine(ii-1,jj-1) + fine(ii-1,jj+1) + & - fine(ii-1,jj+1) + fine(ii+1,jj+1)) -! ... note that we have 9 nodes in total... - crse(i,j) = crse(i,j) / 4.d0 - end do - end do - - end subroutine warpx_sum_fine_to_crse_nodal_2d - subroutine warpx_zero_out_bndry_3d (lo, hi, input_data, bndry_data, mask) & bind(c,name='warpx_zero_out_bndry_3d') diff --git a/Source/WarpX.H b/Source/WarpX.H index f55670cfb..02a366ace 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -250,6 +250,9 @@ public: static amrex::RealBox getRealBox(const amrex::Box& bx, int lev); static std::array<amrex::Real,3> LowerCorner (const amrex::Box& bx, int lev); static std::array<amrex::Real,3> UpperCorner (const amrex::Box& bx, int lev); + // Returns the locations of the lower corner of the box, shifted up + // a half cell if cell centered. + static std::array<amrex::Real,3> LowerCornerWithCentering (const amrex::Box& bx, int lev); static amrex::IntVect RefRatio (int lev); @@ -279,6 +282,7 @@ public: void ClearSliceMultiFabs (); static int num_slice_snapshots_lab; static amrex::Real dt_slice_snapshots_lab; + static amrex::Real particle_slice_width_lab; amrex::RealBox getSliceRealBox() const {return slice_realbox;} // these should be private, but can't due to Cuda limitations @@ -298,6 +302,19 @@ public: const std::array<const amrex::MultiFab*, 3>& B, const std::array<amrex::Real,3>& dx, int ngrow); + + void InitSpaceChargeField (WarpXParticleContainer& pc); + void computePhi (const amrex::Vector<std::unique_ptr<amrex::MultiFab> >& rho, + amrex::Vector<std::unique_ptr<amrex::MultiFab> >& phi, + std::array<amrex::Real, 3> const beta = {{0,0,0}}, + amrex::Real const required_precision=1.e-11 ) const; + void computeE (amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>, 3> >& E, + const amrex::Vector<std::unique_ptr<amrex::MultiFab> >& phi, + std::array<amrex::Real, 3> const beta = {{0,0,0}} ) const; + void computeB (amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>, 3> >& B, + const amrex::Vector<std::unique_ptr<amrex::MultiFab> >& phi, + std::array<amrex::Real, 3> const beta = {{0,0,0}} ) const; + protected: //! Tagging cells for refinement @@ -360,22 +377,6 @@ private: /// void EvolveES(int numsteps); - /// - /// Compute the gravitational potential from rho by solving Poisson's equation. - /// Both rho and phi are assumed to be node-centered. This method is only used - /// in electrostatic mode. - /// - void computePhi(const amrex::Vector<std::unique_ptr<amrex::MultiFab> >& rho, - amrex::Vector<std::unique_ptr<amrex::MultiFab> >& phi) const; - - /// - /// Compute the electric field in each direction by computing the gradient - /// the potential phi using 2nd order centered differences. Both rho and phi - /// are assumed to be node-centered. This method is only used in electrostatic mode. - /// - void computeE(amrex::Vector<std::array<std::unique_ptr<amrex::MultiFab>, 3> >& E, - const amrex::Vector<std::unique_ptr<amrex::MultiFab> >& phi) const; - // // This stuff is needed by the nodal multigrid solver when running in // electrostatic mode. @@ -383,9 +384,6 @@ private: void zeroOutBoundary(amrex::MultiFab& input_data, amrex::MultiFab& bndry_data, const amrex::FabArray<amrex::BaseFab<int> >& mask) const; - void sumFineToCrseNodal(const amrex::MultiFab& fine, amrex::MultiFab& crse, - const amrex::Geometry& cgeom, const amrex::IntVect& ratio); - void fixRHSForSolve(amrex::Vector<std::unique_ptr<amrex::MultiFab> >& rhs, const amrex::Vector<std::unique_ptr<amrex::FabArray<amrex::BaseFab<int> > > >& masks) const ; @@ -411,8 +409,6 @@ private: void InitFromCheckpoint (); void PostRestart (); - void InitOpenbc (); - void InitPML (); void ComputePMLFactors (); @@ -431,29 +427,6 @@ private: std::array<std::unique_ptr<amrex::MultiFab>, 3> getInterpolatedB(int lev) const; - /** \brief Fills the values of the current on the coarse patch by - * averaging the values of the current of the fine patch (on the same level). - * Also fills the guards of the coarse patch. - * - * \param[in] fine fine patches to interpolate from - * \param[out] coarse coarse patches to interpolate to - * \param[in] refinement_ratio integer ratio between the two - */ - void interpolateCurrentFineToCoarse (std::array< amrex::MultiFab const *, 3 > const & fine, - std::array< amrex::MultiFab *, 3 > const & coarse, - int const refinement_ratio); - - /** \brief Fills the values of the charge density on the coarse patch by - * averaging the values of the charge density of the fine patch (on the same level). - * - * \param[in] fine fine patches to interpolate from - * \param[out] coarse coarse patches to interpolate to - * \param[in] refinement_ratio integer ratio between the two - */ - void interpolateDensityFineToCoarse (const amrex::MultiFab& fine, - amrex::MultiFab& coarse, - int const refinement_ratio); - void ExchangeWithPmlB (int lev); void ExchangeWithPmlE (int lev); void ExchangeWithPmlF (int lev); diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index eef033236..5ace10ef4 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -73,6 +73,7 @@ bool WarpX::do_back_transformed_particles = true; int WarpX::num_slice_snapshots_lab = 0; Real WarpX::dt_slice_snapshots_lab; +Real WarpX::particle_slice_width_lab = 0.0; bool WarpX::do_dynamic_scheduling = true; @@ -324,6 +325,9 @@ WarpX::ReadParameters () amrex::Abort(msg.c_str()); } + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(Geom(0).isPeriodic(moving_window_dir) == 0, + "The problem must be non-periodic in the moving window direction"); + moving_window_x = geom[0].ProbLo(moving_window_dir); pp.get("moving_window_v", moving_window_v); @@ -612,6 +616,7 @@ WarpX::ReadParameters () pp.query("num_slice_snapshots_lab", num_slice_snapshots_lab); if (num_slice_snapshots_lab > 0) { pp.get("dt_slice_snapshots_lab", dt_slice_snapshots_lab ); + pp.get("particle_slice_width_lab",particle_slice_width_lab); } } @@ -1062,6 +1067,21 @@ WarpX::UpperCorner(const Box& bx, int lev) #endif } +std::array<Real,3> +WarpX::LowerCornerWithCentering(const Box& bx, int lev) +{ + std::array<Real,3> corner = LowerCorner(bx, lev); + std::array<Real,3> dx = CellSize(lev); + if (!bx.type(0)) corner[0] += 0.5*dx[0]; +#if (AMREX_SPACEDIM == 3) + if (!bx.type(1)) corner[1] += 0.5*dx[1]; + if (!bx.type(2)) corner[2] += 0.5*dx[2]; +#else + if (!bx.type(1)) corner[2] += 0.5*dx[2]; +#endif + return corner; +} + IntVect WarpX::RefRatio (int lev) { diff --git a/Tools/performance_tests/run_automated.py b/Tools/performance_tests/run_automated.py index 19e401a1e..a874542b6 100644 --- a/Tools/performance_tests/run_automated.py +++ b/Tools/performance_tests/run_automated.py @@ -290,8 +290,7 @@ if args.mode=='read' and update_perf_log_repo: git_repo.git.pull() os.chdir( perf_logs_repo ) sys.path.append('./') - import generate_index_html - git_repo.git.add('./index.html') + import write_csv git_repo.git.add('./logs_csv/' + csv_file[machine]) git_repo.git.add('./logs_hdf5/' + perf_database_file) index = git_repo.index |