From 1ce5957cb32a3b7ae39ad68b528e5a370854c58f Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Sun, 7 Jul 2019 11:14:31 -0700 Subject: avoid duplication in current deposition when using buffers --- Source/Particles/PhysicalParticleContainer.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 7e7c9534e..1ed318c13 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1529,8 +1529,14 @@ PhysicalParticleContainer::Evolve (int lev, // // Current Deposition // - DepositCurrent(pti, wp, uxp, uyp, uzp, jx, jy, jz, - cjx, cjy, cjz, np_current, np, thread_num, lev, dt); + // Deposit inside domains + DepositCurrent(pti, wp, uxp, uyp, uzp, &jx, &jy, &jz, + 0, np_current, thread_num, + lev, lev, dt, jx.nGrow()); + // Deposit in buffers + DepositCurrent(pti, wp, uxp, uyp, uzp, cjx, cjy, cjz, + np_current, np-np_current, thread_num, + lev, lev-1, dt, jx.nGrow()); // // copy particle data back -- cgit v1.2.3 From 6d3848aa366d40a3b8f48131da172a6aef5ebae0 Mon Sep 17 00:00:00 2001 From: MaxThevenet Date: Mon, 8 Jul 2019 16:01:37 -0700 Subject: bug when tiling on CPU --- Examples/Tests/SingleParticle/Untitled.ipynb | 114 +++++++++++++++++++++++++ Examples/Tests/SingleParticle/inputs | 10 +-- Source/Particles/PhysicalParticleContainer.cpp | 2 +- Source/Particles/WarpXParticleContainer.cpp | 66 +++++++++++--- 4 files changed, 172 insertions(+), 20 deletions(-) create mode 100644 Examples/Tests/SingleParticle/Untitled.ipynb (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Examples/Tests/SingleParticle/Untitled.ipynb b/Examples/Tests/SingleParticle/Untitled.ipynb new file mode 100644 index 000000000..7a98be71a --- /dev/null +++ b/Examples/Tests/SingleParticle/Untitled.ipynb @@ -0,0 +1,114 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# Import statements\n", + "import yt ; yt.funcs.mylog.setLevel(50)\n", + "import numpy as np\n", + "import scipy.constants as scc\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib notebook" + ] + }, + { + "cell_type": "code", + "execution_count": 82, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/mthevenet/anaconda3/lib/python3.7/site-packages/yt/units/yt_array.py:1394: RuntimeWarning: invalid value encountered in true_divide\n", + " out=out, **kwargs)\n" + ] + }, + { + "data": { + "text/html": [ + "
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "ds = yt.load( './old_diags/plt00001' ) # Create a dataset object\n", + "sl = yt.SlicePlot(ds, 2, 'jz', aspect=.4) # Create a sliceplot object\n", + "sl.annotate_grids() # Show grids\n", + "sl.show() # Show the plot" + ] + }, + { + "cell_type": "code", + "execution_count": 83, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/mthevenet/anaconda3/lib/python3.7/site-packages/yt/units/yt_array.py:1394: RuntimeWarning: invalid value encountered in true_divide\n", + " out=out, **kwargs)\n" + ] + }, + { + "data": { + "text/html": [ + "
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "ds = yt.load( './new_diags/plt00001' ) # Create a dataset object\n", + "sl = yt.SlicePlot(ds, 2, 'jx', aspect=.4) # Create a sliceplot object\n", + "sl.annotate_grids() # Show grids\n", + "sl.show() # Show the plot" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + " " + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.1" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Examples/Tests/SingleParticle/inputs b/Examples/Tests/SingleParticle/inputs index 89e3eb1c6..feecb996c 100644 --- a/Examples/Tests/SingleParticle/inputs +++ b/Examples/Tests/SingleParticle/inputs @@ -1,11 +1,11 @@ -max_step = 1 -amr.n_cell = 8 8 +max_step = 10 +amr.n_cell = 8 16 amr.max_level = 0 amr.blocking_factor = 8 amr.max_grid_size = 16 amr.plot_int = 1 geometry.coord_sys = 0 -geometry.is_periodic = 0 0 +geometry.is_periodic = 1 1 geometry.prob_lo = -8 -12 geometry.prob_hi = 8 12 warpx.do_pml = 0 @@ -23,6 +23,6 @@ particles.species_names = electron electron.charge = -q_e electron.mass = 1.e10 electron.injection_style = "SingleParticle" -electron.single_particle_pos = 0.0 0.0 0.0 -electron.single_particle_vel = 1.e20 0.0 1.e20 # gamma*beta +electron.single_particle_pos = 4 0 6 +electron.single_particle_vel = -2 0 -5 # -2. 1. 4. electron.single_particle_weight = 1.0 diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 43b46ec49..b78d36c87 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1530,6 +1530,7 @@ PhysicalParticleContainer::Evolve (int lev, // Current Deposition // // Deposit inside domains + Print()<<"np_current "<& xyzmin_tile = WarpX::LowerCorner(tilebox, depos_lev); - const std::array& xyzmin = xyzmin_tile; - Print()<<" xyzmin_tile "<& xyzmin_tile = WarpX::LowerCorner(tilebox, depos_lev); + // const std::array& xyzmin = xyzmin_tile; + const std::array& xyzmin = WarpX::LowerCorner(tilebox, depos_lev);; + + // Print()<<" xyzmin_tile "<array(pti); auto jy_arr = jy->array(pti); auto jz_arr = jz->array(pti); - - // auto const& jx_arr = jx->array(); - // auto const& jy_arr = jy->array(); - // auto const& jz_arr = jz->array(); + */ + + // const auto& jx_arr = local_jx[thread_num].array(); + // const auto& jy_arr = local_jy[thread_num].array(); + // const auto& jz_arr = local_jz[thread_num].array(); + + //Array4 const& jx_arr = local_jx[thread_num].array(tbx); + //Array4 const& jy_arr = local_jy[thread_num].array(tby); + //Array4 const& jz_arr = local_jz[thread_num].array(tbz); + Array4 const& jx_arr = local_jx[thread_num].array(); + Array4 const& jy_arr = local_jy[thread_num].array(); + Array4 const& jz_arr = local_jz[thread_num].array(); + + Dim3 lox = lbound(jx_arr); + Dim3 loy = lbound(jy_arr); + Dim3 loz = lbound(jz_arr); + /* + Dim3 loy = lbound(local_jy[thread_num]); + Dim3 loz = lbound(local_jz[thread_num]); + Dim3 hix = hbound(local_jx[thread_num]); + Dim3 hiy = hbound(local_jy[thread_num]); + Dim3 hiz = hbound(local_jz[thread_num]); + */ + Print()<<" lox "< Vx, Vy, Vz; //pti.GetPosition(Vx, Vy, Vz); @@ -446,7 +474,7 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, std::cout<<" vx "< Date: Thu, 11 Jul 2019 09:07:26 -0700 Subject: remove print statements --- Source/Particles/PhysicalParticleContainer.cpp | 1 - Source/Particles/WarpXParticleContainer.cpp | 4 ---- 2 files changed, 5 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index b78d36c87..df08d77a7 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1530,7 +1530,6 @@ PhysicalParticleContainer::Evolve (int lev, // Current Deposition // // Deposit inside domains - Print()<<"np_current "< Date: Thu, 11 Jul 2019 14:26:38 -0700 Subject: option use_picsar_deposition --- Source/Particles/Deposition/CurrentDeposition.H | 3 ++- Source/Particles/PhysicalParticleContainer.cpp | 32 ++++++++++++++++++------- Source/Particles/WarpXParticleContainer.H | 15 ++++++++++++ Source/Particles/WarpXParticleContainer.cpp | 2 +- Source/WarpX.cpp | 6 ++--- 5 files changed, 44 insertions(+), 14 deletions(-) (limited to 'Source/Particles/PhysicalParticleContainer.cpp') diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 006ff05fe..66db003a3 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -109,7 +109,7 @@ void doDepositionShapeN(Real* xp, Real* yp, Real* zp, const amrex::Real q) { BL_PROFILE("PICSAR::CurrentDeposition"); - + Real dxi = 1.0/dx[0]; Real dzi = 1.0/dx[2]; Real dts2dx = 0.5*dt*dxi; @@ -130,6 +130,7 @@ void doDepositionShapeN(Real* xp, Real* yp, Real* zp, compute_shape_factor_lo(shape_lo); Real clightsq = 1.0/PhysConst::c/PhysConst::c; + // Loop over particles and deposit into jx_arr, jy_arr and jz_arr ParallelFor( np_to_depose, [=] AMREX_GPU_DEVICE (long ip) { diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index df08d77a7..86cd0827f 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1529,16 +1529,30 @@ PhysicalParticleContainer::Evolve (int lev, // // Current Deposition // - // Deposit inside domains - DepositCurrent(pti, wp, uxp, uyp, uzp, &jx, &jy, &jz, - 0, np_current, thread_num, - lev, lev, dt); - if (has_buffer){ - // Deposit in buffers - DepositCurrent(pti, wp, uxp, uyp, uzp, cjx, cjy, cjz, - np_current, np-np_current, thread_num, - lev, lev-1, dt); + if (WarpX::use_picsar_deposition) { + // Deposit inside domains + DepositCurrentFortran(pti, wp, uxp, uyp, uzp, &jx, &jy, &jz, + 0, np_current, thread_num, + lev, lev, dt); + if (has_buffer){ + // Deposit in buffers + DepositCurrentFortran(pti, wp, uxp, uyp, uzp, cjx, cjy, cjz, + np_current, np-np_current, thread_num, + lev, lev-1, dt); + } + } else { + // Deposit inside domains + DepositCurrent(pti, wp, uxp, uyp, uzp, &jx, &jy, &jz, + 0, np_current, thread_num, + lev, lev, dt); + if (has_buffer){ + // Deposit in buffers + DepositCurrent(pti, wp, uxp, uyp, uzp, cjx, cjy, cjz, + np_current, np-np_current, thread_num, + lev, lev-1, dt); + } } + // // copy particle data back // diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 6182ec5bd..662b2e1b8 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -189,6 +189,21 @@ public: int depos_lev, amrex::Real dt); + virtual void DepositCurrentFortran(WarpXParIter& pti, + RealVector& wp, + RealVector& uxp, + RealVector& uyp, + RealVector& uzp, + amrex::MultiFab* jx, + amrex::MultiFab* jy, + amrex::MultiFab* jz, + const long offset, + const long np_to_depose, + int thread_num, + int lev, + int depos_lev, + amrex::Real dt); + // If particles start outside of the domain, ContinuousInjection // makes sure that they are initialized when they enter the domain, and // NOT before. Virtual function, overriden by derived classes. diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 42b99efcb..ac123f9b6 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -435,7 +435,7 @@ WarpXParticleContainer::DepositCurrent(WarpXParIter& pti, const long offset, const long np_to_depose, int thread_num, int lev, int depos_lev, Real dt) -{ +{ AMREX_ALWAYS_ASSERT_WITH_MESSAGE((depos_lev==(lev-1)) || (depos_lev==(lev )), "Deposition buffers only work for lev-1"); diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 24011fb2f..fd4840b8d 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -485,11 +485,11 @@ WarpX::ReadParameters () { ParmParse pp("algo"); - pp.query("use_picsar_deposition", use_picsar_deposition) + pp.query("use_picsar_deposition", use_picsar_deposition); current_deposition_algo = GetAlgorithmInteger(pp, "current_deposition"); - if (!use_picsar_algo){ + if (!use_picsar_deposition){ AMREX_ALWAYS_ASSERT_WITH_MESSAGE( current_deposition_algo >= 2, - "if not use_picsar_deposition, cannot use Esirkepov deposition.") + "if not use_picsar_deposition, cannot use Esirkepov deposition."); } charge_deposition_algo = GetAlgorithmInteger(pp, "charge_deposition"); field_gathering_algo = GetAlgorithmInteger(pp, "field_gathering"); -- cgit v1.2.3