Set QED support ON by default (#1529)

* updated doc

* updated cmake

* fixed bug

* updates to take into account modifications in picsar_qed folder structure

* fixed bug

* fixed bug

* picsar/improve_makefile is now used for tests

* use sycl fix with picsar_qed

* update defines for picsar

* Update PICSAR Git Links

Co-authored-by: Axel Huebl <axel.huebl@plasma.ninja>

17 files changed, 62 insertions, 28 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index e38ac2a18..5e6258236 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -49,7 +49,8 @@ option(WarpX_LIB           "Build WarpX as a shared library"            OFF)
 option(WarpX_MPI           "Multi-node support (message-passing)"       ON)
 option(WarpX_OPENPMD       "openPMD I/O (HDF5, ADIOS)"                  OFF)
 option(WarpX_PSATD         "spectral solver support"                    OFF)
-option(WarpX_QED           "PICSAR QED (requires Boost and PICSAR)"     OFF)
+option(WarpX_QED           "QED support (requires PICSAR)"                    ON)
+option(WarpX_QED_TABLE_GEN "QED table generation (requires PICSAR and Boost)" OFF)
 # TODO: sensei, legacy hdf5?
 
 set(WarpX_DIMS_VALUES 2 3 RZ)
@@ -231,8 +232,10 @@ endif()
 
 if(WarpX_QED)
     target_compile_definitions(WarpX PUBLIC WARPX_QED)
-    # FIXME WARPX_QED_TABLE_GEN
-    target_link_libraries(WarpX PUBLIC pxrmp_qed)
+    if(WarpX_QED_TABLE_GEN)
+        target_compile_definitions(WarpX PUBLIC WARPX_QED_TABLE_GEN)
+    endif()
+    target_link_libraries(WarpX PUBLIC PXRMP_QED::PXRMP_QED)
 endif()
 
 # AMReX helper function: propagate CUDA specific target & source properties
diff --git a/Docs/Doxyfile b/Docs/Doxyfile
index efc4270da..9a3b2f078 100644
--- a/Docs/Doxyfile
+++ b/Docs/Doxyfile
@@ -2062,7 +2062,9 @@ PREDEFINED             = AMREX_Linux=1            \
                          WARPX_USE_GPU=1          \
                          WARPX_USE_OPENPMD=1      \
                          WARPX_USE_PSATD=1        \
-                         WARPX_QED=1
+                         WARPX_QED=1              \
+                         WARPX_QED_TABLE_GEN=1    \
+
 
 # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
 # tag can be used to specify a list of macro names that should be expanded. The
diff --git a/Docs/source/building/cmake.rst b/Docs/source/building/cmake.rst
index bf6821c70..9aac1d948 100644
--- a/Docs/source/building/cmake.rst
+++ b/Docs/source/building/cmake.rst
@@ -26,6 +26,7 @@ Please see installation instructions below.
 - a mature `C++14 <https://en.wikipedia.org/wiki/C%2B%2B14>`_ compiler: e.g. GCC 5, Clang 3.6 or newer
 - `CMake 3.15.0+ <https://cmake.org>`_
 - `AMReX <https://amrex-codes.github.io>`_: we automatically download and compile a copy of AMReX
+- `PICSAR <https://github.com/ECP-WarpX/picsar>`_: we automatically download and compile a copy of PICSAR
 
 Optional dependencies include:
 
@@ -33,7 +34,7 @@ Optional dependencies include:
 - `CUDA Toolkit 9.0+ <https://developer.nvidia.com/cuda-downloads>`_: for Nvidia GPU support (see `matching host-compilers <https://gist.github.com/ax3l/9489132>`_)
 - `OpenMP 3.1+ <https://www.openmp.org>`_: for threaded CPU execution (currently not fully accelerated)
 - `FFTW3 <http://www.fftw.org>`_: for spectral solver (PSATD) support
-- `Boost 1.66.0+ <https://www.boost.org/>`_: for QED support
+- `Boost 1.66.0+ <https://www.boost.org/>`_: for QED lookup tables generation support
 - `openPMD-api 0.12.0+ <https://github.com/openPMD/openPMD-api>`_: we automatically download and compile a copy of openPMD-api for openPMD I/O support
 
   - see `optional I/O backends <https://github.com/openPMD/openPMD-api#dependencies>`_
@@ -136,7 +137,8 @@ CMake Option                  Default & Values                             Descr
 ``WarpX_PARSER_DEPTH``        **24**                                       Maximum parser depth for input file functions
 ``WarpX_PRECISION``           SINGLE/**DOUBLE**                            Floating point precision (single/double)
 ``WarpX_PSATD``               ON/**OFF**                                   Spectral solver
-``WarpX_QED``                 ON/**OFF**                                   PICSAR QED (requires Boost and PICSAR)
+``WarpX_QED``                 **ON**/OFF                                   QED support (requires PICSAR)
+``WarpX_QED_TABLE_GEN``       ON/**OFF**                                   QED table generation support (requires PICSAR and Boost)
 ``WarpX_amrex_repo``          ``https://github.com/AMReX-Codes/amrex.git`` Repository URI to pull and build AMReX from
 ``WarpX_amrex_branch``        ``development``                              Repository branch for ``WarpX_amrex_repo``
 ``WarpX_amrex_internal``      **ON**/OFF                                   Needs a pre-installed AMReX library if set to ``OFF``
diff --git a/Docs/source/building/lassen.rst b/Docs/source/building/lassen.rst
index f5a5bcccc..9387d2c9f 100644
--- a/Docs/source/building/lassen.rst
+++ b/Docs/source/building/lassen.rst
@@ -40,7 +40,7 @@ We use the following modules and environments on the system.
    # optional: for PSATD support
    module load fftw/3.3.8
 
-   # optional: for QED support
+   # optional: for QED lookup table generation support
    module load boost/1.70.0
 
    # optional: for openPMD support
diff --git a/Docs/source/building/quartz.rst b/Docs/source/building/quartz.rst
index 139f047d3..baedf345b 100644
--- a/Docs/source/building/quartz.rst
+++ b/Docs/source/building/quartz.rst
@@ -40,7 +40,7 @@ We use the following modules and environments on the system.
    # optional: for PSATD support
    module load fftw/3.3.8
 
-   # optional: for QED support
+   # optional: for QED lookup table generation support
    module load boost/1.73.0
 
    # optional: for openPMD support
diff --git a/Docs/source/building/summit.rst b/Docs/source/building/summit.rst
index e1d8fd3c4..76aaeca11 100644
--- a/Docs/source/building/summit.rst
+++ b/Docs/source/building/summit.rst
@@ -50,7 +50,7 @@ We use the following modules and environments on the system.
    # optional: for PSATD support
    module load fftw
 
-   # optional: for QED support
+   # optional: for QED lookup table generation support
    module load boost/1.66.0
 
    # optional: for openPMD support
diff --git a/Source/Make.WarpX b/Source/Make.WarpX
index bd25adfb0..fb84823b8 100644
--- a/Source/Make.WarpX
+++ b/Source/Make.WarpX
@@ -97,7 +97,7 @@ ifeq ($(USE_EB),TRUE)
 endif
 
 ifeq ($(QED),TRUE)
-  INCLUDE_LOCATIONS += $(PICSAR_HOME)/src/multi_physics/QED/src
+  INCLUDE_LOCATIONS += $(PICSAR_HOME)/multi_physics/QED/include
   CXXFLAGS += -DWARPX_QED
   CFLAGS   += -DWARPX_QED
   FFLAGS   += -DWARPX_QED
diff --git a/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H b/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H
index 0f9d3b625..0598d0b34 100644
--- a/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H
+++ b/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H
@@ -15,8 +15,8 @@
 #include <AMReX_Vector.H>
 #include <AMReX_Gpu.H>
 
-#include <physics/breit_wheeler/breit_wheeler_engine_tables.hpp>
-#include <physics/breit_wheeler/breit_wheeler_engine_core.hpp>
+#include <picsar_qed/physics/breit_wheeler/breit_wheeler_engine_tables.hpp>
+#include <picsar_qed/physics/breit_wheeler/breit_wheeler_engine_core.hpp>
 
 #include <string>
 #include <vector>
diff --git a/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.cpp b/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.cpp
index 4d12e2763..690469810 100644
--- a/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.cpp
+++ b/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.cpp
@@ -6,11 +6,11 @@
  */
 #include "BreitWheelerEngineWrapper.H"
 
-#include <utils/serialization.hpp>
+#include <picsar_qed/utils/serialization.hpp>
 
 //Functions needed to generate a new table
 #ifdef WARPX_QED_TABLE_GEN
-#   include <physics/breit_wheeler/breit_wheeler_engine_tables_generator.hpp>
+#   include <picsar_qed/physics/breit_wheeler/breit_wheeler_engine_tables_generator.hpp>
 #endif
 
 #include <AMReX.H>
diff --git a/Source/Particles/ElementaryProcess/QEDInternals/QedChiFunctions.H b/Source/Particles/ElementaryProcess/QEDInternals/QedChiFunctions.H
index b6ac5c5e4..01852600c 100644
--- a/Source/Particles/ElementaryProcess/QEDInternals/QedChiFunctions.H
+++ b/Source/Particles/ElementaryProcess/QEDInternals/QedChiFunctions.H
@@ -15,7 +15,7 @@
 
 #include "QedWrapperCommons.H"
 
-#include <physics/chi_functions.hpp>
+#include <picsar_qed/physics/chi_functions.hpp>
 
 namespace QedUtils{
     /**
diff --git a/Source/Particles/ElementaryProcess/QEDInternals/QedWrapperCommons.H b/Source/Particles/ElementaryProcess/QEDInternals/QedWrapperCommons.H
index 75fc2fd79..aa95486ae 100644
--- a/Source/Particles/ElementaryProcess/QEDInternals/QedWrapperCommons.H
+++ b/Source/Particles/ElementaryProcess/QEDInternals/QedWrapperCommons.H
@@ -22,7 +22,8 @@
  * compiled for GPU. The user has to set it to the right value
  * (AMREX_GPU_DEVICE in this case).
  */
-#define PXRMP_GPU AMREX_GPU_HOST_DEVICE
+#define PXRMP_WITH_GPU
+#define PXRMP_GPU_QUALIFIER AMREX_GPU_HOST_DEVICE
 //_________________________
 
 /**
diff --git a/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H b/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H
index 3ab0f106f..cc3d4eba1 100644
--- a/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H
+++ b/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H
@@ -15,8 +15,8 @@
 #include <AMReX_Vector.H>
 #include <AMReX_Gpu.H>
 
-#include <physics/quantum_sync/quantum_sync_engine_tables.hpp>
-#include <physics/quantum_sync/quantum_sync_engine_core.hpp>
+#include <picsar_qed/physics/quantum_sync/quantum_sync_engine_tables.hpp>
+#include <picsar_qed/physics/quantum_sync/quantum_sync_engine_core.hpp>
 
 #include <string>
 #include <vector>
diff --git a/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.cpp b/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.cpp
index 3f839ce91..0019f042b 100644
--- a/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.cpp
+++ b/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.cpp
@@ -8,7 +8,7 @@
 
 //Functions needed to generate a new table
 #ifdef WARPX_QED_TABLE_GEN
-#   include <physics/quantum_sync/quantum_sync_engine_tables_generator.hpp>
+#   include <picsar_qed/physics/quantum_sync/quantum_sync_engine_tables_generator.hpp>
 #endif
 
 #include <AMReX.H>
diff --git a/Source/Particles/ElementaryProcess/QEDInternals/SchwingerProcessWrapper.H b/Source/Particles/ElementaryProcess/QEDInternals/SchwingerProcessWrapper.H
index e7466985a..6638e1759 100644
--- a/Source/Particles/ElementaryProcess/QEDInternals/SchwingerProcessWrapper.H
+++ b/Source/Particles/ElementaryProcess/QEDInternals/SchwingerProcessWrapper.H
@@ -10,7 +10,7 @@
 
 #include "QedWrapperCommons.H"
 
-#include <physics/schwinger/schwinger_pair_engine_core.hpp>
+#include <picsar_qed/physics/schwinger/schwinger_pair_engine_core.hpp>
 #include <AMReX_Random.H>
 #include <cmath>
 
diff --git a/cmake/WarpXFunctions.cmake b/cmake/WarpXFunctions.cmake
index b67e0c11a..178cd888a 100644
--- a/cmake/WarpXFunctions.cmake
+++ b/cmake/WarpXFunctions.cmake
@@ -181,11 +181,16 @@ function(set_warpx_binary_name)
             set_property(TARGET ${tgt} APPEND_STRING PROPERTY OUTPUT_NAME ".QED")
         endif()
 
+        if(WarpX_QED_TABLE_GEN)
+            set_property(TARGET ${tgt} APPEND_STRING PROPERTY OUTPUT_NAME ".GENQEDTABLES")
+        endif()
+
+
         if(CMAKE_BUILD_TYPE MATCHES "Debug")
             set_property(TARGET ${tgt} APPEND_STRING PROPERTY OUTPUT_NAME ".DEBUG")
         endif()
     endforeach()
-    
+
     if(WarpX_APP)
         # alias to the latest build, because using the full name is often confusing
         add_custom_command(TARGET app POST_BUILD
@@ -287,5 +292,6 @@ function(warpx_print_summary)
     message("    PRECISION: ${WarpX_PRECISION}")
     message("    OPENPMD: ${WarpX_OPENPMD}")
     message("    QED: ${WarpX_QED}")
+    message("    QED table generation: ${WarpX_QED_TABLE_GEN}")
     message("")
 endfunction()
diff --git a/cmake/dependencies/PICSAR.cmake b/cmake/dependencies/PICSAR.cmake
index ad6f064fc..ef4fa6061 100644
--- a/cmake/dependencies/PICSAR.cmake
+++ b/cmake/dependencies/PICSAR.cmake
@@ -4,7 +4,28 @@ function(find_picsar)
         include(FetchContent)
         set(CMAKE_POLICY_DEFAULT_CMP0077 NEW)
 
-        # FIXME no option to control WarpX_QED_TABLE_GEN / Boost trigger yet
+        # Enable or disable QED lookup tables generation
+
+        # If table generation is enabled, enable or disable
+        # openMP support depending on WarpX_COMPUTE
+        if(WarpX_QED_TABLE_GEN)
+            set(PXRMP_QED_TABLEGEN ON CACHE INTERNAL "")
+            if(WarpX_COMPUTE STREQUAL OMP)
+                set(PXRMP_QED_OMP ON CACHE INTERNAL "")
+            else()
+                set(PXRMP_QED_OMP OFF CACHE INTERNAL "")
+            endif()
+        else()
+            set(PXRMP_QED_TABLEGEN OFF CACHE INTERNAL "")
+            set(PXRMP_QED_OMP OFF CACHE INTERNAL "")
+        endif()
+
+        # Always disable tests
+        set (PXRMP_QED_TEST OFF CACHE INTERNAL "")
+        
+        if(WarpX_COMPUTE STREQUAL SYCL)
+            set (PXRMP_DPCPP_FIX ON CACHE INTERNAL "")
+        endif()
 
         FetchContent_Declare(fetchedpicsar
             GIT_REPOSITORY ${WarpX_picsar_repo}
@@ -15,7 +36,7 @@ function(find_picsar)
 
         if(NOT fetchedpicsar_POPULATED)
             FetchContent_Populate(fetchedpicsar)
-            add_subdirectory(${fetchedpicsar_SOURCE_DIR}/src/multi_physics/QED ${fetchedpicsar_BINARY_DIR})
+            add_subdirectory(${fetchedpicsar_SOURCE_DIR}/multi_physics/QED ${fetchedpicsar_BINARY_DIR})
         endif()
 
         # advanced fetch options
@@ -29,6 +50,9 @@ function(find_picsar)
         # PICSAR options not relevant to WarpX users
         mark_as_advanced(DIM)
         mark_as_advanced(USE_XSDK_DEFAULTS)
+        mark_as_advanced(PXRMP_QED_TABLEGEN)
+        mark_as_advanced(PXRMP_QED_OMP)
+        mark_as_advanced(PXRMP_QED_TEST)
 
         message(STATUS "PICSAR: Using INTERNAL version '${PICSAR_VERSION}'")
     else()
diff --git a/run_test.sh b/run_test.sh
index 428f21517..5b998b751 100755
--- a/run_test.sh
+++ b/run_test.sh
@@ -50,11 +50,7 @@ echo "cd $PWD"
 # Clone PICSAR and AMReX
 git clone --branch development https://github.com/AMReX-Codes/amrex.git
 # Use QED brach for QED tests
-if [ "${WARPX_CI_QED}" = "TRUE" ]; then
-    git clone --branch development https://github.com/ECP-WarpX/picsar.git
-else
-    git clone --branch development https://github.com/ECP-WarpX/picsar.git
-fi
+git clone --branch development https://github.com/ECP-WarpX/picsar.git
 
 # Clone the AMReX regression test utility
 git clone https://github.com/ECP-WarpX/regression_testing.git