path: root/Source/Make.WarpX

AMREX_HOME ?= ../../../amrex
PICSAR_HOME ?= ../../../picsar
OPENBC_HOME ?= ../../../openbc_poisson

USE_MPI       = TRUE
USE_PARTICLES = TRUE
USE_RPATH     = TRUE
USE_GPU_RDC   = FALSE
BL_NO_FORT    = TRUE
CXXSTD        = c++17

# required for AMReX async I/O
MPI_THREAD_MULTIPLE = TRUE

ifeq ($(USE_GPU),TRUE)
  USE_OMP  = FALSE
  USE_CUDA = TRUE
  USE_ACC  = FALSE
  NVCC_HOST_COMP = gnu
endif

ifeq ($(USE_RZ),TRUE)
  # DIM is set here so that it is properly set for
  # all of the included files.
  DIM = 2
endif

ifeq ($(USE_ASCENT_INSITU),TRUE)
  ASCENT_DIR ?= NOT_SET
  ifneq ($(ASCENT_DIR),NOT_SET)
    include $(ASCENT_DIR)/share/ascent/ascent_config.mk
  endif
  USE_CONDUIT = TRUE
  USE_ASCENT = TRUE
  USE_GPU_RDC = TRUE
endif

ifndef USE_PYTHON_MAIN
  USE_PYTHON_MAIN = FALSE
endif

ifeq ($(USE_PYTHON_MAIN),TRUE)
  XTRA_CXXFLAGS += -fPIC
ifeq ($(USE_OMP),TRUE)
  LDFLAGS += -fopenmp
endif
  XTRA_CFLAGS   += -fPIC
  XTRA_FFLAGS   += -fPIC
  XTRA_F90FLAGS += -fPIC
  USERSuffix := .Lib
endif

include $(AMREX_HOME)/Tools/GNUMake/Make.defs

ifeq ($(DIM),3)
  DEFINES += -DWARPX_DIM_3D -DWARPX_ZINDEX=2
endif

ifeq ($(DIM),2)
ifeq ($(USE_RZ),TRUE)
  DEFINES += -DWARPX_DIM_RZ -DWARPX_ZINDEX=1
else
  DEFINES += -DWARPX_DIM_XZ -DWARPX_ZINDEX=1
endif
endif

ifeq ($(DIM),1)
  DEFINES += -DWARPX_DIM_1D_Z -DWARPX_ZINDEX=0
endif

-include Make.package
include $(WARPX_HOME)/Source/Make.package
include $(WARPX_HOME)/Source/ablastr/Make.package
include $(WARPX_HOME)/Source/AcceleratorLattice/Make.package
include $(WARPX_HOME)/Source/BoundaryConditions/Make.package
include $(WARPX_HOME)/Source/Diagnostics/Make.package
include $(WARPX_HOME)/Source/EmbeddedBoundary/Make.package
include $(WARPX_HOME)/Source/FieldSolver/Make.package
include $(WARPX_HOME)/Source/Filter/Make.package
include $(WARPX_HOME)/Source/Initialization/Make.package
include $(WARPX_HOME)/Source/Laser/Make.package
include $(WARPX_HOME)/Source/Parallelization/Make.package
include $(WARPX_HOME)/Source/Particles/Make.package
include $(WARPX_HOME)/Source/Python/Make.package
include $(WARPX_HOME)/Source/Utils/Make.package
include $(WARPX_HOME)/Source/Evolve/Make.package

include $(AMREX_HOME)/Src/Base/Make.package
include $(AMREX_HOME)/Src/Particle/Make.package
include $(AMREX_HOME)/Src/Boundary/Make.package
include $(AMREX_HOME)/Src/AmrCore/Make.package
include $(AMREX_HOME)/Src/LinearSolvers/MLMG/Make.package

ifeq ($(USE_SENSEI_INSITU),TRUE)
  include $(AMREX_HOME)/Src/Amr/Make.package
  include $(AMREX_HOME)/Src/Extern/SENSEI/Make.package
  USE_GPU_RDC = TRUE
endif

ifeq ($(USE_EB),TRUE)
  include $(AMREX_HOME)/Src/EB/Make.package
  USERSuffix := $(USERSuffix).EB
endif

ifeq ($(QED),TRUE)
  INCLUDE_LOCATIONS += $(PICSAR_HOME)/multi_physics/QED/include
  CXXFLAGS += -DWARPX_QED
  CFLAGS   += -DWARPX_QED
  FFLAGS   += -DWARPX_QED
  F90FLAGS += -DWARPX_QED
  USERSuffix := $(USERSuffix).QED

  ifeq ($(QED_TABLE_GEN),TRUE)
    BOOST_ROOT ?= NOT_SET
    ifneq ($(BOOST_ROOT),NOT_SET)
      VPATH_LOCATIONS += $(BOOST_ROOT)
      INCLUDE_LOCATIONS += $(BOOST_ROOT)
    endif
    CXXFLAGS += -DWARPX_QED_TABLE_GEN
    CFLAGS   += -DWARPX_QED_TABLE_GEN
    FFLAGS   += -DWARPX_QED_TABLE_GEN
    F90FLAGS += -DWARPX_QED_TABLE_GEN
     USERSuffix := $(USERSuffix).GENTABLES
  endif
endif

ifeq ($(PRECISION),FLOAT)
  USERSuffix := $(USERSuffix).SP
endif
ifeq ($(USE_SINGLE_PRECISION_PARTICLES),TRUE)
  USERSuffix := $(USERSuffix).pSP
endif

ifeq ($(QED),TRUE)
  include $(PICSAR_HOME)/src/Make.package
endif

WARPX_GIT_VERSION := $(shell cd $(WARPX_HOME); git describe --abbrev=12 --dirty --always --tags)
PICSAR_GIT_VERSION := $(shell cd $(PICSAR_HOME); git describe --abbrev=12 --dirty --always --tags)
WARPX_DEFINES = -DWARPX_GIT_VERSION=\"$(WARPX_GIT_VERSION)\" -DPICSAR_GIT_VERSION=\"$(PICSAR_GIT_VERSION)\"

DEFINES += -DPICSAR_NO_ASSUMED_ALIGNMENT
DEFINES += -DWARPX

ifeq ($(USE_OPENBC_POISSON),TRUE)
  include $(OPENBC_HOME)/Make.package
  DEFINES += -DFFT_FFTW -DMPIPARALLEL -DUSE_OPENBC_POISSON
endif

ifeq ($(USE_OPENPMD), TRUE)
   # try pkg-config query
   ifeq (0, $(shell pkg-config "openPMD >= 0.15.1"; echo $$?))
       CXXFLAGS += $(shell pkg-config --cflags openPMD)
       LIBRARY_LOCATIONS += $(shell pkg-config --variable=libdir openPMD)
       libraries += $(shell pkg-config --libs-only-l openPMD)
   # fallback to manual settings
   else
       OPENPMD_LIB_PATH ?= NOT_SET
       ifneq ($(OPENPMD_LIB_PATH),NOT_SET)
         LIBRARY_LOCATIONS += $(OPENPMD_LIB_PATH)
       endif
       OPENPMD_INCLUDE_PATH ?= NOT_SET
       ifneq ($(OPENPMD_INCLUDE_PATH),NOT_SET)
         INCLUDE_LOCATIONS += $(OPENPMD_INCLUDE_PATH)
       endif
       libraries += -lopenPMD
   endif
   DEFINES += -DWARPX_USE_OPENPMD
   USERSuffix := $(USERSuffix).OPMD
endif


ifeq ($(USE_PSATD),TRUE)
  USERSuffix := $(USERSuffix).PSATD
  DEFINES += -DWARPX_USE_PSATD
  ifeq ($(USE_CUDA),TRUE)
    # Use cuFFT
    libraries += -lcufft
  else ifeq ($(USE_HIP),TRUE)
    # Use rocFFT.  ROC_PATH is defined in amrex
    INCLUDE_LOCATIONS += $(ROC_PATH)/rocfft/include
    LIBRARY_LOCATIONS += $(ROC_PATH)/rocfft/lib
    libraries += -lrocfft
  else # Running on CPU
     # Use FFTW
     ifeq ($(PRECISION),FLOAT)
          libraries += -lfftw3f_mpi -lfftw3f -lfftw3f_threads
     else
          libraries += -lfftw3_mpi -lfftw3 -lfftw3_threads
     endif
     FFTW_HOME ?= NOT_SET
     ifneq ($(FFTW_HOME),NOT_SET)
       VPATH_LOCATIONS += $(FFTW_HOME)/include
       INCLUDE_LOCATIONS += $(FFTW_HOME)/include
       LIBRARY_LOCATIONS += $(FFTW_HOME)/lib
     endif
  endif
  ifeq ($(USE_RZ),TRUE)
    # Use blas and lapack
    INCLUDE_LOCATIONS += $(LAPACKPP_HOME)/include
    INCLUDE_LOCATIONS += $(BLASPP_HOME)/include
    LIBRARY_LOCATIONS += $(LAPACKPP_HOME)/lib
    LIBRARY_LOCATIONS += $(BLASPP_HOME)/lib
    libraries += -llapackpp -lblaspp $(BLAS_LIB) $(LAPACK_LIB)
  endif
endif

ifeq ($(USE_RZ),TRUE)
  USERSuffix := $(USERSuffix).rz
endif

ifeq ($(USE_HDF5),TRUE)
    DEFINES += -DWARPX_USE_HDF5
endif

ifeq ($(USE_GPUCLOCK),TRUE)
  USERSuffix := $(USERSuffix).GPUCLOCK
  DEFINES += -DWARPX_USE_GPUCLOCK
endif

# job_info support
CEXE_sources += AMReX_buildInfo.cpp
INCLUDE_LOCATIONS += $(AMREX_HOME)/Tools/C_scripts

ifeq ($(USE_PYTHON_MAIN),TRUE)

ifeq ($(shell uname),Darwin)
  SHARED_OPTION = -dynamiclib
else
  SHARED_OPTION = -shared
endif

ifeq ($(USE_RZ),TRUE)
  PYDIM = rz
else
  PYDIM = $(DIM)d
endif

installwarpx: libwarpx.$(PYDIM).so
	cp libwarpx.$(PYDIM).so Python/pywarpx
	cd Python; python3 setup.py install --force --with-libwarpx $(PYDIM) $(PYINSTALLOPTIONS)

libwarpx.$(PYDIM).a: $(objForExecs)
	@echo Making static library $@ ...
	$(SILENT) $(AR) -crv $@ $^
	$(SILENT) $(RM) AMReX_buildInfo.cpp
	@echo SUCCESS

libwarpx.$(PYDIM).so: $(objForExecs)
	@echo Making dynamic library $@ ...
ifeq ($(USE_CUDA),TRUE)
	$(SILENT) $(CXX) $(LINKFLAGS) $(SHARED_OPTION) -Xlinker=$(SHARED_OPTION) -Xlinker=-fPIC -o $@ $^ $(LDFLAGS) $(libraries)
else
	$(SILENT) $(CXX) $(LINKFLAGS) $(SHARED_OPTION) -fPIC -o $@ $^ $(LDFLAGS) $(libraries)
endif
	$(SILENT) $(RM) AMReX_buildInfo.cpp
	@echo SUCCESS

clean::
	$(SILENT) $(RM) -rf build
	$(SILENT) $(RM) -f libwarpx.?d.a
	$(SILENT) $(RM) -f libwarpx.?d.so
	$(SILENT) $(RM) -f pywarpx/libwarpx.?d.so

else

ifdef WarpxBinDir

all: $(executable)
	$(SILENT) $(RM) AMReX_buildInfo.cpp
	@if [ ! -d $(WarpxBinDir) ]; then mkdir -p $(WarpxBinDir); fi
	$(SILENT) mv -f $(executable) $(WarpxBinDir)/
	@if [ -d $(executable).dSYM ]; then $(RM) -r $(WarpxBinDir)/$(executable).dSYM; mv -f $(executable).dSYM $(WarpxBinDir)/; fi
	@echo SUCCESS

else

all: $(executable)
	$(SILENT) $(RM) AMReX_buildInfo.cpp
	@echo SUCCESS

endif

endif

AMReX_buildInfo.cpp:
	$(AMREX_HOME)/Tools/C_scripts/makebuildinfo_C.py \
          --amrex_home "$(AMREX_HOME)" \
          --COMP "$(COMP)" --COMP_VERSION "$(COMP_VERSION)" \
          --CXX_comp_name "$(CXX)" --CXX_flags "$(CXXFLAGS) $(CPPFLAGS) $(includes)" \
          --F_comp_name "$(F90)" --F_flags "$(F90FLAGS)" \
          --link_flags "$(LDFLAGS)" --libraries "$(libraries)" \
          --GIT ". $(AMREX_HOME) $(PICSAR_HOME)"

# Generate WarpXVersion.H
vpath WarpXVersion.H $(srcTempDir)/Utils

AUTO_BUILD_SOURCES += $(srcTempDir)/Utils/WarpXVersion.H

$(srcTempDir)/Utils/WarpXVersion.H:
	@echo Generating WarpXVersion.H ...
	@if [ ! -d $(srcTempDir)/Utils ]; then mkdir -p $(srcTempDir)/Utils; fi
	@ $(MKVERSIONHEADER) --code="WARPX" --defines="$(WARPX_DEFINES)" > $@

# Generate ablastr/export.H
vpath export.H $(srcTempDir)/ablastr

AUTO_BUILD_SOURCES += $(srcTempDir)/ablastr/export.H

$(srcTempDir)/ablastr/export.H:
	@echo Generating ablastr/export.H ...
	@if [ ! -d $(srcTempDir)/ablastr ]; then mkdir -p $(srcTempDir)/ablastr; fi
	@echo "#ifndef ABLASTR_EXPORT_H_" > $@
	@echo "#define ABLASTR_EXPORT_H_" >> $@
	@echo "#define ABLASTR_EXPORT" >> $@
	@echo "#endif" >> $@

# Generate Utils/export.H
vpath export.H $(srcTempDir)/Utils

AUTO_BUILD_SOURCES += $(srcTempDir)/Utils/export.H

$(srcTempDir)/Utils/export.H:
	@echo Generating Utils/export.H ...
	@if [ ! -d $(srcTempDir)/Utils ]; then mkdir -p $(srcTempDir)/Utils; fi
	@echo "#ifndef WARPX_EXPORT_H_" > $@
	@echo "#define WARPX_EXPORT_H_" >> $@
	@echo "#define WARPX_EXPORT" >> $@
	@echo "#endif" >> $@

cleanconfig::
	$(SILENT) $(RM) -r $(srcTempDir)/Utils

include $(AMREX_HOME)/Tools/GNUMake/Make.rules

clean::
	$(SILENT) $(RM) AMReX_buildInfo.cpp