diff options
Diffstat (limited to 'Examples/batchScripts')
-rw-r--r-- | Examples/batchScripts/batch_cori.sh | 27 | ||||
-rw-r--r-- | Examples/batchScripts/batch_summit.sh | 3 |
2 files changed, 21 insertions, 9 deletions
diff --git a/Examples/batchScripts/batch_cori.sh b/Examples/batchScripts/batch_cori.sh index bfadc6e0a..e6cd5e1ef 100644 --- a/Examples/batchScripts/batch_cori.sh +++ b/Examples/batchScripts/batch_cori.sh @@ -4,19 +4,30 @@ #SBATCH -t 01:00:00 #SBATCH -q regular #SBATCH -C knl +#SBATCH -S 4 #SBATCH -J <job name> #SBATCH -A <allocation ID> #SBATCH -e error.txt #SBATCH -o output.txt -export OMP_NUM_THREADS=8 export OMP_PLACES=threads export OMP_PROC_BIND=spread -# Typically use OMP_NUM_THREADS=8 and 8 MPI ranks per node, -# without hyperthreading (HT=1) -srun --cpu_bind=cores -n 16 -c 32 <path/to/executable> <input file> -# 4 OpenMP threads, 16 MPI ranks per node, HT=1 -# srun --cpu_bind=cores -n 32 -c 16 <path/to/executable> <inputs file> -# 8 OpenMP threads, 16 MPI ranks per node, HT=2 -# srun --cpu_bind=cores -n 32 -c 16 <path/to/executable> <inputs file> +# KNLs have 4 hyperthreads max +export CORI_MAX_HYPETHREAD_LEVEL=4 +# We use 64 cores out of the 68 available on Cori KNL, +# and leave 4 to the system (see "#SBATCH -S 4" above). +export CORI_NCORES_PER_NODE=64 + +# Typically use 8 MPI ranks per node without hyperthreading, +# i.e., OMP_NUM_THREADS=8 +export WARPX_NMPI_PER_NODE=8 +export WARPX_HYPERTHREAD_LEVEL=1 + +# Compute OMP_NUM_THREADS and the thread count (-c option) +export CORI_NHYPERTHREADS_MAX=$(( ${CORI_MAX_HYPETHREAD_LEVEL} * ${CORI_NCORES_PER_NODE} )) +export WARPX_NTHREADS_PER_NODE=$(( ${WARPX_HYPERTHREAD_LEVEL} * ${CORI_NCORES_PER_NODE} )) +export OMP_NUM_THREADS=$(( ${WARPX_NTHREADS_PER_NODE} / ${WARPX_NMPI_PER_NODE} )) +export WARPX_THREAD_COUNT=$(( ${CORI_NHYPERTHREADS_MAX} / ${WARPX_NMPI_PER_NODE} )) + +srun --cpu_bind=cores -n $(( ${SLURM_JOB_NUM_NODES} * ${WARPX_NMPI_PER_NODE} )) -c ${WARPX_THREAD_COUNT} <path/to/executable> <input file> diff --git a/Examples/batchScripts/batch_summit.sh b/Examples/batchScripts/batch_summit.sh index a2679510f..002660b91 100644 --- a/Examples/batchScripts/batch_summit.sh +++ b/Examples/batchScripts/batch_summit.sh @@ -12,4 +12,5 @@ module load cuda omp=1 export OMP_NUM_THREADS=${omp} -jsrun -n 2 -a 6 -g 6 -c 6 --bind=packed:${omp} <path/to/executable> <input file> > output.txt +num_nodes=$(( $(printf '%s\n' ${LSB_HOSTS} | sort -u | wc -l) - 1 )) +jsrun -n ${num_nodes} -a 6 -g 6 -c 6 --bind=packed:${omp} <path/to/executable> <input file> > output.txt |