blob: 65777f30441b4eb8d2376f7871bb86adcc15a7d4 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
|
#!/bin/bash -l
# Copyright 2021 Axel Huebl, Kevin Gott
#
# This file is part of WarpX.
#
# License: BSD-3-Clause-LBNL
#SBATCH -t 01:00:00
#SBATCH -N 4
#SBATCH -J WarpX
# note: <proj> must end on _g
#SBATCH -A <proj>
# for m3906_g LBNL/AMP users: for large runs, comment in
##SBATCH -q early_science
#SBATCH -C gpu
#SBATCH -c 32
#SBATCH --ntasks-per-node=4
#SBATCH --gpus-per-node=4
#SBATCH -o WarpX.o%j
#SBATCH -e WarpX.e%j
# ============
# -N = nodes
# -n = tasks (MPI ranks, usually = G)
# -G = GPUs (full Perlmutter node, 4)
# -c = CPU per task (128 total threads on CPU, 32 per GPU)
#
# --ntasks-per-node= number of tasks (MPI ranks) per node (full node, 4)
# --gpus-per-task= number of GPUs per task (MPI rank) (full node, 4)
# --gpus-per-node= number of GPUs per node (full node, 4)
#
# --gpu-bind=single:1 sets only one GPU to be visible to each MPI rank
# (quiets AMReX init warnings)
#
# Recommend using --ntasks-per-node=4, --gpus-per-task=1 and --gpu-bind=single:1,
# as they are fixed values and allow for easy scaling with less adjustments.
#
# ============
# GPU-aware MPI
export MPICH_GPU_SUPPORT_ENABLED=1
# expose one GPU per MPI rank
export CUDA_VISIBLE_DEVICES=$SLURM_LOCALID
EXE=./warpx
#EXE=../WarpX/build/bin/warpx.3d.MPI.CUDA.DP.OPMD.QED
#EXE=./main3d.gnu.TPROF.MPI.CUDA.ex
INPUTS=inputs_small
srun ${EXE} ${INPUTS} \
> output.txt
|
