#!/bin/bash
#PBS -S /bin/bash
#PBS -N genesis_capsule_unst
#PBS -q gpu_normal
#PBS -W group_list=<your project group ID>
#PBS -l select=4:ncpus=72:mpiprocs=1:model=gh200:ngpus=1:mem=460gb
#PBS -l walltime=08:00:00
#PBS -j oe
#PBS -k oed
#PBS -r n

#BENCH reference genesis_capsule_unst.data.iter
#BENCH tolerance_abs 5.0e-3
#BENCH tolerance_rel 5.0e-3
#BENCH compare time_avg 500
#BENCH exclude_vars maxcfl, maxmach, minpressure, maxpressure, minuvel, maxuvel, minvvel, maxvvel, minwvel, maxwvel, mintemperature, maxtemperature, cx_*, cy_*, cz_*, mx_*, my_*, mz_*, fx_*, fy_*, fz_*, fl_*, fd_*, fs_*, cl_*, cs_*, nt
#BENCH save_vars cd_capsule

log_file="run.log"

# Load LAVA module
module purge
module use /swbuild/LAVA/LAVA_modules
module load LAVA

# Set environment
export OMP_NUM_THREADS=72
export CUDA_VISIBLE_DEVICES=0

# Change to working directory
cd $PBS_O_WORKDIR

# Run LAVA with NUMA binding (1 MPI rank per node, 1 GPU per node)
mpiexec -N 1 --report-bindings --hostfile $PBS_NODEFILE \
    -x PATH -x LD_LIBRARY_PATH -x CUDA_VISIBLE_DEVICES \
    numactl --cpunodebind 0 --membind 0 \
    mbind.x -t $OMP_NUM_THREADS \
    lava.3D.NS1.NVIDIA.ARM.HPCX unst_genesis_capsule.inputs 2>&1 |& tee -a ${log_file}