# Intel MPI Benchmark

## Without RDMA
- [Without RDMA](https://hackmd.io/@htting/rJktoVy7Y)
## With RDMA

#!/bin/bash
#/bin/bash
#PBS -P 50000041
#PBS -N NTHU_stmv-768
#PBS -q normal
#PBS -l select=32:ncpus=24:mpiprocs=24:ompthreads=1:mem=96gb
#PBS -l walltime=02:00:00
#PBS -o out_stmv_768_tune.txt
#PBS -e err_stmv_768_tune.txt

PBS_O_WORKDIR=$HOME/kerwin/bench
VTUNE_PTH=/app/intel/xe2019/vtune_amplifier/bin64
cd ${PBS_O_WORKDIR}

module load composerxe/2018.0.128
module unload gcc/4.9.3
module load gcc/5.1.0
module load python/3.6.0

module list

source /app/intel/xe2018/vtune_amplifier/amplxe-vars.sh

export I_MPI_STATS=all

NPE=768
GromacsBin="$HOME/kerwin/program/gromacs-intel18/bin/mdrun_mpi"
TestbenchPath="/home/users/industry/ai-hpc/apacsc27/kerwin/bench/bench-file/stmv.tpr"


TRANSPORT=DAPL
# TRANSPORT=OFA

HCA=mlx5_0

MPIFLAGS=""
MPIFLAGS+="-genv DAT_OVERRIDE $HOME/dat.conf "
MPIFLAGS+="-genv I_MPI_DAT_LIBRARY /usr/lib64/libdat2.so "

if [[ "$TRANSPORT" == "DAPL" ]]; then
MPIFLAGS+="-DAPL "
MPIFLAGS+="-genv I_MPI_FABRICS shm:dapl "
MPIFLAGS+="-genv I_MPI_DAPL_UD off "
MPIFLAGS+="-genv I_MPI_DAPL_PROVIDER ofa-v2-$HCA-1u "
MPIFLAGS+="-genv DAPL_MAX_INLINE 256 "
MPIFLAGS+="-genv I_MPI_DAPL_RDMA_RNDV_WRITE on "
MPIFLAGS+="-genv DAPL_IB_MTU 4096 "

elif [[ "$TRANSPORT" == "OFA" ]]; then
MPIFLAGS+="-IB "
MPIFLAGS+="-genv MV2_USE_APM 0 "
MPIFLAGS+="-genv I_MPI_FABRICS shm:ofa "
MPIFLAGS+="-genv I_MPI_OFA_USE_XRC 1 "
MPIFLAGS+="-genv I_MPI_OFA_NUM_ADAPTERS 1 "
MPIFLAGS+="-genv I_MPI_OFA_ADAPTER_NAME $HCA "
MPIFLAGS+="-genv I_MPI_OFA_NUM_PORTS 1 "
fi
MPIFLAGS+="-genv I_MPI_PIN on "
MPIFLAGS+="-genv I_MPI_DEBUG 4 "
#MPIFLAGS+="-genv I_MPI_PIN_PROCESSOR_LIST map=scatter"

mpiexec $MPIFLAGS -ppn 24 -envall -np $NPE aps $GromacsBin -s $TestbenchPath -v -noconfout -maxh 2 -ntomp 1 -nsteps 10000 -dlb yes -notunepme