---
# System prepended metadata

title: Intel MPI

---

# Intel MPI

### FFTW3 using icc 19
```bash=
#/bin/bash
#PBS -P 50000041
#PBS -N install_fftw
#PBS -l select=1:ncpus=24:mpiprocs=1:mem=10G
#PBS -l walltime=02:00:00
#PBS -q normal
#PBS -o out_install_fftw-icc.txt
#PBS -e err_install_fftw-icc.txt

FFTWSOURCE=/home/users/industry/ai-hpc/apacsc27/kerwin/source/fftw-3.3.9
FFTWDIR=/home/users/industry/ai-hpc/apacsc27/kerwin/program

module load intel/19.0.0.117

cd $FFTWSOURCE
rm -r build
mkdir build && cd build
CC="mpiicc" ../configure --enable-mpi --enable-avx2 --enable-sse2 --prefix=$FFTWDIR/fftw3-icc --enable-shared --disable-static --enable-float
make -j 24
make install
```

### GROMACS mpiicc icc (icc 19)
```bash=
#/bin/bash
#PBS -P 50000041
#PBS -N install_gromacs
#PBS -l select=1:ncpus=24:mpiprocs=1:mem=10G
#PBS -l walltime=02:00:00
#PBS -q normal
#PBS -o out_install_gromace-icc.txt
#PBS -e err_install_gromace-icc.txt

FFTWLIB=/home/users/industry/ai-hpc/apacsc27/kerwin/program/fftw3-icc/lib
FFTWINCLUDE=/home/users/industry/ai-hpc/apacsc27/kerwin/program/fftw3-icc/include
GROMACE_IN=/home/users/industry/ai-hpc/apacsc27/kerwin/program
GROMACE_SU=/home/users/industry/ai-hpc/apacsc27/kerwin/source/gromacs-2020
FLAGS="-xCORE-AVX2 -g -static-intel"

module load intel/19.0.0.117
module load cmake/3.14.4
module unload gcc/4.9.3
module load gcc/5.1.0
module load python/3.6.0

module list

cd $GROMACE_SU
rm -r build
mkdir build && cd build

time -p CC="icc" CXX="icpc" CFLAGS=$FLAGS CXXFLAGS=$FLAGS cmake  .. \
	-DGMX_FFT_LIBRARY=fftw3 \
	-DFFTWF_LIBRARY=$FFTWLIB/libfftw3f.so \
	-DFFTWF_INCLUDE_DIR=$FFTWINCLUDE \
	-DGMX_SIMD=AVX2_256 \
	-DGMX_MPI=ON \
	-DGMX_OPENMP=ON \
	-DGMX_BUILD_MDRUN_ONLY=ON \
	-DBUILD_SHARED_LIBS=OFF \
	-DGMX_DOUBLE=OFF \
	-DGMX_GPU=OFF \
	-DGMX_HWLOC=OFF \
	-DCMAKE_INSTALL_PREFIX=$GROMACE_IN/gromacs-intel-1 \
	-DCMAKE_C_COMPILER=mpiicc -DCMAKE_CXX_COMPILER=mpiicpc \
	-DCMAKE_BUILD_TYPE=Release -DCMAKE_COLOR_MAKEFILE=ON -DCMAKE_VERBOSE_MAKEFILE=TRUE

time -p make -j 48 install
```
Using  ``FLAGS="-xCORE-AVX2 -g -static-intel"`` for icc compile flag

**BUT Intel MPI 19 not work on RDMA**

### FFTW3 (icc 18)
```bash=
#/bin/bash
#PBS -P 50000041
#PBS -N install_fftw
#PBS -l select=1:ncpus=24:mpiprocs=1:mem=10G
#PBS -l walltime=02:00:00
#PBS -q normal
#PBS -o out_install_fftw-icc.txt
#PBS -e err_install_fftw-icc.txt
FLAGS="-march=core-avx2 -g -static-intel"

FFTWSOURCE=/home/users/industry/ai-hpc/apacsc27/kerwin/source/fftw-3.3.9
FFTWDIR=/home/users/industry/ai-hpc/apacsc27/kerwin/program

module load composerxe/2018.0.128


cd $FFTWSOURCE
rm -r build
mkdir build && cd build
CC="mpiicc" CFLAGS=$FLAGS ../configure --enable-mpi --enable-avx2 --enable-sse2 --prefix=$FFTWDIR/fftw3-icc18 --enable-shared --disable-static --enable-float
make -j 24
make install
```
### GROMACS Intel MPI (icc 18)
```bash=
#/bin/bash
#PBS -P 50000041
#PBS -N install_gromacs
#PBS -l select=1:ncpus=24:mpiprocs=1:mem=10G
#PBS -l walltime=02:00:00
#PBS -q normal
#PBS -o out_install_gromace-icc.txt
#PBS -e err_install_gromace-icc.txt

FFTWLIB=/home/users/industry/ai-hpc/apacsc27/kerwin/program/fftw3-icc18/lib
FFTWINCLUDE=/home/users/industry/ai-hpc/apacsc27/kerwin/program/fftw3-icc18/include
GROMACE_IN=/home/users/industry/ai-hpc/apacsc27/kerwin/program
GROMACE_SU=/home/users/industry/ai-hpc/apacsc27/kerwin/source/gromacs-2020
FLAGS="-xCORE-AVX2 -g -static-intel"

module load composerxe/2018.0.128
module load cmake/3.14.4
module unload gcc/4.9.3
module load gcc/5.1.0
module load python/3.6.0

module list
# Build gromacs code

cd $GROMACE_SU
rm -r build
mkdir build && cd build

time -p CC="icc" CXX="icpc" CFLAGS=$FLAGS CXXFLAGS=$FLAGS cmake  .. \
	-DGMX_FFT_LIBRARY=fftw3 \
	-DFFTWF_LIBRARY=$FFTWLIB/libfftw3f.so \
	-DFFTWF_INCLUDE_DIR=$FFTWINCLUDE \
	-DGMX_SIMD=AVX2_256 \
	-DGMX_MPI=ON \
	-DGMX_OPENMP=ON \
	-DGMX_BUILD_MDRUN_ONLY=ON \
	-DBUILD_SHARED_LIBS=OFF \
	-DGMX_DOUBLE=OFF \
	-DGMX_GPU=OFF \
	-DGMX_HWLOC=OFF \
	-DCMAKE_INSTALL_PREFIX=$GROMACE_IN/gromacs-intel18 \
	-DCMAKE_C_COMPILER=mpiicc -DCMAKE_CXX_COMPILER=mpiicpc \
	-DCMAKE_BUILD_TYPE=Release -DCMAKE_COLOR_MAKEFILE=ON -DCMAKE_VERBOSE_MAKEFILE=TRUE

time -p make -j 48 install
```

### RUN SCRIPT
```bash=
#!/bin/bash
#/bin/bash
#PBS -P 50000041
#PBS -N NTHU_stmv-768
#PBS -q normal
#PBS -l select=32:ncpus=24:mpiprocs=24:ompthreads=1
#PBS -l walltime=02:00:00
#PBS -o out_stmv_768_tune.txt
#PBS -e err_stmv_768_tune.txt

PBS_O_WORKDIR=$HOME/kerwin/bench
VTUNE_PTH=/app/intel/xe2019/vtune_amplifier/bin64
cd ${PBS_O_WORKDIR}

module load composerxe/2018.0.128
module unload gcc/4.9.3
module load gcc/5.1.0
module load python/3.6.0

module list

export I_MPI_STATS=2-4
export I_MPI_STATS_SCOPE="coll:Allreduce,Reduce"

NPE=768
GromacsBin="$HOME/kerwin/program/gromacs-intel18/bin/mdrun_mpi"
TestbenchPath="/home/users/industry/ai-hpc/apacsc27/kerwin/bench/bench-file/stmv.tpr"

mpiexec -IB -genv I_MPI_DEBUG 5 \
            -genv MV2_USE_APM 0 \
            -genv I_MPI_OFA_ADAPTER_NAME mlx5_0 \
            -genv I_MPI_OFA_NUM_PORTS 1 \
            -ppn 24 \
            -envall \
            -np $NPE \
            $GromacsBin -s $TestbenchPath \
            -v -noconfout -maxh 2 -ntomp 1 -nsteps 10000 -dlb yes
```


