# Intel MPI
### FFTW3 using icc 19
```bash=
#/bin/bash
#PBS -P 50000041
#PBS -N install_fftw
#PBS -l select=1:ncpus=24:mpiprocs=1:mem=10G
#PBS -l walltime=02:00:00
#PBS -q normal
#PBS -o out_install_fftw-icc.txt
#PBS -e err_install_fftw-icc.txt
FFTWSOURCE=/home/users/industry/ai-hpc/apacsc27/kerwin/source/fftw-3.3.9
FFTWDIR=/home/users/industry/ai-hpc/apacsc27/kerwin/program
module load intel/19.0.0.117
cd $FFTWSOURCE
rm -r build
mkdir build && cd build
CC="mpiicc" ../configure --enable-mpi --enable-avx2 --enable-sse2 --prefix=$FFTWDIR/fftw3-icc --enable-shared --disable-static --enable-float
make -j 24
make install
```
### GROMACS mpiicc icc (icc 19)
```bash=
#/bin/bash
#PBS -P 50000041
#PBS -N install_gromacs
#PBS -l select=1:ncpus=24:mpiprocs=1:mem=10G
#PBS -l walltime=02:00:00
#PBS -q normal
#PBS -o out_install_gromace-icc.txt
#PBS -e err_install_gromace-icc.txt
FFTWLIB=/home/users/industry/ai-hpc/apacsc27/kerwin/program/fftw3-icc/lib
FFTWINCLUDE=/home/users/industry/ai-hpc/apacsc27/kerwin/program/fftw3-icc/include
GROMACE_IN=/home/users/industry/ai-hpc/apacsc27/kerwin/program
GROMACE_SU=/home/users/industry/ai-hpc/apacsc27/kerwin/source/gromacs-2020
FLAGS="-xCORE-AVX2 -g -static-intel"
module load intel/19.0.0.117
module load cmake/3.14.4
module unload gcc/4.9.3
module load gcc/5.1.0
module load python/3.6.0
module list
cd $GROMACE_SU
rm -r build
mkdir build && cd build
time -p CC="icc" CXX="icpc" CFLAGS=$FLAGS CXXFLAGS=$FLAGS cmake .. \
-DGMX_FFT_LIBRARY=fftw3 \
-DFFTWF_LIBRARY=$FFTWLIB/libfftw3f.so \
-DFFTWF_INCLUDE_DIR=$FFTWINCLUDE \
-DGMX_SIMD=AVX2_256 \
-DGMX_MPI=ON \
-DGMX_OPENMP=ON \
-DGMX_BUILD_MDRUN_ONLY=ON \
-DBUILD_SHARED_LIBS=OFF \
-DGMX_DOUBLE=OFF \
-DGMX_GPU=OFF \
-DGMX_HWLOC=OFF \
-DCMAKE_INSTALL_PREFIX=$GROMACE_IN/gromacs-intel-1 \
-DCMAKE_C_COMPILER=mpiicc -DCMAKE_CXX_COMPILER=mpiicpc \
-DCMAKE_BUILD_TYPE=Release -DCMAKE_COLOR_MAKEFILE=ON -DCMAKE_VERBOSE_MAKEFILE=TRUE
time -p make -j 48 install
```
Using ``FLAGS="-xCORE-AVX2 -g -static-intel"`` for icc compile flag
**BUT Intel MPI 19 not work on RDMA**
### FFTW3 (icc 18)
```bash=
#/bin/bash
#PBS -P 50000041
#PBS -N install_fftw
#PBS -l select=1:ncpus=24:mpiprocs=1:mem=10G
#PBS -l walltime=02:00:00
#PBS -q normal
#PBS -o out_install_fftw-icc.txt
#PBS -e err_install_fftw-icc.txt
FLAGS="-march=core-avx2 -g -static-intel"
FFTWSOURCE=/home/users/industry/ai-hpc/apacsc27/kerwin/source/fftw-3.3.9
FFTWDIR=/home/users/industry/ai-hpc/apacsc27/kerwin/program
module load composerxe/2018.0.128
cd $FFTWSOURCE
rm -r build
mkdir build && cd build
CC="mpiicc" CFLAGS=$FLAGS ../configure --enable-mpi --enable-avx2 --enable-sse2 --prefix=$FFTWDIR/fftw3-icc18 --enable-shared --disable-static --enable-float
make -j 24
make install
```
### GROMACS Intel MPI (icc 18)
```bash=
#/bin/bash
#PBS -P 50000041
#PBS -N install_gromacs
#PBS -l select=1:ncpus=24:mpiprocs=1:mem=10G
#PBS -l walltime=02:00:00
#PBS -q normal
#PBS -o out_install_gromace-icc.txt
#PBS -e err_install_gromace-icc.txt
FFTWLIB=/home/users/industry/ai-hpc/apacsc27/kerwin/program/fftw3-icc18/lib
FFTWINCLUDE=/home/users/industry/ai-hpc/apacsc27/kerwin/program/fftw3-icc18/include
GROMACE_IN=/home/users/industry/ai-hpc/apacsc27/kerwin/program
GROMACE_SU=/home/users/industry/ai-hpc/apacsc27/kerwin/source/gromacs-2020
FLAGS="-xCORE-AVX2 -g -static-intel"
module load composerxe/2018.0.128
module load cmake/3.14.4
module unload gcc/4.9.3
module load gcc/5.1.0
module load python/3.6.0
module list
# Build gromacs code
cd $GROMACE_SU
rm -r build
mkdir build && cd build
time -p CC="icc" CXX="icpc" CFLAGS=$FLAGS CXXFLAGS=$FLAGS cmake .. \
-DGMX_FFT_LIBRARY=fftw3 \
-DFFTWF_LIBRARY=$FFTWLIB/libfftw3f.so \
-DFFTWF_INCLUDE_DIR=$FFTWINCLUDE \
-DGMX_SIMD=AVX2_256 \
-DGMX_MPI=ON \
-DGMX_OPENMP=ON \
-DGMX_BUILD_MDRUN_ONLY=ON \
-DBUILD_SHARED_LIBS=OFF \
-DGMX_DOUBLE=OFF \
-DGMX_GPU=OFF \
-DGMX_HWLOC=OFF \
-DCMAKE_INSTALL_PREFIX=$GROMACE_IN/gromacs-intel18 \
-DCMAKE_C_COMPILER=mpiicc -DCMAKE_CXX_COMPILER=mpiicpc \
-DCMAKE_BUILD_TYPE=Release -DCMAKE_COLOR_MAKEFILE=ON -DCMAKE_VERBOSE_MAKEFILE=TRUE
time -p make -j 48 install
```
### RUN SCRIPT
```bash=
#!/bin/bash
#/bin/bash
#PBS -P 50000041
#PBS -N NTHU_stmv-768
#PBS -q normal
#PBS -l select=32:ncpus=24:mpiprocs=24:ompthreads=1
#PBS -l walltime=02:00:00
#PBS -o out_stmv_768_tune.txt
#PBS -e err_stmv_768_tune.txt
PBS_O_WORKDIR=$HOME/kerwin/bench
VTUNE_PTH=/app/intel/xe2019/vtune_amplifier/bin64
cd ${PBS_O_WORKDIR}
module load composerxe/2018.0.128
module unload gcc/4.9.3
module load gcc/5.1.0
module load python/3.6.0
module list
export I_MPI_STATS=2-4
export I_MPI_STATS_SCOPE="coll:Allreduce,Reduce"
NPE=768
GromacsBin="$HOME/kerwin/program/gromacs-intel18/bin/mdrun_mpi"
TestbenchPath="/home/users/industry/ai-hpc/apacsc27/kerwin/bench/bench-file/stmv.tpr"
mpiexec -IB -genv I_MPI_DEBUG 5 \
-genv MV2_USE_APM 0 \
-genv I_MPI_OFA_ADAPTER_NAME mlx5_0 \
-genv I_MPI_OFA_NUM_PORTS 1 \
-ppn 24 \
-envall \
-np $NPE \
$GromacsBin -s $TestbenchPath \
-v -noconfout -maxh 2 -ntomp 1 -nsteps 10000 -dlb yes
```
{"metaMigratedAt":"2023-06-16T10:22:37.530Z","metaMigratedFrom":"Content","title":"Intel MPI","breaks":true,"contributors":"[{\"id\":\"7d0cfd00-7db0-47fd-8db4-70e0b5c11c0d\",\"add\":8181,\"del\":2998}]"}