# HPC-AI Competition NAMD Benchmark Guideline
## Run our benchmark
For the NAMD part, We installed NAMD and other necessary software in the NSCC computer. and submit a PBS script “namd-pbs-script.sh” to run the benchmark.
## Necessary software build in NSCC
### gcc(v8.4.0)

cd /home/users/industry/ai-hpc/apacsc29/scratch/app
export version=8.4.0
wget https://ftp.gnu.org/gnu/gcc/gcc-${version}/gcc-${version}.tar.gz
tar vxf gcc-${version}.tar.gz 
cd gcc-8.4.0
./contrib/download_prerequisites
mkdir build
cd build
../configure --enable-checking=release \
--enable-languages=c,c++,fortran --disable-multilib \
--prefix=/root/gcc
make -j$(nproc)
make install -j$(nproc)

#### gcc modulefile
:::info

#%Module

set          BASEPATH           /home/users/industry/ai-hpc/apacsc29/scratch/app/lib/gcc
prepend-path PATH               $BASEPATH/bin
prepend-path LD_LIBRARY_PATH    $BASEPATH/lib:$BASEPATH/lib64

>vim /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0
:::
### Intel mpi(v2019.5.281)

cd /home/users/industry/ai-hpc/apacsc29/scratch/app
wget http://registrationcenter-download.intel.com/akdlm/irc_nas/tec/15809/parallel_studio_xe_2019_update5_cluster_edition_online.tgz
tar xvf parallel_studio_xe_2019_update5_cluster_edition_online.tgz
cd parallel_studio_xe_2019_update5_cluster_edition_online
vim silent.cfg
---------------------------------------------------------
ACCEPT_EULA=accept
PSET_INSTALL_DIR=/home/users/industry/ai-hpc/apacsc29/scratch/app/intel
ACTIVATION_SERIAL_NUMBER=S477-Z28VJXXB
ACTIVATION_TYPE=serial_number
ARCH_SELECTED=INTEL64
---------------------------------------------------------
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0
./install.sh -s silent.cfg

### cmake(v3.18.2) build with icc

cd /home/users/industry/ai-hpc/apacsc29/scratch/app
wget https://github.com/Kitware/CMake/releases/download/v3.18.2/cmake-3.18.2.tar.gz
tar xvf cmake-3.18.2.tar.gz
cd cmake-3.18.2
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0
source /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/bin/compilervars.sh --arch linux --platform intel64
./bootstrap --parallel=$(nproc) --prefix=/home/users/industry/ai-hpc/apacsc29/scratch/app/cmake-icc CC=icc CXX=icpc  
make -j24
make -j24 install

#%Module

set             TOP_DIR         /home/users/industry/ai-hpc/apacsc29/scratch/app/cmake-icc
prepend-path    PATH            ${TOP_DIR}/bin

>vim /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/cmake/3.18.2
### UCX(v1.9.0) build with icc

cd /home/users/industry/ai-hpc/apacsc29/scratch/app
wget https://github.com/openucx/ucx/releases/download/v1.9.0/ucx-1.9.0.tar.gz
tar xvf ucx-1.9.0.tar.gz
cd ucx-1.9.0
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0
source /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/bin/compilervars.sh --arch linux --platform intel64
./configure CC=icc CXX=icpc F77=ifort FC=ifort     --disable-logging --disable-debug --disable-assertions     --disable-params-check    --without-java --with-knem=/opt/knem-1.1.3.90mlnx1 --enable-mt --with-avx --with-march=avx2  --without-xpmem  --prefix=/home/users/industry/ai-hpc/apacsc29/scratch/app/ucx1.9.0
make -j24
make -j24 install

:::info

#%Module

set             TOP_DIR         /home/users/industry/ai-hpc/apacsc29/scratch/app/ucx1.9.0
prepend-path    PATH            ${TOP_DIR}/bin
prepend-path    LD_LIBRARY_PATH ${TOP_DIR}/lib
prepend-path LIBRARY_PATH       ${TOP_DIR}/lib
prepend-path    CPATH           ${TOP_DIR}/include

>vim /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/ucx/1.9.0
:::
### event(v2.1.12)

module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0
cd /home/users/industry/ai-hpc/apacsc29/scratch/app/lib
wget https://github.com/libevent/libevent/releases/download/release-2.1.12-stable/libevent-2.1.12-stable.tar.gz
tar xvf libevent-2.1.12-stable.tar.gz
cd libevent-2.1.12-stable
./configure --prefix=/home/users/industry/ai-hpc/apacsc29/scratch/app/lib/event --disable-openssl
make -j24
make -j24 install

### openmpi(v4.0.4) build with gcc+ucx

cd /home/users/industry/ai-hpc/apacsc29/scratch/app
wget https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.4.tar.gz
tar xvf openmpi-4.0.4.tar.gz
wget http://content.mellanox.com/hpc/hpc-x/v2.7/hpcx-v2.7.0-gcc-MLNX_OFED_LINUX-5.0-1.0.0.0-redhat6.10-x86_64.tbz
tar xvf hpcx-v2.7.0-gcc-MLNX_OFED_LINUX-5.0-1.0.0.0-redhat6.10-x86_64.tbz
cd openmpi-4.0.4
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0
./configure CC=gcc CXX=g++ F77=gfortran FC=gfortran --enable-install-libpmix --with-libevent=/home/users/industry/ai-hpc/apacsc29/scratch/app/lib/event  --with-ucx=/home/users/industry/ai-hpc/apacsc29/scratch/app/hpcx-v2.7.0-gcc-MLNX_OFED_LINUX-5.0-1.0.0.0-redhat6.10-x86_64/ucx --prefix=/home/users/industry/ai-hpc/apacsc29/scratch/app/openmpi-4.0.4-gcc-ucx

:::info

#%Module

set             TOP_DIR         /home/users/industry/ai-hpc/apacsc29/scratch/app/openmpi-4.0.4-gcc-ucx
prepend-path    PATH            ${TOP_DIR}/bin
prepend-path    LD_LIBRARY_PATH ${TOP_DIR}/lib
prepend-path    MANPATH         ${TOP_DIR}/share/man
prepend-path    CPATH           ${TOP_DIR}/include
prepend-path    LIBRARY_PATH    ${TOP_DIR}/lib

>vim /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/openmpi/4.0.4-ucx-gcc
:::
### openmpi(v4.0.4) build with icc+ucx

cd /home/users/industry/ai-hpc/apacsc29/scratch/app/openmpi-4.0.4
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0
source /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/bin/compilervars.sh --arch linux --platform intel64
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/ucx/1.9.0
./configure  CC=icc CXX=icpc F77=ifort FC=ifort --prefix=/home/users/industry/ai-hpc/apacsc29/scratch/app/openmpi-4.0.4-ucx-icc --with-ucx=/home/users/industry/ai-hpc/apacsc29/scratch/app/ucx1.9.0 --enable-install-libpmix --with-libevent=/home/users/industry/ai-hpc/apacsc29/scratch/app/lib/event

:::info

#%Module

set             TOP_DIR         /home/users/industry/ai-hpc/apacsc29/scratch/app/openmpi-4.0.4-ucx-icc
prepend-path    PATH            ${TOP_DIR}/bin
prepend-path    LD_LIBRARY_PATH ${TOP_DIR}/lib
prepend-path    MANPATH         ${TOP_DIR}/share/man
prepend-path    CPATH           ${TOP_DIR}/include
prepend-path    LIBRARY_PATH    ${TOP_DIR}/lib

>vim /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/openmpi/4.0.4-ucx-icc
:::
### Tcl(v8.5.19) build with icc

cd /home/users/industry/ai-hpc/apacsc29/scratch/app
wget https://prdownloads.sourceforge.net/tcl/tcl8.5.19-src.tar.gz
tar xvf tcl8.5.19-src.tar.gz
cd tcl8.5.19-src
source /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/bin/compilervars.sh --arch linux --platform intel64
./configure --prefix=/home/users/industry/ai-hpc/apacsc29/scratch/app/lib/tcl-icc --enable-single --disable-fortran CC=icc --enable-threads
make -j24
make -j24 install

### fftw3(v3.3.8)

wget http://www.fftw.org/fftw-3.3.8.tar.gz
tar xvf fftw-3.3.8.tar.gz
cd fftw-3.3.8
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0
source /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/bin/compilervars.sh --arch linux --platform intel64 
bash -c '
CODE_NAME=fftw \
CODE_TAG=3.3.8 \
CODE_DIR=/home/users/industry/ai-hpc/apacsc29/scratch/NAMD/fftw-3.3.8 \
CMAKE_PATH=/home/users/industry/ai-hpc/apacsc29/scratch/app/cmake-icc/bin/cmake \
INSTALL_DIR=/home/users/industry/ai-hpc/apacsc29/scratch/NAMD/fftw-build-icc \
ICC_PATH=/home/users/industry/ai-hpc/apacsc29/app/intel/compilers_and_libraries_2019.5.281/linux/bin/intel64/icc 
ICC_FLAGS='"-xBROADWELL -axBROADWELL,CORE-AVX2,SSE4.2 -O3 -DNDEBUG"' \
### To build shared library (single precision) with Intel C Compiler
BUILD_LABEL=$CODE_TAG-shared-icc20-avx2-broadwell \
CMD_BUILD_SHARED_ICC=" \
mkdir -p /home/users/industry/ai-hpc/apacsc29/scratch/NAMD/fftw-3.3.8/build-$BUILD_LABEL \
&& cd /home/users/industry/ai-hpc/apacsc29/scratch/NAMD/fftw-3.3.8/build-$BUILD_LABEL \
&& $CMAKE_PATH .. \
-DBUILD_SHARED_LIBS=ON -DENABLE_FLOAT=ON \
-DENABLE_OPENMP=OFF -DENABLE_THREADS=OFF \
-DCMAKE_C_COMPILER=/scratch/users/industry/ai-hpc/apacsc29/app/intel/compilers_and_libraries_2019.5.281/linux/bin/intel64/icc -DCMAKE_CXX_COMPILER=/scratch/users/industry/ai-hpc/apacsc29/app/intel/compilers_and_libraries_2019.5.281/linux/bin/intel64/icpc \
-DENABLE_AVX2=ON -DENABLE_AVX=ON \
-DENABLE_SSE2=ON -DENABLE_SSE=ON \
-DCMAKE_INSTALL_PREFIX=$INSTALL_DIR/$BUILD_LABEL \
-DCMAKE_C_FLAGS_RELEASE=$ICC_FLAGS \
-DCMAKE_CXX_FLAGS_RELEASE=$ICC_FLAGS \
&& time -p make VERBOSE=1 V=1 install  \
&& cd $INSTALL_DIR/$BUILD_LABEL && ln -s lib64 lib | tee $BUILD_LABEL.log "

eval $CMD_BUILD_SHARED_ICC ;
wait
echo $CMD_BUILD_SHARED_ICC ;
' | tee fftw3buildlog 2>&1

:::info

#%Module

set FFTW_DIR /home/users/industry/ai-hpc/apacsc29/scratch/NAMD/fftw-build-icc/3.3.8-shared-icc20-avx2-broadwell

prepend-path CPLUS_INCLUDE_PATH $FFTW_DIR/include
prepend-path LD_LIBRARY_PATH $FFTW_DIR/lib
prepend-path LIBRARY_PATH $FFTW_DIR/lib

>vim /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/fftw-icc
:::
### charm(v6.10.2) build with icc

cd /home/users/industry/ai-hpc/apacsc29/scratch/NAMD
git clone  https://github.com/UIUC-PPL/charm.git charm-openmpi-icc
cd charm-openmpi-icc
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0
source /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/bin/compilervars.sh --arch linux --platform intel64
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/cmake/3.18.2
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/openmpi/4.0.4-ucx-icc
./build charm++ mpi-linux-x86_64  --with-production --incdir=/home/users/industry/ai-hpc/apacsc29/scratch/app/openmpi-4.0.4-ucx-icc/include  --libdir=/home/users/industry/ai-hpc/apacsc29/scratch/app/ucx1.9.0/lib  icc ifort -O3

### charm(v6.10.2) build with icc+ucx

cd /home/users/industry/ai-hpc/apacsc29/scratch/NAMD
git clone  https://github.com/UIUC-PPL/charm.git charm-ucx-icc
cd charm-ucx-icc
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0
source /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/bin/compilervars.sh --arch linux --platform intel64
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/cmake/3.18.2
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/openmpi/4.0.4-ucx-icc
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/ucx/1.9.0
./build charm++ ucx-linux-x86_64 ompipmix --with-production --incdir=/home/users/industry/ai-hpc/apacsc29/scratch/app/openmpi-4.0.4-ucx-icc/include --incdir=/home/users/industry/ai-hpc/apacsc29/scratch/app/ucx1.9.0/include --libdir=/home/users/industry/ai-hpc/apacsc29/scratch/app/openmpi-4.0.4-ucx-icc/lib:/home/users/industry/ai-hpc/apacsc29/scratch/app/ucx1.9.0/lib  icc ifort -O3

Now we can build NAMD in NSCC.
## Build NAMD(v2.14) in NSCC
### Namd build with icc+openmpi+fftw3(without-memopt)

cd /home/users/industry/ai-hpc/apacsc29/scratch/NAMD
wget https://www.ks.uiuc.edu/Research/namd/2.14/download/946183/NAMD_2.14_Source.tar.gz
tar xvf NAMD_2.14_Source.tar.gz
cp -r NAMD_2.14_Source/ namd2.14-openmpi-icc
cd namd2.14-openmpi-icc
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0 
source /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/bin/compilervars.sh --arch linux --platform intel64
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/openmpi/4.0.4-ucx-icc
./config Linux-x86_64-icc  --charm-base /home/users/industry/ai-hpc/apacsc29/scratch/NAMD/charm-openmpi-icc --charm-arch mpi-linux-x86_64-icc-ifort --with-mkl --mkl-prefix /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/mkl --cc /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/bin/intel64/icc --cc-opts '-static-intel -axBROADWELL,CORE-AVX2,SSE4.2 -O3 -DNDEBUG' --cxx /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/bin/intel64/icpc --cxx-opts '-static-intel -axBROADWELL,CORE-AVX2,SSE4.2 -O3 -DNDEBUG' --tcl-prefix /home/users/industry/ai-hpc/apacsc29/scratch/app/lib/tcl-icc

### Namd build with icc+openmpi+fftw3(with-memopt)

cd /home/users/industry/ai-hpc/apacsc29/scratch/NAMD
cp -r NAMD_2.14_Source/ namd2.14-openmpi-icc-memopt
cd namd2.14-openmpi-icc-memopt
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0 
source /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/bin/compilervars.sh --arch linux --platform intel64
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/openmpi/4.0.4-ucx-icc
./config Linux-x86_64-icc --with-memopt --charm-base /home/users/industry/ai-hpc/apacsc29/scratch/NAMD/charm-openmpi-icc --charm-arch mpi-linux-x86_64-icc-ifort --with-mkl --mkl-prefix /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/mkl --cc /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/bin/intel64/icc --cc-opts '-static-intel -axBROADWELL,CORE-AVX2,SSE4.2 -O3 -DNDEBUG' --cxx /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/bin/intel64/icpc --cxx-opts '-static-intel -axBROADWELL,CORE-AVX2,SSE4.2 -O3 -DNDEBUG' --tcl-prefix /home/users/industry/ai-hpc/apacsc29/scratch/app/lib/tcl-icc

### Namd build with icc+ucx+fftw3(with-memopt)

cd /home/users/industry/ai-hpc/apacsc29/scratch/NAMD
cp -r NAMD_2.14_Source/ namd2.14-openmpi-icc-ucx
cd namd2.14-openmpi-icc-ucx
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0 
source /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/bin/compilervars.sh --arch linux --platform intel64
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/openmpi/4.0.4-ucx-icc
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/ucx/1.9.0
./config Linux-x86_64-icc --with-memopt --charm-base /home/users/industry/ai-hpc/apacsc29/scratch/NAMD/charm-ucx-icc --charm-arch ucx-linux-x86_64-ompipmix-icc-ifort  --with-fftw3 --fftw-prefix /home/users/industry/ai-hpc/apacsc29/scratch/NAMD/fftw-build-icc/3.3.8-shared-icc20-avx2-broadwell --cc /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/bin/intel64/icc --cc-opts '-static-intel -axBROADWELL,CORE-AVX2,SSE4.2 -O3 -DNDEBUG' --cxx /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/bin/intel64/icpc --cxx-opts '-static-intel -axBROADWELL,CORE-AVX2,SSE4.2 -O3 -DNDEBUG' --tcl-prefix /home/users/industry/ai-hpc/apacsc29/scratch/app/lib/tcl-icc
cd Linux-x86_64-icc
make -j24

We have built three kinds of NAMD. The first one NAMD is without memopt. It is used to run compress_210stmv.namd. and other kinds of NAMD is used to compare which one runs better.
## Run stmv benchmark

cd /home/users/industry/ai-hpc/apacsc29/scratch/NAMD/stmv
mkdir 210stmv2fs
wget https://www.ks.uiuc.edu/Research/namd/utilities/stmv_sc14.tar.gz --no-check-certificate
tar xvf stmv_sc14.tar.gz
wget https://www.ks.uiuc.edu/Research/namd/utilities/stmv.tar.gz --no-check-certificate
tar xvf stmv.tar.gz
cd 210stmv2fs
cp -r ../stmv/* .
cp -r ../stmv_sc14/make_210tmv.pgn .
cp -r ../stmv_sc14/210stmv2fs .
/home/users/industry/ai-hpc/apacsc29/scratch/NAMD/namd2.14-openmpi-icc/Linux-x86_64-icc/psfgen make_210stmv.pgn
cp -r ../stmv_sc14/compress_210stmv.namd .
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0 
source /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/bin/compilervars.sh --arch linux --platform intel64
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/fftw-icc
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/openmpi/4.0.4-ucx-icc
/home/users/industry/ai-hpc/apacsc29/scratch/NAMD/namd2.14-openmpi-icc/Linux-x86_64-icc/charmrun --allow-run-as-root +p4 /home/users/industry/ai-hpc/apacsc29/scratch/NAMD/namd2.14-openmpi-icc/Linux-x86_64-icc/namd2 compress_210stmv.namd

:::info

minAtomsPerPatch 1000
#ldBalancer hybrid
ldBalancer none
numinputprocs 24
numoutputprocs 50
numoutputwriters 24

>vim /home/users/industry/ai-hpc/apacsc29/scratch/NAMD/stmv/210stmv2fs/210stmv2fs.namd
:::
I have changed some of the 210stmv2fs.namd adjustable parameters. And now we can run 210stmv2fs.namd with NAMD.
## PBS script
### run 210stmv2fs with NAMD build with icc+openmpi+fftw3

#!/bin/bash
#PBS -N namd-icc
#PBS -q normal
#PBS -l select=32:ncpus=24:mpiprocs=24:ompthreads=1:mem=96gb
#PBS -l walltime=20:00
#PBS -P 50000043
#PBS -j oe
#PBS -o result-openmpi-icc-32-24.txt

cd /home/users/industry/ai-hpc/apacsc29/scratch/NAMD/stmv/210stmv2fs


node=32
ncpu=24

rm -f hostfile1
for (( i=1; i<=$node; i=i+1 ))
do
        S[${i}]=`sed -n "$(($i*$ncpu))p" $PBS_NODEFILE`
        echo "${S[$i]} slots=$ncpu" >> hostfile1
done


bash -c '
cd /home/users/industry/ai-hpc/apacsc29/scratch/NAMD/stmv/210stmv2fs \
&& module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0 \
&& source /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/bin/compilervars.sh --arch linux --platform intel64  \
&& module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/openmpi/4.0.4-ucx-gcc \
&& /home/users/industry/ai-hpc/apacsc29/scratch/app/openmpi-4.0.4-gcc-ucx/bin/mpirun   -x LD_LIBRARY_PATH --hostfile hostfile1 /home/users/industry/ai-hpc/apacsc29/scratch/NAMD/namd2.14-openmpi-icc-memopt/Linux-x86_64-icc/namd2  /home/users/industry/ai-hpc/apacsc29/scratch/NAMD/stmv/210stmv2fs/210stmv2fs.namd'

### run 210stmv2fs with NAMD build with icc+ucx+fftw3

#!/bin/bash
#PBS -N namd-openmpi-ucx
#PBS -q normal
#PBS -l select=32:ncpus=24:mpiprocs=24:ompthreads=1:mem=96gb
#PBS -l walltime=20:00
#PBS -P 50000043
#PBS -j oe
#PBS -o result-openmpi-icc-32-24-ucx-24-50-24-flags.txt

cd /home/users/industry/ai-hpc/apacsc29/scratch/NAMD/stmv/210stmv2fs

node=32
ncpu=24

rm -f hostfile1
for (( i=1; i<=$node; i=i+1 ))
do
        S[${i}]=`sed -n "$(($i*$ncpu))p" $PBS_NODEFILE`
        echo "${S[$i]} slots=$ncpu" >> hostfile1
done


bash -c '
cd /home/users/industry/ai-hpc/apacsc29/scratch/NAMD/stmv/210stmv2fs \
&& module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0  \
&& source /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/bin/compilervars.sh --arch linux --platform intel64 \
&& module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/ucx/1.9.0 \
&& module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/fftw-icc \
&& module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/openmpi/4.0.4-ucx-gcc  \
&& /home/users/industry/ai-hpc/apacsc29/scratch/app/openmpi-4.0.4-gcc-ucx/bin/mpirun -x UCX_NET_DEVICES=mlx5_0:1  -x LD_LIBRARY_PATH --hostfile hostfile1 /home/users/industry/ai-hpc/apacsc29/scratch/NAMD/namd2.14-openmpi-icc-ucx/Linux-x86_64-icc/namd2  /home/users/industry/ai-hpc/apacsc29/scratch/NAMD/stmv/210stmv2fs/210stmv2fs.namd'