---
# System prepended metadata

title: NSCC外部環境

---

# NSCC外部環境
## gcc
```shell=
export version=8.4.0
wget https://ftp.gnu.org/gnu/gcc/gcc-${version}/gcc-${version}.tar.gz
tar vxf gcc-${version}.tar.gz 
cd gcc-8.4.0
./contrib/download_prerequisites

mkdir build
cd build

../configure --enable-checking=release \
--enable-languages=c,c++,fortran --disable-multilib \
--prefix=/root/gcc
make -j$(nproc)
make install -j$(nproc)
```
:::info
```shell=
#%Module

set          BASEPATH           /home/users/industry/ai-hpc/apacsc29/scratch/app/lib/gcc
prepend-path PATH               $BASEPATH/bin
prepend-path LD_LIBRARY_PATH    $BASEPATH/lib:$BASEPATH/lib64
```
>/home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0

:::

## cmake
### gcc
```shell=
wget https://github.com/Kitware/CMake/releases/download/v3.17.3/cmake-3.17.3.tar.gz
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0

./bootstrap --parallel=$(nproc) --prefix=/home/users/industry/ai-hpc/apacsc29/scratch/app/cmake
make -j$(nproc)
make install -j$(nproc)
```
:::info
```shell=
#%Module

set             TOP_DIR         /home/users/industry/ai-hpc/apacsc29/scratch/app/cmake
prepend-path    PATH            ${TOP_DIR}/bin
```
>/home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/cmake
:::
### icc
```shell=
wget https://github.com/Kitware/CMake/releases/download/v3.18.2/cmake-3.18.2.tar.gz
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0
source /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/bin/compilervars.sh --arch linux --platform intel64
./bootstrap --parallel=$(nproc) --prefix=/home/users/industry/ai-hpc/apacsc29/scratch/app/cmake-icc CC=icc CXX=icpc  
make -j24
make -j24 install
```
## tcl
### gcc
```shell=
wget https://prdownloads.sourceforge.net/tcl/tcl8.5.19-src.tar.gz
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0
./configure --prefix=/home/users/industry/ai-hpc/apacsc29/scratch/app/lib/tcl --enable-single --disable-fortran CC=gcc --enable-threads
``` 
### icc
```shell=
source /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/bin/compilervars.sh --arch linux --platform intel64
./configure --prefix=/home/users/industry/ai-hpc/apacsc29/scratch/app/lib/tcl-icc --enable-single --disable-fortran CC=icc --enable-threads
```
## ucx
```shell=
git clone https://github.com/openucx/ucx.git ucx1.9.0
./autogen.sh 
```
>./autogen.sh要在共用資料夾外面
### gcc
```shell=
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0
./configure CC=gcc CXX=g++     --disable-logging --disable-debug --disable-assertions     --disable-params-check --enable-devel-headers     --without-java --with-knem=/opt/knem-1.1.3.90mlnx1 --enable-mt --with-avx --with-march=avx2    --prefix=/home/users/industry/ai-hpc/apacsc29/scratch/app/lib/ucx-1.8.0
./configure CC=gcc CXX=g++  --enable-optimizations --disable-logging --disable-debug --disable-assertions --disable-params-check --without-java --with-knem=/opt/knem-1.1.3.90mlnx1 --enable-mt --without-xpmem --prefix=/home/users/industry/ai-hpc/apacsc29/scratch/app/lib/ucx-1.6.0
```
### icc
```shell=
source /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/bin/compilervars.sh --arch linux --platform intel64

./configure CC=icc CXX=icpc F77=ifort FC=ifort     --disable-logging --disable-debug --disable-assertions     --disable-params-check    --without-java --with-knem=/opt/knem-1.1.3.90mlnx1 --enable-mt --with-avx --with-march=avx2  --without-xpmem  --prefix=/home/users/industry/ai-hpc/apacsc29/scratch/app/hpcx-v2.4.0-gcc-MLNX_OFED_LINUX-4.4-2.0.7.0-redhat6.9-x86_64/ucx-icc

./configure CC=icc CXX=icpc F77=ifort FC=ifort     --disable-logging --disable-debug --disable-assertions     --disable-params-check    --without-java --with-knem=/opt/knem-1.1.3.90mlnx1 --enable-mt --with-avx --with-march=avx2  --without-xpmem  --prefix=/home/users/industry/ai-hpc/apacsc29/scratch/app/hpcx-v2.7.0-gcc-MLNX_OFED_LINUX-4.7-1.0.0.1-redhat6.10-x86_64/ucx-icc
```
:::info
```shell=
#%Module

set             TOP_DIR         /home/users/industry/ai-hpc/apacsc29/scratch/app/lib/ucx-1.6.0
prepend-path    PATH            ${TOP_DIR}/bin
prepend-path    LD_LIBRARY_PATH ${TOP_DIR}/lib
prepend-path LIBRARY_PATH       ${TOP_DIR}/lib
prepend-path    CPATH           ${TOP_DIR}/include
```
>module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/ucx/1.6.0
:::
## openmpi
### event
```shell=
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0
wget https://github.com/libevent/libevent/releases/download/release-2.1.12-stable/libevent-2.1.12-stable.tar.gz
./configure --prefix=/home/users/industry/ai-hpc/apacsc29/scratch/app/lib/event --disable-openssl
```
### pmix
```shell=
cd /home/users/industry/ai-hpc/apacsc29/scratch/app/openmpi-3.1.6/opal/mca/pmix/pmix2x/pmix
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0
./configure --prefix=/home/users/industry/ai-hpc/apacsc29/scratch/app/lib/pmix --with-libevent=/home/users/industry/ai-hpc/apacsc29/scratch/app/lib/event
```
:::info
pmix modulefile
```shell=
#%Module

set          BASEPATH           /home/users/industry/ai-hpc/apacsc29/scratch/app/lib/pmix
prepend-path LD_LIBRARY_PATH    $BASEPATH/lib
prepend-path LIBRARY_PATH       $BASEPATH/lib
prepend-path    CPATH           $BASEPATH/include
```
>module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/pmix
:::
### build openmpi

#### 4.0.4(without ucx)
```shell=
wget https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.4.tar.gz

module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0
./configure  --prefix=/home/users/industry/ai-hpc/apacsc29/scratch/app/openmpi-4.0.4-general
```

#### 4.0.4(with icc)
```shell=
source /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/bin/compilervars.sh --arch linux --platform intel64
./configure  CC=icc CXX=icpc F77=ifort FC=ifort --prefix=/home/users/industry/ai-hpc/apacsc29/scratch/app/openmpi-4.0.4-icc 
```
:::info
```shell=
--with-platform=/home/users/industry/ai-hpc/apacsc29/scratch/app/openmpi-4.0.4/contrib/platform/intel/bend/linux-optimized
會有問題
```
:::

#### 4.0.4(with ucx)
```shell=
wget https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.4.tar.gz

module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0
./configure --with-ucx=/home/users/industry/ai-hpc/apacsc29/scratch/app/lib/ucx-1.6.0 --prefix=/home/users/industry/ai-hpc/apacsc29/scratch/app/openmpi-4.0.4-ucx --with-pmix=/home/users/industry/ai-hpc/apacsc29/scratch/app/lib/pmix3 --with-libevent=/home/users/industry/ai-hpc/apacsc29/scratch/app/lib/event
```
```shell=
#%Module

set             TOP_DIR         /home/users/industry/ai-hpc/apacsc29/scratch/app/openmpi-4.0.4-ucx
prepend-path    PATH            ${TOP_DIR}/bin
prepend-path    LD_LIBRARY_PATH ${TOP_DIR}/lib
prepend-path    MANPATH         ${TOP_DIR}/share/man
prepend-path    CPATH           ${TOP_DIR}/include
prepend-path    LIBRARY_PATH    ${TOP_DIR}/lib
```
#### 4.0.4(with ucx pmix) 
```shell=
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0
./configure --enable-install-libpmix --with-libevent=/home/users/industry/ai-hpc/apacsc29/scratch/app/lib/event  --with-ucx=/home/users/industry/ai-hpc/apacsc29/scratch/app/lib/ucx-1.6.0 --prefix=/home/users/industry/ai-hpc/apacsc29/scratch/app/openmpi-4.0.4-ucx-pmix
```
## hpcx-ompi
### pmix3
```shell=
cd /home/users/industry/ai-hpc/apacsc29/scratch/app/hpcx-v2.4.0-gcc-MLNX_OFED_LINUX-4.4-2.0.7.0-redhat6.9-x86_64/sources/openmpi-gitclone/opal/mca/pmix/pmix3x/pmix
./configure --prefix=/home/users/industry/ai-hpc/apacsc29/scratch/app/lib/pmix3 --with-libevent=/home/users/industry/ai-hpc/apacsc29/scratch/app/lib/event
```
[hpcx-2.4](https://docs.mellanox.com/pages/viewpage.action?pageId=12006256)
### gcc
```shell=
wget http://content.mellanox.com/hpc/hpc-x/v2.4/hpcx-v2.4.0-gcc-MLNX_OFED_LINUX-4.4-2.0.7.0-redhat6.9-x86_64.tbz

wget http://content.mellanox.com/hpc/hpc-x/v2.7/hpcx-v2.7.0-gcc-MLNX_OFED_LINUX-4.7-1.0.0.1-redhat6.10-x86_64.tbz

module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0
./configure  --prefix=/home/users/industry/ai-hpc/apacsc29/scratch/app/hpcx-v2.4.0-gcc-MLNX_OFED_LINUX-4.4-2.0.7.0-redhat6.9-x86_64/ompi-rebuild --with-ucx=/home/users/industry/ai-hpc/apacsc29/scratch/app/hpcx-v2.4.0-gcc-MLNX_OFED_LINUX-4.4-2.0.7.0-redhat6.9-x86_64/ucx-rebuild  --with-pmix=/home/users/industry/ai-hpc/apacsc29/scratch/app/lib/pmix3 --with-libevent=/home/users/industry/ai-hpc/apacsc29/scratch/app/lib/event --enable-openib-rdmacm-ibaddr  --with-knem=/opt/knem-1.1.3.90mlnx1 --with-hcoll=/opt/mellanox/hcoll
```
### icc
```shell=
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0
source /home/users/industry/ai-hpc/apacsc29/scratch/app/intel/compilers_and_libraries_2019.5.281/linux/bin/compilervars.sh --arch linux --platform intel64
./configure CC=icc CXX=icpc F77=ifort FC=ifort  --prefix=/home/users/industry/ai-hpc/apacsc29/scratch/app/hpcx-v2.4.0-gcc-MLNX_OFED_LINUX-4.4-2.0.7.0-redhat6.9-x86_64/ompi-icc --with-ucx=/home/users/industry/ai-hpc/apacsc29/scratch/app/hpcx-v2.4.0-gcc-MLNX_OFED_LINUX-4.4-2.0.7.0-redhat6.9-x86_64/ucx-icc  --enable-install-libpmix --with-libevent=/home/users/industry/ai-hpc/apacsc29/scratch/app/lib/event --enable-openib-rdmacm-ibaddr  --with-knem=/opt/knem-1.1.3.90mlnx1 --with-hcoll=/opt/mellanox/hcoll --enable-orterun-prefix-by-default
```
>>--enable-orterun-prefix-by-default
>
>module load /home/users/industry/ai-hpc/apacsc29/scratch/app/hpcx-v2.4.0-gcc-MLNX_OFED_LINUX-4.4-2.0.7.0-redhat6.9-x86_64/modulefiles/hpcx-icc
:::warning
```shell=
--with-platform=contrib/platform/mellanox/optimized
###這行會導致verbs出不來
--with-mxm=/opt/mellanox/mxm
###這行在make會有error
一定要加--with-platform 不然不能跨機
>>./configure CC=icc CXX=icpc F77=ifort FC=ifort  --prefix=/home/users/industry/ai-hpc/apacsc29/scratch/app/hpcx-v2.4.0-gcc-MLNX_OFED_LINUX-4.4-2.0.7.0-redhat6.9-x86_64/ompi-icc --with-ucx=/home/users/industry/ai-hpc/apacsc29/scratch/app/hpcx-v2.4.0-gcc-MLNX_OFED_LINUX-4.4-2.0.7.0-redhat6.9-x86_64/ucx-icc  --enable-install-libpmix --with-libevent=/home/users/industry/ai-hpc/apacsc29/scratch/app/lib/event --enable-openib-rdmacm-ibaddr  --with-knem=/opt/knem-1.1.3.90mlnx1 --with-hcoll=/opt/mellanox/hcoll --with-platform=contrib/platform/mellanox/optimized
```
:::
### intel-mpi
```shell=
wget http://registrationcenter-download.intel.com/akdlm/irc_nas/tec/15809/parallel_studio_xe_2019_update5_cluster_edition_online.tgz
vim silent.cfg
-------------------------
ACCEPT_EULA=accept
PSET_INSTALL_DIR=/home/users/industry/ai-hpc/apacsc29/scratch/app/intel
ACTIVATION_SERIAL_NUMBER=S477-Z28VJXXB
ACTIVATION_TYPE=serial_number
ARCH_SELECTED=INTEL64

-------------------------
module load /home/users/industry/ai-hpc/apacsc29/scratch/modulefiles/gcc/8.4.0
./install.sh -s silent.cfg
```