# Setting CTSM on Puhti
[](https://hackmd.io/2PcRR76PSNGfu5jM7Y1Sng)
## Load available modules for CTSM (Not for user):
- These information has been included in dotcime.
```
module load netcdf-fortran/4.5.4
module load parallel-netcdf/1.12.2
module load python-data/3.10-22.09
module load perl/5.34.1
module load netcdf-c/4.8.1
```
## PIO library is required for ESMF building. So install PIO first (Not for user)
- Load corresponding modules
```
# intel
module load intel-oneapi-compilers-classic/2021.6.0
module load intel-oneapi-mpi/2021.6.0
module load intel-oneapi-mkl/2022.1.0
module load netcdf-fortran/4.5.4
module load netcdf-c/4.8.1
module load parallel-netcdf/1.12.2
# GNU
module purge
module load gcc/13.2.0
module load openmpi/5.0.5
module load netcdf-c/4.9.2
module load netcdf-fortran/4.6.1
module load intel-oneapi-mkl/2024.0.0
```
- Download the library
```
wget -q https://github.com/NCAR/ParallelIO/archive/refs/tags/pio2_5_10.tar.gz
tar zxvf pio2_5_10.tar.gz
cd ParallelIO-pio2_5_10
```
- Configure and build the library
```
autoreconf --install --force
#CC=mpiicc FC=mpiifort ./configure --prefix=/projappl/project_2006422/pio-2.5.2 --enable-fortran
export CC=mpiicc
export FC=mpiifort
#export CPPFLAGS='-I/appl/spack/v018/install-tree/intel-2021.6.0/netcdf-c-4.8.1-x5mttm/include -I/appl/spack/v018/install-tree/intel-2021.6.0/curl-7.83.0-dajhhg/include -I/appl/spack/v018/install-tree/intel-2021.6.0/hdf5-1.12.2-rzlc7b/include -I/appl/spack/v018/install-tree/intel-2021.6.0/parallel-netcdf-1.12.2-qp3uqs/include -I/appl/spack/v018/install-tree/intel-2021.6.0/hdf-4.2.15-qa7qer/include -I/appl/spack/v018/install-tree/intel-2021.6.0/netcdf-fortran-4.5.4-nucfze/include'
#export LDFLAGS='-L/appl/spack/v018/install-tree/intel-2021.6.0/netcdf-c-4.8.1-x5mttm/lib -L/appl/spack/v018/install-tree/intel-2021.6.0/netcdf-fortran-4.5.4-nucfze/lib -L/appl/spack/v018/install-tree/intel-2021.6.0/parallel-netcdf-1.12.2-qp3uqs/lib'
# Intel
CC=mpiicc FC=mpiifort ./configure --prefix=/projappl/project_2006422/pio-2.5.10 --enable-fortran
# GNU:
CC=mpicc FC=mpif90 ./configure --prefix=/projappl/project_2006422/pio-2.5.10 --enable-fortran
make check
make install
```
## Install ESMF library (Not for user)
- Version 8.4.1 is required (due to updates of some functionalities)
```
cd /fmi/projappl/project_2006422
wget -q https://github.com/esmf-org/esmf/archive/refs/tags/v8.4.1.tar.gz
tar zxf ESMF_8_4_1.tar.gz
cd esmf-ESMF_8_4_1
```
- ~~gfortran version~~ (not needed for ctsm, intel compiler is much faster on puhti)
-- MKL library is recomended to use. Library linker options can be found [here](https://www.intel.com/content/www/us/en/developer/tools/oneapi/onemkl-link-line-advisor.html#gs.vqrrhg)
-- Compiler options on puhti can be found [here](https://docs.csc.fi/computing/compiling-puhti/)
-- Available setting for ESMF installation can be found [here](https://earthsystemmodeling.org/docs/release/ESMF_8_2_0/ESMF_usrdoc/node10.html)
```
# This part is updated on 20.02, for GNU compiler
module purge
module load gcc/13.2.0
module load openmpi/5.0.5
module load netcdf-c/4.9.2
module load netcdf-fortran/4.6.1
module load intel-oneapi-mkl/2024.0.0
## ESMF should compiled using openmpi (ESMF_MeshCreat() needs MPI functions)
export ESMF_DIR='/projappl/project_2006422/esmf'
export ESMF_INSTALL_PREFIX='/projappl/project_2006422/esmf_gfortran_v8.8.0'
export ESMF_COMPILER="gfortran"
export ESMF_LAPACK="mkl"
export ESMF_LAPACK_LIBS="-mkl -Wl,--no-as-needed -lmkl_gf_ilp64 -lmkl_sequential -lmkl_core -lpthread -lm -ldl"
export ESMF_LAPACK_LIBPATH="${MKLROOT}/lib/intel64"
export ESMF_F90COMPILEOPTS="-w -fallow-argument-mismatch -O2" #-fdefault-integer-8 -m64 -I"${MKLROOT}/include" (no need to do this)
export ESMF_CXXCOMPILEOPTS="-w -fallow-argument-mismatch -O2"
export ESMF_NETCDF="nc-config"
export ESMF_PNETCDF="pnetcdf-config"
export ESMF_PIO="external"
export ESMF_PIO_INCLUDE="/fmi/projappl/project_2006422/pio-2.5.10/include"
export ESMF_PIO_LIBPATH="/fmi/projappl/project_2006422/pio-2.5.10/lib"
export ESMF_COMM=openmpi
# When using ESMF library or function, need to include following library:
export LD_LIBRARY_PATH=/appl/spack/v022/install-tree/gcc-13.2.0/intel-oneapi-mkl-2024.0.0-rm55cl/mkl/2024.0/lib::/appl/spack/v022/install-tree/gcc-13.2.0/openmpi-5.0.5-tpby2m/lib:/appl/spack/v022/install-tree/gcc-13.2.0/hdf5-1.14.3-infqqv/lib:/appl/spack/v022/install-tree/gcc-13.2.0/parallel-netcdf-1.12.3-swrs3u/lib:/appl/spack/v022/install-tree/gcc-13.2.0/netcdf-fortran-4.6.1-wwft3x/lib:/appl/spack/v022/install-tree/gcc-13.2.0/netcdf-c-4.9.2-6lfhys/lib:/projappl/project_2006422/esmf_gfortran_v8.8.0/lib/libO/Linux.gfortran.64.openmpi.default:/projappl/project_2006422/pio-2.5.10/lib
```
- ifort version
```
module purge
module load intel-oneapi-compilers-classic/2021.6.0
module load intel-oneapi-mpi/2021.6.0
module load intel-oneapi-mkl/2022.1.0
module load netcdf-fortran/4.5.4
module load netcdf-c/4.8.1
module load parallel-netcdf/1.12.2
export ESMF_DIR='/projappl/project_2006422/esmf'
export ESMF_INSTALL_PREFIX='/projappl/project_2006422/esmf_ifort_v8.4.1'
export ESMF_COMPILER="intel"
#export ESMF_CXX="icpc"
#export ESMF_F90="ifort"
export ESMF_LAPACK="mkl"
export ESMF_LAPACK_LIBS="-qmkl -lmkl_intel_ilp64 -lmkl_sequential -lmkl_core -lpthread -lm -ldl"
export ESMF_LAPACK_LIBPATH="${MKLROOT}/lib/intel64"
export ESMF_F90COMPILEOPTS="-I"${MKLROOT}/include"" # -i8 (no need to do this, can cause type mismatch problem)
#export ESMF_CXXCOMPILEOPTS="-w -fallow-argument-mismatch -O2"
export ESMF_NETCDF="nc-config"
export ESMF_PNETCDF="pnetcdf-config"
export ESMF_PIO="external"
export ESMF_PIO_INCLUDE="/fmi/projappl/project_2006422/pio-2.5.10/include"
export ESMF_PIO_LIBPATH="/fmi/projappl/project_2006422/pio-2.5.10/lib"
#export ESMF_NETCDF_INCLUDE=""
#export ESMF_NETCDF_LIBS=""
#export ESMF_PIO="internal"
#export ESMF_PIO_INCLUDE=""
#export ESMF_PIO_LIBPATH=""
```
```
gmake -f makefile
gmake install
#ldconfig
#gmake clean # only needed if you want to re-install
```
## Prepare input data
- All inputdata for FiON sites (haltiala, qvidja, yoni & viikki) have been kept on puhti at: /fmi/projappl/project_2006422/FiON_input
- Create surface data:
```
# Use tools/site_and_regional/subset_data
## Load python packages
source /projappl/project_2006422/python/venv/bin/activate
module load geoconda
## Download the input data to /scratch/project_2007574/hui_tmp/inputdata/lnd/clm2/surfdata_esmf/ctsm5.3.0/. All the needed input data are kept in: default_data_2000.cfg
svn export https://svn-ccsm-inputdata.cgd.ucar.edu/trunk/inputdata/lnd/clm2/surfdata_esmf/ctsm5.3.0/surfdata_0.9x1.25_hist_2000_16pfts_c240908.nc
## Run the script
./subset_data point --lat 60.70084 --lon 5.092566 --site lygra --create-surface --create-domain
./subset_data point --lat 60.70084 --lon 5.092566 --site lygra --create-surface --create-domain --overwrite
```
## Prepare .cime folder on home directory (e.g., /users/<your_user_name>/)
- You need to clone the **puhti branch**.
`git clone -b puhti https://github.com/huitang-earth/dotcime.git`
- mv **dotcime** to **.cime**
`mv dotcime .cime`
## Setting ctsm-fates experiment (test!!!)
- Dowload the codes
```
git clone https://github.com/ESCOMP/CTSM.git
# Go to CTSM folder
cd CTSM
# Creat a twinwin branch from specific tag of ctsm
git checkout -b twinwin ctsm5.1.dev122
~~# Donwload the external components
~~./manage_externals/checkout_externals~~
# version 5.3: externals are kepted in a different way
module load netcdf-fortran/4.5.4
module load parallel-netcdf/1.12.2
module load python-data/3.10-22.09
module load perl/5.34.1
module load netcdf-c/4.8.1
# different submoduls are now kept in: .gitmodules
# To load the submodules:
./bin/git-fleximod update -f -v
```
- Creat a single site test-case
```
# Go to cime/scripts folder under CTSM
cd cime/scripts
# Great a new case
# case folder can be whatever name and folder
# --compset can be different (need to explore which one is more suitable)
# --res, --machine, --project need to be the same as the example
./create_newcase --case ../../../ctsm_cases/test4 --compset I1PtClm51Fates --res CLM_USRDAT --machine puhti --run-unsupported --project project_2006422``
```
-- LYGRA site
```
# create the case
./create_newcase --case ../../../ctsm_cases/test_lygra2_gfortran --compset 2000_DATM%1PT_CLM60%FATES_SICE_SOCN_SROF_SGLC_SWAV_SESP --res CLM_USRDAT --machine puhti --run-unsupported --project project_2006422
# Setup and build the case:
./case.setup
# Modify use_nl_clm:
fsurdat = "/scratch/project_2007574/hui_tmp/FiON/lnd/clm2/surfdata_map/lygra/surfdata_lygra_hist_2000_16pfts_c250219.nc"
# Copy 'stream_definition_datm.xml' to 'CTSM/components/cdeps/datm/cime_config' replacing the default file there.
# Build the case
./case.build
# Modify env_run.xlm setting
./xmlchange --file env_run.xml --id PTS_LAT --val 60.70084
./xmlchange --file env_run.xml --id PTS_LON --val 5.092566
./xmlchange --file env_run.xml --id DATM_YR_START --val 2024
./xmlchange --file env_run.xml --id DATM_YR_END --val 2024
./xmlchange --file env_run.xml --id CLM_USRDAT_NAME --val lygra
# Run the case
## Need to add the netcdf library path (don't know why)
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/appl/spack/v022/install-tree/gcc-13.2.0/parallel-netcdf-1.12.3-swrs3u/lib:/appl/spack/v022/install-tree/gcc-13.2.0/netcdf-fortran-4.6.1-wwft3x/lib:/appl/spack/v022/install-tree/gcc-13.2.0/netcdf-c-4.9.2-6lfhys/lib
## ESMF should compiled using openmpi (ESMF_MeshCreat() needs MPI functions)
## Submit the job
./case.submit
```
- Compile the case (in case folder)
```
# Go to the case folder (where you have set for --case when creating the case)
./case.setup
```
- **surface datafile is required for building ctsm**, this can be set in **user_nl_clm**. For example, copy the following line into "user_nl_clm"
`fsurdat = "/fmi/projappl/project_2006422/FiON_input/inputdata/lnd/clm2/surfdata_map/viikki/surfdata_1x1_viikki_hist_78pfts_CMIP6_simyr1980_c230413.nc"`
- Build the case
`./case.build`
- Run the case (in case folder)
-- need to change datm stream file to have the proper input for **CLMUSRDAT**. This is done in **CTSM source code folder** "CTSM/components/cdep/datm". The files need to be changed can be found [here](https://github.com/huitang-earth/scripts_ctsm_FiON/tree/twinwin/twinwin/overwrites). You can replace the old ones with the files in this folder.
-- ~~projects have to be **project_2006442**~~
-- ~~partition needs to be **fmi** or **fmitest**~~
-- Set **PTS_LAT** and **PTS_LON** for twinwin site
```
./xmlchange --file env_run.xml --id PTS_LAT --val 60.224806904
./xmlchange --file env_run.xml --id PTS_LON --val 25.019959847
./xmlchange --file env_run.xml --id DATM_YR_START --val 2000
./xmlchange --file env_run.xml --id DATM_YR_END --val 2001
./xmlchange --file env_run.xml --id CLM_USRDAT_NAME --val viikki
```
- ~~ATM_DOMAIN_MESH does not to be set, if the above step is done~~
```
# This part is not needed.
# Creat mesh file for the site
/fmi/projappl/project_2006422/esmf_ifort_v8.4.1/bin/binO/Linux.intel.64.mpiuni.default/ESMF_Scrip2Unstruct SCRIPgrid_viikki_nomask_c230413.nc mesh_test.nc 0 ESMF
# Set ATM_DOMAIN_MESH
./xmlchange --file env_run.xml --id ATM_DOMAIN_MESH --val "/fmi/projappl/project_2006422/FiON_input/inputdata/share/scripgrids/viikki/mesh_test.nc"
```
- Submit the case
`./case.submit`
Sign in with Wallet
Connect another wallet