## 1. Obvious changes

### 1.1. Add correct handling of NVHPC toolset in the compiler flags' selection code
- affected files:
- `source/ifs-source/nemo/UTIL/build/cmake/arch_configuration.cmake`
- `source/ifs-source/nemo/UTIL/build/cmake/compiler_flags.cmake`
- `source/ifs-source/nemo/UTIL/build/cmake/tools_configuration.cmake`
- symptom: wrong keys for nvfortran
- solution: replace `MATCHES "PGI"` -> `MATCHES "PGI|NVHPC"`
- **exception:** in `compiler_flags.cmake` we try to include `compiler_flags_PGI_CXX.cmake` but we do not have this file in `ifs-source/nemo/UTIL/build/cmake`, so we temporarely avoid this

--- ifs-bundle-feature.mn5-d797d63.BAK/source/ifs-source/nemo/UTIL/build/cmake/arch_configuration.cmake 2024-05-23 12:26:51.107947564 +0200
+++ ifs-bundle-feature.mn5-d797d63/source/ifs-source/nemo/UTIL/build/cmake/arch_configuration.cmake     2024-08-21 17:31:53.454886095 +0200
@@ -14,7 +14,7 @@
   include( arch_keys_Intel )
 elseif( CMAKE_Fortran_COMPILER_ID MATCHES "Cray" )
   include( arch_keys_Cray )
-elseif( CMAKE_Fortran_COMPILER_ID MATCHES "PGI" )
+elseif( CMAKE_Fortran_COMPILER_ID MATCHES "PGI|NVHPC" )
   include( arch_keys_PGI )
 else()
   message( STATUS "Compiler not supported")
--- ifs-bundle-feature.mn5-d797d63.BAK/source/ifs-source/nemo/UTIL/build/cmake/compiler_flags.cmake     2024-05-23 12:26:51.107947564 +0200
+++ ifs-bundle-feature.mn5-d797d63/source/ifs-source/nemo/UTIL/build/cmake/compiler_flags.cmake 2024-08-21 17:32:53.119303307 +0200
@@ -23,7 +23,7 @@
   include( compiler_flags_XL_Fortran )
 elseif( CMAKE_Fortran_COMPILER_ID MATCHES "Cray" )
   include( compiler_flags_Cray_Fortran )
-elseif( CMAKE_Fortran_COMPILER_ID MATCHES "PGI" )
+elseif( CMAKE_Fortran_COMPILER_ID MATCHES "PGI|NVHPC" )
   include( compiler_flags_PGI_Fortran )
 else()
   message( STATUS "Fortran compiler with ID ${CMAKE_Fortran_COMPILER_ID} will be used with CMake default options")
@@ -49,7 +49,7 @@
   include( compiler_flags_Cray_CXX )
 elseif( CMAKE_CXX_COMPILER_ID MATCHES "Clang" )
   include( compiler_flags_Clang_CXX )
-elseif( CMAKE_CXX_COMPILER_ID MATCHES "PGI" )
+elseif( CMAKE_CXX_COMPILER_ID MATCHES "PGI"  )
   include( compiler_flags_PGI_CXX )
 else()
   message( STATUS "C++ compiler with ID ${CMAKE_CXX_COMPILER_ID} will be used with CMake default options")
--- ifs-bundle-feature.mn5-d797d63.BAK/source/ifs-source/nemo/UTIL/build/cmake/tools_configuration.cmake        2024-05-23 12:26:51.107947564 +0200
+++ ifs-bundle-feature.mn5-d797d63/source/ifs-source/nemo/UTIL/build/cmake/tools_configuration.cmake    2024-08-21 17:34:44.144283004 +0200
@@ -14,7 +14,7 @@
   include( tools_keys_Intel )
 elseif( CMAKE_Fortran_COMPILER_ID MATCHES "Cray" )
   include( tools_keys_Cray )
-elseif( CMAKE_Fortran_COMPILER_ID MATCHES "PGI" )
+elseif( CMAKE_Fortran_COMPILER_ID MATCHES "PGI|NVHPC" )
   include( tools_keys_PGI )
 else()
   message( STATUS "Compiler not supported")   

---

### 1.2. Linking stage issues: missing libs
- affected files:
- `source/ifs-source/nemo/NEMOGCM/CMakeLists.txt`
- solution: add `${NETCDF_LIBRARIES}` `${DRHOOK_LIBRARIES}` to the `LIBS` section in `ecbuild_add_executable`

--- ifs-bundle-feature.mn5-d797d63.BAK/source/ifs-source/nemo/NEMOGCM/CMakeLists.txt    2024-05-23 12:26:49.803932185 +0200
+++ ifs-bundle-feature.mn5-d797d63/source/ifs-source/nemo/NEMOGCM/CMakeLists.txt        2024-08-21 17:36:04.981678000 +0200
@@ -169,6 +169,8 @@
                           NEMO/OPA_SRC/nemo.f90
                         LIBS
                           nemo_${NEMO_VERSION}.${PREC}
+                         ${NETCDF_LIBRARIES} 
+                         ${DRHOOK_LIBRARIES}
                         DEFINITIONS
                           ${BUILD_DEFINITIONS_NEMOGCM_${NEMO_VERSION}}
                           ${BUILD_DEFINITIONS_GRID}

---

### 1.3. Correct some compilation flags
- affected files:
- `source/ifs-source/nemo/tools/CMakeLists.txt`
- solution: add `PUBLIC_INCLUDES $<BUILD_INTERFACE:${NEMO_INCLUDE_DIRS}>` to `ecbuild_add_library` for `tools_matchup`

--- ifs-bundle-feature.mn5-d797d63.BAK/source/ifs-source/nemo/tools/CMakeLists.txt      2024-05-23 12:26:51.139947942 +0200
+++ ifs-bundle-feature.mn5-d797d63/source/ifs-source/nemo/tools/CMakeLists.txt  2024-08-21 17:37:13.196376842 +0200
@@ -321,6 +321,8 @@
                         ${NEMO_INCLUDE_DIRS}
                         ${MPI_Fortran_INCLUDE_PATH}
                         ${NETCDF_INCLUDE_DIRS}
+                     PUBLIC_INCLUDES 
+                       $<BUILD_INTERFACE:${NEMO_INCLUDE_DIRS}>                 
                      DEFINITIONS
                         ${BUILD_DEFINITIONS_TOOLS}
                   )

---

### 1.4. Add correct handling of NVHPC toolset in the compiler flags' selection code
- affected files:
- `source/ifs-source/nemo/CMakeLists.txt`
- solution: replace `MATCHES "PGI"` -> `MATCHES "PGI|NVHPC"` in the conditional below `include(compiler_flags)`

--- ifs-bundle-feature.mn5-d797d63.BAK/source/ifs-source/nemo/CMakeLists.txt    2024-05-23 12:26:49.743931477 +0200
+++ ifs-bundle-feature.mn5-d797d63/source/ifs-source/nemo/CMakeLists.txt        2024-08-21 17:37:52.822785922 +0200
@@ -263,7 +263,7 @@

 # Set compiler flags
 include( compiler_flags )
-if( CMAKE_Fortran_COMPILER_ID MATCHES "PGI" )
+if( CMAKE_Fortran_COMPILER_ID MATCHES "PGI|NVHPC" )
   include_directories( ${MPI_Fortran_INCLUDE_PATH} ${NETCDF_INCLUDE_DIRS} )
 endif()

---

### 1.5. Architecture files
- affected files:
- `arch/eurohpc/mn5-acc/env.sh`
- `arch/eurohpc/mn5-acc/toolchain.cmake`
- syptom: we need some basic architecture-dependent files (to select of modulles, compilers etc.) for MN5 Accelerated partition
- solution: added new files to support the MN5 Accelerated partition system software and libs
- comment: those files were created by Operations Dept. of BSC, we didn't really took care about the contents. As a memo note, we could say that `nvidia-hpc-sdk/24.5` seems to be also available

--- /dev/null   2024-08-07 11:18:28.953000000 +0200
+++ ifs-bundle-feature.mn5-d797d63/arch/eurohpc/mn5-acc/toolchain.cmake 2024-07-23 08:34:23.000000000 +0200
@@ -0,0 +1,2 @@
+set( IFS_Fortran_FLAGS " ${IFS_Fortran_FLAGS}" )
+set( CMAKE_CXX_STANDARD 11 )
--- /dev/null   2024-08-07 11:18:28.953000000 +0200
+++ ifs-bundle-feature.mn5-d797d63/arch/eurohpc/mn5-acc/env.sh  2024-06-19 13:31:30.000000000 +0200
@@ -0,0 +1,44 @@
+# Source me to get the correct configure/build/run environment
+
+# Store tracing and disable (module is *way* too verbose)
+{ tracing_=${-//[^x]/}; set +x; } 2>/dev/null
+
+module_load() {
+  echo "+ module load $*"
+  module load $*
+}
+module_unload() {
+  echo "+ module unload $*"
+  module unload $*
+}
+module_purge() {
+  echo "+ module purge"
+  module purge
+}
+
+module load nvidia-hpc-sdk/24.3 cmake/3.29.2 mkl/2024.0 hdf5/1.14.1-2-nvidia-nvhpcx fftw/3.3.10-gcc-nvhpcx pnetcdf/1.12.3-nvidia-nvhpcx netcdf/c-4.9.2_fortran-4.6.1_cxx4-4.3.1-nvidia-nvhpcx aec/1.1.2-gcc python/3.12.1-gcc eccodes/2.34.1-gcc
+
+export TBBROOT=/apps/GPP/ONEAPI/2024.0.1/tbb/latest/
+export TBBMALLOC_DIR=${TBBROOT}/lib/
+export LD_LIBRARY_PATH=$TBBROOT/lib/:$LD_LIBRARY_PATH
+export aec_ROOT=$AEC_DIR
+export AEC_ROOT=$AEC_DIR
+
+#export EIGEN_DIR=/apps/EIGEN/3.3.9/INTEL
+#export eigen_DIR=/apps/EIGEN/3.3.9/INTEL
+#export Eigen3_DIR=/apps/EIGEN/3.3.9/INTEL/
+
+export FC=nvfortran
+export F90=nvfortran
+export F77=nvfortran
+export CC=nvc
+export CXX=nvc++
+
+export I_MPI_SHM=spr_avx512
+export I_MPI_PLATFORM=spr
+
+export MPIFC=mpif90
+export MPIF90=mpif90
+export MPIF77=mpif90
+export MPICC=mpicc
+export MPICXX=mpicxx

---

## 2. Discussion points

### 2.1. Roll back to `-O1` for some of the NEMOQC sources due to endless compilation process of `NemoQcMod_OceanSound/Ops_OceanXBTCorrect.f90` (nvfortran bug?)
- affected files:
- `source/ifs-source/nemo/UTIL/build/cmake/NemoQc_configuration.cmake`
- symptom: compilation of this file takes forever; switching to `-O1` for this file only doesn't work: compiler says that with `-Mvect` we automatically turn on `-O3`. Adding `-Mnovect` doesn't help
- solution: replace the compilation flags: `-O3` -> `-O1` in NVHPC case for the whole `NEMOQC` project

--- ifs-bundle-feature.mn5-d797d63.BAK/source/ifs-source/nemo/UTIL/build/cmake/NemoQc_configuration.cmake       2024-05-23 12:26:51.107947564 +0200
+++ ifs-bundle-feature.mn5-d797d63/source/ifs-source/nemo/UTIL/build/cmake/NemoQc_configuration.cmake   2024-08-21 13:17:58.733840637 +0200
@@ -54,6 +54,15 @@
   set( CMAKE_Fortran_FLAGS_DEBUG "-O0" )
   set( CMAKE_Fortran_FLAGS_BIT   "-O3" )

+elseif( CMAKE_Fortran_COMPILER_ID MATCHES "NVHPC" )
+  set( CMAKE_Fortran_FLAGS "-g -i8 -r8" )
+  set( CMAKE_Fortran_FLAGS_DEBUG "-O0" )
+  set( CMAKE_Fortran_FLAGS_BIT   "-O1" )
+  set_source_files_properties(
+      NemoQcMod_OceanSound/Ops_OceanXBTCorrect.f90
+      PROPERTIES OVERRIDE_COMPILE_FLAGS "-Mnovect -O1"
+  )
+
 else()
   message( ERROR "${CMAKE_Fortran_COMPILER_ID} compiler not supported" )
 endif()

---

### 2.2. Deactivate the workaround for a nvfortran bug in the `sugfl1.F90` (is it general for NVHPC?)
- affected files:
- `source/ifs-source/arpifs/setup/sugfl1.F90` and `ifs_sp/arpifs/setup/sugfl1.F90`
- symptom: seems the `ifdef __PGI` blocks are activated, but this leads to syntax errors
- solution: we replace `#if defined(__PGI)` with `#if 0` to turn off the compiler issue workaround

--- ifs-bundle-feature.mn5-d797d63.BAK/source/ifs-source/arpifs/setup/sugfl1.F90        2024-05-23 12:26:49.619930015 +0200
+++ ifs-bundle-feature.mn5-d797d63/source/ifs-source/arpifs/setup/sugfl1.F90    2024-08-21 12:30:51.181568089 +0200
@@ -67,7 +67,11 @@
 USE YOMANEB     , ONLY : NGRBCHEM, NGRBGHGASSIM
 USE YOE_AERODIAG, ONLY : TYPE_AERO_WVL_DIAG
 USE ARPCLIM_CHEM_MODULE
-#if defined(__PGI)
+
+
+
+!!#if defined(__PGI)
+#if 0
 USE PAR_GFL
 USE YOE_AERODIAG, ONLY : NPAERAOT, NPAERLISI_VAR, NPAERLISI_WVL, NPAERLISI
 #endif
@@ -114,7 +118,10 @@

 !     ------------------------------------------------------------------

-#if defined(__PGI)
+
+
+!!#if defined(__PGI)
+#if 0
 !PGI compiler bug workaround. Namelist members cannot be pointers to derived types because the read fails or segfaults
 TYPE(TYPE_GFL_NAML) :: &
  & YGHG_NL(JPGHG), YCHEM_NL(JPCHEM), YAERO_NL(JPAERO), YERA40_NL(JPERA40), &
@@ -131,10 +138,13 @@
  & YFORC_NL(:), YEZDIAG_NL(:), YEXT_NL(:), YLIMA_NL(:), YEDRP_NL(:)
 #endif

+
+
 ! If you just wrap the #if around the first line of the statement to avoid duplicating
 ! the list of variables, then "git ifsnorms" bails out entirely when trying to
 ! parse any branch touching this file. See IGT-133.
-#if defined(__PGI)
+!!#if defined(__PGI)
#if 0
 TYPE(TYPE_GFL_NAML) :: &
  & YQ_NL, YI_NL, YL_NL, &
  & YLCONV_NL, YICONV_NL, YRCONV_NL, YSCONV_NL, YIRAD_NL, YLRAD_NL, &
@@ -156,7 +166,11 @@
  & YEFB1_NL, YEFB2_NL, YEFB3_NL, YPHYCTY_NL, YFSD_NL
 #endif

-#if defined(__PGI)
+
+
+
+!!#if defined(__PGI)
+#if 0
 TYPE(TYPE_AERO_WVL_DIAG) :: YAERO_WVL_DIAG_NL(20)
 #else
 TYPE(TYPE_AERO_WVL_DIAG), POINTER :: YAERO_WVL_DIAG_NL(:)
@@ -258,8 +272,12 @@
 ATOL_ASIS => YDCHEM%ATOL_ASIS
 N_F_ATOL_ASIS => YDCHEM%N_F_ATOL_ASIS

+
+
+
 ! Associate pointers for variables in namelist NAMGFL
-#if defined(__PGI)
+!!#if defined(__PGI)
+#if 0
 #else
 YQ_NL          => YGFL%YQ_NL
 YI_NL          => YGFL%YI_NL
@@ -438,7 +456,9 @@
 !*       2. READ NAMELIST.
 !           --------------

-#if defined(__PGI)
+
+!!#if defined(__PGI)
+#if 0
 !-- Set local variables to defaults
 YQ_NL          = YGFL%YQ_NL
 YI_NL          = YGFL%YI_NL
@@ -503,7 +523,10 @@
 #endif
 CALL POSNAM(NULNAM,'NAMGFL')
 READ(NULNAM,NAMGFL)
-#if defined(__PGI)
+
+
+!!#if defined(__PGI)
+#if 0
 !-- A temporary PGI bug fix : namelist read fails in PGI 17.1 => hardcoding (see fort.4 &NAMGFL-namelist)
 YGFL%YQ_NL          = YQ_NL
 YGFL%YI_NL          = YI_NL

---

### 2.3. Replace `ISNAN()` calls (better standard complience?)
- affected files:
- `source/ifs-source/nemo/NEMOGCM_V40/src/OCE/OBS/obsinter_h2d.h90`
- `source/ifs-source/nemo/NEMOGCM_V40/src/OCE/OBS/obs_inter_h2d.F90`
- `source/ifs_sp/nemo/NEMOGCM_V40/src/OCE/stpctl.F90`
- symptom: `ISNAN()` is not recognized by nvfortran compiler (non-standard?)
- solution: replace with `IEEE_IS_NAN()` and `USE ieee_arithmetic`

--- ifs-bundle-feature.mn5-d797d63.BAK/source/ifs-source/nemo/NEMOGCM_V40/src/OCE/OBS/obs_inter_h2d.F90 2024-05-23 12:26:50.987946149 +0200
+++ ifs-bundle-feature.mn5-d797d63/source/ifs-source/nemo/NEMOGCM_V40/src/OCE/OBS/obs_inter_h2d.F90     2024-08-21 20:06:26.828378875 +0200
@@ -26,6 +26,7 @@
    USE in_out_manager
    USE obs_const, ONLY : &
       & obfillflt              ! Fillvalue
+   USE ieee_arithmetic
    USE obs_utils           ! Utility functions
       USE lib_mpp,ONLY : &
       & ctl_warn, ctl_stop, mpprank
--- ifs-bundle-feature.mn5-d797d63.BAK/source/ifs-source/nemo/NEMOGCM_V40/src/OCE/OBS/obsinter_h2d.h90  2024-05-23 12:26:50.987946149 +0200
+++ ifs-bundle-feature.mn5-d797d63/source/ifs-source/nemo/NEMOGCM_V40/src/OCE/OBS/obsinter_h2d.h90      2024-08-21 20:04:53.152509527 +0200
@@ -315,12 +315,12 @@
             pweig(1,2,jk) = z2dmp(jk)
             pweig(2,1,jk) = z2dpm(jk)
             pweig(2,2,jk) = z2dpp(jk)
-            IF (ISNAN(z2dpm(jk))) THEN
+            IF (IEEE_IS_NAN(z2dpm(jk))) THEN
                WRITE(mpprank+1000,*) "NaNs in z2dpm(jk) OUTSIDE IF at", jk
                CALL FLUSH(mpprank+1000)
                CALL ctl_stop( 'obs_int_h2d_init: NaNs encountered in z2dpm' )
             ENDIF
-            IF (ISNAN(z2dpp(jk))) THEN
+            IF (IEEE_IS_NAN(z2dpp(jk))) THEN
                WRITE(mpprank+1000,*) "NaNs in z2dpp(jk) OUTSIDE IF at", jk
                CALL FLUSH(mpprank+1000)
                CALL ctl_stop( 'obs_int_h2d_init: NaNs encountered in z2dpp' )
@@ -358,12 +358,12 @@
                   pweig(2,1,jk) = z2dpmt(jk)
                   pweig(2,2,jk) = z2dppt(jk)

-                  IF (ISNAN(z2dpmt(jk))) THEN
+                  IF (IEEE_IS_NAN(z2dpmt(jk))) THEN
                      WRITE(mpprank+1000,*) "NaNs in z2dpmt(jk) INSIDE IF at", jk
                      CALL FLUSH(mpprank+1000)
                      CALL ctl_stop( 'obs_int_h2d_init: NaNs encountered in z2dpmt' )
                   ENDIF
-                  IF (ISNAN(z2dppt(jk))) THEN
+                  IF (IEEE_IS_NAN(z2dppt(jk))) THEN
                      WRITE(mpprank+1000,*) "NaNs in z2dppt(jk) INSIDE IF at", jk
                      CALL FLUSH(mpprank+1000)
                      CALL ctl_stop( 'obs_int_h2d_init: NaNs encountered in z2dppt' )
--- ifs-bundle-feature.mn5-d797d63.BAK/source/ifs_sp/nemo/NEMOGCM_V40/src/OCE/stpctl.F90        2024-05-23 12:26:50.995946243 +0200
+++ ifs-bundle-feature.mn5-d797d63/source/ifs_sp/nemo/NEMOGCM_V40/src/OCE/stpctl.F90    2024-08-21 20:34:03.323320170 +0200
@@ -28,6 +28,7 @@
    USE timing
    USE netcdf          ! NetCDF library
    USE lib_fortran     ! For glob_sum for nemo.stat
+   USE ieee_arithmetic
    IMPLICIT NONE
    PRIVATE

@@ -176,7 +177,7 @@
          &  zmax(3) >=   0._wp .OR.   &                    ! negative or zero sea surface salinity
          &  zmax(4) >= 100._wp .OR.   &                    ! too large sea surface salinity ( > 100 )
          &  zmax(4) <    0._wp .OR.   &                    ! too large sea surface salinity (keep this line for sea-ice)
-         &  ISNAN( zmax(1) + zmax(2) + zmax(3) ) .OR.  &   ! NaN encounter in the tests
+         &  IEEE_IS_NAN( zmax(1) + zmax(2) + zmax(3) ) .OR.  &   ! NaN encounter in the tests
          &  ABS(   zmax(1) + zmax(2) + zmax(3) ) > HUGE(1._wp) ) THEN    ! Infinity encounter in the tests
          IF( ll_colruns ) THEN
             ! first: close the netcdf file, so we can read it

---

### 2.4. Compiler flags check (nvfortran or cmake bug?)
- affected files:
- `source/multio/cmake/fortran_compile_flags.cmake`
- symptom: for some unknown reason, cmake reports failure while checking ability of nvfortran to take `-Mfree` and `-Mextend`
- solution: added `NO_FAIL` to fortran flags selection procedure

--- ifs-bundle-feature.mn5-d797d63.BAK/source/multio/cmake/fortran_compile_flags.cmake  2024-05-23 12:26:37.351785327 +0200
+++ ifs-bundle-feature.mn5-d797d63/source/multio/cmake/fortran_compile_flags.cmake      2024-08-21 17:38:59.755207615 +0200
@@ -62,7 +62,7 @@

 foreach( flag    fpe initsnan checkbounds convert linelength nofma )
   if( ${flag}_flags )
-    ecbuild_add_fortran_flags( "${${flag}_flags}" NAME ${flag} )
+         ecbuild_add_fortran_flags( "${${flag}_flags}" NAME ${flag} NO_FAIL)
   endif()
 endforeach()