Merge pull request #56 from bbakernoaa/feature/modulefiles

Use spack-stack modulefiles

CMakeLists.txt

@@ -1,11 +1,21 @@
-# Check environment
 cmake_minimum_required(VERSION 3.5)
 
 # Set the project name and version
 project(NEXUS
         VERSION 0.2.0
         LANGUAGES Fortran)
 
+# FC should be used to set compiler
+if(NOT DEFINED ENV{FC})
+  message(WARNING "FC is not set. You may want to use it to specify a specific compiler.")
+else()
+  if("$ENV{FC}" STREQUAL "")
+    message(WARNING "FC is set to empty string. You may want to use it to specify a specific compiler.")
+  else()
+    message(STATUS "FC is set: '$ENV{FC}'")
+  endif()
+endif()
+
 # Include local macros
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
 

README.md

@@ -23,15 +23,19 @@ pre-commit install --install-hooks
 
 ### Setup
 
-#### NOAA Hera
+#### Supported UFS machines
 
-Use the official setup.
+UFS spack-stack module files are included for Hera, WCOSS2, Orion, etc.
 ```
 module use ./modulefiles
-module load hera.intel
 ```
 
-Input data:
+Then, on Hera, for example:
+```
+module load ufs_hera.intel
+```
+
+Hera input data:
 ```
 /scratch1/NCEPDEV/rstprod/nexus_emissions
 ```
@@ -41,6 +45,7 @@ Input data:
 
 #### GMU Hopper
 
+Custom modules.
 ```
 . /groups/ESS3/zmoon/nexus/env5
 ```
@@ -52,58 +57,10 @@ Input data:
 
 #### Ubuntu
 
-Tested with Ubuntu 22.04.
-Using GCC v12 available via `apt`.
-
-Dependencies:
-```bash
-sudo apt install build-essential gfortran-12 libnetcdf-dev libnetcdff-dev liblapack-dev libopenblas-dev mpi-default-dev mpi-default-bin
-```
-
-Build ESMF and prepare for NEXUS build:
-```bash
-v="8.3.1"  # ESMF
-gcc="12"
-esmf_base=$HOME/esmf
-
-export ESMF_DIR=${esmf_base}/${v}-gcc-${gcc}
-cd $esmf_base
-mkdir -p $ESMF_DIR
-wget https://github.com/esmf-org/esmf/archive/refs/tags/v${v}.tar.gz
-tar xzvf v${v}.tar.gz --directory=/tmp && mv /tmp/esmf-${v}/* $ESMF_DIR
-
-export ESMF_COMPILER=gfortran
-export ESMF_LAPACK=netlib
-export ESMF_COMM=mpi
-export ESMF_PIO=internal
-export ESMF_NETCDF=nc-config
-
-export OMPI_FC=gfortran-${gcc}
-export OMPI_CC=gcc-${gcc}
-export OMPI_CXX=g++-${gcc}
-export ESMF_F90COMPILER=mpifort
-export ESMF_CCOMPILER=mpicc
-export ESMF_CXXCOMPILER=mpic++
-export ESMF_F90LINKER=/usr/bin/ld
-export ESMF_CLINKER=/usr/bin/ld
-export ESMF_CXXLINKER=/usr/bin/ld
-
-cd $ESMF_DIR
-make lib
-
-# Location of the corresponding `esmf.mk` is needed for CMake to find the lib later
-# You can find these with, e.g., `find . -name 'esmf.mk' -exec realpath {} \;`,
-# but it should be the following:
-ESMFMKFILE=${ESMF_DIR}/lib/libO/Linux.gfortran.64.mpi.default/esmf.mk
-
-# For NEXUS
-export CMAKE_Fortran_COMPILER=$ESMF_F90COMPILER
-```
-
-To build NEXUS, at least these env vars should be set:
-- `ESMFMKFILE`
-- `OMPI_FC`
-- `CMAKE_Fortran_COMPILER`
+See https://github.com/noaa-oar-arl/gha-esmf for examples of how to build ESMF,
+or follow the steps to download and use a pre-built ESMF.
+
+Remember to set `ESMFMKFILE` to point to the `esmf.mk` file of your ESMF build.
 
 ### Build
 

modulefiles/ufs_common.lua

@@ -0,0 +1,21 @@
+whatis("Description: UFS build environment common libraries")
+
+help([[Load UFS Model common libraries]])
+
+local ufs_modules = {
+  {["jasper"]          = "2.0.32"},
+  {["zlib"]            = "1.2.13"},
+  {["libpng"]          = "1.6.37"},
+  {["hdf5"]            = "1.14.0"},
+  {["netcdf-c"]        = "4.9.2"},
+  {["netcdf-fortran"]  = "4.6.1"},
+  {["parallelio"]      = "2.5.10"},
+  {["esmf"]            = "8.6.0"},
+}
+
+for i = 1, #ufs_modules do
+  for name, default_version in pairs(ufs_modules[i]) do
+    local env_version_name = string.gsub(name, "-", "_") .. "_ver"
+    load(pathJoin(name, os.getenv(env_version_name) or default_version))
+  end
+end

modulefiles/ufs_derecho.intel.lua

@@ -0,0 +1,30 @@
+help([[
+loads UFS Model prerequisites for NOAA Parallelworks/Intel
+]])
+
+setenv("LMOD_TMOD_FIND_FIRST","yes")
+prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/derecho/modulefiles")
+load("ecflow/5.8.4")
+load("mysql/8.0.33")
+
+setenv("LMOD_TMOD_FIND_FIRST","yes")
+prepend_path("MODULEPATH", "/lustre/desc1/scratch/epicufsrt/contrib/modulefiles_extra")
+prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core")
+
+unload("ncarcompilers")
+stack_intel_ver=os.getenv("stack_intel_ver") or "2021.10.0"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+stack_cray_mpich_ver=os.getenv("stack-cray-mpich_ver") or "8.1.25"
+load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
+
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+load(pathJoin("cmake", cmake_ver))
+
+stack_python_ver=os.getenv("stack_python_ver") or "3.10.13"
+load(pathJoin("stack-python", stack_python_ver))
+
+setenv("CMAKE_Platform", "derecho.intel")
+load("ufs-weather-model-env")
+
+whatis("Description: UFS build environment")

modulefiles/ufs_gaea.intel.lua

@@ -0,0 +1,33 @@
+help([[
+  This module loads libraries required for building and running UFS Weather Model
+  on the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0.
+]])
+
+whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea ]===])
+
+prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core")
+
+stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.28"
+load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
+
+stack_python_ver=os.getenv("stack_python_ver") or "3.10.13"
+load(pathJoin("stack-python", stack_python_ver))
+
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+load(pathJoin("cmake", cmake_ver))
+
+load("ufs_common")
+
+nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
+load(pathJoin("nccmp", nccmp_ver))
+
+unload("darshan-runtime")
+unload("cray-libsci")
+
+setenv("CC","cc")
+setenv("CXX","CC")
+setenv("FC","ftn")
+setenv("CMAKE_Platform","gaea.intel")

modulefiles/ufs_hera.intel.lua

@@ -0,0 +1,26 @@
+help([[
+loads UFS Model prerequisites for Hera/Intel
+]])
+
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core")
+
+stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1"
+load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
+
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+load(pathJoin("cmake", cmake_ver))
+
+load("ufs_common")
+
+nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
+load(pathJoin("nccmp", nccmp_ver))
+
+setenv("CC", "mpiicc")
+setenv("CXX", "mpiicpc")
+setenv("FC", "mpiifort")
+setenv("CMAKE_Platform", "hera.intel")
+
+whatis("Description: UFS build environment")

modulefiles/ufs_hera.intelllvm.lua

@@ -0,0 +1,33 @@
+help([[
+loads UFS Model prerequisites for Hera/IntelLLVM
+]])
+
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core")
+
+stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1"
+load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
+
+load("gnu")
+load("intel/2023.2.0")
+
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+load(pathJoin("cmake", cmake_ver))
+
+load("ufs_common")
+
+nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
+load(pathJoin("nccmp", nccmp_ver))
+
+setenv("I_MPI_CC", "icx")
+setenv("I_MPI_CXX", "icpx")
+setenv("I_MPI_FC", "ifx")
+
+setenv("CC", "mpicc")
+setenv("CXX", "mpiicpc")
+setenv("FC", "mpifc")
+setenv("CMAKE_Platform", "hera.intel")
+
+whatis("Description: UFS build environment")

modulefiles/ufs_hercules.intel.lua

@@ -0,0 +1,26 @@
+help([[
+loads UFS Model prerequisites for Hercules/Intel
+]])
+
+prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core")
+
+stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0"
+load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
+
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+load(pathJoin("cmake", cmake_ver))
+
+load("ufs_common")
+
+nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
+load(pathJoin("nccmp", nccmp_ver))
+
+setenv("CC", "mpiicc")
+setenv("CXX", "mpiicpc")
+setenv("FC", "mpiifort")
+setenv("CMAKE_Platform", "hercules.intel")
+
+whatis("Description: UFS build environment")

modulefiles/ufs_hercules.intelllvm.lua

@@ -0,0 +1,30 @@
+help([[
+loads UFS Model prerequisites for Hercules/IntelLLVM
+]])
+
+prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core")
+
+stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0"
+load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
+
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+load(pathJoin("cmake", cmake_ver))
+
+load("ufs_common")
+
+nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
+load(pathJoin("nccmp", nccmp_ver))
+
+setenv("I_MPI_CC", "icx")
+setenv("I_MPI_CXX", "icpx")
+setenv("I_MPI_F90", "ifx")
+
+setenv("CC", "mpiicc")
+setenv("CXX", "mpiicpc")
+setenv("FC", "mpiifort")
+setenv("CMAKE_Platform", "hercules.intel")
+
+whatis("Description: UFS build environment")