diff --git a/guide.sh b/guide.sh old mode 100755 new mode 100644 index 2cbcbdb..9c9d577 --- a/guide.sh +++ b/guide.sh @@ -1,4 +1,5 @@ #!/bin/bash + # Color definitions BBlack='\033[1;30m' BRed='\033[1;31m' @@ -21,6 +22,7 @@ Color_Off='\033[0m' set -e CONFIG_FILE="$HOME/.host_data_path.conf" +TETHYS_SCRIPT="./viewOnTethys.sh" echo -e "\n=========================================================" echo -e "${UWhite} Welcome to CIROH-UA:NextGen National Water Model App! ${Color_Off}" @@ -46,6 +48,12 @@ else read -erp "Enter your input data directory path (use absolute path): " HOST_DATA_PATH fi +# Check the directory exists +if [ ! -d "$HOST_DATA_PATH" ]; then + echo -e "${BRed}Directory does not exist. Exiting the program.${Color_Off}" + exit 0 +fi + # Save the new path to the config file echo "$HOST_DATA_PATH" > "$CONFIG_FILE" echo -e "The Directory you've given is:\n$HOST_DATA_PATH\n" @@ -56,7 +64,7 @@ validate_directory() { local color=$3 if [ -d "$dir" ]; then - local count=$(ls "$dir" | wc -l) + local count=$(find $dir -type f | uniq | wc -l) echo -e "${color}${name}${Color_Off} exists. $count ${name} found." else echo -e "Error: Directory $dir does not exist." @@ -74,7 +82,7 @@ cleanup_folder() { local folder_name="$3" # Construct the find command - local find_cmd="find \"$folder_path\" -maxdepth 1 -type f \( $file_types \)" + local find_cmd="find \"$folder_path\" -maxdepth 2 -type f \( $file_types \)" # Execute the find command and count the results local file_count=$(eval "$find_cmd" 2> /dev/null | wc -l) @@ -98,7 +106,7 @@ choose_option() { echo "Cleaning folder for fresh run" # Construct the find delete command - local find_delete_cmd="find \"$folder_path\" -maxdepth 1 -type f \( $file_types \) -delete" + local find_delete_cmd="find \"$folder_path\" -maxdepth 2 -type f \( $file_types \) -delete" # Execute the find delete command eval "$find_delete_cmd" @@ -122,8 +130,9 @@ choose_option() { # Cleanup Process for Outputs Folder cleanup_folder "$HOST_DATA_PATH/outputs/" "-name '*' " "Outputs" -# Cleanup Process for ngen/data Folder -cleanup_folder "$HOST_DATA_PATH/" "-name '*.parquet' -o -name '*.csv' -o -name '*.cn'" "ngen/data" +# Cleanup Process for restarts Folder +cleanup_folder "$HOST_DATA_PATH/restarts/" "-name '*' " "Restarts" + # File discovery @@ -131,16 +140,16 @@ echo -e "\nLooking in the provided directory gives us:" find_files() { local path=$1 local name=$2 - local color=$3 + local regex=$3 + local color=$4 - local files=$(find "$path" -iname "*$name*.*") + local files=$(find "$path" -iname "$regex") echo -e "${color}Found these $name files:${Color_Off}" echo "$files" || echo "No $name files found." } -find_files "$HOST_DATA_PATH" "datastream" "$UGreen" -find_files "$HOST_DATA_PATH" "datastream" "$UGreen" -find_files "$HOST_DATA_PATH" "realization" "$UGreen" +find_files "$HOST_DATA_PATH" "hydrofabric" "*.gpkg" "$UGreen" +find_files "$HOST_DATA_PATH" "realization" "realization.json" "$UGreen" # Detect Arch and Singularity echo -e "\nDetected ISA = $(uname -a)" @@ -157,41 +166,50 @@ if uname -a | grep arm64 || uname -a | grep aarch64 ; then IMAGE_NAME=ciroh-ngen-singularity_latest.sif else ARCH=amd64 - IMAGE_URL=library://trupeshkumarpatel/awiciroh/ciroh-ngen-singularity:latest_x86 + IMAGE_URL=library://awiciroh/ngiab/ciroh-ngen-singularity:latest IMAGE_NAME=ciroh-ngen-singularity_latest.sif fi - # Model run options echo -e "${UYellow}Select an option (type a number): ${Color_Off}" -options=("Run NextGen Model using local singularity image" "Run Nextgen using remote singularity image" "Exit") +options=("Run NextGen using existing local singularity image" "Run NextGen after updating to latest singularity image" "Exit") select option in "${options[@]}"; do case $option in - "Run NextGen Model using local singularity image") + "Run NextGen using existing local singularity image") echo "running the model" break ;; - "Run Nextgen using remote singularity image") + "Run NextGen after updating to latest singularity image") echo "pulling container and running the model" - singularity pull --arch $ARCH $IMAGE_NAME $IMAGE_URL + singularity pull -F --arch $ARCH $IMAGE_NAME $IMAGE_URL break ;; Exit) echo "Have a nice day!" exit 0 ;; - *) echo "Invalid option $REPLY, 1 to continue and 2 to exit" + *) echo "Invalid option $REPLY, 1 to continue with existing local image, 2 to update and run, and 3 to exit" ;; esac done + echo -e "\nRunning NextGen singularity container..." echo -e "Mounting local host directory $HOST_DATA_PATH to /ngen/ngen/data within the container." singularity run --bind $HOST_DATA_PATH:/ngen/ngen/data $IMAGE_NAME /ngen/ngen/data # Final output count -Final_Outputs_Count=$(ls "$HOST_DATA_PATH/outputs" | wc -l) +Final_Outputs_Count=$(ls "$HOST_DATA_PATH/outputs/*/*" | wc -l) echo -e "$Final_Outputs_Count new outputs created." echo -e "Any copied files can be found here: $HOST_DATA_PATH/outputs" + +# visualize with Tethys +if [ $Final_Outputs_Count -gt 0 ]; then + ARG1="$HOST_DATA_PATH" + if ! "$TETHYS_SCRIPT" "$ARG1"; then + printf "Failed to visualize outputs in Tethys:" + fi +fi + echo -e "Thank you for running NextGen In A Box: National Water Model! Have a nice day!" exit 0 diff --git a/singularity/singularity_ngen.def b/singularity/singularity_ngen.def index a41bab3..47ad5fa 100644 --- a/singularity/singularity_ngen.def +++ b/singularity/singularity_ngen.def @@ -23,6 +23,8 @@ From: rockylinux:9.2 dnf install libstdc++ libstdc++-devel glibc glibc-devel libgfortran openmpi openmpi-devel -y dnf --enablerepo=devel install texinfo -y dnf install netcdf netcdf-devel netcdf-fortran netcdf-fortran-devel netcdf-openmpi netcdf-openmpi-devel netcdf-fortran-openmpi netcdf-fortran-openmpi-devel -y + + # Boost installation curl -L -o boost_1_79_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.79.0/boost_1_79_0.tar.bz2/download tar -xjf boost_1_79_0.tar.bz2 -C /tmp/ cp -r /tmp/boost_1_79_0/boost /usr/include/ @@ -34,6 +36,8 @@ From: rockylinux:9.2 /tmp/t-route/install_t_route.sh /tmp/ngen/install_ngen.sh + cp -R /usr/share/modulefiles/mpi /usr/share/lmod/lmod/modulefiles/Core/ + rm -rf /tmp/ngen /tmp/t-route /tmp/netcdf /tmp/extern /tmp/guide %runscript diff --git a/singularity/templates/extern/install_extern_libraries.sh b/singularity/templates/extern/install_extern_libraries.sh index c4ff296..2377a36 100644 --- a/singularity/templates/extern/install_extern_libraries.sh +++ b/singularity/templates/extern/install_extern_libraries.sh @@ -27,7 +27,7 @@ echo "-- Now Installing External Library noah-owp-modular ..." echo "-----------------------------------------------------------" cd /ngen/extern/noah-owp-modular # cmake -B cmake_build -S . -cmake -DnetCDF_MOD_PATH=/usr/include/openmpi-x86_64/ -B cmake_build -S . +cmake -DNETCDF_MODULE_DIR=/usr/lib64/gfortran/modules/openmpi/ -B cmake_build -S . cmake --build cmake_build --target surfacebmi -- -j 2 echo "===========================================================" @@ -87,7 +87,9 @@ cp /tmp/guide/HelloNGEN.sh /ngen/HelloNGEN.sh chown -R root /dmod /ngen /root chmod a+x /dmod/bin/* /ngen/HelloNGEN.sh /root find /ngen/extern/ -name "*.so*" -exec ln -s "{}" /dmod/shared_libs/ \; - +find /ngen/extern/noah-owp-modular -type f -iname "*.TBL" -exec cp '{}' /dmod/datasets/static \; ls -ahl /dmod/shared_libs echo "===========================================================" +ls -ahl /dmod/datasets/static +echo "===========================================================" echo "===========================================================" \ No newline at end of file diff --git a/singularity/templates/guide/HelloNGEN.sh b/singularity/templates/guide/HelloNGEN.sh index 0b06615..1a3dae7 100644 --- a/singularity/templates/guide/HelloNGEN.sh +++ b/singularity/templates/guide/HelloNGEN.sh @@ -9,7 +9,7 @@ CYAN='\e[36m' RESET='\e[0m' # Increasing `ulimit` to Open files -ulimit -n 1000000 +ulimit -n 500000 # Loading Lmod source /etc/profile.d/modules.sh @@ -24,6 +24,8 @@ echo -e "${CYAN}Working directory is:${RESET}" pwd echo -e "\n" + + # Function to automatically select file if only one is found auto_select_file() { local files=($1) @@ -35,8 +37,8 @@ auto_select_file() { } # Finding files -HYDRO_FABRIC_CATCHMENTS=$(find . -name "*datastream*.gpkg") -HYDRO_FABRIC_NEXUS=$(find . -name "*datastream*.gpkg") +HYDRO_FABRIC_CATCHMENTS=$(find . -name "*.gpkg") +HYDRO_FABRIC_NEXUS=$(find . -name "*.gpkg") NGEN_REALIZATIONS=$(find . -name "*realization*.json") # Auto-selecting files if only one is found @@ -78,8 +80,7 @@ if [ "$2" == "auto" ] echo "Run completed successfully, exiting, have a nice day!" exit 0 else - echo "Entering Interactive Mode" - continue + echo "Entering Interactive Mode" fi echo -e "${YELLOW}Select an option (type a number): ${RESET}" @@ -91,12 +92,15 @@ select option in "${options[@]}"; do n1=${selected_catchment:-$(read -p "Enter the hydrofabric catchment file path: " n1; echo "$n1")} n2=${selected_nexus:-$(read -p "Enter the hydrofabric nexus file path: " n2; echo "$n2")} n3=${selected_realization:-$(read -p "Enter the Realization file path: " n3; echo "$n3")} - + echo -e "${GREEN}Selected files:\nCatchment: $n1\nNexus: $n2\nRealization: $n3${RESET}\n" if [ "$option" == "Run NextGen model framework in parallel mode" ]; then procs=$(nproc) - procs=2 # Temporary fixed value + num_catchments=$(find forcings -name *.csv | wc -l) + if [ $num_catchments -lt $procs ]; then + procs=$num_catchments + fi generate_partition "$n1" "$n2" "$procs" run_command="mpirun -n $procs /dmod/bin/ngen-parallel $n1 all $n2 all $n3 $(pwd)/partitions_$procs.json" else @@ -104,7 +108,6 @@ select option in "${options[@]}"; do fi echo -e "${YELLOW}Your NGEN run command is $run_command${RESET}" - sleep 3 break ;; "Run Bash shell") @@ -114,7 +117,7 @@ select option in "${options[@]}"; do "Exit") exit 0 ;; - *) + *) echo -e "${RED}Invalid option $REPLY${RESET}" ;; esac @@ -159,9 +162,9 @@ select option in "${options[@]}"; do echo -e "${GREEN}Have a nice day.${RESET}" break ;; - *) + *) echo -e "${RED}Invalid option $REPLY${RESET}" ;; esac done -exit \ No newline at end of file +exit diff --git a/singularity/templates/ngen/install_ngen.sh b/singularity/templates/ngen/install_ngen.sh index bbbce8e..e58cab4 100755 --- a/singularity/templates/ngen/install_ngen.sh +++ b/singularity/templates/ngen/install_ngen.sh @@ -46,7 +46,9 @@ echo "-- Now Building NGen at /ngen/serialbuild ..." echo "-----------------------------------------------------------" cd /ngen # cmake -B /ngen/mpibuild -S . -DNGEN_WITH_MPI=ON -DNGEN_WITH_PYTHON=ON -DNGEN_WITH_ROUTING=ON -DNGEN_WITH_NETCDF=ON -DNGEN_QUIET=ON -cmake -B $current_path/ngen/serialbuild -S . -DNGEN_WITH_MPI:BOOL=OFF -DNGEN_WITH_NETCDF:BOOL=ON -DNGEN_WITH_SQLITE:BOOL=ON -DNGEN_WITH_UDUNITS:BOOL=ON -DNGEN_WITH_BMI_FORTRAN:BOOL=ON -DNGEN_WITH_BMI_C:BOOL=ON -DNGEN_WITH_PYTHON:BOOL=ON -DNGEN_WITH_ROUTING:BOOL=ON -DNGEN_WITH_TESTS:BOOL=ON -DNGEN_QUIET:BOOL=ON -DNETCDF_CXX_INCLUDE_DIR=/usr/local/include -DNETCDF_CXX_LIBRARY=/usr/local/lib64/libnetcdf-cxx4.so # -DNETCDF_INCLUDE_DIR=/usr/include -DNETCDF_LIBRARY=/usr/lib64/libnetcdf.so +cmake -B $current_path/ngen/serialbuild -S . -DCMAKE_INSTALL_PREFIX=/ngen/ngen -DNGEN_WITH_EXTERN_ALL=ON -DNGEN_WITH_MPI:BOOL=OFF -DNGEN_WITH_NETCDF:BOOL=ON -DNGEN_WITH_SQLITE:BOOL=ON -DNGEN_WITH_UDUNITS:BOOL=ON -DNGEN_QUIET:BOOL=ON -DNGEN_WITH_BMI_FORTRAN:BOOL=ON -DNGEN_WITH_BMI_C:BOOL=ON -DNGEN_WITH_PYTHON:BOOL=ON -DNGEN_WITH_ROUTING:BOOL=ON -DNGEN_WITH_TESTS:BOOL=ON -DNETCDF_CXX_INCLUDE_DIR=/usr/local/include -DNETCDF_CXX_LIBRARY=/usr/local/lib64/libnetcdf-cxx4.so +# -DNETCDF_INCLUDE_DIR=/usr/include -DNETCDF_LIBRARY=/usr/lib64/libnetcdf.so + echo "===========================================================" echo "===========================================================" @@ -65,7 +67,7 @@ echo "-- Now Building NGen at /ngen/parallelbuild ..." echo "-----------------------------------------------------------" cd /ngen # cmake -B /ngen/mpibuild -S . -DNGEN_WITH_MPI=ON -DNGEN_WITH_PYTHON=ON -DNGEN_WITH_ROUTING=ON -DNGEN_WITH_NETCDF=ON -DNGEN_QUIET=ON -cmake -B $current_path/ngen/parallelbuild -S . -DNGEN_WITH_MPI:BOOL=ON -DNGEN_WITH_NETCDF:BOOL=ON -DNGEN_WITH_SQLITE:BOOL=ON -DNGEN_WITH_UDUNITS:BOOL=ON -DNGEN_WITH_BMI_FORTRAN:BOOL=ON -DNGEN_WITH_BMI_C:BOOL=ON -DNGEN_WITH_PYTHON:BOOL=ON -DNGEN_WITH_ROUTING:BOOL=ON -DNGEN_WITH_TESTS:BOOL=ON -DNGEN_QUIET:BOOL=ON -DNETCDF_CXX_INCLUDE_DIR=/usr/local/include -DNETCDF_CXX_LIBRARY=/usr/local/lib64/libnetcdf-cxx4.so # -DNETCDF_INCLUDE_DIR=/usr/include -DNETCDF_LIBRARY=/usr/lib64/libnetcdf.so +cmake -B $current_path/ngen/parallelbuild -S . -DCMAKE_INSTALL_PREFIX=/ngen/ngen -DNGEN_WITH_EXTERN_ALL=ON -DNGEN_WITH_MPI:BOOL=ON -DNGEN_WITH_NETCDF:BOOL=ON -DNGEN_WITH_SQLITE:BOOL=ON -DNGEN_WITH_UDUNITS:BOOL=ON -DNGEN_QUIET:BOOL=ON -DNGEN_WITH_BMI_FORTRAN:BOOL=ON -DNGEN_WITH_BMI_C:BOOL=ON -DNGEN_WITH_PYTHON:BOOL=ON -DNGEN_WITH_ROUTING:BOOL=ON -DNGEN_WITH_TESTS:BOOL=ON -DNETCDF_CXX_INCLUDE_DIR=/usr/local/include -DNETCDF_CXX_LIBRARY=/usr/local/lib64/libnetcdf-cxx4.so # -DNETCDF_INCLUDE_DIR=/usr/include -DNETCDF_LIBRARY=/usr/lib64/libnetcdf.so echo "===========================================================" echo "===========================================================" diff --git a/singularity/templates/t-route/install_t_route.sh b/singularity/templates/t-route/install_t_route.sh index be070ac..5904f79 100755 --- a/singularity/templates/t-route/install_t_route.sh +++ b/singularity/templates/t-route/install_t_route.sh @@ -19,6 +19,8 @@ echo "-----------------------------------------------------------" export FC=gfortran NETCDF=/usr/include git clone --progress --single-branch --branch master https://github.com/NOAA-OWP/t-route cd t-route +git submodule update --init + ./compiler.sh no-e cd /t-route/src/troute-network @@ -27,10 +29,15 @@ python setup.py --use-cython bdist_wheel cd /t-route/src/troute-routing python setup.py --use-cython bdist_wheel +# troute-config doesn't use setup.py, use build to make the wheel +cd /t-route/src/troute-config +python -m build . \ + cd /t-route/src/troute-nwm python setup.py bdist_wheel cd /t-route cp /t-route/src/troute-network/dist/*.whl /t-route/wheels cp /t-route/src/troute-routing/dist/*.whl /t-route/wheels +cp /t-route/src/troute-config/dist/*.whl /t-route/wheels cp /t-route/src/troute-nwm/dist/*.whl /t-route/wheels