diff --git a/src/common/common_utils.cpp b/src/common/common_utils.cpp index 9473e85c..b688be9d 100644 --- a/src/common/common_utils.cpp +++ b/src/common/common_utils.cpp @@ -47,7 +47,7 @@ namespace yask { // https://semver.org/. // Format: "major.minor.patch[-alpha|-beta]". - const string version = "4.05.01"; + const string version = "4.05.02"; string yask_get_version_string() { return version; diff --git a/src/kernel/yask.sh b/src/kernel/yask.sh index 93d2f1a8..138045f3 100755 --- a/src/kernel/yask.sh +++ b/src/kernel/yask.sh @@ -58,7 +58,7 @@ arch_is_def_cpu=1 # arch has been set for CPU default only. # Default nodes. nnodes=1 -if [[ ! -z ${SLURM_NNODES:+x} ]]; then +if [[ -n "$SLURM_NNODES" ]]; then nnodes=$SLURM_NNODES fi @@ -74,25 +74,24 @@ elif command -v nvidia-smi >/dev/null; then fi is_offload=0 -# Default MPI ranks. -# Try Slurm var, then numactl, then lscpu. -# For latter two, the goal is to count only NUMA nodes with CPUs. +# Get NUMA nodes. +# The goal is to count only NUMA nodes with CPUs. # (Systems with HBM may have NUMA nodes without CPUs.) +if command -v numactl >/dev/null; then + nnumas=`numactl -s | awk '/^cpubind:/ { print NF-1 }'` +elif command -v lscpu >/dev/null; then + nnumas=`lscpu | grep -c '^NUMA node.*CPU'` +fi + +# Default MPI ranks. +# Try Slurm var, then num NUMA nodes. nranks=$nnodes nranks_is_def_cpu=1 # nranks has been set for CPU default only. -if [[ ! -z ${SLURM_NTASKS:+x} && $SLURM_NTASKS > $nnodes ]]; then +if [[ -n "$SLURM_NTASKS" && $SLURM_NTASKS > $nnodes ]]; then nranks=$SLURM_NTASKS nranks_is_def_cpu=0 -elif command -v numactl >/dev/null; then - ncpubinds=`numactl -s | awk '/^cpubind:/ { print NF-1 }'` - if [[ -n "$ncpubinds" ]]; then - nranks=$(( $ncpubinds * $nnodes )) - fi -elif command -v lscpu >/dev/null; then - nnumas=`lscpu | grep -c '^NUMA node.*CPU'` - if [[ -n "$nnumas" ]]; then - nranks=$(( $nnumas * $nnodes )) - fi +elif [[ -n "$nnumas" ]]; then + nranks=$(( $nnumas * $nnodes )) fi nranks_offload=$nnodes if [[ $ngpus > 0 ]]; then @@ -368,7 +367,7 @@ done # parsing options. echo $invo # Check required opt (yes, it's an oxymoron). -if [[ -z ${stencil:+x} ]]; then +if [[ -z "$stencil" ]]; then echo "error: missing required option: -stencil " show_stencils fi @@ -394,7 +393,12 @@ if [[ $nranks > 1 || $force_mpi == 1 ]]; then # Add default Intel MPI settings. envs+=" I_MPI_PRINT_VERSION=1 I_MPI_DEBUG=5" - envs+=" I_MPI_PIN_DOMAIN=numa" + + # Add NUMA pinning if number of discovered NUMA nodes + # equals what is being used. + if [[ -n "$nnumas" && $nnumas == $ppn ]]; then + envs+=" I_MPI_PIN_DOMAIN=numa" + fi # Check whether HBM policy setting is allowed. if [[ `I_MPI_HBW_POLICY=hbw_preferred,hbw_preferred mpirun -np 1 /bin/date |& grep -c 'Unknown memory policy'` == 0 ]]; then @@ -532,7 +536,7 @@ if [[ $is_offload == 1 ]]; then if command -v nvidia-smi >/dev/null; then config_cmds+="; nvidia-smi"; fi - if [[ ! -z "$mpi_cmd" ]]; then + if [[ -n "$mpi_cmd" ]]; then envs+=" I_MPI_OFFLOAD=2" fi fi