Merge branch 'master' into filterapi2.tmp

.github/workflows/run_tests_osx.yml

@@ -108,7 +108,7 @@ jobs:
       ###
       - name: Install autoconf
         shell: bash -l {0}
-        run: brew install automake
+        run: brew install automake m4
 
       - name: Run autoconf
         shell: bash -l {0}

CMakeLists.txt

@@ -954,6 +954,11 @@ if(NETCDF_ENABLE_PARALLEL4 AND NETCDF_ENABLE_HDF5)
     file(COPY "${netCDF_BINARY_DIR}/tmp/run_par_test.sh"
       DESTINATION ${netCDF_BINARY_DIR}/nc_test4
       FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
+    configure_file("${netCDF_SOURCE_DIR}/nc_test4/run_par_warn_test.sh.in"
+      "${netCDF_BINARY_DIR}/tmp/run_par_warn_test.sh" @ONLY NEWLINE_STYLE LF)
+    file(COPY "${netCDF_BINARY_DIR}/tmp/run_par_warn_test.sh"
+      DESTINATION ${netCDF_BINARY_DIR}/nc_test4
+      FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
     configure_file("${netCDF_SOURCE_DIR}/h5_test/run_par_tests.sh.in"
       "${netCDF_BINARY_DIR}/tmp/run_par_tests.sh" @ONLY NEWLINE_STYLE LF)
     file(COPY "${netCDF_BINARY_DIR}/tmp/run_par_tests.sh"
@@ -1759,6 +1764,21 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/libnetcdf.settings.in"
   "${CMAKE_CURRENT_BINARY_DIR}/libnetcdf.settings"
   @ONLY)
 
+if(NETCDF_ENABLE_PARALLEL4 AND NETCDF_ENABLE_HDF5)
+  if(HDF5_PARALLEL)
+    configure_file("${netCDF_SOURCE_DIR}/nc_test4/run_par_test.sh.in"
+      "${netCDF_BINARY_DIR}/tmp/run_par_test.sh" @ONLY NEWLINE_STYLE LF)
+    file(COPY "${netCDF_BINARY_DIR}/tmp/run_par_test.sh"
+      DESTINATION ${netCDF_BINARY_DIR}/nc_test4
+      FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
+    configure_file("${netCDF_SOURCE_DIR}/h5_test/run_par_tests.sh.in"
+      "${netCDF_BINARY_DIR}/tmp/run_par_tests.sh" @ONLY NEWLINE_STYLE LF)
+    file(COPY "${netCDF_BINARY_DIR}/tmp/run_par_tests.sh"
+      DESTINATION ${netCDF_BINARY_DIR}/h5_test
+      FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
+  endif()
+endif()
+
 # Read in settings file, print out.
 # Avoid using system-specific calls so that this
 # might also work on Windows.
@@ -1841,7 +1861,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/s3gc.in ${CMAKE_CURRENT_BINARY_DIR}/s
 #####
 # Build and copy nc_test4/findplugin.sh to various places
 #####
-foreach(CC nc_test4 nczarr_test v3_nczarr_test plugins examples/C)
+foreach(CC nc_test4 nczarr_test v3_nczarr_test plugins h5_test examples/C)
     configure_file(${CMAKE_CURRENT_SOURCE_DIR}/plugins/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/${CC}/findplugin.sh @ONLY NEWLINE_STYLE LF)
 endforeach()
 

INSTALL.md

@@ -80,7 +80,7 @@ full functionality. (See \ref architecture).
 Requirements {#netcdf_requirements}
 ----------------------------------
 
-* HDF5 1.8.9 (netcdf-4 support)
+* HDF5 1.8.15 (netcdf-4 support)
 * zlib 1.2.5
 * curl 7.18.0 (DAP support)
 
@@ -100,7 +100,7 @@ Building with netCDF-4 and the Remote Data Client {#build_default}
 
 The usual way of building netCDF requires the HDF5, zlib, and curl
 libraries. (And, optionally, the szlib library). Versions required are
-at least HDF5 1.8.9, zlib 1.2.5, and curl 7.18.0 or later.
+at least HDF5 1.8.15, zlib 1.2.5, and curl 7.18.0 or later.
 (Optionally, if building with szlib, get szip 2.0 or later.)
 
 These packages are available at:
@@ -332,7 +332,7 @@ For example, one user reports that she can build other applications
 with netCDF-4 by setting the LIBS environment variable:
 
 ~~~
-LIBS='-L/X/netcdf-4.0/lib -lnetcdf -L/X/hdf5-1.8.9/lib -lhdf5_hl -lhdf5 -lz -lm -L/X/szip-2.1/lib -lsz'
+LIBS='-L/X/netcdf-4.0/lib -lnetcdf -L/X/hdf5-1.8.15/lib -lhdf5_hl -lhdf5 -lz -lm -L/X/szip-2.1/lib -lsz'
 ~~~
 
 For shared builds, only -lnetcdf is needed. All other libraries will

cmake/dependencies.cmake

@@ -114,7 +114,7 @@ if(USE_HDF5)
   ##
   # Assert HDF5 version meets minimum required version.
   ##
-  set(HDF5_VERSION_REQUIRED 1.8.10)
+  set(HDF5_VERSION_REQUIRED 1.8.15)
 
   ###
   # For now we assume that if we are building netcdf

configure.ac

@@ -1,4 +1,4 @@
-#                                               -*- Autoconf -*-
+o#                                               -*- Autoconf -*-
 ## Process this file with autoconf to produce a configure script.
 
 # This is part of Unidata's netCDF package. Copyright 2005-2018, see
@@ -784,6 +784,10 @@ if test "x$enable_filter_zstd" = "xyes" ; then
 else
    have_zstd=no
 fi
+AC_MSG_CHECKING([whether we are going to build with zstd])
+AC_SUBST([HAS_ZSTD], [$have_zstd])
+AC_MSG_RESULT([$have_zstd])
+
 ##
 # End zstd checks
 ##
@@ -1299,7 +1303,6 @@ AC_MSG_NOTICE([checking types, headers, and functions])
 AC_CHECK_HEADERS([sys/param.h])
 AC_CHECK_HEADERS([libgen.h])
 #AC_CHECK_HEADERS([locale.h])
-AC_HEADER_STDC
 AC_CHECK_HEADERS([locale.h stdio.h stdarg.h fcntl.h malloc.h stdlib.h string.h strings.h unistd.h sys/stat.h getopt.h sys/time.h sys/types.h time.h dirent.h stdint.h ctype.h sys/xattr_h])
 
 # Do sys/resource.h separately
@@ -1454,7 +1457,7 @@ AC_SUBST(USEPLUGINS, [${enable_plugins}])
 
 AC_FUNC_ALLOCA
 AC_CHECK_DECLS([isnan, isinf, isfinite],,,[#include <math.h>])
-AC_STRUCT_ST_BLKSIZE
+AC_CHECK_MEMBERS([struct stat.st_blksize])
 UD_CHECK_IEEE
 AC_CHECK_TYPES([size_t, ssize_t, schar, uchar, longlong, ushort, uint, int64, uint64, size64_t, ssize64_t, _off64_t, uint64_t, ptrdiff_t])
 AC_TYPE_OFF_T
@@ -2245,7 +2248,7 @@ AC_MSG_NOTICE([generating header files and makefiles])
 AC_CONFIG_FILES(test_common.sh:test_common.in)
 AC_CONFIG_FILES(s3cleanup.sh:s3cleanup.in, [chmod ugo+x s3cleanup.sh])
 AC_CONFIG_FILES(s3gc.sh:s3gc.in, [chmod ugo+x s3gc.sh])
-for FP in plugins nc_test4 nczarr_test examples/C ; do
+for FP in plugins nc_test4 nczarr_test h5_test examples/C ; do
 AC_CONFIG_FILES(${FP}/findplugin.sh:plugins/findplugin.in, [chmod ugo+x ${FP}/findplugin.sh])
 done
 AC_CONFIG_FILES(ncdap_test/findtestserver.c:ncdap_test/findtestserver.c.in, [chmod ugo+x ncdap_test/findtestserver.c])
@@ -2254,6 +2257,7 @@ AC_CONFIG_FILES(dap4_test/findtestserver4.c:ncdap_test/findtestserver.c.in)
 AC_CONFIG_FILES(dap4_test/pingurl4.c:ncdap_test/pingurl.c)
 AC_CONFIG_FILES([h5_test/run_par_tests.sh], [chmod ugo+x h5_test/run_par_tests.sh])
 AC_CONFIG_FILES([nc_test4/run_par_test.sh], [chmod ugo+x nc_test4/run_par_test.sh])
+AC_CONFIG_FILES([nc_test4/run_par_warn_test.sh], [chmod ugo+x nc_test4/run_par_warn_test.sh])
 AC_CONFIG_FILES([nc_perf/run_par_bm_test.sh], [chmod ugo+x nc_perf/run_par_bm_test.sh])
 AC_CONFIG_FILES([nc_perf/run_gfs_test.sh], [chmod ugo+x nc_perf/run_gfs_test.sh])
 AC_CONFIG_FILES(nczarr_test/timer_utils.h:unit_test/timer_utils.h)

h5_test/CMakeLists.txt

@@ -24,3 +24,17 @@ IF(TEST_PARALLEL4)
   build_bin_test(tst_h_par_compress)
   add_sh_test(h5_test run_par_tests)
 ENDIF()
+
+# If zstd is enabled, run a test for it.
+if(NETCDF_ENABLE_PLUGINS)
+  if(NETCDF_ENABLE_FILTER_ZSTD)
+    build_bin_test(tst_h_zstd)
+    add_sh_test(h5_test run_h5_zstd_tests)    
+  endif()
+endif()
+
+# Copy the shell file to build directory.
+file(COPY run_h5_zstd_tests.sh DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE)
+
+
+

h5_test/Makefile.am

@@ -2,16 +2,24 @@
 # Copyright 2018 University Corporation for Atmospheric Research/Unidata
 # See COPYRIGHT file for conditions of use.
 #
-# This entire directory will be skipped, unless the configure script
-# is run with --enable-netcdf-4. This directory contains tests that
-# only use HDF5; these tests don't use netCDF at all.
+# This directory contains tests that only use HDF5; these tests don't
+# use netCDF at all. These tests will only be run if netcdf-4 is
+# enabled.
 #
 # If one of these tests fails, then netCDF-4 will not work correctly.
 # Ed Hartnett, Ward Fisher
 
 # Set AM_CPPFLAGS and AM_LDFLAGS based on user choices.
 include $(top_srcdir)/lib_flags.am
 
+# Un comment to use a more verbose test driver
+# SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
+# sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
+# LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
+# TESTS_ENVIRONMENT = export SETX=1;
+
+TEST_EXTENSIONS = .sh
+
 # These programs test HDF5 to make sure it will do (some of) the stuff
 # that netCDF-4 needs.
 H5TESTS = tst_h_files tst_h_files2 tst_h_files4 tst_h_atts		\
@@ -21,7 +29,7 @@ tst_h_strings tst_h_strings1 tst_h_strings2 tst_h_ints			\
 tst_h_dimscales tst_h_dimscales1 tst_h_dimscales2 tst_h_dimscales3	\
 tst_h_enums tst_h_dimscales4 tst_h_rename
 
-# If benchmarks were turned on, build and run a bunch more tests.
+# If benchmarks were turned on, build and run more tests.
 if BUILD_BENCHMARKS
 H5TESTS += tst_h_mem
 endif # BUILD_BENCHMARKS
@@ -32,6 +40,14 @@ check_PROGRAMS = $(H5TESTS)
 # List of tests to run.
 TESTS = $(H5TESTS)
 
+# If zstd is enabled, run a test for it.
+if NETCDF_ENABLE_PLUGINS
+if HAVE_ZSTD
+check_PROGRAMS += tst_h_zstd
+TESTS += run_h5_zstd_tests.sh
+endif # HAVE_ZSTD
+endif # NETCDF_ENABLE_PLUGINS
+
 # The parallel program is run from a script.
 if TEST_PARALLEL4
 check_PROGRAMS += tst_h_par tst_h_par_compress
@@ -41,7 +57,7 @@ endif
 # We must include these files in the distribution.
 EXTRA_DIST = run_par_tests.sh.in ref_tst_h_compounds.h5		\
 ref_tst_h_compounds2.h5 run_par_tests.sh ref_tst_compounds.nc	\
-h5_err_macros.h CMakeLists.txt
+h5_err_macros.h CMakeLists.txt run_h5_zstd_tests.sh
 
 # Clean up test results.
 CLEANFILES = tst_h_*.h5

h5_test/run_h5_zstd_tests.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+# Copyright 2024 University Corporation for Atmospheric
+# Research/Unidata. See netcdf-c/COPYRIGHT file for more info.
+
+# This shell runs the zstandard tests.
+
+# Ed Hartnett
+
+if test "x$srcdir" = x ; then srcdir=`pwd`; fi
+. ../test_common.sh
+
+# Load the findplugins function
+. ${builddir}/findplugin.sh
+export HDF5_PLUGIN_PATH="${HDF5_PLUGIN_DIR}"
+echo "HDF5_PLUGIN_PATH=$HDF5_PLUGIN_PATH"
+
+findplugin h5zstd
+echo ""
+echo "Testing zstandard I/O with HDF5..."
+
+./tst_h_zstd
+echo "SUCCESS!!!"
+
+exit 0