ckp

.github/workflows/run_tests_osx.yml

@@ -6,7 +6,7 @@
 
 name: Run macOS-based netCDF Tests
 
-on: [pull_request,workflow_dispatch]
+on: [push,pull_request,workflow_dispatch]
 
 concurrency:  
   group: ${{ github.workflow}}-${{ github.head_ref }}  

.github/workflows/run_tests_s3.yml

@@ -9,7 +9,7 @@
 
 name: Run S3 netCDF Tests (under Ubuntu Linux)
 
-on: [workflow_dispatch]
+on: [push,workflow_dispatch]
 
 concurrency:  
   group: ${{ github.workflow}}-${{ github.head_ref }}  

.github/workflows/run_tests_ubuntu.yml

@@ -4,7 +4,7 @@
 
 name: Run Ubuntu/Linux netCDF Tests
 
-on: [pull_request, workflow_dispatch]
+on: [push,pull_request, workflow_dispatch]
 
 concurrency:  
   group: ${{ github.workflow}}-${{ github.head_ref }}  

.github/workflows/run_tests_win_cygwin.yml

@@ -1,6 +1,6 @@
 name: Run Cygwin-based tests
 
-on: [pull_request,workflow_dispatch]
+on: [push,pull_request,workflow_dispatch]
 
 concurrency:  
   group: ${{ github.workflow}}-${{ github.head_ref }}  

.github/workflows/run_tests_win_mingw.yml

@@ -9,7 +9,7 @@ name: Run MSYS2, MinGW64-based Tests (Not Visual Studio)
 env:
   CPPFLAGS: "-D_BSD_SOURCE"
 
-on: [pull_request,workflow_dispatch]
+on: [push,pull_request,workflow_dispatch]
 
 concurrency:  
   group: ${{ github.workflow}}-${{ github.head_ref }}  

docs/filters.md

@@ -14,8 +14,8 @@ should be aware.
 An option is now provided to automatically install
 HDF5 filters into a default location, or optionally
 into a user-specified location. This is described in
-[Appendix H](#filters_appendixh)
-(with supporting information in [Appendix G](#filters_appendixg)). 
+[Appendix I](#filters_appendixi)
+(with supporting information in [Appendix H](#filters_appendixh)). 
 
 * ***NCZarr Filter Support***<br>
 [NCZarr filters](#filters_nczarr) are now supported.
@@ -900,11 +900,11 @@ typedef struct NCZ_codec_t {
                  Currently always NCZ_CODEC_HDF5 */
     const char* codecid;            /* The name/id of the codec */
     unsigned int hdf5id; /* corresponding hdf5 id */
-    void (*NCZ_codec_initialize)(void);
-    void (*NCZ_codec_finalize)(void);
-    int (*NCZ_codec_to_hdf5)(const char* codec, int* nparamsp, unsigned** paramsp);
-    int (*NCZ_hdf5_to_codec)(size_t nparams, const unsigned* params, char** codecp);
-    int (*NCZ_modify_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* nparamsp, unsigned** paramsp);
+    void (*NCZ_codec_initialize)(NCproplist* env);
+    void (*NCZ_codec_finalize)(NCproplist* env);
+    int (*NCZ_codec_to_hdf5)(NCproplist* env, const char* codec, int* h5idp, int* nparamsp, unsigned** paramsp);
+    int (*NCZ_hdf5_to_codec)(NCproplist* env, int h5id, size_t nparams, const unsigned* params, char** codecp);
+    int (*NCZ_modify_parameters)(NCproplist* env, int* h5idp, size_t* vnparamsp, unsigned** vparamsp, size_t* nparamsp, unsigned** paramsp);
 } NCZ_codec_t;
 ````
 
@@ -922,12 +922,14 @@ visible parameters.
 
 ##### Signature
 ````
-    int NCZ_codec_to_hdf(const char* codec, int* nparamsp, unsigned** paramsp);
+    int NCZ_codec_to_hdf(NCproplist* env, const char* codec, int* h5idp, int* nparamsp, unsigned** paramsp);
 ````
 ##### Arguments
-1. codec &mdash; (in) ptr to JSON string representing the codec.
-2. nparamsp &mdash; (out) store the length of the converted HDF5 unsigned vector
-3. paramsp &mdash; (out) store a pointer to the converted HDF5 unsigned vector; caller must free the returned vector. Note the double indirection.
+1. env $mdash; (in) ptr to a property list of key+value pairs.
+2. codec &mdash; (in) ptr to JSON string representing the codec.
+3. h5idp &mdash; (in/out) the hdf5 filter id.
+4. nparamsp &mdash; (out) store the length of the converted HDF5 unsigned vector
+5. paramsp &mdash; (out) store a pointer to the converted HDF5 unsigned vector; caller must free the returned vector. Note the double indirection.
 
 Return Value: a netcdf-c error code.
 
@@ -938,12 +940,12 @@ return a corresponding JSON codec representation of those visible parameters.
 
 ##### Signature
 ````
-    int NCZ_hdf5_to_codec)(int ncid, int varid, size_t nparams, const unsigned* params, char** codecp);
+    int NCZ_hdf5_to_codec)(NCproplist* env, int id, size_t nparams, const unsigned* params, char** codecp);
 ````
 ##### Arguments
 
-1. ncid    &mdash; the variables' containing group
-2. varid   &mdash; the containing variable
+1. env &mdash; property list of key+value pairs.
+2. id &mdash; the hdf5 id.
 3. nparams &mdash; (in) the length of the HDF5 visible parameters vector
 4. params &mdash; (in) pointer to the HDF5 visible parameters vector.
 5. codecp &mdash; (out) store the string representation of the codec; caller must free.
@@ -957,12 +959,12 @@ to a set of working parameters; also provide option to modify visible parameters
 
 ##### Signature
 ````
-    int NCZ_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp);
+    int NCZ_modify_parameters(NCproplist* env, int* idp, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp);
 ````
 ##### Arguments
 
-1. ncid &mdash; (in) group id containing the variable.
-2. varid &mdash; (in) the id of the variable to which this filter is being attached.
+1. env &mdash; (in) property list of key+value pairs.
+2. idp &mdash; (in/out) the hdf5 id.
 3. vnparamsp &mdash; (in/out) the count of visible parameters
 4. vparamsp &mdash; (in/out) the set of visible parameters
 5. wnparamsp &mdash; (out) the count of working parameters
@@ -977,8 +979,12 @@ This function is called as soon as a shared library is loaded and matched with a
 
 ##### Signature
 ````
-    int NCZ_codec_initialize)(void);
+    int NCZ_codec_initialize)(NCproplist* env);
 ````
+##### Arguments
+
+1. env &mdash; (in) property list of key+value pairs.
+
 Return Value: a netcdf-c error code.
 
 #### NCZ\_codec\_finalize
@@ -989,8 +995,12 @@ If the client code does not invoke *nc\_finalize* then memory checkers may compl
 
 ##### Signature
 ````
-    int NCZ_codec_finalize)(void);
+    int NCZ_codec_finalize)(NCproplist* env);
 ````
+##### Arguments
+
+1. env &mdash; (in) property list of key+value pairs.
+
 Return Value: a netcdf-c error code.
 
 ### Multi-Codec API
@@ -1013,7 +1023,27 @@ The list of returned items are used to try to provide defaults
 for any HDF5 filters that have no corresponding Codec.
 This is for internal use only.
 
-## Appendix F. Standard Filters  {#filters_appendixf}
+## Appendix F. Default HDF5 Filter Codecs  {#filters_appendixf}
+
+It is recognized that it will be a while (if ever) until
+HDF5 filters also specify the necessary codec information.
+In order to provide some support for filters that do not have
+corresponding codec support, a "hdf5raw" codec manager is provided.
+
+This hdf5raw codec manager encodes the parameters of the HDF5 filter
+into one of these two codec forms:
+
+* Zarr Version 2
+    ````{"id": "hdf5raw", "hdf5id": "<hdf5-id>, "nparams": <uint>, "0": <uint>...,"<N>": <uint>}````
+* Zarr Version 3
+    ````{"name": "hdf5raw, "configuration": {"hdf5id": <uint>, "nparams": <uint>, "0": <uint>...,"<N>": <uint>}}````
+
+There are couple things to note about hdf5raw:
+1. this cannot be used if a modify_parameters function is required.
+2. this representation will not be usable by other Zarr implementations, unless of course they choose to implement it.
+
+
+## Appendix G. Standard Filters  {#filters_appendixg}
 
 Support for a select set of standard filters is built into the NetCDF API.
 Generally, they are accessed using the following generic API, where XXXX is
@@ -1043,10 +1073,10 @@ Consider the zstandard compressor, which is one of the supported standard filter
 When installing the netcdf library, the following other libraries must be installed.
 
 1. *libzstd.so* | *zstd.dll* | *libzstd.dylib* -- The actual zstandard compressor library; typically installed by using your platform specific package manager.
-2. The HDF5 wrapper for *libzstd.so* -- There are several options for obtaining this (see [Appendix G](#filters_appendixg).)
+2. The HDF5 wrapper for *libzstd.so* -- There are several options for obtaining this (see [Appendix H](#filters_appendixh).)
 3. (Optional) The Zarr wrapper for *libzstd.so* -- you need this if you intend to read/write Zarr datasets that were compressed using zstandard; again see [Appendix G](#filters_appendixg).
 
-## Appendix G. Finding Filters  {#filters_appendixg}
+## Appendix H. Finding Filters  {#filters_appendixh}
 
 A major problem for filter users is finding an implementation of an HDF5 filter wrapper and (optionally)
 its corresponding NCZarr wrapper. There are several ways to do this.
@@ -1067,7 +1097,7 @@ You can install this library to get access to these supported filters.
 It does not currently include the required NCZarr Codec API,
 so they are only usable with netcdf-4. This will change in the future.
 
-## Appendix H. Auto-Install of Filter Wrappers {#filters_appendixh}
+## Appendix I. Auto-Install of Filter Wrappers {#filters_appendixi}
 
 As part of the overall build process, a number of filter wrappers are built as shared libraries in the "plugins" directory.
 These wrappers can be installed as part of the overall netcdf-c installation process.
@@ -1145,7 +1175,7 @@ The important thing to note is that at run-time, there are several cases to cons
 3. HDF5_PLUGIN_DIR is not defined at either run-time or build-time -- no action needed
 4. HDF5_PLUGIN_DIR is not defined at run-time but was defined at build-time -- this will probably fail
 
-## Appendix I. A Warning on Backward Compatibility {#filters_appendixi}
+## Appendix J. A Warning on Backward Compatibility {#filters_appendixj}
 
 The API defined in this document should accurately reflect the
 current state of filters in the netCDF-c library.  Be aware that
@@ -1172,4 +1202,4 @@ For additional information, see [Appendix B](#filters_appendixb).
 *Author*: Dennis Heimbigner<br>
 *Email*: dmh at ucar dot edu<br>
 *Initial Version*: 1/10/2018<br>
-*Last Revised*: 5/18/2022
+*Last Revised*: 10/18/2023

docs/nczarr.md

@@ -736,7 +736,7 @@ ubyte enum _bool_t {FALSE=0, TRUE=1};
 ````
 The type complex64 is supported by by defining this compound type:
 ````
-compound _complex64_t { float64 i; float64 j;}
+compound _Complex64_t { float64 i; float64 j;}
 ````
 
 Strings present a problem because there is a proposal

include/CMakeLists.txt

@@ -45,6 +45,10 @@ INSTALL(FILES ${netCDF_SOURCE_DIR}/include/netcdf_json.h
     DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
     COMPONENT headers)
 
+INSTALL(FILES ${netCDF_SOURCE_DIR}/include/netcdf_proplist.h
+    DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+    COMPONENT headers)
+
 FILE(GLOB CUR_EXTRA_DIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*.h)
 SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} Makefile.am CMakeLists.txt)
 ADD_EXTRA_DIST("${CUR_EXTRA_DIST}")

include/Makefile.am

@@ -7,7 +7,10 @@
 # Ed Hartnett, Dennis Heimbigner, Ward Fisher
 
 include_HEADERS = netcdf.h netcdf_meta.h netcdf_mem.h netcdf_aux.h	\
-netcdf_filter.h netcdf_filter_build.h netcdf_filter_hdf5_build.h netcdf_dispatch.h netcdf_json.h
+netcdf_filter.h netcdf_filter_build.h netcdf_filter_hdf5_build.h netcdf_dispatch.h netcdf_vutils.h
+
+# Built source
+include_HEADERS += netcdf_json.h netcdf_proplist.h
 
 if BUILD_PARALLEL
 include_HEADERS += netcdf_par.h
@@ -20,7 +23,7 @@ nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h ncauth.h		\
 ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h ncexternl.h	\
 ncpathmgr.h ncindex.h hdf4dispatch.h hdf5internal.h nc_provenance.h	\
 hdf5dispatch.h ncmodel.h isnan.h nccrc.h ncexhash.h ncxcache.h          \
-ncjson.h ncxml.h ncs3sdk.h
+ncjson.h ncxml.h ncs3sdk.h ncproplist.h
 
 if USE_DAP
 noinst_HEADERS += ncdap.h
@@ -32,6 +35,8 @@ endif
 
 EXTRA_DIST = CMakeLists.txt XGetopt.h netcdf_meta.h.in netcdf_dispatch.h.in
 
+BUILT_SOURCES = netcdf_json.h netcdf_proplist.h
+
 # netcdf_json.h is constructed as a header-only file for use by
 # nczarr code wrappers in the plugin directory. It is
 # constructed by joining libdispatch/ncjson.c with
@@ -41,6 +46,21 @@ EXTRA_DIST = CMakeLists.txt XGetopt.h netcdf_meta.h.in netcdf_dispatch.h.in
 # having to reference libnetcdf in the nczarr code wrappers.
 
 # Give the recipe for rebuilding netcdf_json.h
-makepluginjson::
-	sed -e 's/NCJSON_H/NETCDF_JSON_H/' -e '/ncjson.h/d' <${srcdir}/ncjson.h > netcdf_json.h
+netcdf_json.h: ${top_srcdir}/libdispatch/ncjson.c ${top_srcdir}/include/ncjson.h
+	sed -e 's/NCJSON_H/NETCDF_JSON_H/' -e '/ncjson.h/d' -e '/#endif[^!]*!NETCDF_JSON_H!/d' <${srcdir}/ncjson.h > netcdf_json.h
 	sed -e '/ncjson.h/d' < ${srcdir}/../libdispatch/ncjson.c >> netcdf_json.h
+	echo '#endif /*NETCDF_JSON_H*/' >> netcdf_json.h
+
+# netcdf_proplist.h is constructed as a header-only file for use by
+# nczarr code wrappers in the plugin directory. It is
+# constructed by joining libdispatch/ncproplist.c with
+# include/ncproplist.h.  Additionally all the entry points are
+# marked with a macro (OPTSTATIC) that marks the entry point as
+# static inside netcdf_proplist.h.  This is an ugly hack to avoid
+# having to reference libnetcdf in the nczarr code wrappers.
+
+# Give the recipe for rebuilding netcdf_proplist.h
+netcdf_proplist.h: ${top_srcdir}/libdispatch/ncproplist.c ${top_srcdir}/include/ncproplist.h
+	sed -e 's/NCPROPLIST_H/NETCDF_PROPLIST_H/' -e '/ncproplist.h/d' -e '/#endif[^!]*!NETCDF_PROPLIST_H!/d' <${srcdir}/ncproplist.h > netcdf_proplist.h
+	sed -e '/ncproplist.h/d' < ${srcdir}/../libdispatch/ncproplist.c >> netcdf_proplist.h
+	echo '#endif /*NETCDF_PROPLIST_H*/' >> netcdf_proplist.h

include/ncjson.h

@@ -29,7 +29,7 @@ and do the command:
 #define OPTEXPORT static
 #else
 #define OPTEXPORT DLLEXPORT
-#endif /*NETCDF_JSON_H*/
+#endif
 
 /**************************************************/
 /* Json object sorts (note use of term sort rather than e.g. type or discriminant) */
@@ -114,6 +114,9 @@ OPTEXPORT int NCJinsert(NCjson* object, const char* key, NCjson* jvalue);
 /* Insert key-value pair into a dict object. key and value will be copied */
 OPTEXPORT int NCJinsertstring(NCjson* object, const char* key, const char* value);
 
+/* Insert key-value pair into a dict object. key and value will be copied */
+OPTEXPORT int NCJinsertint(NCjson* object, const char* key, int n);
+
 /* Unparser to convert NCjson object to text in buffer */
 OPTEXPORT int NCJunparse(const NCjson* json, unsigned flags, char** textp);
 
@@ -149,5 +152,4 @@ OPTEXPORT const char* NCJtotext(const NCjson* json, unsigned flags);
 
 /**************************************************/
 
-#endif /*NCJSON_H*/
-
+#endif /*!NCJSON_H!*/ /* Leave the ! as a tag for sed */