cleanup actions

CMakeLists.txt

@@ -45,7 +45,6 @@ set(PACKAGE "netCDF" CACHE STRING "")
 include(netcdf_functions_macros)
 include(deprecated)
 
-
 # Backport of built-in `PROJECT_IS_TOP_LEVEL` from CMake 3.21
 if (NOT DEFINED NETCDF_IS_TOP_LEVEL)
   set(NETCDF_IS_TOP_LEVEL OFF)
@@ -1763,6 +1762,10 @@ endif()
 # WARNING: this must match the value in configure.ac
 set(S3TESTBUCKET "unidata-zarr-test-data" CACHE STRING "S3 test bucket")
 
+# Additional S3 Test Endpoing
+set(S3ENDPOINT "s3.us-east-1.amazonaws.com" CACHE STRING "S3 endpoint")
+
+
 # The working S3 path tree within the Unidata bucket.
 # WARNING: this must match the value in configure.ac
 set(S3TESTSUBTREE "netcdf-c" CACHE STRING "Working S3 path.")

Makefile.am

@@ -110,9 +110,11 @@ endif
 
 # Build Cloud Storage if desired.
 if NETCDF_ENABLE_NCZARR
+ZARR = libnczarr
 ZARR_TEST_DIRS = nczarr_test
+if NETCDF_ENABLE_NCZARR_V3
 ZARR_TEST_DIRS += v3_nczarr_test
-ZARR = libnczarr
+endif
 endif
 
 # Optionally build test plugins

RELEASE_NOTES.md

@@ -6,7 +6,7 @@ Release Notes       {#RELEASE_NOTES}
 This file contains a high-level description of this package's evolution. Releases are in reverse chronological order (most recent first). Note that, as of netcdf 4.2, the `netcdf-c++` and `netcdf-fortran` libraries have been separated into their own libraries.
 
 ## 4.9.4 - TBD
-* Add experimental support for the Zarr Version 3 storage format. This code willl change as the Zarr Version 3 Specification evolves. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????).
+* Add experimental support for the Zarr Version 3 storage format. This code willl change as the Zarr Version 3 Specification evolves. See [Github #3068](https://github.com/Unidata/netcdf-c/pull/3068).
 
 ## 4.9.3 - TBD
 

config.h.cmake.in

@@ -157,6 +157,9 @@ are set when opening a binary file on Windows. */
 /* if true, enable NCZARR */
 #cmakedefine NETCDF_ENABLE_NCZARR 1
 
+/* if true, enable NCZARR Version 3*/
+#cmakedefine NETCDF_ENABLE_NCZARR_V3 1
+
 /* if true, enable nczarr filter support */
 #cmakedefine NETCDF_ENABLE_NCZARR_FILTERS 1
 
@@ -193,6 +196,11 @@ are set when opening a binary file on Windows. */
 /* S3 Working subtree path prefix*/
 #define S3TESTSUBTREE "${S3TESTSUBTREE}"
 
+/* S3 Test endpoint */
+#define S3ENDPOINT "${S3ENDPOINT}"
+
+/* S3 Test Bucket */
+
 /* if true, run extra tests which may not work yet */
 #cmakedefine EXTRA_TESTS 1
 

configure.ac

@@ -237,6 +237,17 @@ AC_ARG_ENABLE([nczarr],
 test "x$enable_nczarr" = xno || enable_nczarr=yes
 AC_MSG_RESULT($enable_nczarr)
 
+AC_MSG_CHECKING([whether netcdf zarr storage format version 3 should be enabled])
+AC_ARG_ENABLE([nczarr-v3],
+              [AS_HELP_STRING([--enable-nczarr-v3],
+                              [enable netcdf zarr version 3 storage support])]) dnl
+test "x$enable_nczarr_v3" = xyes || enable_nczarr_v3=no
+AC_MSG_RESULT($enable_nczarr_v3)
+if test "x$enable_nczarr_v3" = xyes ; then
+AC_DEFINE([NETCDF_ENABLE_NCZARR_V3], [1], [if true, include NCZarr V3 support])
+fi
+AM_CONDITIONAL([NETCDF_ENABLE_NCZARR_V3],[test "x$enable_nczarr_v3" = xyes])
+
 # HDF5 | HDF4 | NCZarr => netcdf-4
 if test "x$enable_hdf5" = xyes || test "x$enable_hdf4" = xyes || test "x$enable_nczarr" = xyes ; then
    enable_netcdf4=yes
@@ -248,11 +259,14 @@ AC_ARG_ENABLE([default-zarr-format_v3],
               [AS_HELP_STRING([--enable-default-zarr-format_v3],
                               [Specify the default Zarr format.])])
 test "x$enable_default_zarr_format_v3" = xyes | enable_default_zarr_format_v3=no # V2 is the default by default
+if test "x$enable_nczarr_v3" = xno ; then
+  enable_default_zarr_format_v3=no
+fi
 AC_MSG_RESULT([$enable_default_zarr_format_v3}])
 if test "x$enable_default_zarr_format_v3" = xyes; then
    DFALTZARRFORMAT=3
 else
-DFALTZARRFORMAT=2
+   DFALTZARRFORMAT=2
 fi
 AC_DEFINE_UNQUOTED([DFALTZARRFORMAT], [$DFALTZARRFORMAT], [Default Zarr format])
 AC_SUBST([DFALTZARRFORMAT],[$DFALTZARRFORMAT])
@@ -2135,6 +2149,7 @@ AC_SUBST(HAS_S3_AWS,[$enable_s3_aws])
 AC_SUBST(HAS_S3_INTERNAL,[$enable_s3_internal])
 AC_SUBST(HAS_HDF5_ROS3,[$has_hdf5_ros3])
 AC_SUBST(HAS_NCZARR,[$enable_nczarr])
+AC_SUBST(HAS_NCZARR_V3,[$enable_nczarr_v3])
 AC_SUBST(NETCDF_ENABLE_S3_TESTING,[$with_s3_testing])
 AC_SUBST(HAS_NCZARR_ZIP,[$enable_nczarr_zip])
 AC_SUBST(NCZARR_DEFAULT_FORMAT,[$DFALTZARRFORMAT])
@@ -2169,6 +2184,11 @@ AC_SUBST([S3TESTBUCKET],["unidata-zarr-test-data"])
 AC_DEFINE([S3TESTSUBTREE], ["netcdf-c"], [S3 test path prefix])
 AC_SUBST([S3TESTSUBTREE],[netcdf-c])
 
+# Additional S3 Test Endpoint
+# WARNING: this must match the value in CMakeLists.txt
+AC_DEFINE([S3ENDPOINT], ["s3.us-east-1.amazonaws.com"], [S3 test endpoint])
+AC_SUBST([S3ENDPOINT],["s3.us-east-1.amazonaws.com"])
+
 # Build a small unique id to avoid interference on same platform
 PLATFORMUID="$RANDOM"
 # Make sure uid > 0
@@ -2334,6 +2354,7 @@ AX_SET_META([NC_HAS_S3_AWS],[$enable_s3_aws],[yes])
 AX_SET_META([NC_HAS_S3_INTERNAL],[$enable_s3_internal],[yes])
 AX_SET_META([NC_HAS_HDF5_ROS3],[$has_hdf5_ros3],[yes])
 AX_SET_META([NC_HAS_NCZARR],[$enable_nczarr],[yes])
+AX_SET_META([NC_HAS_NCZARR_V3],[$enable_nczarr_v3],[yes])
 AX_SET_META([NC_HAS_LOGGING],[$enable_logging],[yes])
 AX_SET_META([NC_HAS_QUANTIZE],[$enable_quantize],[yes])
 AX_SET_META([NC_HAS_SZIP],[$enable_hdf5_szip],[yes])
@@ -2432,19 +2453,8 @@ AC_CONFIG_FILES([Makefile
                  dap4_test/Makefile
                  plugins/Makefile
 		 nczarr_test/Makefile
- 		 v3_nczarr_test/Makefile
+		 v3_nczarr_test/Makefile
                  ])
 AC_OUTPUT()
 
-# Initialize v3_nczarr_test
-AC_MSG_CHECKING([@@@ Initializing v3_nczarr_test])
-set -x
-mkdir -p v3_nczarr_test
-cd v3_nczarr_test
-find . -maxdepth 0 -type f -not \( -name CMakeLists.txt -or -name '*.am' -or -name Makefile.in \) -delete
-cp v3only/* .
-cd ..
-set +x
-AC_MSG_RESULT([done])
-
 cat libnetcdf.settings

docs/nczarr.md

@@ -35,8 +35,9 @@ NCZarr uses a data model that, by design, extends the Zarr Version 2 Specificati
 
 __Note Carefully__: a legal _NCZarr_ dataset is expected to also be a legal _Zarr_ dataset.
 The inverse is true also. A legal _Zarr_ dataset is expected to also be a legal _NCZarr_ dataset, where "legal" means it conforms to the Zarr version 2 or 3 specification.
-In addition, certain non-Zarr features are allowed and used.
-Specifically the XArray [7] ''\_ARRAY\_DIMENSIONS'' attribute is one such.
+In addition, certain extra-Zarr features are allowed and used, namely:
+1. the XArray [7] ''\_ARRAY\_DIMENSIONS'' attribute.
+2. the .zmetadata conventions where all the JSON metadata is held in a single object.
 
 There are two other, secondary assumptions:
 
@@ -151,11 +152,13 @@ the netcdf-c library was built.
 
 As an aside, it should be the case that zipping a _file_
 format directory tree will produce a file readable by the
-_zip_ storage format, and vice-versa.
+_zip_ storage format, and vice-versa. This may change depending
+on the outcome of current deliberations by the Zarr committee.
 
 By default, the XArray convention is supported for Zarr Version 2
 and used for both NCZarr files and pure Zarr files. It is not
-needed for Version 3 and is ignored.
+needed for Version 3, which has an equivalent array metadata key
+called "dimension_names"
 This means that every variable in the root group whose named dimensions
 are also in the root group will have an attribute called
 *\_ARRAY\_DIMENSIONS* that stores those dimension names.
@@ -443,6 +446,39 @@ Any of the following conditions will cause ''_ARRAY_DIMENSIONS'' not to be writt
 
 Note that this attribute is not needed for Zarr Version 3, and is ignored.
 
+## The ".zmetdata" Mode
+The NCzarr implementation of Version 2 also support the ".zmetadata" convention.
+This convention adds an extra, root-level object called ".zmetadata".
+This object is a JSON dictionary with this form:
+````
+{"metadata":
+    {
+        "<key>": <contents>,
+        "<key>": <contents>,
+	...
+        "<key>": <contents>
+    },
+"zarr_consolidated_format":1
+}
+````
+Each &lt;key&gt; refers to a content-bearing object and the &lt;contents&gt; is the JSON content of that object.
+An example might look as follows:
+````
+{
+  "metadata":
+    {
+        ".zgroup": {"zarr_format": 2},
+        ".zattr":  {"globalfloat": 1},
+        "v/.zarray": {"zarr_format": 2, "shape": [1], "dtype": "<i4", "chunks": [1], "fill_value": -2147483647, "order": "C", "compressor": null, "filters": null}
+    },
+  "zarr_consolidated_format":1
+}
+````
+The .zmetadata object is used by default. If necessary, it can be disabled
+in either of two ways.
+1. The _nozmetadata_ mode tells the library to disable the use of .zmetadata for a file.
+2. The environment variable "NCNOZMETADATA", if set to any non-null value, will disable the use of .zmetadata for all files.
+
 # Examples {#nczarr_examples}
 
 Here are a couple of examples using the _ncgen_ and _ncdump_ utilities.

include/Makefile.am

@@ -52,7 +52,7 @@ BUILT_SOURCES = netcdf_json.h netcdf_proplist.h
 # is out of source.
 
 # Give the recipe for building netcdf_json.h
-netcdf_json.h: ${top_srcdir}/libdispatch/ncjson.c ${top_srcdir}/include/ncjson.h
+netcdf_json.h: ${top_srcdir}/libdispatch/ncjson.c ${top_srcdir}/include/ncjson.h ${top_srcdir}/include/ncexternl.h
 	rm -fr ${builddir}/netcdf_json.h
 	cat ${srcdir}/ncjson.h \
 	    | sed -e '/!NCJSON_H/d' \
@@ -63,7 +63,7 @@ netcdf_json.h: ${top_srcdir}/libdispatch/ncjson.c ${top_srcdir}/include/ncjson.h
 
 # netcdf_proplist is analogous to netcdf_json but, of course, using libdispatch/ncproplist.c and include/ncproplist.h
 # Give the recipe for building netcdf_proplist.h. Basically same as for netcdf_json.h
-netcdf_proplist.h: ${top_srcdir}/libdispatch/ncproplist.c ${top_srcdir}/include/ncproplist.h
+netcdf_proplist.h: ${top_srcdir}/libdispatch/ncproplist.c ${top_srcdir}/include/ncproplist.h ${top_srcdir}/include/ncexternl.h
 	rm -fr ${builddir}/netcdf_proplist.h
 	cat ${srcdir}/ncproplist.h \
 	    | sed -e '/!NCPROPLIST_H/d' \

include/ncjson.h

@@ -113,7 +113,7 @@ OPTEXPORT int NCJinsert(NCjson* object, const char* key, NCjson* jvalue);
 /* Insert key-value pair into a dict object. key and value will be copied */
 OPTEXPORT int NCJinsertstring(NCjson* object, const char* key, const char* value);
 
-/* Overwrite key-value pair into a dict object. Act like NCJinsert if key not found */
+/* Overwrite key-value pair in a dict object. Act like NCJinsert if key not found */
 OPTEXPORT int NCJoverwrite(NCjson* object, const char* key, NCjson* value);
 
 /* Insert key-value pair into a dict object. key and value will be copied */
@@ -164,7 +164,19 @@ OPTEXPORT void NCJdictsort(NCjson* jdict);
 
 /**************************************************/
 /* Error detection helper */
-#define NCJcheck(expr) do{if((expr) < 0) {stat = (NCJ_ERR); goto done;}}while(0)
+#undef NCJDEBUG
+#ifdef NCJDEBUG
+static int
+NCJBREAKPOINT(int err)
+{
+    (void)NCJBREAKPOINT;
+    return err;
+}
+#else
+#define NCJBREAKPOINT(err) (err)
+#endif /*NCJDEBUG*/
+#define NCJcheck(expr) do{if((expr) < 0) {stat = NCJBREAKPOINT(NCJ_ERR); goto done;}}while(0)
+
 /**************************************************/
 
 #endif /*!NCJSON_H*/ /* Leave the ! as a tag for sed */

include/nclog.h

@@ -11,7 +11,7 @@
 #include <stdio.h>
 #include "ncexternl.h"
 
-#undef NCCATCH
+#define NCCATCH
 
 #define NCENVLOGGING "NCLOGGING"
 #define NCENVTRACING "NCTRACING"

include/netcdf.h

@@ -541,8 +541,9 @@ by the desired type. */
 #define NC_ENOOBJECT     (-141)    /**< Some object not found */
 #define NC_EPLUGIN       (-142)    /**< Unclassified failure in accessing a dynamically loaded plugin> */
 #define NC_ENOTZARR      (-143)    /**< Malformed (NC)Zarr file */
-
-#define NC4_LAST_ERROR   (-143)    /**< @internal All netCDF errors > this. */
+#define NC_EZARRMETA     (-144)    /**< Malformed (NC)Zarr file consolidated metadata */
+
+#define NC4_LAST_ERROR   (-144)    /**< @internal All netCDF errors > this. */
 
 /* Errors for all remote access methods(e.g. DAP and CDMREMOTE)*/
 #define NC_EURL         (NC_EDAPURL)   /**< Malformed URL */

libdispatch/derror.c

@@ -287,7 +287,9 @@ const char *nc_strerror(int ncerr1)
 	 return "NetCDF: Unclassified failure in accessing a dynamically loaded plugin";
       case NC_ENOTZARR:
 	 return "Malformed (NC)Zarr file";
-     default:
+      case NC_EZARRMETA:
+         return "Malformed (NC)Zarr file consolidated metadata";
+      default:
 #ifdef USE_PNETCDF
         /* The behavior of ncmpi_strerror here is to return
            NULL, not a string.  This causes problems in (at least)

libdispatch/dinfermodel.c

@@ -170,6 +170,7 @@ static const struct MODEINFER modenegations[] = {
 {"bytes","nczarr"}, /* bytes negates (nc)zarr */
 {"bytes","zarr"},
 {"noxarray","xarray"},
+{"nozmetadata","zmetadata"},
 {NULL,NULL}
 };
 
@@ -409,6 +410,7 @@ envvlist2string(NClist* envv, const char* delim)
     NCbytes* buf = NULL;
     char* result = NULL;
 
+    NC_UNUSED(delim);
     if(envv == NULL || nclistlength(envv) == 0) return NULL;
     buf = ncbytesnew();
     for(i=0;i<nclistlength(envv);i+=2) {
@@ -719,6 +721,9 @@ cleanfragments(NClist* fraglenv, NClist* newlist)
 static int
 processfragmentkeys(const char* key, const char* value, NCmodel* model)
 {
+    NC_UNUSED(key);
+    NC_UNUSED(value);
+    NC_UNUSED(model);
     return NC_NOERR;
 }
 

libdispatch/ncjson.c

@@ -883,6 +883,8 @@ NCJclone(const NCjson* json, NCjson** clonep)
 {
     int stat = NCJ_OK;
     NCjson* clone = NULL;
+
+    if(clonep) *clonep = NULL;
     if(json == NULL) goto done;
     switch(NCJsort(json)) {
     case NCJ_INT:
@@ -967,21 +969,44 @@ NCJaddstring(NCjson* json, int sort, const char* s)
     return NCJTHROW(stat);
 }
 
-/* Insert key-value pair into a dict object. key will be strdup'd, jvalue will be claimed */
+/* Insert key-value pair into a dict object. key will be strdup'd, jvalue will be used without copying.
+   If key already exists, then it's value is overwritten
+*/
 OPTSTATIC int
-NCJinsert(NCjson* dict, const char* key, NCjson* jvalue)
+NCJinsert(NCjson* jdict, const char* key, NCjson* jvalue)
 {
     int stat = NCJ_OK;
+    size_t i;
     NCjson* jkey = NULL;
-    if(dict == NULL
-	|| dict->sort != NCJ_DICT
+    NCjson* jprev = NULL;
+    int found;
+
+    if(jdict == NULL
+	|| NCJsort(jdict) != NCJ_DICT
 	|| key == NULL
 	|| jvalue == NULL) {stat = NCJTHROW(NCJ_ERR); goto done;}
-    if((stat = NCJnewstring(NCJ_STRING,key,&jkey))==NCJ_ERR) goto done;
-    if((stat=listsetalloc(&dict->list,dict->list.len+2))<0) goto done;
-    if((stat = NCJappend(dict,jkey))==NCJ_ERR) goto done;
-    if((stat = NCJappend(dict,jvalue))==NCJ_ERR) goto done;
+    for(found=(-1),i=0;i < NCJdictlength(jdict); i++) {
+	jkey = NCJdictkey(jdict,i);
+	if (jkey != NULL && strcmp(NCJstring(jkey), key) == 0) {
+	    found = (int)i;
+	    break;
+	}
+    }
+    if(found >= 0) {
+	jprev = NCJdictvalue(jdict,found);
+	// replace existing values for new key
+	NCJreclaim(jprev); // free old value
+	NCJdictvalue(jdict,found) = jvalue; jvalue = NULL;
+	jkey = NULL; /* avoid reclamation */
+    } else { /* not found */
+        if((stat=listsetalloc(&jdict->list,jdict->list.len+2))<0) goto done;
+	NCJcheck(NCJnewstring(NCJ_STRING, key, (NCjson**)&jkey));
+	NCJcheck(NCJappend(jdict,jkey)); jkey = NULL;
+	NCJcheck(NCJappend(jdict,jvalue)); jvalue = NULL;
+    }
 done:
+    NCJreclaim(jkey);
+    NCJreclaim(jvalue);
     return NCJTHROW(stat);
 }