diff --git a/.github/workflows/run_tests_osx.yml b/.github/workflows/run_tests_osx.yml index 8f2d9d62af..3293dcc7cb 100644 --- a/.github/workflows/run_tests_osx.yml +++ b/.github/workflows/run_tests_osx.yml @@ -6,7 +6,7 @@ name: Run macOS-based netCDF Tests -on: [pull_request,workflow_dispatch] +on: [push,pull_request,workflow_dispatch] concurrency: group: ${{ github.workflow}}-${{ github.head_ref }} diff --git a/.github/workflows/run_tests_s3.yml b/.github/workflows/run_tests_s3.yml index ddbb353c8e..cf45621955 100644 --- a/.github/workflows/run_tests_s3.yml +++ b/.github/workflows/run_tests_s3.yml @@ -9,7 +9,7 @@ name: Run S3 netCDF Tests (under Ubuntu Linux) -on: [workflow_dispatch] +on: [push,workflow_dispatch] concurrency: group: ${{ github.workflow}}-${{ github.head_ref }} diff --git a/.github/workflows/run_tests_ubuntu.yml b/.github/workflows/run_tests_ubuntu.yml index 64b91bb99a..ab8ce5dcfa 100644 --- a/.github/workflows/run_tests_ubuntu.yml +++ b/.github/workflows/run_tests_ubuntu.yml @@ -4,7 +4,7 @@ name: Run Ubuntu/Linux netCDF Tests -on: [pull_request, workflow_dispatch] +on: [push,pull_request, workflow_dispatch] concurrency: group: ${{ github.workflow}}-${{ github.head_ref }} diff --git a/.github/workflows/run_tests_win_cygwin.yml b/.github/workflows/run_tests_win_cygwin.yml index 6e124448ed..27486bbc59 100644 --- a/.github/workflows/run_tests_win_cygwin.yml +++ b/.github/workflows/run_tests_win_cygwin.yml @@ -1,6 +1,6 @@ name: Run Cygwin-based tests -on: [pull_request,workflow_dispatch] +on: [push,pull_request,workflow_dispatch] concurrency: group: ${{ github.workflow}}-${{ github.head_ref }} diff --git a/.github/workflows/run_tests_win_mingw.yml b/.github/workflows/run_tests_win_mingw.yml index 978275cf6c..75ac3174cd 100644 --- a/.github/workflows/run_tests_win_mingw.yml +++ b/.github/workflows/run_tests_win_mingw.yml @@ -9,7 +9,7 @@ name: Run MSYS2, MinGW64-based Tests (Not Visual Studio) env: CPPFLAGS: "-D_BSD_SOURCE" -on: [pull_request,workflow_dispatch] +on: [push,pull_request,workflow_dispatch] concurrency: group: ${{ github.workflow}}-${{ github.head_ref }} diff --git a/docs/filters.md b/docs/filters.md index 995544b82d..e2bb58493f 100644 --- a/docs/filters.md +++ b/docs/filters.md @@ -14,8 +14,8 @@ should be aware. An option is now provided to automatically install HDF5 filters into a default location, or optionally into a user-specified location. This is described in -[Appendix H](#filters_appendixh) -(with supporting information in [Appendix G](#filters_appendixg)). +[Appendix I](#filters_appendixi) +(with supporting information in [Appendix H](#filters_appendixh)). * ***NCZarr Filter Support***
[NCZarr filters](#filters_nczarr) are now supported. @@ -900,11 +900,11 @@ typedef struct NCZ_codec_t { Currently always NCZ_CODEC_HDF5 */ const char* codecid; /* The name/id of the codec */ unsigned int hdf5id; /* corresponding hdf5 id */ - void (*NCZ_codec_initialize)(void); - void (*NCZ_codec_finalize)(void); - int (*NCZ_codec_to_hdf5)(const char* codec, int* nparamsp, unsigned** paramsp); - int (*NCZ_hdf5_to_codec)(size_t nparams, const unsigned* params, char** codecp); - int (*NCZ_modify_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* nparamsp, unsigned** paramsp); + void (*NCZ_codec_initialize)(NCproplist* env); + void (*NCZ_codec_finalize)(NCproplist* env); + int (*NCZ_codec_to_hdf5)(NCproplist* env, const char* codec, int* h5idp, int* nparamsp, unsigned** paramsp); + int (*NCZ_hdf5_to_codec)(NCproplist* env, int h5id, size_t nparams, const unsigned* params, char** codecp); + int (*NCZ_modify_parameters)(NCproplist* env, int* h5idp, size_t* vnparamsp, unsigned** vparamsp, size_t* nparamsp, unsigned** paramsp); } NCZ_codec_t; ```` @@ -922,12 +922,14 @@ visible parameters. ##### Signature ```` - int NCZ_codec_to_hdf(const char* codec, int* nparamsp, unsigned** paramsp); + int NCZ_codec_to_hdf(NCproplist* env, const char* codec, int* h5idp, int* nparamsp, unsigned** paramsp); ```` ##### Arguments -1. codec — (in) ptr to JSON string representing the codec. -2. nparamsp — (out) store the length of the converted HDF5 unsigned vector -3. paramsp — (out) store a pointer to the converted HDF5 unsigned vector; caller must free the returned vector. Note the double indirection. +1. env $mdash; (in) ptr to a property list of key+value pairs. +2. codec — (in) ptr to JSON string representing the codec. +3. h5idp — (in/out) the hdf5 filter id. +4. nparamsp — (out) store the length of the converted HDF5 unsigned vector +5. paramsp — (out) store a pointer to the converted HDF5 unsigned vector; caller must free the returned vector. Note the double indirection. Return Value: a netcdf-c error code. @@ -938,12 +940,12 @@ return a corresponding JSON codec representation of those visible parameters. ##### Signature ```` - int NCZ_hdf5_to_codec)(int ncid, int varid, size_t nparams, const unsigned* params, char** codecp); + int NCZ_hdf5_to_codec)(NCproplist* env, int id, size_t nparams, const unsigned* params, char** codecp); ```` ##### Arguments -1. ncid — the variables' containing group -2. varid — the containing variable +1. env — property list of key+value pairs. +2. id — the hdf5 id. 3. nparams — (in) the length of the HDF5 visible parameters vector 4. params — (in) pointer to the HDF5 visible parameters vector. 5. codecp — (out) store the string representation of the codec; caller must free. @@ -957,12 +959,12 @@ to a set of working parameters; also provide option to modify visible parameters ##### Signature ```` - int NCZ_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); + int NCZ_modify_parameters(NCproplist* env, int* idp, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); ```` ##### Arguments -1. ncid — (in) group id containing the variable. -2. varid — (in) the id of the variable to which this filter is being attached. +1. env — (in) property list of key+value pairs. +2. idp — (in/out) the hdf5 id. 3. vnparamsp — (in/out) the count of visible parameters 4. vparamsp — (in/out) the set of visible parameters 5. wnparamsp — (out) the count of working parameters @@ -977,8 +979,12 @@ This function is called as soon as a shared library is loaded and matched with a ##### Signature ```` - int NCZ_codec_initialize)(void); + int NCZ_codec_initialize)(NCproplist* env); ```` +##### Arguments + +1. env — (in) property list of key+value pairs. + Return Value: a netcdf-c error code. #### NCZ\_codec\_finalize @@ -989,8 +995,12 @@ If the client code does not invoke *nc\_finalize* then memory checkers may compl ##### Signature ```` - int NCZ_codec_finalize)(void); + int NCZ_codec_finalize)(NCproplist* env); ```` +##### Arguments + +1. env — (in) property list of key+value pairs. + Return Value: a netcdf-c error code. ### Multi-Codec API @@ -1013,7 +1023,27 @@ The list of returned items are used to try to provide defaults for any HDF5 filters that have no corresponding Codec. This is for internal use only. -## Appendix F. Standard Filters {#filters_appendixf} +## Appendix F. Default HDF5 Filter Codecs {#filters_appendixf} + +It is recognized that it will be a while (if ever) until +HDF5 filters also specify the necessary codec information. +In order to provide some support for filters that do not have +corresponding codec support, a "hdf5raw" codec manager is provided. + +This hdf5raw codec manager encodes the parameters of the HDF5 filter +into one of these two codec forms: + +* Zarr Version 2 + ````{"id": "hdf5raw", "hdf5id": ", "nparams": , "0": ...,"": }```` +* Zarr Version 3 + ````{"name": "hdf5raw, "configuration": {"hdf5id": , "nparams": , "0": ...,"": }}```` + +There are couple things to note about hdf5raw: +1. this cannot be used if a modify_parameters function is required. +2. this representation will not be usable by other Zarr implementations, unless of course they choose to implement it. + + +## Appendix G. Standard Filters {#filters_appendixg} Support for a select set of standard filters is built into the NetCDF API. Generally, they are accessed using the following generic API, where XXXX is @@ -1043,10 +1073,10 @@ Consider the zstandard compressor, which is one of the supported standard filter When installing the netcdf library, the following other libraries must be installed. 1. *libzstd.so* | *zstd.dll* | *libzstd.dylib* -- The actual zstandard compressor library; typically installed by using your platform specific package manager. -2. The HDF5 wrapper for *libzstd.so* -- There are several options for obtaining this (see [Appendix G](#filters_appendixg).) +2. The HDF5 wrapper for *libzstd.so* -- There are several options for obtaining this (see [Appendix H](#filters_appendixh).) 3. (Optional) The Zarr wrapper for *libzstd.so* -- you need this if you intend to read/write Zarr datasets that were compressed using zstandard; again see [Appendix G](#filters_appendixg). -## Appendix G. Finding Filters {#filters_appendixg} +## Appendix H. Finding Filters {#filters_appendixh} A major problem for filter users is finding an implementation of an HDF5 filter wrapper and (optionally) its corresponding NCZarr wrapper. There are several ways to do this. @@ -1067,7 +1097,7 @@ You can install this library to get access to these supported filters. It does not currently include the required NCZarr Codec API, so they are only usable with netcdf-4. This will change in the future. -## Appendix H. Auto-Install of Filter Wrappers {#filters_appendixh} +## Appendix I. Auto-Install of Filter Wrappers {#filters_appendixi} As part of the overall build process, a number of filter wrappers are built as shared libraries in the "plugins" directory. These wrappers can be installed as part of the overall netcdf-c installation process. @@ -1145,7 +1175,7 @@ The important thing to note is that at run-time, there are several cases to cons 3. HDF5_PLUGIN_DIR is not defined at either run-time or build-time -- no action needed 4. HDF5_PLUGIN_DIR is not defined at run-time but was defined at build-time -- this will probably fail -## Appendix I. A Warning on Backward Compatibility {#filters_appendixi} +## Appendix J. A Warning on Backward Compatibility {#filters_appendixj} The API defined in this document should accurately reflect the current state of filters in the netCDF-c library. Be aware that @@ -1172,4 +1202,4 @@ For additional information, see [Appendix B](#filters_appendixb). *Author*: Dennis Heimbigner
*Email*: dmh at ucar dot edu
*Initial Version*: 1/10/2018
-*Last Revised*: 5/18/2022 +*Last Revised*: 10/18/2023 diff --git a/docs/nczarr.md b/docs/nczarr.md index f18f58f007..35d3fb8659 100644 --- a/docs/nczarr.md +++ b/docs/nczarr.md @@ -736,7 +736,7 @@ ubyte enum _bool_t {FALSE=0, TRUE=1}; ```` The type complex64 is supported by by defining this compound type: ```` -compound _complex64_t { float64 i; float64 j;} +compound _Complex64_t { float64 i; float64 j;} ```` Strings present a problem because there is a proposal diff --git a/include/CMakeLists.txt b/include/CMakeLists.txt index 224bd2bb38..b9696df7ea 100644 --- a/include/CMakeLists.txt +++ b/include/CMakeLists.txt @@ -45,6 +45,10 @@ INSTALL(FILES ${netCDF_SOURCE_DIR}/include/netcdf_json.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT headers) +INSTALL(FILES ${netCDF_SOURCE_DIR}/include/netcdf_proplist.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + COMPONENT headers) + FILE(GLOB CUR_EXTRA_DIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*.h) SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} Makefile.am CMakeLists.txt) ADD_EXTRA_DIST("${CUR_EXTRA_DIST}") diff --git a/include/Makefile.am b/include/Makefile.am index 3b7e13839e..e49d8266ea 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -7,7 +7,10 @@ # Ed Hartnett, Dennis Heimbigner, Ward Fisher include_HEADERS = netcdf.h netcdf_meta.h netcdf_mem.h netcdf_aux.h \ -netcdf_filter.h netcdf_filter_build.h netcdf_filter_hdf5_build.h netcdf_dispatch.h netcdf_json.h +netcdf_filter.h netcdf_filter_build.h netcdf_filter_hdf5_build.h netcdf_dispatch.h netcdf_vutils.h + +# Built source +include_HEADERS += netcdf_json.h netcdf_proplist.h if BUILD_PARALLEL include_HEADERS += netcdf_par.h @@ -20,7 +23,7 @@ nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h ncauth.h \ ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h ncexternl.h \ ncpathmgr.h ncindex.h hdf4dispatch.h hdf5internal.h nc_provenance.h \ hdf5dispatch.h ncmodel.h isnan.h nccrc.h ncexhash.h ncxcache.h \ -ncjson.h ncxml.h ncs3sdk.h +ncjson.h ncxml.h ncs3sdk.h ncproplist.h if USE_DAP noinst_HEADERS += ncdap.h @@ -32,6 +35,8 @@ endif EXTRA_DIST = CMakeLists.txt XGetopt.h netcdf_meta.h.in netcdf_dispatch.h.in +BUILT_SOURCES = netcdf_json.h netcdf_proplist.h + # netcdf_json.h is constructed as a header-only file for use by # nczarr code wrappers in the plugin directory. It is # constructed by joining libdispatch/ncjson.c with @@ -41,6 +46,21 @@ EXTRA_DIST = CMakeLists.txt XGetopt.h netcdf_meta.h.in netcdf_dispatch.h.in # having to reference libnetcdf in the nczarr code wrappers. # Give the recipe for rebuilding netcdf_json.h -makepluginjson:: - sed -e 's/NCJSON_H/NETCDF_JSON_H/' -e '/ncjson.h/d' <${srcdir}/ncjson.h > netcdf_json.h +netcdf_json.h: ${top_srcdir}/libdispatch/ncjson.c ${top_srcdir}/include/ncjson.h + sed -e 's/NCJSON_H/NETCDF_JSON_H/' -e '/ncjson.h/d' -e '/#endif[^!]*!NETCDF_JSON_H!/d' <${srcdir}/ncjson.h > netcdf_json.h sed -e '/ncjson.h/d' < ${srcdir}/../libdispatch/ncjson.c >> netcdf_json.h + echo '#endif /*NETCDF_JSON_H*/' >> netcdf_json.h + +# netcdf_proplist.h is constructed as a header-only file for use by +# nczarr code wrappers in the plugin directory. It is +# constructed by joining libdispatch/ncproplist.c with +# include/ncproplist.h. Additionally all the entry points are +# marked with a macro (OPTSTATIC) that marks the entry point as +# static inside netcdf_proplist.h. This is an ugly hack to avoid +# having to reference libnetcdf in the nczarr code wrappers. + +# Give the recipe for rebuilding netcdf_proplist.h +netcdf_proplist.h: ${top_srcdir}/libdispatch/ncproplist.c ${top_srcdir}/include/ncproplist.h + sed -e 's/NCPROPLIST_H/NETCDF_PROPLIST_H/' -e '/ncproplist.h/d' -e '/#endif[^!]*!NETCDF_PROPLIST_H!/d' <${srcdir}/ncproplist.h > netcdf_proplist.h + sed -e '/ncproplist.h/d' < ${srcdir}/../libdispatch/ncproplist.c >> netcdf_proplist.h + echo '#endif /*NETCDF_PROPLIST_H*/' >> netcdf_proplist.h diff --git a/include/nc_hashmap.h b/include/nc_hashmap.h deleted file mode 100644 index c92b7ab4e4..0000000000 --- a/include/nc_hashmap.h +++ /dev/null @@ -1,22 +0,0 @@ -/*! \file - -Copyright 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, -2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, -2015, 2016, 2017, 2018 -University Corporation for Atmospheric Research/Unidata. - -See \ref copyright file for more info. - -*/ - -#ifndef HASHMAP_H_INCLUDED -#define HASHMAP_H_INCLUDED -#include - -/** Hashmap structure (forward declaration) */ -struct s_hashmap; -typedef struct s_hashmap NC_hashmap; - - - -#endif diff --git a/include/ncjson.h b/include/ncjson.h index 1862897405..23117fda49 100644 --- a/include/ncjson.h +++ b/include/ncjson.h @@ -29,7 +29,7 @@ and do the command: #define OPTEXPORT static #else #define OPTEXPORT DLLEXPORT -#endif /*NETCDF_JSON_H*/ +#endif /**************************************************/ /* Json object sorts (note use of term sort rather than e.g. type or discriminant) */ @@ -114,6 +114,9 @@ OPTEXPORT int NCJinsert(NCjson* object, const char* key, NCjson* jvalue); /* Insert key-value pair into a dict object. key and value will be copied */ OPTEXPORT int NCJinsertstring(NCjson* object, const char* key, const char* value); +/* Insert key-value pair into a dict object. key and value will be copied */ +OPTEXPORT int NCJinsertint(NCjson* object, const char* key, int n); + /* Unparser to convert NCjson object to text in buffer */ OPTEXPORT int NCJunparse(const NCjson* json, unsigned flags, char** textp); @@ -149,5 +152,4 @@ OPTEXPORT const char* NCJtotext(const NCjson* json, unsigned flags); /**************************************************/ -#endif /*NCJSON_H*/ - +#endif /*!NCJSON_H!*/ /* Leave the ! as a tag for sed */ diff --git a/include/ncproplist.h b/include/ncproplist.h new file mode 100644 index 0000000000..c1e4f89453 --- /dev/null +++ b/include/ncproplist.h @@ -0,0 +1,124 @@ +/********************************************************************* + * Copyright 2018, UCAR/Unidata + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + * $Header$ + *********************************************************************/ +#ifndef NCPROPLIST_H +#define NCPROPLIST_H + +#if HAVE_CONFIG_H +#include "config.h" +#endif +#include +#include "ncexternl.h" + +/* +WARNING: +If you modify this file, +then you need to got to +the include/ directory +and do the command: + make makepluginjson +*/ + +/* Inside libnetcdf and for plugins, export the json symbols */ +#ifndef DLLEXPORT +#ifdef _WIN32 +#define DLLEXPORT __declspec(dllexport) +#else +#define DLLEXPORT +#endif +#endif + +/* Override for plugins */ +#ifdef NETCDF_PROPLIST_H +#define OPTEXPORT static +#else +#define OPTEXPORT DLLEXPORT +#endif + +/**************************************************/ +/* +This is used to store a property list mapping a small number of +fixed-sized key strings to an arbitrary uintptr_t value. The +uintptr_t type is used to ensure that the value can be a pointer or a +small string upto sizeof(uintptr_t) - 1 (for trailing nul). The big +problem is reclaiming the value if it a pointer. The fact that the +number of keys is small makes it feasible to use linear search. +This is currently only used for plugins, but may be extended to other uses. +*/ + +/*! Proplist-related structs. + NOTES: + 1. 'value' is the an arbitrary uintptr_t integer or void* pointer. + + WARNINGS: + 1. It is critical that |uintptr_t| == |void*| +*/ + +#define NCPROPSMAXKEY 31 /* characters assert (NCPROPSMAXKEY+1)/8 == 0*/ + +/* Returns 0 => error; 1 => success */ +typedef int (*NCPreclaimfcn)(uintptr_t userdata, const char* key, void* value, uintptr_t size); + +/* The property list proper is a sequence of these objects */ +typedef struct NCProperty { + char key[NCPROPSMAXKEY+1]; /* copy of the key string; +1 for trailing nul */ + uintptr_t flags; +# define NCPF_SIMPLE (1<<0) /* non-reclaimable */ +# define NCPF_BYTES (1<<1) /* reclaimable bytes */ +# define NCPF_COMPLEX (1<<2) /* extended case */ + uintptr_t value; + uintptr_t size; /* size = |value| as ptr to memory, if string, then include trailing nul */ + uintptr_t userdata; /* extra data for following functions */ + NCPreclaimfcn reclaim; +} NCProperty; + +/* +The property list object. +*/ +typedef struct NCproplist { + size_t alloc; /* allocated space to hold properties */ + size_t count; /* # of defined properties */ + NCProperty* properties; +} NCproplist; + +#if defined(_CPLUSPLUS_) || defined(__CPLUSPLUS__) +extern "C" { +#endif + +#ifndef NETCDF_PROPLIST_H + +/* Suppress create/modify function for plugin use */ + +/* Create, free, etc. */ +OPTEXPORT NCproplist* ncplistnew(void); +OPTEXPORT int ncplistfree(NCproplist*); + +/* Insert properties */ +OPTEXPORT int ncplistadd(NCproplist* plist,const char* key, uintptr_t value); /* use when reclaim not needed */ +OPTEXPORT int ncplistaddstring(NCproplist* plist, const char* key, const char* str); /* use when value is simple string (char*) */ +OPTEXPORT int ncplistaddbytes(NCproplist* plist, const char* key, void* value, uintptr_t size); /* use when value is simple ptr and reclaim is simple free function */ +OPTEXPORT int ncplistaddx(NCproplist* plist, const char* key, void* value, uintptr_t size, uintptr_t userdata, NCPreclaimfcn); /* fully extended case */ + +#endif /*NETCDF_PROPLIST_H*/ + +/* +Lookup key and return value. +Return 1 if found, 0 otherwise; returns the data in datap if !null +*/ +OPTEXPORT int ncplistget(const NCproplist*, const char* key, uintptr_t* datap, uintptr_t* sizep); + +/* Iteration support */ + +/* Return the number of properties in the property list */ +#define ncplistlen(plist) (((NCproplist)(plist))->count) + +/* get the ith key+value */ +OPTEXPORT int ncplistith(const NCproplist*, size_t i, char* const * keyp, uintptr_t const * valuep, uintptr_t* sizep); + +#if defined(_CPLUSPLUS_) || defined(__CPLUSPLUS__) +} +#endif + +#endif /*!NCPROPLIST_H!*/ /* Leave the ! as a tag for sed */ diff --git a/include/netcdf_filter_build.h b/include/netcdf_filter_build.h index a38143e97f..4d3924d313 100644 --- a/include/netcdf_filter_build.h +++ b/include/netcdf_filter_build.h @@ -21,6 +21,14 @@ #define NETCDF_FILTER_BUILD_H 1 #include "netcdf_filter_hdf5_build.h" +/* Avoid inclusion of both netcf_json.h and ncjson.h */ +#ifndef NCJSON_H +#include "netcdf_json.h" +#endif +/* Ditto */ +#ifndef NCPROPLIST_H +#include "netcdf_proplist.h" +#endif /**************************************************/ /* Build To a NumCodecs-style C-API for Filters */ @@ -28,9 +36,6 @@ /* Version of the NCZ_codec_t structure */ #define NCZ_CODEC_CLASS_VER 1 -/* Version of the NCZ_codec_env_t structure */ -#define NCZ_CODEC_ENV_VER 1 - /* List of the kinds of NCZ_codec_t formats */ #define NCZ_CODEC_HDF5 1 /* HDF5 <-> Codec converter */ @@ -87,20 +92,21 @@ The function pointers defined in NCZ_codec_t manipulate HDF5 parameters and NumC * Initialize use of the filter. This is invoked when a filter is loaded. -void (*NCZ_codec_initialize)(void); +void (*NCZ_codec_initialize)(struct NCproplist*); * Finalize use of the filter. Since HDF5 does not provide this functionality, the codec may need to do it. See H5Zblosc.c for an example. This function is invoked when a filter is unloaded. -void (*NCZ_codec_finalize)(void); +void (*NCZ_codec_finalize)(struct NCproplist*); * Convert a JSON representation to an HDF5 representation. Invoked when a NumCodec JSON Codec is extracted from Zarr metadata. -int (*NCZ_codec_to_hdf5)(void* codec_env, const char* codec, int* nparamsp, unsigned** paramsp); +int (*NCZ_codec_to_hdf5)(struct NCproplist* env, const char* codec, int* idp, int* nparamsp, unsigned** paramsp); -@param codec_env -- (in) extra environmental information +@param env -- (in) extra environmental information @param codec -- (in) ptr to JSON string representing the codec. +@param id -- the hdf5 filter id number; @param nparamsp -- (out) store the length of the converted HDF5 unsigned vector @param paramsp -- (out) store a pointer to the converted HDF5 unsigned vector; caller frees. Note the double indirection. @@ -109,9 +115,10 @@ int (*NCZ_codec_to_hdf5)(void* codec_env, const char* codec, int* nparamsp, unsi * Convert an HDF5 vector of visible parameters to a JSON representation. -int (*NCZ_hdf5_to_codec)(void* codec_env, size_t nparams, const unsigned* params, char** codecp); +int (*NCZ_hdf5_to_codec)(const struct NCproplist* env, int id, size_t nparams, const unsigned* params, char** codecp); -@param codec_env -- (in) extra environmental information +@param env -- (in) extra environmental information +@param id -- the hdf5 filter id number; @param nparams -- (in) the length of the HDF5 unsigned vector @param params -- (in) pointer to the HDF5 unsigned vector. @param codecp -- (out) store the string representation of the codec; caller must free. @@ -120,10 +127,10 @@ int (*NCZ_hdf5_to_codec)(void* codec_env, size_t nparams, const unsigned* params * Convert a set of visible parameters to a set of working parameters using extra environmental information. Also allows for changes to the visible parameters. Invoked before filter is actually used. -int (*NCZ_modify_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); +int (*NCZ_modify_parameters)(const struct NCproplist* env, int* idp, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); -@param ncid -- (in) ncid of the variable's group -@param varid -- (in) varid of the variable +@param env -- (in) properties, including file ncid and the variable varid +@param idp -- (in/out) the hdf5 filter id number; @params vnparamsp -- (in/out) number of visible parameters @params vparamsp -- (in/out) vector of visible parameters @params wnparamsp -- (out) number of working parameters @@ -132,9 +139,10 @@ int (*NCZ_modify_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** * Convert an HDF5 vector of visible parameters to a JSON representation. -int (*NCZ_hdf5_to_codec)(void* codec_env, size_t nparams, const unsigned* params, char** codecp); +int (*NCZ_hdf5_to_codec)(const struct NCproplist* env, int id, size_t nparams, const unsigned* params, char** codecp); -@param codec_env -- (in) extra environmental information +@param env -- (in) extra environmental information +@param id -- (in) the hdf5 filter id number; @param nparams -- (in) the length of the HDF5 unsigned vector @param params -- (in) pointer to the HDF5 unsigned vector. @param codecp -- (out) store the string representation of the codec; caller must free. @@ -142,16 +150,8 @@ int (*NCZ_hdf5_to_codec)(void* codec_env, size_t nparams, const unsigned* params */ -/* -The following struct is passed to all NCZ_codec_t functions -to provide extra environmental information. -*/ -typedef struct NCZ_codec_env_t { - int codec_env_version; - int zarrformat; -} NCZ_codec_env_t; - -#define NCZ_CODEC_ENV_EMPTY {NCZ_CODEC_ENV_VER, 0} +/* Opaque */ +struct NCproplist; /* The struct that provides the necessary filter info. @@ -164,11 +164,11 @@ typedef struct NCZ_codec_t { Currently always NCZ_CODEC_HDF5 */ const char* codecid; /* The name/id of the codec */ unsigned int hdf5id; /* corresponding hdf5 id */ - void (*NCZ_codec_initialize)(void); - void (*NCZ_codec_finalize)(void); - int (*NCZ_codec_to_hdf5)(void* codec_env, const char* codec, size_t* nparamsp, unsigned** paramsp); - int (*NCZ_hdf5_to_codec)(void* codec_env, size_t nparams, const unsigned* params, char** codecp); - int (*NCZ_modify_parameters)(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); + void (*NCZ_codec_initialize)(const struct NCproplist* env); + void (*NCZ_codec_finalize)(const struct NCproplist* env); + int (*NCZ_codec_to_hdf5)(const struct NCproplist* env, const char* codec, int* idp, size_t* nparamsp, unsigned** paramsp); + int (*NCZ_hdf5_to_codec)(const struct NCproplist* env, int id, size_t nparams, const unsigned* params, char** codecp); + int (*NCZ_modify_parameters)(const struct NCproplist* env, int* idp, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); } NCZ_codec_t; #ifndef NC_UNUSED diff --git a/include/netcdf_json.h b/include/netcdf_json.h index 46657548a6..75f7c1f620 100644 --- a/include/netcdf_json.h +++ b/include/netcdf_json.h @@ -29,7 +29,7 @@ and do the command: #define OPTEXPORT static #else #define OPTEXPORT DLLEXPORT -#endif /*NETCDF_JSON_H*/ +#endif /**************************************************/ /* Json object sorts (note use of term sort rather than e.g. type or discriminant) */ @@ -44,7 +44,10 @@ and do the command: #define NCJ_NSORTS 8 -/* No flags are currently defined, but the argument is a placeholder */ +/* Dump/text/unparse flags */ +#define NCJFLAG_NONE 0 +#define NCJFLAG_INDENTED 1 + /* Define a struct to store primitive values as unquoted strings. The sort will provide more info. Do not bother with @@ -106,7 +109,13 @@ OPTEXPORT int NCJaddstring(NCjson* json, int sort, const char* s); OPTEXPORT int NCJappend(NCjson* object, NCjson* value); /* Insert key-value pair into a dict object. key will be copied */ -OPTEXPORT int NCJinsert(NCjson* object, char* key, NCjson* value); +OPTEXPORT int NCJinsert(NCjson* object, const char* key, NCjson* jvalue); + +/* Insert key-value pair into a dict object. key and value will be copied */ +OPTEXPORT int NCJinsertstring(NCjson* object, const char* key, const char* value); + +/* Insert key-value pair into a dict object. key and value will be copied */ +OPTEXPORT int NCJinsertint(NCjson* object, const char* key, int n); /* Unparser to convert NCjson object to text in buffer */ OPTEXPORT int NCJunparse(const NCjson* json, unsigned flags, char** textp); @@ -118,7 +127,7 @@ OPTEXPORT int NCJclone(const NCjson* json, NCjson** clonep); /* dump NCjson* object to output file */ OPTEXPORT void NCJdump(const NCjson* json, unsigned flags, FILE*); /* convert NCjson* object to output string */ -OPTEXPORT const char* NCJtotext(const NCjson* json); +OPTEXPORT const char* NCJtotext(const NCjson* json, unsigned flags); #endif #if defined(__cplusplus) @@ -143,8 +152,6 @@ OPTEXPORT const char* NCJtotext(const NCjson* json); /**************************************************/ -#endif /*NETCDF_JSON_H*/ - /* Copyright 2018, UCAR/Unidata. See the COPYRIGHT file for more information. */ @@ -171,8 +178,11 @@ and do the command: #include +#undef NCJCATCH #undef NCJDEBUG -#ifdef NCJDEBUG +#undef NCJTRACE + +#ifdef NCJCATCH /* Warning: do not evaluate err more than once */ #define NCJTHROW(err) ncjbreakpoint(err) static int ncjbreakpoint(int err) {return err;} @@ -210,6 +220,8 @@ typedef struct NCJparser { long long num; int tf; int status; /* NCJ_ERR|NCJ_OK */ + unsigned flags; +# define NCJ_TRACE 1 } NCJparser; typedef struct NCJbuf { @@ -233,7 +245,7 @@ typedef struct NCJbuf { #define nulldup(x) ((x)?strdup(x):(x)) #endif -#ifdef NCJDEBUG +#if defined NCJDEBUG || defined NCJTRACE static char* tokenname(int token); #endif @@ -264,15 +276,6 @@ static int bytesappendc(NCJbuf* bufp, const char c); /* Hide everything for plugins */ #ifdef NETCDF_JSON_H #define OPTSTATIC static -static int NCJparsen(size_t len, const char* text, unsigned flags, NCjson** jsonp); -static int NCJnew(int sort, NCjson** objectp); -static int NCJnewstring(int sort, const char* value, NCjson** jsonp); -static int NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp); -static int NCJclone(const NCjson* json, NCjson** clonep); -static int NCJaddstring(NCjson* json, int sort, const char* s); -static int NCJinsert(NCjson* object, char* key, NCjson* jvalue); -static int NCJappend(NCjson* object, NCjson* value); -static int NCJunparse(const NCjson* json, unsigned flags, char** textp); #else /*!NETCDF_JSON_H*/ #define OPTSTATIC #endif /*NETCDF_JSON_H*/ @@ -295,6 +298,7 @@ NCJparsen(size_t len, const char* text, unsigned flags, NCjson** jsonp) parser = calloc(1,sizeof(NCJparser)); if(parser == NULL) {stat = NCJTHROW(NCJ_ERR); goto done;} + parser->flags = flags; parser->text = (char*)malloc(len+1+1); if(parser->text == NULL) {stat = NCJTHROW(NCJ_ERR); goto done;} @@ -537,14 +541,14 @@ NCJlex(NCJparser* parser) count = ((parser->pos) - start); if(NCJyytext(parser,start,count)) goto done; /* Discriminate the word string to get the proper sort */ - if(testbool(parser->yytext) == NCJ_OK) + if(testbool(parser->yytext)) token = NCJ_BOOLEAN; /* do int test first since double subsumes int */ - else if(testint(parser->yytext) == NCJ_OK) + else if(testint(parser->yytext)) token = NCJ_INT; - else if(testdouble(parser->yytext) == NCJ_OK) + else if(testdouble(parser->yytext)) token = NCJ_DOUBLE; - else if(testnull(parser->yytext) == NCJ_OK) + else if(testnull(parser->yytext)) token = NCJ_NULL; else token = NCJ_STRING; @@ -576,15 +580,24 @@ fprintf(stderr,"%s(%d): |%s|\n",tokenname(token),token,parser->yytext); done: if(parser->status == NCJ_ERR) token = NCJ_UNDEF; +#ifdef NCJTRACE + if(parser->flags & NCJ_TRACE) { + const char* txt = NULL; + switch(token) { + case NCJ_STRING: case NCJ_INT: case NCJ_DOUBLE: case NCJ_BOOLEAN: txt = parser->yytext; break; + default: break; + } + fprintf(stderr,">>>> token=%s:'%s'\n",tokenname(token),(txt?txt:"")); + } +#endif return token; } static int testnull(const char* word) { - if(strcasecmp(word,NCJ_TAG_NULL)==0) - return NCJTHROW(NCJ_OK); - return NCJTHROW(NCJ_ERR); + if(strcasecmp(word,NCJ_TAG_NULL)==0) return 1; + return 0; } static int @@ -592,8 +605,8 @@ testbool(const char* word) { if(strcasecmp(word,NCJ_TAG_TRUE)==0 || strcasecmp(word,NCJ_TAG_FALSE)==0) - return NCJTHROW(NCJ_OK); - return NCJTHROW(NCJ_ERR); + return 1; + return 0; } static int @@ -604,7 +617,7 @@ testint(const char* word) int count = 0; /* Try to convert to number */ ncvt = sscanf(word,"%lld%n",&i,&count); - return NCJTHROW((ncvt == 1 && strlen(word)==count ? NCJ_OK : NCJ_ERR)); + return (ncvt == 1 && strlen(word)==((size_t)count) ? 1 : 0); } static int @@ -614,16 +627,16 @@ testdouble(const char* word) double d; int count = 0; /* Check for Nan and Infinity */ - if(0==(int)strcasecmp("nan",word)) return NCJTHROW(NCJ_OK); - if(0==(int)strcasecmp("infinity",word)) return NCJTHROW(NCJ_OK); - if(0==(int)strcasecmp("-infinity",word)) return NCJTHROW(NCJ_OK); + if(0==(int)strcasecmp("nan",word)) return 1; + if(0==(int)strcasecmp("infinity",word)) return 1; + if(0==(int)strcasecmp("-infinity",word)) return 1; /* Allow the XXXf versions as well */ - if(0==(int)strcasecmp("nanf",word)) return NCJTHROW(NCJ_OK); - if(0==(int)strcasecmp("infinityf",word)) return NCJTHROW(NCJ_OK); - if(0==(int)strcasecmp("-infinityf",word)) return NCJTHROW(NCJ_OK); + if(0==(int)strcasecmp("nanf",word)) return 1; + if(0==(int)strcasecmp("infinityf",word)) return 1; + if(0==(int)strcasecmp("-infinityf",word)) return 1; /* Try to convert to number */ - ncvt = sscanf(word,"%lg%n",&d,&count); - return NCJTHROW((ncvt == 1 && strlen(word)==count ? NCJ_OK : NCJ_ERR)); + ncvt = sscanf(word,"%lg%n",&d,&count); + return (ncvt == 1 && strlen(word)==((size_t)count) ? 1 : 0); } static int @@ -808,7 +821,7 @@ unescape1(int c) return c; } -#ifdef NCJDEBUG +#if defined NCJDEBUG || defined NCJTRACE static char* tokenname(int token) { @@ -861,7 +874,7 @@ NCJcvt(const NCjson* jvalue, int outsort, struct NCJconst* output) break; case CASE(NCJ_INT,NCJ_BOOLEAN): - sscanf(jvalue->string,"%lldd",&output->ival); + sscanf(jvalue->string,"%lld",&output->ival); output->bval = (output->ival?1:0); break; case CASE(NCJ_INT,NCJ_INT): @@ -1032,7 +1045,7 @@ NCJaddstring(NCjson* json, int sort, const char* s) /* Insert key-value pair into a dict object. key will be strdup'd */ OPTSTATIC int -NCJinsert(NCjson* object, char* key, NCjson* jvalue) +NCJinsert(NCjson* object, const char* key, NCjson* jvalue) { int stat = NCJ_OK; NCjson* jkey = NULL; @@ -1045,6 +1058,43 @@ NCJinsert(NCjson* object, char* key, NCjson* jvalue) return NCJTHROW(stat); } +/* Insert key-value pair into a dict object. key will be strdup'd */ +OPTSTATIC int +NCJinsertstring(NCjson* object, const char* key, const char* value) +{ + int stat = NCJ_OK; + NCjson* jkey = NULL; + NCjson* jvalue = NULL; + if(key == NULL || value == NULL) + {stat = NCJTHROW(NCJ_ERR); goto done;} + if((stat = NCJnewstring(NCJ_STRING,key,&jkey))==NCJ_ERR) goto done; + if((stat = NCJnewstring(NCJ_STRING,value,&jvalue))==NCJ_ERR) goto done; + if((stat = NCJappend(object,jkey))==NCJ_ERR) goto done; + if((stat = NCJappend(object,jvalue))==NCJ_ERR) goto done; +done: + return NCJTHROW(stat); +} + +/* Insert key-value pair into a dict object. key will be strdup'd */ +OPTSTATIC int +NCJinsertint(NCjson* object, const char* key, int value) +{ + int stat = NCJ_OK; + NCjson* jkey = NULL; + NCjson* jvalue = NULL; + char digits[64]; + + if(key == NULL) + {stat = NCJTHROW(NCJ_ERR); goto done;} + if((stat = NCJnewstring(NCJ_STRING,key,&jkey))==NCJ_ERR) goto done; + snprintf(digits,sizeof(digits),"%d",value); + if((stat = NCJnewstring(NCJ_INT,digits,&jvalue))==NCJ_ERR) goto done; + if((stat = NCJappend(object,jkey))==NCJ_ERR) goto done; + if((stat = NCJappend(object,jvalue))==NCJ_ERR) goto done; +done: + return NCJTHROW(stat); +} + /* Append value to an array or dict object. */ OPTSTATIC int NCJappend(NCjson* object, NCjson* value) @@ -1209,19 +1259,8 @@ bytesappendc(NCJbuf* bufp, const char c) return bytesappend(bufp,s); } -OPTSTATIC void -NCJdump(const NCjson* json, unsigned flags, FILE* out) -{ - char* text = NULL; - (void)NCJunparse(json,0,&text); - if(out == NULL) out = stderr; - fprintf(out,"%s\n",text); - fflush(out); - nullfree(text); -} - OPTSTATIC const char* -NCJtotext(const NCjson* json) +NCJtotext(const NCjson* json, unsigned flags) { static char outtext[4096]; char* text = NULL; @@ -1233,18 +1272,37 @@ NCJtotext(const NCjson* json) return outtext; } +OPTSTATIC void +NCJdump(const NCjson* json, unsigned flags, FILE* out) +{ + const char* text = NCJtotext(json,flags); + if(out == NULL) out = stderr; + fprintf(out,"%s\n",text); + fflush(out); +} + /* Hack to avoid static unused warning */ static void netcdf_supresswarnings(void) { - void* ignore; + void* ignore = NULL; ignore = (void*)netcdf_supresswarnings; - ignore = (void*)NCJinsert; - ignore = (void*)NCJaddstring; - ignore = (void*)NCJcvt; - ignore = (void*)NCJdictget; ignore = (void*)NCJparse; + ignore = (void*)NCJparsen; + ignore = (void*)NCJreclaim; + ignore = (void*)NCJnew; + ignore = (void*)NCJnewstring; + ignore = (void*)NCJnewstringn; + ignore = (void*)NCJdictget; + ignore = (void*)NCJcvt; + ignore = (void*)NCJaddstring; + ignore = (void*)NCJappend; + ignore = (void*)NCJinsert; + ignore = (void*)NCJinsertstring; + ignore = (void*)NCJinsertint; + ignore = (void*)NCJunparse; + ignore = (void*)NCJclone; ignore = (void*)NCJdump; - ignore = (void*)NCJtotext; ignore = ignore; } +#endif /*NETCDF_JSON_H*/ diff --git a/libdispatch/ncutil.h b/include/netcdf_vutils.h similarity index 98% rename from libdispatch/ncutil.h rename to include/netcdf_vutils.h index 44ab508b36..9f362782f1 100644 --- a/libdispatch/ncutil.h +++ b/include/netcdf_vutils.h @@ -1,8 +1,8 @@ -/* Copyright 2018, UCAR/Unidata and OPeNDAP, Inc. +/* Copyright 2018, UCAR/Unidata See the COPYRIGHT file for more information. */ -#ifndef UTILS_H -#define UTILS_H 1 +#ifndef NCVUTILS_H +#define NCVUTILS_H 1 /* Define a header-only simple version of a dynamically expandable list and byte buffer */ /* To be used in code that should be independent of libnetcdf */ @@ -239,4 +239,4 @@ util_initialize(void) #define vsclear(vs) vssetlength(vs,0) #define vssetlength(vs,len) do{if((vs)!=NULL) (vs)->length=len;} while(0) -#endif /*UTILS_H*/ +#endif /*NCVUTIL_H*/ diff --git a/libdispatch/CMakeLists.txt b/libdispatch/CMakeLists.txt index 98b90bc96c..5f961bd80f 100644 --- a/libdispatch/CMakeLists.txt +++ b/libdispatch/CMakeLists.txt @@ -6,7 +6,7 @@ # See netcdf-c/COPYRIGHT file for more info. SET(libdispatch_SOURCES dcopy.c dfile.c ddim.c datt.c dattinq.c dattput.c dattget.c derror.c dvar.c dvarget.c dvarput.c dvarinq.c ddispatch.c nclog.c dstring.c dutf8.c dinternal.c doffsets.c ncuri.c nclist.c ncbytes.c nchashmap.c nctime.c nc.c nclistmgr.c utf8proc.h utf8proc.c dpathmgr.c dutil.c drc.c dauth.c dreadonly.c dnotnc4.c dnotnc3.c dinfermodel.c daux.c dinstance.c dinstance_intern.c -dcrc32.c dcrc32.h dcrc64.c ncexhash.c ncxcache.c ncjson.c ds3util.c dparallel.c dmissing.c) +dcrc32.c dcrc32.h dcrc64.c ncexhash.c ncxcache.c ncjson.c ds3util.c dparallel.c dmissing.c ncproplist.c ) # Netcdf-4 only functions. Must be defined even if not used SET(libdispatch_SOURCES ${libdispatch_SOURCES} dgroup.c dvlen.c dcompound.c dtype.c denum.c dopaque.c dfilter.c) diff --git a/libdispatch/Makefile.am b/libdispatch/Makefile.am index b49d4f6152..5da0a9f205 100644 --- a/libdispatch/Makefile.am +++ b/libdispatch/Makefile.am @@ -21,7 +21,8 @@ dinternal.c ddispatch.c dutf8.c nclog.c dstring.c ncuri.c nclist.c \ ncbytes.c nchashmap.c nctime.c nc.c nclistmgr.c dauth.c doffsets.c \ dpathmgr.c dutil.c dreadonly.c dnotnc4.c dnotnc3.c dinfermodel.c \ daux.c dinstance.c dcrc32.c dcrc32.h dcrc64.c ncexhash.c ncxcache.c \ -ncjson.c ds3util.c dparallel.c dmissing.c dinstance_intern.c +ncjson.c ds3util.c dparallel.c dmissing.c dinstance_intern.c \ +ncproplist.c # Add the utf8 codebase libdispatch_la_SOURCES += utf8proc.c utf8proc.h @@ -50,7 +51,7 @@ endif # ENABLE_BYTERANGE if ENABLE_S3 if ENABLE_S3_INTERNAL # Renamed to avoid conflicts with the HDF5 files -libdispatch_la_SOURCES += ncs3sdk_h5.c nch5s3comms.c nch5s3comms.h ncutil.h nccurl_setup.h \ +libdispatch_la_SOURCES += ncs3sdk_h5.c nch5s3comms.c nch5s3comms.h ncvstring.h nccurl_setup.h \ nccurl_sha256.c nccurl_sha256.h nccurl_hmac.c nccurl_hmac.h AM_CPPFLAGS += -I$(top_srcdir)/libncxml libdispatch_la_CPPFLAGS += ${AM_CPPFLAGS} diff --git a/libdispatch/nch5s3comms.c b/libdispatch/nch5s3comms.c index ba09b931b0..5cb5714241 100644 --- a/libdispatch/nch5s3comms.c +++ b/libdispatch/nch5s3comms.c @@ -92,7 +92,7 @@ #include "netcdf.h" #include "ncuri.h" -#include "ncutil.h" +#include "netcdf_vutils.h" /*****************/ @@ -1152,7 +1152,7 @@ NCH5_s3comms_s3r_open(const char* root, NCS3SVC svc, const char *region, const c /* Compute the signing key */ if (SUCCEED != NCH5_s3comms_signing_key(&signing_key, access_key, region, iso8601now)) HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "problem in NCH5_s3comms_s3comms_signing_key."); - if (nulllen(signing_key)==0) + if (signing_key == NULL) HGOTO_ERROR(H5E_ARGS, NC_EAUTH, NULL, "signing key cannot be null."); handle->signing_key = signing_key; signing_key = NULL; @@ -2033,7 +2033,7 @@ NCH5_s3comms_signing_key(unsigned char **mdp, const char *secret, const char *re if ((size_t)ret != (AWS4_secret_len - 1)) HGOTO_ERRORVA(H5E_ARGS, NC_EINVAL, FAIL, "problem writing AWS4+secret `%s`", secret); - if((md = (unsigned char*)malloc(SHA256_DIGEST_LENGTH))==NULL) + if((md = (unsigned char*)calloc(1,SHA256_DIGEST_LENGTH))==NULL) HGOTO_ERROR(H5E_ARGS, NC_ENOMEM, NULL, "could not malloc space for signing key ."); /* hash_func, key, len(key), msg, len(msg), digest_dest, digest_len_dest diff --git a/libdispatch/ncjson.c b/libdispatch/ncjson.c index ffc5350fcc..a2a4eace38 100644 --- a/libdispatch/ncjson.c +++ b/libdispatch/ncjson.c @@ -25,10 +25,11 @@ and do the command: #include "ncjson.h" +#undef NCJCATCH #undef NCJDEBUG -#define NCJTRACE +#undef NCJTRACE -#ifdef NCJDEBUG +#ifdef NCJCATCH /* Warning: do not evaluate err more than once */ #define NCJTHROW(err) ncjbreakpoint(err) static int ncjbreakpoint(int err) {return err;} @@ -122,16 +123,6 @@ static int bytesappendc(NCJbuf* bufp, const char c); /* Hide everything for plugins */ #ifdef NETCDF_JSON_H #define OPTSTATIC static -static int NCJparsen(size_t len, const char* text, unsigned flags, NCjson** jsonp); -static int NCJnew(int sort, NCjson** objectp); -static int NCJnewstring(int sort, const char* value, NCjson** jsonp); -static int NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp); -static int NCJclone(const NCjson* json, NCjson** clonep); -static int NCJaddstring(NCjson* json, int sort, const char* s); -static int NCJinsert(NCjson* object, const char* key, NCjson* jvalue); -static int NCJinsertstring(NCjson* object, const char* key, const char* value); -static int NCJappend(NCjson* object, NCjson* value); -static int NCJunparse(const NCjson* json, unsigned flags, char** textp); #else /*!NETCDF_JSON_H*/ #define OPTSTATIC #endif /*NETCDF_JSON_H*/ @@ -397,14 +388,14 @@ NCJlex(NCJparser* parser) count = ((parser->pos) - start); if(NCJyytext(parser,start,count)) goto done; /* Discriminate the word string to get the proper sort */ - if(testbool(parser->yytext) == NCJ_OK) + if(testbool(parser->yytext)) token = NCJ_BOOLEAN; /* do int test first since double subsumes int */ - else if(testint(parser->yytext) == NCJ_OK) + else if(testint(parser->yytext)) token = NCJ_INT; - else if(testdouble(parser->yytext) == NCJ_OK) + else if(testdouble(parser->yytext)) token = NCJ_DOUBLE; - else if(testnull(parser->yytext) == NCJ_OK) + else if(testnull(parser->yytext)) token = NCJ_NULL; else token = NCJ_STRING; @@ -452,9 +443,8 @@ fprintf(stderr,"%s(%d): |%s|\n",tokenname(token),token,parser->yytext); static int testnull(const char* word) { - if(strcasecmp(word,NCJ_TAG_NULL)==0) - return NCJTHROW(NCJ_OK); - return NCJTHROW(NCJ_ERR); + if(strcasecmp(word,NCJ_TAG_NULL)==0) return 1; + return 0; } static int @@ -462,8 +452,8 @@ testbool(const char* word) { if(strcasecmp(word,NCJ_TAG_TRUE)==0 || strcasecmp(word,NCJ_TAG_FALSE)==0) - return NCJTHROW(NCJ_OK); - return NCJTHROW(NCJ_ERR); + return 1; + return 0; } static int @@ -474,7 +464,7 @@ testint(const char* word) int count = 0; /* Try to convert to number */ ncvt = sscanf(word,"%lld%n",&i,&count); - return NCJTHROW((ncvt == 1 && strlen(word)==count ? NCJ_OK : NCJ_ERR)); + return (ncvt == 1 && strlen(word)==((size_t)count) ? 1 : 0); } static int @@ -484,16 +474,16 @@ testdouble(const char* word) double d; int count = 0; /* Check for Nan and Infinity */ - if(0==(int)strcasecmp("nan",word)) return NCJTHROW(NCJ_OK); - if(0==(int)strcasecmp("infinity",word)) return NCJTHROW(NCJ_OK); - if(0==(int)strcasecmp("-infinity",word)) return NCJTHROW(NCJ_OK); + if(0==(int)strcasecmp("nan",word)) return 1; + if(0==(int)strcasecmp("infinity",word)) return 1; + if(0==(int)strcasecmp("-infinity",word)) return 1; /* Allow the XXXf versions as well */ - if(0==(int)strcasecmp("nanf",word)) return NCJTHROW(NCJ_OK); - if(0==(int)strcasecmp("infinityf",word)) return NCJTHROW(NCJ_OK); - if(0==(int)strcasecmp("-infinityf",word)) return NCJTHROW(NCJ_OK); + if(0==(int)strcasecmp("nanf",word)) return 1; + if(0==(int)strcasecmp("infinityf",word)) return 1; + if(0==(int)strcasecmp("-infinityf",word)) return 1; /* Try to convert to number */ - ncvt = sscanf(word,"%lg%n",&d,&count); - return NCJTHROW((ncvt == 1 && strlen(word)==count ? NCJ_OK : NCJ_ERR)); + ncvt = sscanf(word,"%lg%n",&d,&count); + return (ncvt == 1 && strlen(word)==((size_t)count) ? 1 : 0); } static int @@ -731,7 +721,7 @@ NCJcvt(const NCjson* jvalue, int outsort, struct NCJconst* output) break; case CASE(NCJ_INT,NCJ_BOOLEAN): - sscanf(jvalue->string,"%lldd",&output->ival); + sscanf(jvalue->string,"%lld",&output->ival); output->bval = (output->ival?1:0); break; case CASE(NCJ_INT,NCJ_INT): @@ -932,6 +922,26 @@ NCJinsertstring(NCjson* object, const char* key, const char* value) return NCJTHROW(stat); } +/* Insert key-value pair into a dict object. key will be strdup'd */ +OPTSTATIC int +NCJinsertint(NCjson* object, const char* key, int value) +{ + int stat = NCJ_OK; + NCjson* jkey = NULL; + NCjson* jvalue = NULL; + char digits[64]; + + if(key == NULL) + {stat = NCJTHROW(NCJ_ERR); goto done;} + if((stat = NCJnewstring(NCJ_STRING,key,&jkey))==NCJ_ERR) goto done; + snprintf(digits,sizeof(digits),"%d",value); + if((stat = NCJnewstring(NCJ_INT,digits,&jvalue))==NCJ_ERR) goto done; + if((stat = NCJappend(object,jkey))==NCJ_ERR) goto done; + if((stat = NCJappend(object,jvalue))==NCJ_ERR) goto done; +done: + return NCJTHROW(stat); +} + /* Append value to an array or dict object. */ OPTSTATIC int NCJappend(NCjson* object, NCjson* value) @@ -1096,15 +1106,6 @@ bytesappendc(NCJbuf* bufp, const char c) return bytesappend(bufp,s); } -OPTSTATIC void -NCJdump(const NCjson* json, unsigned flags, FILE* out) -{ - const char* text = NCJtotext(json,flags); - if(out == NULL) out = stderr; - fprintf(out,"%s\n",text); - fflush(out); -} - OPTSTATIC const char* NCJtotext(const NCjson* json, unsigned flags) { @@ -1118,18 +1119,36 @@ NCJtotext(const NCjson* json, unsigned flags) return outtext; } +OPTSTATIC void +NCJdump(const NCjson* json, unsigned flags, FILE* out) +{ + const char* text = NCJtotext(json,flags); + if(out == NULL) out = stderr; + fprintf(out,"%s\n",text); + fflush(out); +} + /* Hack to avoid static unused warning */ static void netcdf_supresswarnings(void) { - void* ignore; + void* ignore = NULL; ignore = (void*)netcdf_supresswarnings; - ignore = (void*)NCJinsert; - ignore = (void*)NCJaddstring; - ignore = (void*)NCJcvt; - ignore = (void*)NCJdictget; ignore = (void*)NCJparse; + ignore = (void*)NCJparsen; + ignore = (void*)NCJreclaim; + ignore = (void*)NCJnew; + ignore = (void*)NCJnewstring; + ignore = (void*)NCJnewstringn; + ignore = (void*)NCJdictget; + ignore = (void*)NCJcvt; + ignore = (void*)NCJaddstring; + ignore = (void*)NCJappend; + ignore = (void*)NCJinsert; + ignore = (void*)NCJinsertstring; + ignore = (void*)NCJinsertint; + ignore = (void*)NCJunparse; + ignore = (void*)NCJclone; ignore = (void*)NCJdump; - ignore = (void*)NCJtotext; ignore = ignore; } diff --git a/libdispatch/nclist.c b/libdispatch/nclist.c index f2c3f4d47b..5ed21753eb 100644 --- a/libdispatch/nclist.c +++ b/libdispatch/nclist.c @@ -31,7 +31,7 @@ NClist* nclistnew(void) ncinitialized = 1; } */ - l = (NClist*)malloc(sizeof(NClist)); + l = (NClist*)calloc(1,sizeof(NClist)); if(l) { l->alloc=0; l->length=0; @@ -286,10 +286,13 @@ nclistclone(const NClist* l, int deep) void* nclistextract(NClist* l) { - void* result = l->content; + void* result = NULL; + if(l) { + result = l->content; l->alloc = 0; l->length = 0; l->content = NULL; + } return result; } diff --git a/libdispatch/ncproplist.c b/libdispatch/ncproplist.c new file mode 100644 index 0000000000..45d3880644 --- /dev/null +++ b/libdispatch/ncproplist.c @@ -0,0 +1,295 @@ +/* +Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata +See LICENSE.txt for license information. +*/ + +#include "config.h" +#include +#include +#include +#include +#ifdef HAVE_STDINT_H +#include +#endif +#include "ncdispatch.h" +#include "nccrc.h" +#include "ncproplist.h" + +#undef DEBUG +#define ASSERTIONS + +#ifdef ASSERTIONS +#define ASSERT(x) assert(x) +#else +#define ASSERT(x) +#endif + +/**************************************************/ +/* Hide everything for plugins */ +#ifdef NETCDF_PROPLIST_H +#define OPTSTATIC static +#else /*!NETCDF_PROPLIST_H*/ +#define OPTSTATIC +#endif /*NETCDF_PROPLIST_H*/ + +/**************************************************/ + +#define MINPROPS 2 +#define EXPANDFACTOR 1 + +#define hasspace(plist,nelems) ((plist)->alloc >= ((plist)->count + (nelems))) + +#define emptyprop {" ",0,0,0,NULL} + +/**************************************************/ + +#ifndef NETCDF_PROPLIST_H + +/* The non-read-only functions are elided in the unified header */ + +/* Forward */ +static int extendplist(NCproplist* plist, size_t nprops); + +/** + * Create new property list + * @return pointer to the created property list. + */ +OPTSTATIC NCproplist* +ncplistnew(void) +{ + NCproplist* plist = NULL; + plist = calloc(1,sizeof(NCproplist)); + /* Assume property list will hold at lease MINPROPS properties */ + plist->alloc = MINPROPS; + plist->count = 0; + plist->properties = (NCProperty*)calloc(MINPROPS,sizeof(NCProperty)); + return plist; +} + +/** + * Reclaim memory used by a property list + * @param plist to reclaim + * @return NC_NOERR if succeed, NC_EXXX otherwise. + */ +OPTSTATIC int +ncplistfree(NCproplist* plist) +{ + int i,stat = NC_NOERR; + if(plist == NULL) goto done; + if(plist->properties != NULL) { + for(i=0;icount;i++) { + NCProperty* prop = &plist->properties[i]; + void* ptr = (void*)prop->value; /* convert to ptr */ + assert(prop->flags & (NCPF_SIMPLE|NCPF_BYTES|NCPF_COMPLEX)); + if(prop->flags & NCPF_SIMPLE) continue; /* no reclaim needed */ + if(prop->flags & NCPF_BYTES) { + if(ptr != NULL) free(ptr); + } else { /* (prop->flags & NCPF_COMPLEX) */ + int ok; + assert(prop->reclaim != NULL); + ok = prop->reclaim(prop->userdata, prop->key, ptr, prop->size); + if(!ok && stat == NC_NOERR) stat = NC_EINVAL; + } + } + free(plist->properties); + } + free(plist); +done: + return stat; +} + +/** + * Add a non-reclaimable entry to the property list + * @param plist into which the value is be inserted. + * @param key + * @param value + * @return NC_NOERR if succeed, NC_EXXX otherwise. + */ +OPTSTATIC int +ncplistadd(NCproplist* plist, const char* key, uintptr_t value) +{ + int stat = NC_NOERR; + NCProperty* prop = NULL; + size_t keylen; + if(plist == NULL) goto done; + if(!hasspace(plist,1)) {if((stat = extendplist(plist,(plist->count+1)*EXPANDFACTOR))) goto done;} /* extra space */ + prop = &plist->properties[plist->count]; + keylen = strlen(key); + if(keylen > NCPROPSMAXKEY) keylen = NCPROPSMAXKEY; /* truncate */ + memcpy(prop->key,key,keylen); + prop->key[keylen] = '\0'; + prop->value = value; + prop->flags = NCPF_SIMPLE; + plist->count++; +done: + return stat; +} + +/** + * Add a reclaimable entry to the property list, where the value + * can be reclaimed using a simple free(); + * @param plist into which the value is be inserted. + * @param key + * @param value ptr to memory chunk + * @param size |*value| + * @return NC_NOERR if succeed, NC_EXXX otherwise. + */ +OPTSTATIC int +ncplistaddbytes(NCproplist* plist, const char* key, void* value, uintptr_t size) +{ + int stat = NC_NOERR; + NCProperty* prop = NULL; + size_t keylen; + if(plist == NULL) goto done; + if(!hasspace(plist,1)) {if((stat = extendplist(plist,(plist->count+1)*EXPANDFACTOR))) goto done;} /* extra space */ + prop = &plist->properties[plist->count]; + keylen = strlen(key); + if(keylen > NCPROPSMAXKEY) keylen = NCPROPSMAXKEY; /* truncate */ + memcpy(prop->key,key,keylen); + prop->key[keylen] = '\0'; + prop->value = (uintptr_t)value; + prop->flags = NCPF_BYTES; + plist->count++; +done: + return stat; +} + +/** + * Add a reclaimable entry to the property list, where the value + * can be reclaimed using a simple free(); + * @param plist into which the value is be inserted. + * @param key + * @param value ptr to memory chunk + * @param size |*value| + * @return NC_NOERR if succeed, NC_EXXX otherwise. + */ +OPTSTATIC int +ncplistaddstring(NCproplist* plist, const char* key, const char* str) +{ + uintptr_t size = 0; + if(str) size = (uintptr_t)strlen(str); + return ncplistaddbytes(plist,key,(void*)str,size); +} + +/** + * Most general case for adding a property. + * @param plist into which the value is be inserted. + * @param key + * @param value + * @param size + * @param userdata extra environment data for the reclaim function. + * @param fcn the reclaim function + * @return NC_NOERR if succeed, NC_EXXX otherwise. + */ +OPTSTATIC int +ncplistaddx(NCproplist* plist, const char* key, void* value, uintptr_t size, uintptr_t userdata, NCPreclaimfcn fcn) +{ + int stat = NC_NOERR; + NCProperty* prop = NULL; + size_t keylen; + if(plist == NULL) goto done; + if(!hasspace(plist,1)) {if((stat = extendplist(plist,(plist->count+1)*EXPANDFACTOR))) goto done;} /* extra space */ + prop = &plist->properties[plist->count]; + keylen = strlen(key); + if(keylen > NCPROPSMAXKEY) keylen = NCPROPSMAXKEY; /* truncate */ + memcpy(prop->key,key,keylen); + prop->key[keylen] = '\0'; + prop->value = (uintptr_t)value; + prop->size = size; + prop->reclaim = fcn; + prop->userdata = userdata; + prop->flags = NCPF_COMPLEX; + plist->count++; +done: + return stat; +} + +/* Increase size of a plist to be at lease nprops properties */ +static int +extendplist(NCproplist* plist, size_t nprops) +{ + int stat = NC_NOERR; + size_t newsize = plist->count + nprops; + NCProperty* newlist = NULL; + if((plist->alloc >= newsize) || (nprops == 0)) + goto done; /* Already enough space */ + newlist = realloc(plist->properties,newsize*sizeof(NCProperty)); + if(newlist == NULL) {stat = NC_ENOMEM; goto done;} + plist->properties = newlist; newlist = NULL; + plist->alloc = newsize; +done: + return stat; +} + +#endif /*NETCDF_PROPLIST_H*/ + +/** + * Lookup key and return value and size + * @param plist to search + * @param key for which to search + * @param valuep returned value + * @param sizep returned size + * @return NC_NOERR if key found, NC_EMPTY if key not found; NC_EXXX otherwise + */ +OPTSTATIC int +ncplistget(const NCproplist* plist, const char* key, uintptr_t* valuep, uintptr_t* sizep) +{ + int stat = NC_EEMPTY; /* assume not found til proven otherwise */ + size_t i; + NCProperty* props; + uintptr_t value = 0; + uintptr_t size = 0; + if(plist == NULL || key == NULL) goto done; + for(i=0,props=plist->properties;icount;i++,props++) { + if(strcmp(props->key,key)==0) { + value = props->value; + size = props->size; + stat = NC_NOERR; /* found */ + break; + } + } + if(valuep) *valuep = value; + if(sizep) *sizep = size; +done: + return stat; +} + +/* Iteration support */ + +/** + * Get the ith key+value.a + * @param plist to search + * @param i which property to get. + * @param keyp return i'th key + * @param valuep return i'th value + * @param valuep return i'th size + * @return NC_NOERR if success, NC_EINVAL otherwise + */ +OPTSTATIC int +ncplistith(const NCproplist* plist, size_t i, char* const * keyp, uintptr_t const * valuep, uintptr_t* sizep) +{ + int stat = NC_NOERR; + NCProperty* prop = NULL; + if(plist == NULL) goto done; + if(i >= plist->count) {stat = NC_EINVAL; goto done;} + prop = &plist->properties[i]; + if(keyp) *((char**)keyp) = (char*)prop->key; + if(valuep) *((uintptr_t*)valuep) = (uintptr_t)prop->value; + if(sizep) *sizep = prop->size; +done: + return stat; +} + +/**************************************************/ +/* Support Functions */ + +/* Hack to avoid static unused warning */ +static void +ncp_supresswarnings(void) +{ + void* ignore; + ignore = (void*)ncp_supresswarnings; + ignore = (void*)ncplistget; + ignore = (void*)ncplistith; + ignore = ignore; +} diff --git a/libncpoco/cp_unix.c b/libncpoco/cp_unix.c index 3ccea2ed03..7ff01f79ec 100755 --- a/libncpoco/cp_unix.c +++ b/libncpoco/cp_unix.c @@ -46,7 +46,7 @@ static void ncperr(const char* fcn, NCPSharedLib* lib) { const char* msg = dlerror(); - lib->err.msg[0] = '\0'; + memset(lib->err.msg,0,sizeof(lib->err.msg)); if(msg != NULL) { strlcat(lib->err.msg,fcn,sizeof(lib->err.msg)); strlcat(lib->err.msg,": ",sizeof(lib->err.msg)); diff --git a/libncpoco/cp_win32.c b/libncpoco/cp_win32.c index b4284ac101..896344f638 100755 --- a/libncpoco/cp_win32.c +++ b/libncpoco/cp_win32.c @@ -105,10 +105,9 @@ load(NCPSharedLib* lib , const char* path0, int flags) char* msg = NULL; FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, errcode, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), &msg, 0, NULL); - if(msg) { + memset(lib->err.msg,0,sizeof(lib->err.msg)); + if(msg) strncpy(lib->err.msg,msg,sizeof(lib->err.msg)); - } else - lib->err.msg[0] = '\0'; ret = NC_ENOTFOUND; goto ldone; } diff --git a/libncpoco/ncpoco.c b/libncpoco/ncpoco.c index 12d4b41eec..b12b2e0a9b 100755 --- a/libncpoco/ncpoco.c +++ b/libncpoco/ncpoco.c @@ -68,6 +68,7 @@ EXTERNL int ncpload(NCPSharedLib* lib, const char* path, int flags) { if(lib == NULL || path == NULL) return NC_EINVAL; + ncpclearerrmsg(lib); return lib->api.load(lib,path,flags); } @@ -75,6 +76,7 @@ EXTERNL int ncpunload(NCPSharedLib* lib) /* Unloads a shared library. */ { if(lib == NULL) return NC_EINVAL; + ncpclearerrmsg(lib); return lib->api.unload(lib); } @@ -93,6 +95,7 @@ EXTERNL void* ncpgetsymbol(NCPSharedLib* lib,const char* name) { if(lib == NULL) return NULL; + ncpclearerrmsg(lib); return lib->api.getsymbol(lib,name); } @@ -113,3 +116,11 @@ ncpgeterrmsg(NCPSharedLib* lib) if(lib == NULL) return NULL; return (lib->err.msg[0] == '\0' ? NULL : lib->err.msg); } + +/* Clear the last err msg. */ +EXTERNL void +ncpclearerrmsg(NCPSharedLib* lib) +{ + if(lib == NULL) return; + memset(lib->err.msg,0,sizeof(lib->err.msg)); +} diff --git a/libncpoco/ncpoco.h b/libncpoco/ncpoco.h index 43fb077bab..fc1e993b99 100755 --- a/libncpoco/ncpoco.h +++ b/libncpoco/ncpoco.h @@ -71,6 +71,9 @@ EXTERNL const char* ncpgetpath(NCPSharedLib*); /* Return last err msg */ EXTERNL const char* ncpgeterrmsg(NCPSharedLib* lib); +/* Clear the last err msg. */ +EXTERNL void ncpclearerrmsg(NCPSharedLib* lib); + EXTERNL const char* intstr(int err1); #endif /*NCPOCO_H*/ diff --git a/libnczarr/zarr.h b/libnczarr/zarr.h index 6b11648fc9..d250aea388 100644 --- a/libnczarr/zarr.h +++ b/libnczarr/zarr.h @@ -65,8 +65,12 @@ EXTERNL int ncz_splitkey(const char* path, NClist* segments); EXTERNL int NCZ_readdict(NCZMAP* zmap, const char* key, NCjson** jsonp); EXTERNL int NCZ_readarray(NCZMAP* zmap, const char* key, NCjson** jsonp); EXTERNL int ncz_nctypedecode(const char* snctype, nc_type* nctypep); -EXTERNL int ncz_nctype2dtype(nc_type nctype, int endianness, int purezarr,int len, char** dnamep); -EXTERNL int ncz_dtype2nctype(const char* dtype, nc_type typehint, int purezarr, nc_type* nctypep, int* endianp, int* typelenp); + +EXTERNL int ncz2_nctype2dtype(nc_type nctype, int endianness, int purezarr,int len, char** dnamep); +EXTERNL int ncz2_dtype2nctype(const char* dtype, nc_type typehint, int purezarr, nc_type* nctypep, int* endianp, int* typelenp); +EXTERNL int ncz3_nctype2dtype(nc_type nctype, int purezarr, int strlen, char** dnamep); +EXTERNL int ncz3_dtype2nctype(const char* dtype, int purezarr, nc_type* nctypep, int* typelenp); + EXTERNL int NCZ_inferattrtype(NCjson* value, nc_type typehint, nc_type* typeidp); EXTERNL int NCZ_inferinttype(unsigned long long u64, int negative); EXTERNL int ncz_fill_value_sort(nc_type nctype, int*); @@ -90,7 +94,10 @@ EXTERNL int NCZ_char2fixed(const char** charp, void* fixed, size_t count, int ma EXTERNL int NCZ_copy_data(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const void* memory, size_t count, int reading, void* copy); EXTERNL int NCZ_iscomplexjson(NCjson* value, nc_type typehint); EXTERNL int NCZ_makeFQN(NC_GRP_INFO_T* parent, NC_OBJ* object, NCbytes* fqn); -EXTERNL int NCZ_parseFQN(NC_FILE_INFO_T* file, const char* fqn0, NClist* path); +EXTERNL int NCZ_locateFQN(NC_GRP_INFO_T* parent, const char* fqn, NC_SORT sort, NC_OBJ** objectp); +EXTERNL char* NCZ_deescape(const char* s); +EXTERNL char* NCZ_backslashescape(const char* s); +EXTERNL int NCZ_sort(void* vec, size_t count, int (*compare)(const void*, const void*)); /* zwalk.c */ EXTERNL int NCZ_read_chunk(int ncid, int varid, size64_t* zindices, void* chunkdata); diff --git a/libnczarr/zclose.c b/libnczarr/zclose.c index a24b5e1997..1554d97e0f 100644 --- a/libnczarr/zclose.c +++ b/libnczarr/zclose.c @@ -51,6 +51,7 @@ ncz_close_file(NC_FILE_INFO_T* file, int abort) goto done; NCZ_freestringvec(0,zinfo->envv_controls); NC_authfree(zinfo->auth); + nullfree(zinfo); done: diff --git a/libnczarr/zfilter.c b/libnczarr/zfilter.c index b4e8b9b7ce..ce3fa4dbeb 100644 --- a/libnczarr/zfilter.c +++ b/libnczarr/zfilter.c @@ -62,6 +62,9 @@ #define NULLIFY(x) ((x)?(x):"NULL") +#define H5Z_FILTER_RAW 0 +#define H5Z_CODEC_RAW "hdf5raw" + static NCZ_Codec codec_empty = {NULL, NULL, 0}; static NCZ_HDF5 hdf5_empty = {0, {0,NULL}, {0, NULL}}; @@ -72,13 +75,16 @@ static const char* NCZ_KNOWN_PSEUDO_FILTERS[] = {"bytes",NULL}; /* WARNING: GLOBAL DATA */ /* TODO: move to common global state */ -/* All possible HDF5 filter plugins */ -/* Consider onverting to linked list or hash table or equivalent since very sparse */ -NCZ_Plugin* loaded_plugins[H5Z_FILTER_MAX]; -int loaded_plugins_max = -1; - -static NClist* codec_defaults = NULL; /* NClist */ -static NClist* default_libs = NULL; /* NClist; sources of the defaults */ +/* Plugin global data */ +static struct PluginGlobal { + /* All possible HDF5 filter plugins */ + /* Consider onverting to linked list or hash table or equivalent since very sparse */ + NCZ_Plugin* loaded_plugins[H5Z_FILTER_MAX]; + int loaded_plugins_max; /* max define plugin id */ + const struct CodecAPI* hdf5raw; + NClist* codec_defaults; /* NClist */ + NClist* default_libs; /* NClist; sources of the defaults */ +} plugins; static int NCZ_filter_initialized = 0; @@ -575,7 +581,7 @@ NCZ_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int* ids flist = var->filters; - nfilters = nclistlength(flist); /* including incomplets */ + nfilters = nclistlength(flist); /* including incomplete filters */ if(nfilters > 0 && ids != NULL) { int k; for(k=0;k>> DEBUGL: finalize reclaim:\n"); #endif - for(i=0;i<=loaded_plugins_max;i++) { - if(loaded_plugins[i]) { - NCZ_unload_plugin(loaded_plugins[i]); - loaded_plugins[i] = NULL; + for(i=0;i<=plugins.loaded_plugins_max;i++) { + if(plugins.loaded_plugins[i]) { + NCZ_unload_plugin(plugins.loaded_plugins[i]); + plugins.loaded_plugins[i] = NULL; } } /* Reclaim the codec defaults */ - if(nclistlength(codec_defaults) > 0) { - for(i=0;i 0) { + for(i=0;i 0) { - for(i=0;i 0) { + for(i=0;i>> DEBUGL: NCZ_filter_finalize: reclaim default_lib[i]=%p\n",l); #endif @@ -731,8 +739,8 @@ NCZ_filter_finalize(void) #else memset(loaded_plugins,0,sizeof(loaded_plugins)); #endif - nclistfree(default_libs); default_libs = NULL; - nclistfree(codec_defaults); codec_defaults = NULL; + nclistfree(plugins.default_libs); plugins.default_libs = NULL; + nclistfree(plugins.codec_defaults); plugins.codec_defaults = NULL; done: NCZ_filter_initialized = 0; return ZUNTRACE(stat); @@ -743,10 +751,10 @@ NCZ_plugin_save(int filterid, NCZ_Plugin* p) { int stat = NC_NOERR; ZTRACE(6,"filterid=%d p=%p",filterid,p); - if(filterid <= 0 || filterid >= H5Z_FILTER_MAX) + if(filterid < 0 || filterid >= H5Z_FILTER_MAX) {stat = NC_EINVAL; goto done;} - if(filterid > loaded_plugins_max) loaded_plugins_max = filterid; - loaded_plugins[filterid] = p; + if(filterid > plugins.loaded_plugins_max) plugins.loaded_plugins_max = filterid; + plugins.loaded_plugins[filterid] = p; done: return ZUNTRACE(stat); } @@ -759,8 +767,8 @@ NCZ_plugin_loaded(int filterid, NCZ_Plugin** pp) ZTRACE(6,"filterid=%d",filterid); if(filterid <= 0 || filterid >= H5Z_FILTER_MAX) {stat = NC_EINVAL; goto done;} - if(filterid <= loaded_plugins_max) - plug = loaded_plugins[filterid]; + if(filterid <= plugins.loaded_plugins_max) + plug = plugins.loaded_plugins[filterid]; if(pp) *pp = plug; done: return ZUNTRACEX(stat,"plugin=%p",*pp); @@ -893,6 +901,8 @@ NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* j NCZ_Filter* filter = NULL; NCZ_Plugin* plugin = NULL; NCZ_VAR_INFO_T* zvar = (NCZ_VAR_INFO_T*)var->format_var_info; + char digits[64]; + const char* trueid = NULL; ZTRACE(6,"file=%s var=%s jfilter=%s",file->hdr.name,var->hdr.name,NCJtrace(jfilter)); @@ -906,15 +916,24 @@ NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* j if((stat = NCZF_codec2hdf(file,var,jfilter,filter,NULL))) goto done; /* Find the plugin for this filter */ - for(i=0;i<=loaded_plugins_max;i++) { - if (!loaded_plugins[i]) continue; - if(!loaded_plugins[i] || !loaded_plugins[i]->codec.codec) continue; /* no plugin or no codec */ - if(strcmp(filter->codec.id, loaded_plugins[i]->codec.codec->codecid) == 0) - {plugin = loaded_plugins[i]; break;} + for(i=1;i<=plugins.loaded_plugins_max;i++) { + NCZ_Plugin* p = plugins.loaded_plugins[i]; + if(p == NULL) continue; + if(p == NULL|| p->codec.codec == NULL) continue; /* no plugin or no codec */ + if(p->codec.ishdf5raw) { + /* get true id */ + snprintf(digits,sizeof(digits),"%d",p->hdf5.filter->id); + trueid = digits; + } else { + trueid = p->codec.codec->codecid; + } + if(strcmp(filter->codec.id, trueid) == 0) + {plugin = p; break;} } if(plugin != NULL) { /* Save the hdf5 id */ - filter->hdf5.id = plugin->codec.codec->hdf5id; + assert(plugin->hdf5.filter != NULL); + filter->hdf5.id = plugin->hdf5.filter->id; /* Convert codec to parameters */ if((stat = NCZF_codec2hdf(file,var,jfilter,filter,plugin))) goto done; filter->flags |= FLAG_VISIBLE; @@ -1048,8 +1067,9 @@ NCZ_load_all_plugins(void) struct stat buf; NClist* dirs = nclistnew(); char* defaultpluginpath = NULL; + - ZTRACE(6,""); + ZTRACE(6,""); #ifdef DEBUGL fprintf(stderr,">>> DEBUGL: NCZ_load_all_plugins\n"); @@ -1104,69 +1124,95 @@ NCZ_load_all_plugins(void) if((ret = NCZ_load_plugin_dir(dir))) goto done; } #ifdef DEBUGL - { int i; - fprintf(stderr,"codec_defaults:"); - for(i=0;icodec->hdf5id); } fprintf(stderr,"\n"); } #endif - if(nclistlength(codec_defaults)) { /* Try to provide default for any HDF5 filters without matching Codec. */ - /* Search the defaults */ - for(j=0;jcodec != NULL) { - const NCZ_codec_t* codec = dfalt->codec; - int hdf5id = codec->hdf5id; - NCZ_Plugin* p = NULL; - if(hdf5id < 0 || hdf5id > loaded_plugins_max) {ret = NC_EFILTER; goto done;} - p = loaded_plugins[hdf5id]; /* get candidate */ - if(p != NULL && p->hdf5.filter != NULL - && p->codec.codec == NULL) { + { /* See if we have the hdf5raw codec */ + size_t i; + for(i=0;icodec->hdf5id == H5Z_FILTER_RAW && strcasecmp(codec->codec->codecid,H5Z_CODEC_RAW)==0) { + plugins.hdf5raw = codec; + } + } + } + + if(nclistlength(plugins.codec_defaults)) { /* Try to provide default for any HDF5 filters without matching Codec. */ + int hdf5id; + /* Search all the plugins */ + for(hdf5id=1;hdf5id<=plugins.loaded_plugins_max;hdf5id++) { + NCZ_Plugin* p = plugins.loaded_plugins[hdf5id]; + if(p == NULL) continue; + if(p->hdf5.filter == NULL) continue; /* plugin has no hdf5 filter defined */ + if(p->codec.codec != NULL) continue; /* plugin already complete */ + /* Search the defaults for a matching codec */ + for(j=0;jcodec->hdf5id != hdf5id) continue; /* not a matching codec */ + /* we have a matching codec */ #ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: plugin defaulted: id=%u, codec=%s src=%s\n",hdf5id,codec->codecid,dfalt->codeclib->path); + fprintf(stderr,">>> DEBUGL: plugin defaulted: id=%u, codec=%s src=%s\n",hdf5id,dfalt->codec->codecid,dfalt->codeclib->path); #endif - p->codec.codec = codec; - p->codec.codeclib = dfalt->codeclib; - p->codec.defaulted = 1; - } + p->codec.codec = dfalt->codec; + p->codec.codeclib = dfalt->codeclib; + p->codec.defaulted = 1; + break; /* Do not search for other matches */ } } } - /* Mark all plugins for which we do not have both HDF5 and codec */ + /* Make a second pass over plugins to (1) apply hdf5raw codec if possible or (2) mark as incomplete */ { - int i; + int hdf5id; NCZ_Plugin* p; - for(i=0;ihdf5.filter == NULL || p->codec.codec == NULL) { - /* mark this entry as incomplete */ + for(hdf5id=1;hdf5id<=plugins.loaded_plugins_max;hdf5id++) { + p = plugins.loaded_plugins[hdf5id]; + if(p != NULL) { + if(p->hdf5.filter == NULL) { + /* no matching hdf5 filter, so mark this entry as incomplete */ p->incomplete = 1; + } else if(p->codec.codec == NULL) { + if(plugins.hdf5raw) { + /* We can default to using the hdf5raw codec manager */ + p->codec.codec = plugins.hdf5raw->codec; + p->codec.codeclib = plugins.hdf5raw->codeclib; + p->codec.defaulted = 1; + p->codec.ishdf5raw = 1; #ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: Incomplete plugin: id=%u; reasons: %s %s\n",i, - (p->hdf5.filter==NULL?"hdf5":""),(p->codec.codec==NULL?"codec":"")); + fprintf(stderr,">>> DEBUGL: plugin accepted via hdf5raw: id=%u\n",hdf5id); #endif + } else /*plugins.hdf5raw == NULL*/ + p->incomplete = 1; /* oh well */ } #ifdef DEBUGL - else - fprintf(stderr,">>> DEBUGL: plugin accepted: id=%u\n",i); + if(p->incomplete) + fprintf(stderr,">>> DEBUGL: Incomplete plugin: id=%u; reasons: %s %s\n",hdf5id, + (p->hdf5.filter==NULL?"hdf5":""),(p->codec.codec==NULL?"codec":"")); +#endif +#ifdef DEBUGL + if(p->hdf5.filter != NULL && p->codec.codec != NULL) + fprintf(stderr,">>> DEBUGL: plugin accepted: id=%u\n",hdf5id); #endif } } } - /* Iniitalize all remaining plugins */ + + /* Initialize all accepted plugins */ { int i; NCZ_Plugin* p; - for(i=0;iincomplete) continue; if(p->hdf5.filter != NULL && p->codec.codec != NULL) { if(p->codec.codec && p->codec.codec->NCZ_codec_initialize) - p->codec.codec->NCZ_codec_initialize(); + p->codec.codec->NCZ_codec_initialize(NULL); #ifdef DEBUGL fprintf(stderr,">>> DEBUGL: plugin initialized: id=%u\n",p->hdf5.filter->id); #endif @@ -1252,6 +1298,10 @@ NCZ_load_plugin_dir(const char* path) strlcat(file,path,flen); strlcat(file,"/",flen); strlcat(file,name,flen); +#if defined NAMEOPT || defined _WIN32 + /*triage because visual studio does a popup if the file will not load*/ + if(!pluginnamecheck(file)) continue; +#endif /* See if can load the file */ stat = NCZ_load_plugin(file,&plugin); switch (stat) { @@ -1263,9 +1313,8 @@ NCZ_load_plugin_dir(const char* path) } if(plugin != NULL) { id = plugin->hdf5.filter->id; - if(loaded_plugins[id] == NULL) { - loaded_plugins[id] = plugin; - if(id > loaded_plugins_max) loaded_plugins_max = id; + if(plugins.loaded_plugins[id] == NULL) { + if((stat = NCZ_plugin_save(id,plugin))) goto done; #ifdef DEBUGL fprintf(stderr,">>> DEBUGL: plugin loaded: %s\n",printplugin(plugin)); #endif @@ -1291,7 +1340,7 @@ loadcodecdefaults(const char* path, const NCZ_codec_t** cp, NCPSharedLib* lib, i int stat = NC_NOERR; int lib_used = 0; - nclistpush(default_libs,lib); + nclistpush(plugins.default_libs,lib); for(;*cp;cp++) { struct CodecAPI* c0; #ifdef DEBUGL @@ -1302,7 +1351,7 @@ loadcodecdefaults(const char* path, const NCZ_codec_t** cp, NCPSharedLib* lib, i c0->codec = *cp; c0->codeclib = lib; lib_used = 1; /* remember */ - nclistpush(codec_defaults,c0); c0 = NULL; + nclistpush(plugins.codec_defaults,c0); c0 = NULL; } done: if(lib_usedp) *lib_usedp = lib_used; @@ -1328,11 +1377,6 @@ NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp) if(plugp) *plugp = NULL; -#if defined NAMEOPT || defined _WIN32 - /*triage because visual studio does a popup if the file will not load*/ - if(!pluginnamecheck(path)) {stat = NC_ENOFILTER; goto done;} -#endif - /* load the shared library */ if((stat = ncpsharedlibnew(&lib))) goto done; if((stat = ncpload(lib,path,flags))) goto done; @@ -1452,8 +1496,8 @@ NCZ_unload_plugin(NCZ_Plugin* plugin) fprintf(stderr,">>> DEBUGL: unload: %s\n",printplugin(plugin)); #endif if(plugin->codec.codec && plugin->codec.codec->NCZ_codec_finalize) - plugin->codec.codec->NCZ_codec_finalize(); - if(plugin->hdf5.filter != NULL) loaded_plugins[plugin->hdf5.filter->id] = NULL; + plugin->codec.codec->NCZ_codec_finalize(NULL); + if(plugin->hdf5.filter != NULL) plugins.loaded_plugins[plugin->hdf5.filter->id] = NULL; if(plugin->hdf5.hdf5lib != NULL) (void)ncpsharedlibfree(plugin->hdf5.hdf5lib); if(!plugin->codec.defaulted && plugin->codec.codeclib != NULL) (void)ncpsharedlibfree(plugin->codec.codeclib); memset(plugin,0,sizeof(NCZ_Plugin)); @@ -1482,9 +1526,10 @@ pluginnamecheck(const char* name) char c = *p; if(c == '/') break; if(c == '.') { - if(count >= 3 && memcmp(p,".so",3)==0) return 1; - if(count >= 4 && memcmp(p,".dll",4)==0) return 1; - if(count >= 6 && memcmp(p,".dylib",6)==0) return 1; + /* exclude e.g xxx.so.1.2.3 in favor of xxx.so */ + if(count >= 3 && strcmp(p,".so")==0) return 1; + if(count >= 4 && strcmp(p,".dll")==0) return 1; + if(count >= 6 && strcmp(p,".dylib")==0) return 1; } } return 0; @@ -1554,6 +1599,7 @@ static int ensure_working(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, NCZ_Filter* filter) { int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; if(FILTERINCOMPLETE(filter)) {stat = THROW(NC_ENOFILTER); goto done;} if(!(filter->flags & FLAG_WORKING)) { @@ -1568,9 +1614,14 @@ ensure_working(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, NCZ_Filter* filter) printparams(filter->hdf5.working.nparams,filter->hdf5.working.params)); #endif if(filter->plugin && filter->plugin->codec.codec->NCZ_modify_parameters) { - stat = filter->plugin->codec.codec->NCZ_modify_parameters(ncidfor(var),var->hdr.id, + NCproplist* props = ncplistnew(); + ncplistadd(props,"zarr_format",zfile->zarr.zarr_format); + ncplistadd(props,"fileid",ncidfor(var)); + ncplistadd(props,"varid",var->hdr.id); + stat = filter->plugin->codec.codec->NCZ_modify_parameters(props,&filter->hdf5.id, &filter->hdf5.visible.nparams, &filter->hdf5.visible.params, &filter->hdf5.working.nparams, &filter->hdf5.working.params); + ncplistfree(props); #ifdef DEBUGF fprintf(stderr,">>> DEBUGF: NCZ_modify_parameters: stat=%d ncid=%d varid=%d filter=%s\n",stat, (int)ncidfor(var),(int)var->hdr.id, printfilter(filter)); diff --git a/libnczarr/zfilter.h b/libnczarr/zfilter.h index 975c50b642..d562dbc27e 100644 --- a/libnczarr/zfilter.h +++ b/libnczarr/zfilter.h @@ -71,8 +71,9 @@ typedef struct NCZ_Plugin { } hdf5; struct CodecAPI { int defaulted; /* codeclib was a defaulting library */ + int ishdf5raw; /* The codec is the hdf5raw codec */ const struct NCZ_codec_t* codec; - struct NCPSharedLib* codeclib; /* of the source codec; null if same as hdf5 */ + struct NCPSharedLib* codeclib; /* of the codec; null if same as hdf5 */ } codec; } NCZ_Plugin; @@ -83,7 +84,7 @@ typedef struct NCZ_Params { /* HDF5 Info */ typedef struct NCZ_HDF5 { - unsigned id; /**< HDF5 id corresponding to filterid. */ + int id; /**< HDF5 id corresponding to filterid. */ NCZ_Params visible; NCZ_Params working; } NCZ_HDF5; @@ -97,13 +98,13 @@ typedef struct NCZ_Codec { typedef struct NCZ_Filter { int flags; /**< Flags describing state of this filter. */ -# define FLAG_VISIBLE 1 /* If set, then visible parameters are defined */ -# define FLAG_WORKING 2 /* If set, then WORKING parameters are defined */ -# define FLAG_CODEC 4 /* If set, then visbile parameters come from an existing codec string */ -# define FLAG_HDF5 8 /* If set, => visible parameters came from nc_def_var_filter */ -# define FLAG_NEWVISIBLE 16 /* If set, => visible parameters were modified */ -# define FLAG_INCOMPLETE 32 /* If set, => filter has no complete matching plugin */ -# define FLAG_SUPPRESS 64 /* If set, => filter should not be used (probably because variable is not fixed size */ +# define FLAG_VISIBLE 1 /* If set, then visible parameters are defined */ +# define FLAG_WORKING 2 /* If set, then WORKING parameters are defined */ +# define FLAG_CODEC 4 /* If set, then visbile parameters come from an existing codec string */ +# define FLAG_HDF5 8 /* If set, => visible parameters came from nc_def_var_filter */ +# define FLAG_NEWVISIBLE 16 /* If set, => visible parameters were modified */ +# define FLAG_INCOMPLETE 32 /* If set, => filter has no complete matching plugin */ +# define FLAG_SUPPRESS 64 /* If set, => filter should not be used (probably because variable is not fixed size */ NCZ_HDF5 hdf5; NCZ_Codec codec; struct NCZ_Plugin* plugin; /**< Implementation of this filter. */ diff --git a/libnczarr/zformat.c b/libnczarr/zformat.c index 366660f84a..c92963dc0a 100644 --- a/libnczarr/zformat.c +++ b/libnczarr/zformat.c @@ -7,7 +7,6 @@ #include "zformat.h" #ifdef ENABLE_NCZARR_FILTERS #include "zfilter.h" -#include "netcdf_filter_build.h" #endif /**************************************************/ @@ -142,3 +141,37 @@ NCZF_codec2hdf(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, const NCjso return THROW(stat); } +/**************************************************/ +/* +From Zarr V2 Specification: +"The compressed sequence of bytes for each chunk is stored under +a key formed from the index of the chunk within the grid of +chunks representing the array. To form a string key for a +chunk, the indices are converted to strings and concatenated +with the dimension_separator character ('.' or '/') separating +each index. For example, given an array with shape (10000, +10000) and chunk shape (1000, 1000) there will be 100 chunks +laid out in a 10 by 10 grid. The chunk with indices (0, 0) +provides data for rows 0-1000 and columns 0-1000 and is stored +under the key "0.0"; the chunk with indices (2, 4) provides data +for rows 2000-3000 and columns 4000-5000 and is stored under the +key "2.4"; etc." +*/ + +/** + * @param R Rank + * @param chunkindices The chunk indices + * @param dimsep the dimension separator + * @param keyp Return the chunk key string + */ +int +NCZF_buildchunkkey(const NC_FILE_INFO_T* file, size_t rank, const size64_t* chunkindices, char dimsep, char** keyp) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = NULL; + + zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + assert(zfile != NULL); + stat = zfile->dispatcher->buildchunkkey(rank,chunkindices,dimsep,keyp); + return THROW(stat); +} diff --git a/libnczarr/zformat.h b/libnczarr/zformat.h index 9d3c1ee9c9..1fa2e2feaf 100644 --- a/libnczarr/zformat.h +++ b/libnczarr/zformat.h @@ -31,12 +31,13 @@ typedef struct NCZ_Formatter { int (*create) (NC_FILE_INFO_T* file, NCURI* uri, NCZMAP* map); int (*open) (NC_FILE_INFO_T* file, NCURI* uri, NCZMAP* map); + int (*close) (NC_FILE_INFO_T* file); int (*readmeta) (NC_FILE_INFO_T* file); int (*writemeta) (NC_FILE_INFO_T* file); int (*readattrs) (NC_FILE_INFO_T* file, NC_OBJ* container); /* Support lazy read */ + int (*buildchunkkey)(size_t rank, const size64_t* chunkindices, char dimsep, char** keyp); int (*codec2hdf) (const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, const NCjson* jfilter, NCZ_Filter* filter, struct NCZ_Plugin* plugin); int (*hdf2codec) (const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, NCZ_Filter* filter); - int (*close) (NC_FILE_INFO_T* file); } NCZ_Formatter; #if defined(__cplusplus) @@ -54,12 +55,14 @@ extern int NCZF_open(NC_FILE_INFO_T* file, NCURI* uri, NCZMAP* map); extern int NCZF_readmeta(NC_FILE_INFO_T* file); extern int NCZF_writemeta(NC_FILE_INFO_T* file); extern int NCZF_close(NC_FILE_INFO_T* file); - + extern int NCZF_readattrs(NC_FILE_INFO_T* file, NC_OBJ* container); /* Support lazy read */ extern int NCZF_codec2hdf(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, const NCjson* jfilter, NCZ_Filter* filter, struct NCZ_Plugin* plugin); extern int NCZF_hdf2codec(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, NCZ_Filter* filter); +extern int NCZF_buildchunkkey(const NC_FILE_INFO_T* file, size_t rank, const size64_t* chunkindices, char dimsep, char** keyp); + /* Define known dispatch tables and initializers */ /* Each handles a specific NCZarr format + Pure Zarr */ /* WARNING: there is a lot of similar code in the dispatchers, diff --git a/libnczarr/zformat1.c b/libnczarr/zformat1.c index 73ca80aef7..ec8b9d25c7 100644 --- a/libnczarr/zformat1.c +++ b/libnczarr/zformat1.c @@ -14,10 +14,15 @@ /*Forward*/ static int ZF1_create(NC_FILE_INFO_T* file, NCURI* uri, NCZMAP* map); static int ZF1_open(NC_FILE_INFO_T* file, NCURI* uri, NCZMAP* map); +static int ZF1_close(NC_FILE_INFO_T* file); static int ZF1_writemeta(NC_FILE_INFO_T* file); static int ZF1_readmeta(NC_FILE_INFO_T* file); static int ZF1_readattrs(NC_FILE_INFO_T* file, NC_OBJ* container); -static int ZF1_close(NC_FILE_INFO_T* file); +static int ZF1_buildchunkkey(size_t rank, const size64_t* chunkindices, char dimsep, char** keyp); +#ifdef ENABLE_NCZARR_FILTERS +static int ZF1_hdf2codec(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, NCZ_Filter* filter); +static int ZF1_codec2hdf(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, const NCjson* jfilter, NCZ_Filter* filter, NCZ_Plugin* plugin); +#endif static int read_superblock(NC_FILE_INFO_T* file, int* nczarrvp); static int read_grp(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zinfo, NCZMAP* map, NC_GRP_INFO_T* grp); @@ -384,7 +389,7 @@ read_vars(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO int endianness; if((stat = NCJdictget(jvar,"dtype",&jvalue))) goto done; /* Convert dtype to nc_type + endianness */ - if((stat = ncz_dtype2nctype(NCJstring(jvalue),NC_NAT,purezarr,&vtype,&endianness,&vtypelen))) + if((stat = ncz2_dtype2nctype(NCJstring(jvalue),NC_NAT,purezarr,&vtype,&endianness,&vtypelen))) goto done; if(vtype > NC_NAT && vtype <= NC_MAX_ATOMIC_TYPE) { /* Locate the NC_TYPE_INFO_T object */ @@ -487,7 +492,7 @@ read_vars(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO if((stat = NCJdictget(jvar,"dimension_separator",&jvalue))) goto done; if(jvalue != NULL) { /* Verify its value */ - if(NCJsort(jvalue) == NCJ_STRING && NCJstring(jvalue) != NULL && strlen(NCJstring(jvalue)) == 1) + if(NCJisatomic(jvalue) && NCJstring(jvalue) != NULL && strlen(NCJstring(jvalue)) == 1) zvar->dimension_separator = NCJstring(jvalue)[0]; } /* If value is invalid, then use global default */ @@ -769,7 +774,7 @@ ZF1_readattrs(NC_FILE_INFO_T* file, NC_OBJ* container) /* case 1: name = _NCProperties, grp=root, varid==NC_GLOBAL */ if(strcmp(aname,NCPROPS)==0 && grp != NULL && file->root_grp == grp) { /* Setup provenance */ - if(NCJsort(value) != NCJ_STRING) + if(NCJisatomic(value)) {stat = (THROW(NC_ENCZARR)); goto done;} /*malformed*/ if((stat = NCZ_read_provenance(file,aname,NCJstring(value)))) goto done; @@ -783,7 +788,7 @@ ZF1_readattrs(NC_FILE_INFO_T* file, NC_OBJ* container) {stat = NC_ENOMEM; goto done;} for(i=0;ixarray,strdup(NCJstring(k))); } } @@ -1009,7 +1014,7 @@ NCZ_read_atts(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zinfo, NCZMAP* map, NC_OBJ* /* case 1: name = _NCProperties, grp=root, varid==NC_GLOBAL */ if(strcmp(aname,NCPROPS)==0 && grp != NULL && file->root_grp == grp) { /* Setup provenance */ - if(NCJsort(value) != NCJ_STRING) + if(!NCJisatomic(value)) {stat = (THROW(NC_ENCZARR)); goto done;} /*malformed*/ if((stat = NCZ_read_provenance(file,aname,NCJstring(value)))) goto done; @@ -1023,7 +1028,7 @@ NCZ_read_atts(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zinfo, NCZMAP* map, NC_OBJ* {stat = NC_ENOMEM; goto done;} for(i=0;ixarray,strdup(NCJstring(k))); } } @@ -1189,7 +1194,7 @@ NCZ_computeattrinfo(const char* name, NClist* atypes, nc_type typehint, int pure const char* aname = nclistget(atypes,i); if(strcmp(aname,name)==0) { const char* atype = nclistget(atypes,i+1); - if((stat = ncz_dtype2nctype(atype,typehint,purezarr,&typeid,NULL,NULL))) goto done; + if((stat = ncz2_dtype2nctype(atype,typehint,purezarr,&typeid,NULL,NULL))) goto done; break; } } @@ -1544,8 +1549,8 @@ NCZ_jtypes2atypes(NCjson* jtypes, NClist* atypes) for(i=0;i 0) ncbytesappend(key,dimsep); + /* Print as decimal with no leading zeros */ + snprintf(sindex,sizeof(sindex),"%lu",(unsigned long)chunkindices[r]); + ncbytescat(key,sindex); + } + ncbytesnull(key); + if(keyp) *keyp = ncbytesextract(key); + + ncbytesfree(key); + return THROW(stat); +} + /**************************************************/ /* Format Filter Support Functions */ /* JSON Parse/unparse of filter codecs */ #ifdef ENABLE_NCZARR_FILTERS -int +static int ZF1_hdf2codec(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, NCZ_Filter* filter) { int stat = NC_NOERR; - NCZ_codec_env_t env = NCZ_CODEC_ENV_EMPTY_V2; /* Convert the HDF5 id + visible parameters to the codec form */ @@ -1644,7 +1673,7 @@ ZF1_hdf2codec(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, NCZ_Filter* nullfree(filter->codec.codec); filter->codec.codec = NULL; filter->codec.id = strdup(filter->plugin->codec.codec->codecid); if(filter->plugin->codec.codec->NCZ_hdf5_to_codec) { - stat = filter->plugin->codec.codec->NCZ_hdf5_to_codec(&env,filter->hdf5.visible.nparams,filter->hdf5.visible.params,&filter->codec.codec); + stat = filter->plugin->codec.codec->NCZ_hdf5_to_codec(NCplistzarrv2,filter->hdf5.id,filter->hdf5.visible.nparams,filter->hdf5.visible.params,&filter->codec.codec); #ifdef DEBUGF fprintf(stderr,">>> DEBUGF: NCZ_hdf5_to_codec: visible=%s codec=%s\n",printnczparams(filter->hdf5.visible),filter->codec.codec); #endif @@ -1657,12 +1686,11 @@ ZF1_hdf2codec(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, NCZ_Filter* } /* Build filter from parsed Zarr metadata */ -int +static int ZF1_codec2hdf(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, const NCjson* jfilter, NCZ_Filter* filter, NCZ_Plugin* plugin) { int stat = NC_NOERR; NCjson* jvalue = NULL; - NCZ_codec_env_t env = NCZ_CODEC_ENV_EMPTY_V2; assert(jfilter != NULL); assert(filter != NULL); @@ -1670,16 +1698,16 @@ ZF1_codec2hdf(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, const NCjson if(filter->codec.id == NULL) { /* Get the id of this codec filter */ if(NCJdictget(jfilter,"id",&jvalue)<0) {stat = NC_EFILTER; goto done;} - if(NCJsort(jvalue) != NCJ_STRING) {stat = THROW(NC_ENOFILTER); goto done;} + if(!NCJisatomic(jvalue)) {stat = THROW(NC_ENOFILTER); goto done;} filter->codec.id = strdup(NCJstring(jvalue)); } if(plugin != NULL) { /* Save the hdf5 id */ - filter->hdf5.id = plugin->codec.codec->hdf5id; + filter->hdf5.id = plugin->hdf5.filter->id; /* Convert the codec to hdf5 form visible parameters */ if(plugin->codec.codec->NCZ_codec_to_hdf5) { - stat = plugin->codec.codec->NCZ_codec_to_hdf5(&env,filter->codec.codec,&filter->hdf5.visible.nparams,&filter->hdf5.visible.params); + stat = plugin->codec.codec->NCZ_codec_to_hdf5(NCplistzarrv2,filter->codec.codec,&filter->hdf5.id,&filter->hdf5.visible.nparams,&filter->hdf5.visible.params); #ifdef DEBUGF fprintf(stderr,">>> DEBUGF: NCZ_codec_to_hdf5: codec=%s, hdf5=%s\n",printcodec(codec),printhdf5(hdf5)); #endif @@ -1702,9 +1730,11 @@ static const NCZ_Formatter NCZ_formatter1_table = { ZF1_create, ZF1_open, + ZF1_close, ZF1_readmeta, ZF1_writemeta, ZF1_readattrs, + ZF1_buildchunkkey, #ifdef ENABLE_NCZARR_FILTERS ZF1_codec2hdf, ZF1_hdf2codec, @@ -1712,7 +1742,6 @@ static const NCZ_Formatter NCZ_formatter1_table = { NULL, NULL, #endif - ZF1_close }; const NCZ_Formatter* NCZ_formatter1 = &NCZ_formatter1_table; diff --git a/libnczarr/zformat2.c b/libnczarr/zformat2.c index a48916dbe2..286049f814 100644 --- a/libnczarr/zformat2.c +++ b/libnczarr/zformat2.c @@ -14,10 +14,15 @@ /*Forward*/ static int ZF2_create(NC_FILE_INFO_T* file, NCURI* uri, NCZMAP* map); static int ZF2_open(NC_FILE_INFO_T* file, NCURI* uri, NCZMAP* map); +static int ZF2_close(NC_FILE_INFO_T* file); static int ZF2_writemeta(NC_FILE_INFO_T* file); static int ZF2_readmeta(NC_FILE_INFO_T* file); static int ZF2_readattrs(NC_FILE_INFO_T* file, NC_OBJ* container); -static int ZF2_close(NC_FILE_INFO_T* file); +static int ZF2_buildchunkkey(size_t rank, const size64_t* chunkindices, char dimsep, char** keyp); +#ifdef ENABLE_NCZARR_FILTERS +static int ZF2_hdf2codec(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, NCZ_Filter* filter); +static int ZF2_codec2hdf(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, const NCjson* jfilter, NCZ_Filter* filter, NCZ_Plugin* plugin); +#endif static int write_grp(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO_T* grp); static int write_var_meta(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_VAR_INFO_T* var); @@ -345,7 +350,7 @@ write_var_meta(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_VAR int endianness = var->type_info->endianness; int atomictype = var->type_info->hdr.id; assert(atomictype > 0 && atomictype <= NC_MAX_ATOMIC_TYPE); - if((stat = ncz_nctype2dtype(atomictype,endianness,purezarr,NCZ_get_maxstrlen((NC_OBJ*)var),&dtypename))) goto done; + if((stat = ncz2_nctype2dtype(atomictype,endianness,purezarr,NCZ_get_maxstrlen((NC_OBJ*)var),&dtypename))) goto done; if((stat = NCJaddstring(jvar,NCJ_STRING,dtypename))) goto done; nullfree(dtypename); dtypename = NULL; } @@ -628,7 +633,7 @@ write_atts(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_OBJ* co /* Collect the corresponding dtype */ { - if((stat = ncz_nctype2dtype(a->nc_typeid,endianness,purezarr,typesize,&tname))) goto done; + if((stat = ncz2_nctype2dtype(a->nc_typeid,endianness,purezarr,typesize,&tname))) goto done; if((stat = NCJnewstring(NCJ_STRING,tname,&jtype))) goto done; nullfree(tname); tname = NULL; if((stat = NCJinsert(jtypes,a->hdr.name,jtype))) goto done; /* add {name: type} */ @@ -872,7 +877,7 @@ ZF2_readattrs(NC_FILE_INFO_T* file, NC_OBJ* container) /* case 1: name = _NCProperties, grp=root, varid==NC_GLOBAL */ if(strcmp(aname,NCPROPS)==0 && grp != NULL && file->root_grp == grp) { /* Setup provenance */ - if(NCJsort(value) != NCJ_STRING) + if(!NCJisatomic(value)) {stat = (THROW(NC_ENCZARR)); goto done;} /*malformed*/ if((stat = NCZ_read_provenance(file,aname,NCJstring(value)))) goto done; @@ -886,7 +891,7 @@ ZF2_readattrs(NC_FILE_INFO_T* file, NC_OBJ* container) {stat = NC_ENOMEM; goto done;} for(i=0;ixarray,strdup(NCJstring(k))); } } @@ -1198,7 +1203,7 @@ read_vars(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO int endianness; if((stat = NCJdictget(jvar,"dtype",&jvalue))) goto done; /* Convert dtype to nc_type + endianness */ - if((stat = ncz_dtype2nctype(NCJstring(jvalue),NC_NAT,purezarr,&vtype,&endianness,&vtypelen))) + if((stat = ncz2_dtype2nctype(NCJstring(jvalue),NC_NAT,purezarr,&vtype,&endianness,&vtypelen))) goto done; if(vtype > NC_NAT && vtype <= NC_MAX_ATOMIC_TYPE) { /* Locate the NC_TYPE_INFO_T object */ @@ -1246,7 +1251,7 @@ read_vars(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO rank = NCJlength(jdimrefs); for(j=0;jdimension_separator = NCJstring(jvalue)[0]; } /* If value is invalid, then use global default */ @@ -1807,7 +1812,7 @@ NCZ_read_atts(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_OBJ* /* case 1: name = _NCProperties, grp=root, varid==NC_GLOBAL */ if(strcmp(aname,NCPROPS)==0 && grp != NULL && file->root_grp == grp) { /* Setup provenance */ - if(NCJsort(value) != NCJ_STRING) + if(!NCJisatomic(value)) {stat = (THROW(NC_ENCZARR)); goto done;} /*malformed*/ if((stat = NCZ_read_provenance(file,aname,NCJstring(value)))) goto done; @@ -1821,7 +1826,7 @@ NCZ_read_atts(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_OBJ* {stat = NC_ENOMEM; goto done;} for(i=0;ixarray,strdup(NCJstring(k))); } } @@ -1987,7 +1992,7 @@ NCZ_computeattrinfo(const char* name, NClist* atypes, nc_type typehint, int pure const char* aname = nclistget(atypes,i); if(strcmp(aname,name)==0) { const char* atype = nclistget(atypes,i+1); - if((stat = ncz_dtype2nctype(atype,typehint,purezarr,&typeid,NULL,NULL))) goto done; + if((stat = ncz2_dtype2nctype(atype,typehint,purezarr,&typeid,NULL,NULL))) goto done; break; } } @@ -2248,8 +2253,8 @@ NCZ_jtypes2atypes(NCjson* jtypes, NClist* atypes) for(i=0;i 0) ncbytesappend(key,dimsep); + /* Print as decimal with no leading zeros */ + snprintf(sindex,sizeof(sindex),"%lu",(unsigned long)chunkindices[r]); + ncbytescat(key,sindex); + } + ncbytesnull(key); + if(keyp) *keyp = ncbytesextract(key); + + ncbytesfree(key); + return THROW(stat); +} + /**************************************************/ /* Format Filter Support Functions */ @@ -2339,7 +2369,6 @@ int ZF2_hdf2codec(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, NCZ_Filter* filter) { int stat = NC_NOERR; - NCZ_codec_env_t env = NCZ_CODEC_ENV_EMPTY_V2; /* Convert the HDF5 id + visible parameters to the codec form */ @@ -2348,7 +2377,7 @@ ZF2_hdf2codec(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, NCZ_Filter* nullfree(filter->codec.codec); filter->codec.codec = NULL; filter->codec.id = strdup(filter->plugin->codec.codec->codecid); if(filter->plugin->codec.codec->NCZ_hdf5_to_codec) { - stat = filter->plugin->codec.codec->NCZ_hdf5_to_codec(&env,filter->hdf5.visible.nparams,filter->hdf5.visible.params,&filter->codec.codec); + stat = filter->plugin->codec.codec->NCZ_hdf5_to_codec(NCplistzarrv2,filter->hdf5.id,filter->hdf5.visible.nparams,filter->hdf5.visible.params,&filter->codec.codec); #ifdef DEBUGF fprintf(stderr,">>> DEBUGF: NCZ_hdf5_to_codec: visible=%s codec=%s\n",printnczparams(filter->hdf5.visible),filter->codec.codec); #endif @@ -2366,7 +2395,6 @@ ZF2_codec2hdf(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, const NCjson { int stat = NC_NOERR; NCjson* jvalue = NULL; - NCZ_codec_env_t env = NCZ_CODEC_ENV_EMPTY_V2; assert(jfilter != NULL); assert(filter != NULL); @@ -2374,7 +2402,7 @@ ZF2_codec2hdf(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, const NCjson if(filter->codec.id == NULL) { /* Get the id of this codec filter */ if(NCJdictget(jfilter,"id",&jvalue)<0) {stat = NC_EFILTER; goto done;} - if(NCJsort(jvalue) != NCJ_STRING) {stat = THROW(NC_ENOFILTER); goto done;} + if(!NCJisatomic(jvalue)) {stat = THROW(NC_ENOFILTER); goto done;} filter->codec.id = strdup(NCJstring(jvalue)); } @@ -2385,10 +2413,10 @@ ZF2_codec2hdf(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, const NCjson if(plugin != NULL) { /* Save the hdf5 id */ - filter->hdf5.id = plugin->codec.codec->hdf5id; + filter->hdf5.id = plugin->hdf5.filter->id; /* Convert the codec to hdf5 form visible parameters */ if(plugin->codec.codec->NCZ_codec_to_hdf5) { - stat = plugin->codec.codec->NCZ_codec_to_hdf5(&env,filter->codec.codec,&filter->hdf5.visible.nparams,&filter->hdf5.visible.params); + stat = plugin->codec.codec->NCZ_codec_to_hdf5(NCplistzarrv2,filter->codec.codec,&filter->hdf5.id,&filter->hdf5.visible.nparams,&filter->hdf5.visible.params); #ifdef DEBUGF fprintf(stderr,">>> DEBUGF: NCZ_codec_to_hdf5: codec=%s, hdf5=%s\n",printcodec(codec),printhdf5(hdf5)); #endif @@ -2411,9 +2439,11 @@ static const NCZ_Formatter NCZ_formatter2_table = { ZF2_create, ZF2_open, + ZF2_close, ZF2_readmeta, ZF2_writemeta, ZF2_readattrs, + ZF2_buildchunkkey, #ifdef ENABLE_NCZARR_FILTERS ZF2_codec2hdf, ZF2_hdf2codec, @@ -2421,7 +2451,6 @@ static const NCZ_Formatter NCZ_formatter2_table = { NULL, NULL, #endif - ZF2_close }; const NCZ_Formatter* NCZ_formatter2 = &NCZ_formatter2_table; diff --git a/libnczarr/zformat3.c b/libnczarr/zformat3.c index 5459eb96ac..8bab2ea5b6 100644 --- a/libnczarr/zformat3.c +++ b/libnczarr/zformat3.c @@ -1,4 +1,3 @@ - /********************************************************************* * Copyright 2018, UCAR/Unidata * See netcdf/COPYRIGHT file for copying and redistribution conditions. @@ -24,41 +23,47 @@ NCjson* NCZ_Bytes_Little_Json = NULL; /*Forward*/ static int ZF3_create(NC_FILE_INFO_T* file, NCURI* uri, NCZMAP* map); static int ZF3_open(NC_FILE_INFO_T* file, NCURI* uri, NCZMAP* map); +static int ZF3_close(NC_FILE_INFO_T* file); static int ZF3_writemeta(NC_FILE_INFO_T* file); static int ZF3_readmeta(NC_FILE_INFO_T* file); static int ZF3_readattrs(NC_FILE_INFO_T* file, NC_OBJ* container); -static int ZF3_close(NC_FILE_INFO_T* file); +static int ZF3_buildchunkkey(size_t rank, const size64_t* chunkindices, char dimsep, char** keyp); +#ifdef ENABLE_NCZARR_FILTERS +static int ZF3_hdf2codec(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, NCZ_Filter* filter); +static int ZF3_codec2hdf(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, const NCjson* jfilter, NCZ_Filter* filter, NCZ_Plugin* plugin); +#endif static int write_grp(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO_T* grp); static int write_var_meta(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_VAR_INFO_T* var); static int write_var(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_VAR_INFO_T* var); static int build_atts(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NC_OBJ* container, NCindex* attlist, NCjson** jattsp, NCjson** jnczattsp); +static int build_superblock(NC_FILE_INFO_T* file, NCjson** jsuperp); +static int build_group_json(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NCjson** grpp); -static int read_grp(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zinfo, NCZMAP* map, NC_GRP_INFO_T* grp); -static int read_dims(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zinfo, NCZMAP* map, NC_GRP_INFO_T* grp, NClist* diminfo); -static int read_vars(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zinfo, NCZMAP* map, NC_GRP_INFO_T* grp, NClist* varnames); -static int read_subgrps(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zinfo, NCZMAP* map, NC_GRP_INFO_T* grp, NClist* subgrpnames); +static int read_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp); +static int read_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames); +static int read_subgrps(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* subgrpnames); +static int NCZ_read_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCjson* jblock); +static int parse_superblock(NC_FILE_INFO_T* file, NCjson* jsuper); +static int parse_dims(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp); -static int NCZ_collect_dims(NC_FILE_INFO_T* file, NCjson** jdimsp); -static int NCZ_parse_group_content(NCjson* jcontent, NClist* dimdefs, NClist* varnames, NClist* subgrps); -static int NCZ_parse_group_content_pure(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zinfo, NC_GRP_INFO_T* grp, NClist* varnames, NClist* subgrps); -static int NCZ_read_atts(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NC_OBJ* container, NCjson* jblock); +static int NCZ_collect_grps(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, NCjson** jgrpsp); +static int NCZ_collect_arrays(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, NCjson** jarraysp); +static int NCZ_collect_dims(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, NCjson** jdimsp); -static int NCZ_searchvars(NCZ_FILE_INFO_T* zfile, NC_GRP_INFO_T* grp, NClist* varnames); -static int NCZ_searchsubgrps(NCZ_FILE_INFO_T* zfile, NC_GRP_INFO_T* grp, NClist* subgrpnames); static int NCZ_decodeints(NCjson* jshape, size64_t* shapes); static int NCZ_computeattrinfo(const char* name, NClist* atypes, nc_type typehint, int purezarr, NCjson* values, nc_type* typeidp, size_t* typelenp, size_t* lenp, void** datap); static int NCZ_computeattrdata(nc_type typehint, nc_type* typeidp, NCjson* values, size_t* typelenp, size_t* countp, void** datap); -static int NCZ_computedimrefs(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zinfo, NCZMAP* map, NC_VAR_INFO_T* var, int ndims, NClist* dimnames, size64_t* shapes, NC_DIM_INFO_T** dims); +static int NCZ_computedimrefs(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, NCjson* jvar, NClist* dimnames, size64_t* shapes, NC_DIM_INFO_T** dims); static int NCZ_load_jatts(NCjson* block, NCjson** jattrsp, NClist** atypesp); static int NCZ_json_convention_read(NCjson* json, NCjson** jtextp); static int NCZ_attr_convert(NCjson* src, nc_type typeid, size_t typelen, int* countp, NCbytes* dst); -static int NCZ_parsedimrefs(NC_FILE_INFO_T* file, NClist* dimnames, size64_t* shape, NC_DIM_INFO_T** dims, int create); static int NCZ_jtypes2atypes(NCjson* jtypes, NClist* atypes); static int NCZ_charify(NCjson* src, NCbytes* buf); -static int NCZ_locategroup(NC_FILE_INFO_T* file, size_t nsegs, NClist* segments, NC_GRP_INFO_T** grpp); -static int NCZ_createdim(NC_FILE_INFO_T* file, const char* name, size64_t dimlen, NC_DIM_INFO_T** dimp); +static int subobjects_pure(NCZ_FILE_INFO_T* zfile, NC_GRP_INFO_T* grp, NClist* varnames, NClist* grpnames); +static int subobjects(NCZ_FILE_INFO_T* zfile, NC_GRP_INFO_T* grp, NClist* varnames, NClist* grpnames); +static int finddim(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* dimname, size64_t size, NC_DIM_INFO_T** dimp); /**************************************************/ @@ -144,19 +149,14 @@ static int write_grp(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO_T* grp) { int i,stat = NC_NOERR; - char version[1024]; int purezarr = 0; int rootgrp = 0; char* fullpath = NULL; char* key = NULL; NCjson* jgroup = NULL; - NCjson* jdims = NULL; - NCjson* jsubgrps = NULL; NCjson* jsuper = NULL; NCjson* jatts = NULL; NCjson* jtypes = NULL; - NCjson* jarrays = NULL; - NCbytes* fqn = ncbytesnew(); ZTRACE(3,"file=%s grp=%s isclose=%d",file->controller->path,grp->hdr.name,isclose); @@ -167,14 +167,13 @@ write_grp(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO if((stat = NCZ_grpkey(grp,&fullpath))) goto done; - /* If the zarr.info for non-root group has attributes, + /* If the group zarr.info has attributes, or group is root and not purezarr then build Z3GROUP contents */ - if(rootgrp || ncindexsize(grp->att) > 0) { + if((!purezarr && rootgrp) || ncindexsize(grp->att) > 0) { if((stat = NCJnew(NCJ_DICT,&jgroup))) goto done; if((stat = NCJinsertstring(jgroup,"node_type","group"))) goto done; - snprintf(version,sizeof(version),"%d",zfile->zarr.zarr_format); - if((stat = NCJinsertstring(jgroup,"zarr_format",version))) goto done; + if((stat = NCJinsertint(jgroup,"zarr_format",zfile->zarr.zarr_format))) goto done; /* Insert the group attributes */ /* Build the attributes dictionary */ assert(grp->att); @@ -185,52 +184,14 @@ write_grp(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO } if(!purezarr && rootgrp) { + assert(jgroup != NULL); /* Build the superblock */ - /* Track the library version that wrote this */ - strncpy(version,NCZARR_PACKAGE_VERSION,sizeof(version)); - if((stat = NCJnew(NCJ_DICT,&jsuper))) goto done; - if((stat = NCJinsertstring(jsuper,"version",version))) goto done; - snprintf(version,sizeof(version),"%u", (unsigned)zfile->zarr.nczarr_format); - if((stat = NCJinsertstring(jsuper,"format",version))) goto done; - - /* Create dimensions dict */ - if((stat = NCZ_collect_dims(file,&jdims))) goto done; - - /* Create vars list */ - if((stat = NCJnew(NCJ_ARRAY,&jarrays))) goto done; - for(i=0; ivars); i++) { - NC_VAR_INFO_T* var = (NC_VAR_INFO_T*)ncindexith(grp->vars,i); - ncbytesclear(fqn); - if((stat = NCZ_makeFQN(var->container,(NC_OBJ*)var,fqn))) goto done; - if((stat = NCJaddstring(jarrays,NCJ_STRING,ncbytescontents(fqn)))) goto done; - } - - /* Create subgroups list */ - if((stat = NCJnew(NCJ_ARRAY,&jsubgrps))) goto done; - for(i=0; ichildren); i++) { - NC_GRP_INFO_T* g = (NC_GRP_INFO_T*)ncindexith(grp->children,i); - ncbytesclear(fqn); - if((stat = NCZ_makeFQN(g->parent,(NC_OBJ*)g,fqn))) goto done; - if((stat = NCJaddstring(jsubgrps,NCJ_STRING,ncbytescontents(fqn)))) goto done; - } - - /* Assemble the superblock */ - - /* Insert the "dimensions" dict */ - if((stat = NCJinsert(jsuper,"dimensions",jdims))) goto done; - jdims = NULL; - - /* Insert the "arrays" dict */ - if((stat = NCJinsert(jsuper,"arrays",jarrays))) goto done; - jarrays = NULL; - - /* Insert the "groups" dict */ - if((stat = NCJinsert(jsuper,"groups",jsubgrps))) goto done; - jsubgrps = NULL; + if((stat = build_superblock(file,&jsuper))) goto done; } /* Insert superblock into root group */ if(jsuper != NULL) { + assert(jgroup != NULL); /* Disable must_understand */ if((stat = NCJinsertstring(jgroup,"must_understand","false"))) goto done; assert(jgroup != NULL); @@ -238,14 +199,14 @@ write_grp(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO jsuper = NULL; } - /* build Z3GROUP path */ - if((stat = nczm_concat(fullpath,Z3GROUP,&key))) - goto done; - - /* Write to map */ - if((stat=NCZ_uploadjson(map,key,jgroup))) - goto done; - nullfree(key); key = NULL; + if(jgroup != NULL) { + /* build Z3GROUP path */ + if((stat = nczm_concat(fullpath,Z3GROUP,&key))) goto done; + /* Write to map */ + if((stat=NCZ_uploadjson(map,key,jgroup))) goto done; + nullfree(key); key = NULL; + jgroup = NULL; + } /* Now write all the variables */ for(i=0; ivars); i++) { @@ -262,12 +223,10 @@ write_grp(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO done: nullfree(key); nullfree(fullpath); - ncbytesfree(fqn); - NCJreclaim(jsuper); NCJreclaim(jgroup); - NCJreclaim(jarrays); - NCJreclaim(jsubgrps); - NCJreclaim(jdims); + NCJreclaim(jsuper); + NCJreclaim(jatts); + NCJreclaim(jtypes); return ZUNTRACE(THROW(stat)); } @@ -293,6 +252,7 @@ write_var_meta(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_VAR NCjson* jvar = NULL; NCjson* jncvar = NULL; NCjson* jdimrefs = NULL; + NCjson* jdimnames = NULL; NCjson* jtmp = NULL; NCjson* jtmp2 = NULL; NCjson* jtmp3 = NULL; @@ -341,14 +301,10 @@ write_var_meta(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_VAR if(!purezarr) { /* Disable must_understand */ if((stat = NCJinsertstring(jvar,"must_understand","false"))) goto done; - /* create the "_nczarr_array" dict */ - if((stat = NCJnew(NCJ_DICT,&jncvar))) goto done; } /* zarr_format key */ - snprintf(tmpstr,sizeof(tmpstr),"%d",zfile->zarr.zarr_format); - if((stat = NCJaddstring(jvar,NCJ_STRING,"zarr_format"))) goto done; - if((stat = NCJaddstring(jvar,NCJ_INT,tmpstr))) goto done; + if((stat = NCJinsertint(jvar,"zarr_format",zfile->zarr.zarr_format))) goto done; /* Collect the shape vector */ for(i=0;indims;i++) { @@ -363,24 +319,22 @@ write_var_meta(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_VAR /* Integer list defining the length of each dimension of the array.*/ /* Create the list */ if((stat = NCJnew(NCJ_ARRAY,&jtmp))) goto done; - if(zvar->scalar) { - NCJaddstring(jtmp,NCJ_INT,"1"); - } else for(i=0;indims;i++) { - snprintf(tmpstr,sizeof(tmpstr),"%llu",shape[i]); - NCJaddstring(jtmp,NCJ_INT,tmpstr); + if(!zvar->scalar) { + for(i=0;indims;i++) { + snprintf(tmpstr,sizeof(tmpstr),"%llu",shape[i]); + NCJaddstring(jtmp,NCJ_INT,tmpstr); + } } if((stat = NCJinsert(jvar,"shape",jtmp))) goto done; jtmp = NULL; - /* dtype key */ + /* data_type key */ /* A string or list defining a valid data type for the array. */ - if((stat = NCJaddstring(jvar,NCJ_STRING,"dtype"))) goto done; - { /* Add the type name */ - int endianness = var->type_info->endianness; + { /* compute the type name */ int atomictype = var->type_info->hdr.id; assert(atomictype > 0 && atomictype <= NC_MAX_ATOMIC_TYPE); - if((stat = ncz_nctype2dtype(atomictype,endianness,purezarr,NCZ_get_maxstrlen((NC_OBJ*)var),&dtypename))) goto done; - if((stat = NCJaddstring(jvar,NCJ_STRING,dtypename))) goto done; + if((stat = ncz3_nctype2dtype(atomictype,purezarr,NCZ_get_maxstrlen((NC_OBJ*)var),&dtypename))) goto done; + if((stat = NCJinsertstring(jvar,"data_type",dtypename))) goto done; nullfree(dtypename); dtypename = NULL; } @@ -457,9 +411,13 @@ write_var_meta(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_VAR endianness = (NC_isLittleEndian()?NC_ENDIAN_LITTLE:NC_ENDIAN_BIG); if(endianness == NC_ENDIAN_LITTLE) bytescodec = NCZ_Bytes_Little_Json; else {assert(endianness == NC_ENDIAN_LITTLE); bytescodec = NCZ_Bytes_Little_Json;} - if((stat = NCJappend(jcodecs,bytescodec))) goto done; + if(NCJclone(bytescodec,&jtmp) || jtmp == NULL) {stat = NC_ENOTZARR; goto done;} + if(NCJappend(jcodecs,jtmp)) goto done; + jtmp = NULL; } + /* Get chain of filters for this variable */ + filterchain = (NClist*)var->filters; if(nclistlength(filterchain) > 0) { int k; for(k=0;kndims > 0) { + /* build "dimension_names" key */ + if(NCJnew(NCJ_ARRAY,&jdimnames)) {stat = NC_ENOTZARR; goto done;} + for(i=0;indims;i++) { + NC_DIM_INFO_T* dim = var->dim[i]; + if(NCJaddstring(jdimnames,NCJ_STRING, dim->hdr.name)) {stat = NC_ENOTZARR; goto done;} + } + if(NCJinsert(jvar,"dimension_names",jdimnames)) {stat = NC_ENOTZARR; goto done;} + jdimnames = NULL; + + /* Capture dimref names as FQNs; simultaneously collect the simple dim names */ + if(!purezarr && var->ndims > 0) { if((dimrefs = nclistnew())==NULL) {stat = NC_ENOMEM; goto done;} for(i=0;indims;i++) { NC_DIM_INFO_T* dim = var->dim[i]; @@ -485,7 +453,7 @@ write_var_meta(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_VAR } /* Build the NCZ_V3_ARRAY dict entry */ - { + if(!purezarr) { /* Create the dimrefs json object */ if((stat = NCJnew(NCJ_ARRAY,&jdimrefs))) goto done; @@ -493,26 +461,15 @@ write_var_meta(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_VAR const char* dim = nclistget(dimrefs,i); NCJaddstring(jdimrefs,NCJ_STRING,dim); } - if((stat = NCJnew(NCJ_DICT,&jncvar))) - goto done; + + /* create the "_nczarr_array" dict */ + if((stat = NCJnew(NCJ_DICT,&jncvar))) goto done; /* Insert dimrefs */ - if((stat = NCJinsert(jncvar,"dimrefs",jdimrefs))) + if((stat = NCJinsert(jncvar,"dimensions",jdimrefs))) goto done; jdimrefs = NULL; /* Avoid memory problems */ - /* Add the _Storage flag */ - /* Record if this is a scalar */ - if(var->ndims == 0) { - if((stat = NCJnewstring(NCJ_INT,"1",&jtmp)))goto done; - if((stat = NCJinsert(jncvar,"scalar",jtmp))) goto done; - jtmp = NULL; - } - /* everything looks like it is chunked */ - if((stat = NCJnewstring(NCJ_STRING,"chunked",&jtmp)))goto done; - if((stat = NCJinsert(jncvar,"storage",jtmp))) goto done; - jtmp = NULL; - if(!purezarr) { if((stat = NCJinsert(jvar,NCZ_V3_ARRAY,jncvar))) goto done; jncvar = NULL; @@ -527,6 +484,7 @@ write_var_meta(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_VAR if((stat=NCZ_uploadjson(map,key,jvar))) goto done; nullfree(key); key = NULL; + NCJreclaim(jvar); jvar = NULL; var->created = 1; @@ -539,7 +497,11 @@ write_var_meta(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_VAR NCJreclaim(jvar); NCJreclaim(jncvar); NCJreclaim(jtmp); + NCJreclaim(jtmp2); NCJreclaim(jfill); + NCJreclaim(jcodecs); + NCJreclaim(jdimnames); + NCJreclaim(jdimrefs); return ZUNTRACE(THROW(stat)); } @@ -598,25 +560,17 @@ build_atts(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NC_OBJ* container, NCin NCjson* jdimrefs = NULL; NCjson* jint = NULL; NCjson* jdata = NULL; - int isxarray = 0; int purezarr = 0; - int rootvar = 0; NC_VAR_INFO_T* var = NULL; char* tname = NULL; - int endianness = (NC_isLittleEndian()?NC_ENDIAN_LITTLE:NC_ENDIAN_BIG); ZTRACE(3,"file=%s container=%s |attlist|=%u",file->controller->path,container->name,(unsigned)ncindexsize(attlist)); - if(container->sort == NCVAR) { + if(container->sort == NCVAR) var = (NC_VAR_INFO_T*)container; - if(var->container && var->container->parent == NULL) - rootvar = 1; - } purezarr = (zfile->controls.flags & FLAG_PUREZARR)?1:0; - if(zfile->controls.flags & FLAG_XARRAYDIMS) isxarray = 1; - /* Create the attribute dictionary */ if((stat = NCJnew(NCJ_DICT,&jatts))) goto done; @@ -647,7 +601,7 @@ build_atts(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NC_OBJ* container, NCin if(!purezarr) { /* Collect the corresponding dtype */ - if((stat = ncz_nctype2dtype(a->nc_typeid,endianness,purezarr,typesize,&tname))) goto done; + if((stat = ncz3_nctype2dtype(a->nc_typeid,purezarr,typesize,&tname))) goto done; if((stat = NCJnewstring(NCJ_STRING,tname,&jtype))) goto done; nullfree(tname); tname = NULL; if((stat = NCJinsert(jtypes,a->hdr.name,jtype))) goto done; /* add {name: type} */ @@ -679,47 +633,117 @@ build_atts(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NC_OBJ* container, NCin } } - if(container->sort == NCVAR) { - if(rootvar && isxarray) { - int dimsinroot = 1; - /* Insert the XARRAY _ARRAY_ATTRIBUTE attribute */ - if((stat = NCJnew(NCJ_ARRAY,&jdimrefs))) - goto done; - /* Fake the scalar case */ - if(var->ndims == 0) { - NCJaddstring(jdimrefs,NCJ_STRING,XARRAYSCALAR); - } else /* Walk the dimensions and capture the names */ - for(i=0;indims;i++) { - NC_DIM_INFO_T* dim = var->dim[i]; - /* Verify that the dimension is in the root group */ - if(dim->container && dim->container->parent != NULL) { - dimsinroot = 0; /* dimension is not in root */ - break; - } - } - if(dimsinroot) { - /* Walk the dimensions and capture the names */ - for(i=0;indims;i++) { - char* dimname; - NC_DIM_INFO_T* dim = var->dim[i]; - dimname = strdup(dim->hdr.name); - if(dimname == NULL) {stat = NC_ENOMEM; goto done;} - NCJaddstring(jdimrefs,NCJ_STRING,dimname); - nullfree(dimname); dimname = NULL; - } - /* Add the _ARRAY_DIMENSIONS attribute */ - if((stat = NCJinsert(jatts,NC_XARRAY_DIMS,jdimrefs))) goto done; - jdimrefs = NULL; - } - } - } if(jattsp) {*jattsp = jatts; jatts = NULL;} if(jnczattsp) {*jnczattsp = jnczatts; jnczatts = NULL;} done: + NCJreclaim(jtypes); + NCJreclaim(jtype); + NCJreclaim(jdimrefs); + NCJreclaim(jint); + NCJreclaim(jdata); return THROW(stat); } -/**************************************************/ +/** +The super block contains a JSON tree representing part of the +so-called combined metadata information. +It contains the group names, the dimension objects, +and the array names. +Eventually,if the Zarr V3 spec defines the combined metadata, +then that will be used instead of this information. + +The general form is this, +where the topmost dictionary node represents the root group: +_nczarr_superblock: { + version: 3.0.0, + format: 3, + root: { + dimensions: {name: , size: , unlimited: 1|0}, + arrays: [{name: },...], + children: [ + {name: , + dimensions: {name: , size: , unlimited: 1|0}, + arrays: [{name: },...], + children: [{name: , subgrps: [...]},{name: , subgrps: [...]},...] + }, + ... + ], + } +} + +*/ + +static int +build_superblock(NC_FILE_INFO_T* file, NCjson** jsuperp) +{ + int stat = NC_NOERR; + NCjson* jroot = NULL; + NCjson* jsuper = NULL; + char version[1024]; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + + if((stat = NCJnew(NCJ_DICT,&jsuper))) goto done; + + /* Fill in superblock */ + + /* Track the library version that wrote this */ + strncpy(version,NCZARR_PACKAGE_VERSION,sizeof(version)); + NCJinsertstring(jsuper,"version",version); + NCJinsertint(jsuper,"format",zfile->zarr.nczarr_format); + + /* Capture and insert the subgroup skeleton */ + if((stat = build_group_json(file,file->root_grp,&jroot))) goto done; + NCJinsert(jsuper, "root", jroot); + jroot = NULL; + + if(jsuperp) {*jsuperp = jsuper; jsuper = NULL;} +done: + NCJreclaim(jroot); + NCJreclaim(jsuper); + return THROW(stat); +} + +static int +build_group_json(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NCjson** grpp) +{ + int stat = NC_NOERR; + NCjson* jgrp = NULL; + NCjson* jsubgrps = NULL; + NCjson* jdims = NULL; + NCjson* jarrays = NULL; + NCjson* jname = NULL; + + /* Collect the dimensions in this group */ + if((stat = NCZ_collect_dims(file, grp, &jdims))) goto done; + + /* Collect the arrays in this group */ + if((stat = NCZ_collect_arrays(file, grp, &jarrays))) goto done; + + /* Collect the subgroups in this group */ + if((stat = NCZ_collect_grps(file, grp, &jsubgrps))) goto done; + + /* Fill in the grp dict */ + NCJnew(NCJ_DICT,&jgrp); + NCJnewstring(NCJ_STRING,grp->hdr.name,&jname); + NCJinsert(jgrp,"name",jname); + jname = NULL; + NCJinsert(jgrp,"dimensions",jdims); + jdims = NULL; + NCJinsert(jgrp,"arrays",jarrays); + jarrays = NULL; + NCJinsert(jgrp,"children",jsubgrps); + jsubgrps = NULL; + + if(grpp) {*grpp = jgrp; jgrp = NULL;} + +done: + NCJreclaim(jgrp); + NCJreclaim(jsubgrps); + NCJreclaim(jdims); + NCJreclaim(jarrays); + NCJreclaim(jname); + return THROW(stat); +} /**************************************************/ /* Map->Internal */ @@ -739,44 +763,33 @@ static int ZF3_readmeta(NC_FILE_INFO_T* file) { int stat = NC_NOERR; - int nczarr_format = 0; NCZ_FILE_INFO_T* zfile = NULL; - NCZMAP* map = NULL; NCjson* jblock = NULL; NCjson* jsuper = NULL; - NCjson* jtmp = NULL; ZTRACE(3,"file=%s",file->controller->path); zfile = file->format_file_info; - map = zfile->map; /* Read the root group's metadata */ switch(stat = NCZ_downloadjson(zfile->map, Z3METAROOT, &jblock)) { case NC_EEMPTY: /* not there */ - nczarr_format = NCZARRFORMAT0; /* apparently pure zarr */ zfile->controls.flags |= FLAG_PUREZARR; stat = NC_NOERR; /* reset */ goto done; case NC_NOERR: /* See if _nczarr_superblock key exists */ - if((stat = NCJdictget(jblock,NCZ_V3_SUPERBLOCK,&jsuper))) goto done; + NCJdictget(jblock,NCZ_V3_SUPERBLOCK,&jsuper); if(jsuper != NULL) { /* in any case this is nczarr format 3 */ - nczarr_format = 3; - /* See if superblock has version and format */ - if((stat = NCJdictget(jsuper,"version",&jtmp))) goto done; - if(jtmp == NULL) {stat = NC_ENCZARR; goto done;} /* Malformed */ - if((stat = NCJdictget(jsuper,"format",&jtmp))) goto done; - if(jtmp == NULL) {stat = NC_ENCZARR; goto done;} /* Malformed */ - sscanf(NCJstring(jtmp),"%d",&nczarr_format); + if((stat = parse_superblock(file,jsuper))) goto done; } break; default: goto done; } - /* Now load the groups starting with root */ - if((stat = read_grp(file,zfile,map,file->root_grp))) + /* Now load the groups contents starting with root */ + if((stat = read_grp(file,file->root_grp))) goto done; done: @@ -805,6 +818,33 @@ ZF3_readattrs(NC_FILE_INFO_T* file, NC_OBJ* container) return THROW(stat); } +static int +ZF3_buildchunkkey(size_t rank, const size64_t* chunkindices, char dimsep, char** keyp) +{ + int stat = NC_NOERR; + int r; + NCbytes* key = ncbytesnew(); + + if(keyp) *keyp = NULL; + + assert(islegaldimsep(dimsep)); + + ncbytescat(key,"c"); + ncbytesappend(key,dimsep); + for(r=0;r 0) ncbytesappend(key,dimsep); + /* Print as decimal with no leading zeros */ + snprintf(sindex,sizeof(sindex),"%lu",(unsigned long)chunkindices[r]); + ncbytescat(key,sindex); + } + ncbytesnull(key); + if(keyp) *keyp = ncbytesextract(key); + + ncbytesfree(key); + return THROW(stat); +} + static int ZF3_close(NC_FILE_INFO_T* file) { @@ -812,6 +852,36 @@ ZF3_close(NC_FILE_INFO_T* file) return THROW(stat); } +/**************************************************/ + +static int +parse_superblock(NC_FILE_INFO_T* file, NCjson* jsuper) +{ + int stat = NC_NOERR; + NCjson* jvalue = NULL; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + NC_GRP_INFO_T* root = (NC_GRP_INFO_T*)file->root_grp; + NCZ_GRP_INFO_T* zroot = (NCZ_GRP_INFO_T*)root->format_grp_info; + + NCJdictget(jsuper,"version",&jvalue); + if(jvalue != NULL) { + sscanf(NCJstring(jvalue),"%d.0.0",&zfile->zarr.nczarr_format); + assert(zfile->zarr.nczarr_format == NCZARRFORMAT3); + } + NCJdictget(jsuper,"format",&jvalue); + if(jvalue != NULL) { + sscanf(NCJstring(jvalue),"%d",&zfile->zarr.zarr_format); + assert(zfile->zarr.zarr_format == ZARRFORMAT3); + } + zfile->superblock = jsuper; + NCJdictget(jsuper,"root",&jvalue); + if(jvalue == NULL) {stat = NC_ENCZARR; goto done;} + zroot->grpsuper = jvalue; + +done: + return THROW(stat); +} + /** * @internal Read group data from map to memory * @@ -822,16 +892,16 @@ ZF3_close(NC_FILE_INFO_T* file) * @author Dennis Heimbigner */ static int -read_grp(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO_T* grp) +read_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp) { int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + NCZMAP* map = zfile->map; char* fullpath = NULL; char* key = NULL; int purezarr = 0; - NCjson* json = NULL; NCjson* jgroup = NULL; - NClist* dimdefs = nclistnew(); - NClist* varnames = nclistnew(); + NClist* subvars = nclistnew(); NClist* subgrps = nclistnew(); ZTRACE(3,"file=%s grp=%s",file->controller->path,grp->hdr.name); @@ -850,31 +920,27 @@ read_grp(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO_ stat=NCZ_downloadjson(map,key,&jgroup); nullfree(key); key = NULL; if(!purezarr && !jgroup) {stat = NC_ENCZARR; goto done;} - - if(purezarr) { - /* have to search the metadata tree to find vars and groups */ - if((stat = NCZ_parse_group_content_pure(file,zfile,grp,varnames,subgrps))) goto done; - } else { /*!purezarr*/ - /* Pull out lists about group content */ - if((stat = NCZ_parse_group_content(jgroup,dimdefs,varnames,subgrps))) goto done; - /* Define dimensions */ - if((stat = read_dims(file,zfile,map,grp,dimdefs))) goto done; - } + + /* Define dimensions */ + if((stat = parse_dims(file,grp))) goto done; /* Define attributes */ - if((stat = NCZ_read_atts(file,zfile,(NC_OBJ*)grp,jgroup))) goto done; + if((stat = NCZ_read_atts(file,(NC_OBJ*)grp,jgroup))) goto done; + + /* Pull out lists about groups and vars */ + if(purezarr) + {if((stat = subobjects_pure(zfile,grp,subvars,subgrps))) goto done;} + else + {if((stat = subobjects(zfile,grp,subvars,subgrps))) goto done;} - /* Define vars taking xarray into account */ - if((stat = read_vars(file,zfile,map,grp,varnames))) goto done; + /* Define vars */ + if((stat = read_vars(file,grp,subvars))) goto done; /* Read sub-groups */ - if((stat = read_subgrps(file,zfile,map,grp,subgrps))) goto done; + if((stat = read_subgrps(file,grp,subgrps))) goto done; done: - NCJreclaim(json); - NCJreclaim(jgroup); - nclistfreeall(dimdefs); - nclistfreeall(varnames); + nclistfreeall(subvars); nclistfreeall(subgrps); nullfree(fullpath); nullfree(key); @@ -886,39 +952,62 @@ read_grp(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO_ * * @param file Pointer to file info struct. * @param grp Pointer to grp info struct. - * @param diminfo List of (name,length,isunlimited) triples * * @return ::NC_NOERR No error. * @author Dennis Heimbigner */ static int -read_dims(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO_T* grp, NClist* diminfo) +parse_dims(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp) { int i,stat = NC_NOERR; - + NCZ_GRP_INFO_T* zgrp = (NCZ_GRP_INFO_T*)grp->format_grp_info; + NCjson* jdims = NULL; + NCjson* jdim = NULL; + NCjson* jname = NULL; + NCjson* jsize = NULL; + NCjson* junlim = NULL; + NCjson* jcfg = NULL; + ZTRACE(3,"file=%s grp=%s |diminfo|=%u",file->controller->path,grp->hdr.name,nclistlength(diminfo)); + /* Get dim defs for this group */ + NCJdictget(zgrp->grpsuper,"dimensions",&jdims); + if(jdims == NULL) goto done; /* no dims to create */ + assert(NCJsort(jdims)==NCJ_ARRAY); /* Reify each dim in turn */ - for(i = 0; i < nclistlength(diminfo); i+=3) { - NC_DIM_INFO_T* dim = NULL; - size64_t len = 0; - long long isunlim = 0; - const char* name = nclistget(diminfo,i); - const char* slen = nclistget(diminfo,i+1); - const char* sisunlimited = nclistget(diminfo,i+2); - - /* Create the NC_DIM_INFO_T object */ - sscanf(slen,"%lld",&len); /* Get length */ - if(sisunlimited != NULL) - sscanf(sisunlimited,"%lld",&isunlim); /* Get unlimited flag */ - else - isunlim = 0; - if((stat = nc4_dim_list_add(grp, name, (size_t)len, -1, &dim))) - goto done; - dim->unlimited = (isunlim ? 1 : 0); - if((dim->format_dim_info = calloc(1,sizeof(NCZ_DIM_INFO_T))) == NULL) - {stat = NC_ENOMEM; goto done;} - ((NCZ_DIM_INFO_T*)dim->format_dim_info)->common.file = file; + for(i = 0; i < NCJlength(jdims); i++) { + NC_DIM_INFO_T* dim = NULL; + size64_t dimlen = 0; + int isunlim = 0; + + /* Extract info */ + jdim = NCJith(jdims,i); + assert(NCJsort(jdim) == NCJ_DICT); + NCJdictget(jdim,"name",&jname); + assert(jname != NULL); + NCJdictget(jdim,"configuration",&jcfg); + assert(jcfg != NULL); + NCJdictget(jcfg,"size",&jsize); + assert(jsize != NULL); + NCJdictget(jcfg,"unlimited",&junlim); /* might be null */ + + /* Create the NC_DIM_INFO_T object */ + { + const char* name = NCJstring(jname); + unsigned long data; + sscanf(NCJstring(jsize),"%lu",&data); /* Get length */ + dimlen = (size64_t)data; + if(junlim != NULL) { + sscanf(NCJstring(junlim),"%ld",&data); /* Get unlimited flag */ + isunlim = (data == 0 ? (int)0 : (int)1); + } else + isunlim = 0; + if((stat = nc4_dim_list_add(grp, name, (size_t)dimlen, -1, &dim))) goto done; + dim->unlimited = isunlim; + if((dim->format_dim_info = calloc(1,sizeof(NCZ_DIM_INFO_T))) == NULL) + {stat = NC_ENOMEM; goto done;} + ((NCZ_DIM_INFO_T*)dim->format_dim_info)->common.file = file; + } } done: @@ -927,27 +1016,31 @@ read_dims(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO /** * @internal Materialize vars into memory; - * Take xarray and purezarr into account. + * Take purezarr into account. * * @param file Pointer to file info struct. * @param grp Pointer to grp info struct. + * @param varlist list of var names in this group * @param varnames List of names of variables in this group * * @return ::NC_NOERR No error. * @author Dennis Heimbigner */ static int -read_vars(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO_T* grp, NClist* varnames) +read_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) { - int stat = NC_NOERR; - int i,j; + int i,j,k,stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + NCZMAP* map = zfile->map; int purezarr = 0; ZTRACE(3,"file=%s grp=%s |varnames|=%u",file->controller->path,grp->hdr.name,nclistlength(varnames)); if(zfile->controls.flags & FLAG_PUREZARR) purezarr = 1; - /* Load each var in turn */ + if(nclistlength(varnames) == 0) goto done; /* Nothing to create */ + + /* Reify each array in turn */ for(i = 0; i < nclistlength(varnames); i++) { /* per-variable info */ NC_VAR_INFO_T* var = NULL; @@ -956,27 +1049,31 @@ read_vars(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO NCjson* jncvar = NULL; NCjson* jdimrefs = NULL; NCjson* jvalue = NULL; + NCjson* jtmp = NULL; + NCjson* jtmp2 = NULL; + NCjson* jsep = NULL; + NCjson* jcodecs = NULL; + NCjson* jcodec = NULL; char* varpath = NULL; char* key = NULL; const char* varname = NULL; size64_t* shapes = NULL; + size64_t* chunks = NULL; NClist* dimnames = NULL; int suppress = 0; /* Abort processing of this variable */ nc_type vtype = NC_NAT; int vtypelen = 0; int rank = 0; - int zarr_rank = 0; /* Need to watch out for scalars */ #ifdef ENABLE_NCZARR_FILTERS int varsized = 0; - NCjson* jfilter = NULL; int chainindex = 0; #endif - varname = nclistget(varnames,i); + varname = (const char*)nclistget(varnames,i); dimnames = nclistnew(); - if((stat = nc4_var_list_add2(grp, varname, &var))) - goto done; + /* Create the var */ + if((stat = nc4_var_list_add2(grp, varname, &var))) goto done; /* And its annotation */ if((zvar = calloc(1,sizeof(NCZ_VAR_INFO_T)))==NULL) @@ -991,15 +1088,13 @@ read_vars(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO var->quantize_mode = -1; /* Construct var path */ - if((stat = NCZ_varkey(var,&varpath))) - goto done; + if((stat = NCZ_varkey(var,&varpath))) goto done; /* Construct the path to the zarray object */ - if((stat = nczm_concat(varpath,Z3ARRAY,&key))) - goto done; + if((stat = nczm_concat(varpath,Z3ARRAY,&key))) goto done; + /* Download the zarray object */ - if((stat=NCZ_readdict(map,key,&jvar))) - goto done; + if((stat=NCZ_readdict(map,key,&jvar))) goto done; nullfree(key); key = NULL; assert(NCJsort(jvar) == NCJ_DICT); @@ -1008,26 +1103,23 @@ read_vars(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO /* Verify the format */ { int version; + if((stat = NCJdictget(jvar,"node_type",&jvalue))) goto done; + if(strcasecmp("array",NCJstring(jvalue))!=0) {stat = THROW(NC_ENOTZARR); goto done;} if((stat = NCJdictget(jvar,"zarr_format",&jvalue))) goto done; sscanf(NCJstring(jvalue),"%d",&version); if(version != zfile->zarr.zarr_format) {stat = (THROW(NC_ENCZARR)); goto done;} } - /* Set the type and endianness of the variable */ + /* Get the type of the variable */ { - int endianness; - if((stat = NCJdictget(jvar,"dtype",&jvalue))) goto done; - /* Convert dtype to nc_type + endianness */ - if((stat = ncz_dtype2nctype(NCJstring(jvalue),NC_NAT,purezarr,&vtype,&endianness,&vtypelen))) - goto done; + if((stat = NCJdictget(jvar,"data_type",&jvalue))) goto done; + /* Convert dtype to nc_type */ + if((stat = ncz3_dtype2nctype(NCJstring(jvalue),purezarr,&vtype,&vtypelen))) goto done; if(vtype > NC_NAT && vtype <= NC_MAX_ATOMIC_TYPE) { /* Locate the NC_TYPE_INFO_T object */ - if((stat = ncz_gettype(file,grp,vtype,&var->type_info))) - goto done; + if((stat = ncz_gettype(file,grp,vtype,&var->type_info))) goto done; } else {stat = NC_EBADTYPE; goto done;} - var->endianness = endianness; - var->type_info->endianness = var->endianness; /* Propagate */ if(vtype == NC_STRING) { zvar->maxstrlen = vtypelen; vtypelen = sizeof(char*); /* in-memory len */ @@ -1035,28 +1127,38 @@ read_vars(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO } } + /* shape */ + { + if((stat = NCJdictget(jvar,"shape",&jvalue))) goto done; + if(NCJsort(jvalue) != NCJ_ARRAY) {stat = (THROW(NC_ENCZARR)); goto done;} + + if(NCJlength(jvalue) == 0) { + zvar->scalar = 1; + rank = 0; + } else { + zvar->scalar = 0; + rank = NCJlength(jvalue); + } + + if(rank > 0) { + /* Save the rank of the variable */ + if((stat = nc4_var_set_ndims(var, rank))) goto done; + /* extract the shapes */ + if((shapes = (size64_t*)malloc(sizeof(size64_t)*rank)) == NULL) + {stat = (THROW(NC_ENOMEM)); goto done;} + if((stat = NCZ_decodeints(jvalue, shapes))) goto done; + } + /* Set storage flag */ + var->storage = (zvar->scalar?NC_CONTIGUOUS:NC_CHUNKED); + } + if(!purezarr) { /* Extract the _NCZARR_ARRAY values */ - /* Do this first so we know about storage esp. scalar */ - /* Extract the NCZ_V3_ARRAY dict */ if((stat = NCJdictget(jvar,NCZ_V3_ARRAY,&jncvar))) goto done; if(jncvar == NULL) {stat = NC_ENCZARR; goto done;} assert((NCJsort(jncvar) == NCJ_DICT)); - /* Extract scalar flag */ - if((stat = NCJdictget(jncvar,"scalar",&jvalue))) - goto done; - if(jvalue != NULL) { - var->storage = NC_CHUNKED; - zvar->scalar = 1; - } - /* Extract storage flag */ - if((stat = NCJdictget(jncvar,"storage",&jvalue))) - goto done; - if(jvalue != NULL) { - var->storage = NC_CHUNKED; - } /* Extract dimrefs list */ - switch ((stat = NCJdictget(jncvar,"dimrefs",&jdimrefs))) { + switch ((stat = NCJdictget(jncvar,"dimensions",&jdimrefs))) { case NC_NOERR: /* Extract the dimref names */ assert((NCJsort(jdimrefs) == NCJ_ARRAY)); if(zvar->scalar) { @@ -1065,11 +1167,10 @@ read_vars(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO rank = NCJlength(jdimrefs); for(j=0;jdimension_separator = 0; - if((stat = NCJdictget(jvar,"dimension_separator",&jvalue))) goto done; - if(jvalue != NULL) { + if((stat = NCJdictget(jvar,"chunk_key_encoding",&jvalue))) goto done; + if(jvalue == NULL) { + /* If value is invalid, then use global default */ + if(!islegaldimsep(zvar->dimension_separator)) + zvar->dimension_separator = ngs->zarr.dimension_separator; /* use global value */ /* Verify its value */ - if(NCJsort(jvalue) == NCJ_STRING && NCJstring(jvalue) != NULL && strlen(NCJstring(jvalue)) == 1) - zvar->dimension_separator = NCJstring(jvalue)[0]; + } else { + if(NCJsort(jvalue) != NCJ_DICT) {stat = NC_ENOTZARR; goto done;} + if((stat = NCJdictget(jvalue,"name",&jtmp))) {stat = NC_ENOTZARR; goto done;} + if(strcasecmp("default",NCJstring(jtmp))==0) { + if((stat = NCJdictget(jvalue,"configuration",&jtmp))) {stat = NC_ENOTZARR; goto done;} + if(jtmp != NULL) { + if((stat = NCJdictget(jtmp,"separator",&jsep))) {stat = NC_ENOTZARR; goto done;} + if(jsep != NULL) { + if(NCJisatomic(jsep) && NCJstring(jsep) != NULL && strlen(NCJstring(jsep)) == 1) + zvar->dimension_separator = NCJstring(jsep)[0]; + } else + zvar->dimension_separator = '/'; + } else + zvar->dimension_separator = '/'; + } else if(strcasecmp("v2",NCJstring(jtmp))==0) { + if((stat = NCJdictget(jvalue,"separator",&jsep))) {stat = NC_ENOTZARR; goto done;} + if(jsep != NULL) { + if(NCJsort(jsep) == NCJ_STRING && NCJstring(jsep) != NULL && strlen(NCJstring(jsep)) == 1) + zvar->dimension_separator = NCJstring(jsep)[0]; + else {stat = NC_ENOTZARR; goto done;} + } else + zvar->dimension_separator = '.'; + } else {stat = NC_ENOTZARR; goto done;} + } - /* If value is invalid, then use global default */ - if(!islegaldimsep(zvar->dimension_separator)) - zvar->dimension_separator = ngs->zarr.dimension_separator; /* use global value */ assert(islegaldimsep(zvar->dimension_separator)); /* we are hosed */ } @@ -1117,127 +1240,91 @@ read_vars(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO } } - /* shape */ - { - if((stat = NCJdictget(jvar,"shape",&jvalue))) goto done; - if(NCJsort(jvalue) != NCJ_ARRAY) {stat = (THROW(NC_ENCZARR)); goto done;} - - /* Process the rank */ - zarr_rank = NCJlength(jvalue); - if(zarr_rank == 0) { - /* suppress variable */ - ZLOG(NCLOGWARN,"Empty shape for variable %s suppressed",var->hdr.name); - suppress = 1; - goto suppressvar; - } - - if(zvar->scalar) { - rank = 0; - zarr_rank = 1; /* Zarr does not support scalars */ - } else - rank = (zarr_rank = NCJlength(jvalue)); - - if(zarr_rank > 0) { - /* Save the rank of the variable */ - if((stat = nc4_var_set_ndims(var, rank))) goto done; - /* extract the shapes */ - if((shapes = (size64_t*)malloc(sizeof(size64_t)*zarr_rank)) == NULL) - {stat = (THROW(NC_ENOMEM)); goto done;} - if((stat = NCZ_decodeints(jvalue, shapes))) goto done; - } - } - /* chunks */ { - size64_t chunks[NC_MAX_VAR_DIMS]; - if((stat = NCJdictget(jvar,"chunks",&jvalue))) goto done; - if(jvalue != NULL && NCJsort(jvalue) != NCJ_ARRAY) - {stat = (THROW(NC_ENCZARR)); goto done;} - /* Verify the rank */ - if(zvar->scalar || zarr_rank == 0) { - if(var->ndims != 0) - {stat = (THROW(NC_ENCZARR)); goto done;} - zvar->chunkproduct = 1; - zvar->chunksize = zvar->chunkproduct * var->type_info->size; - /* Create the cache */ - if((stat = NCZ_create_chunk_cache(var,var->type_info->size*zvar->chunkproduct,zvar->dimension_separator,&zvar->cache))) - goto done; - } else {/* !zvar->scalar */ - if(zarr_rank == 0) {stat = NC_ENCZARR; goto done;} - var->storage = NC_CHUNKED; - if(var->ndims != rank) - {stat = (THROW(NC_ENCZARR)); goto done;} - if((var->chunksizes = malloc(sizeof(size_t)*zarr_rank)) == NULL) - {stat = NC_ENOMEM; goto done;} - if((stat = NCZ_decodeints(jvalue, chunks))) goto done; - /* validate the chunk sizes */ - zvar->chunkproduct = 1; - for(j=0;jchunksizes[j] = (size_t)chunks[j]; - zvar->chunkproduct *= chunks[j]; - } - zvar->chunksize = zvar->chunkproduct * var->type_info->size; - /* Create the cache */ - if((stat = NCZ_create_chunk_cache(var,var->type_info->size*zvar->chunkproduct,zvar->dimension_separator,&zvar->cache))) - goto done; - } - if((stat = NCZ_adjust_var_cache(var))) goto done; - } - /* Capture row vs column major; currently, column major not used*/ - { - if((stat = NCJdictget(jvar,"order",&jvalue))) goto done; - if(strcmp(NCJstring(jvalue),"C")==1) - ((NCZ_VAR_INFO_T*)var->format_var_info)->order = 1; - else ((NCZ_VAR_INFO_T*)var->format_var_info)->order = 0; + if(!zvar->scalar) { + if((stat = NCJdictget(jvar,"chunk_grid",&jvalue))) goto done; + if(jvalue == NULL) {stat = (THROW(NC_ENOTBUILT)); goto done;} + if((stat = NCJdictget(jvalue,"name",&jtmp))) goto done; + if(jtmp == NULL) {stat = (THROW(NC_ENOTZARR)); goto done;} + if(strcasecmp("regular",NCJstring(jtmp))!=0) {stat = THROW(NC_ENOTZARR); goto done;} + if((stat = NCJdictget(jvalue,"configuration",&jtmp))) goto done; + if(jtmp == NULL) {stat = (THROW(NC_ENOTZARR)); goto done;} + if((stat = NCJdictget(jtmp,"chunk_shape",&jtmp2))) goto done; + if(jtmp2 == NULL) {stat = (THROW(NC_ENOTZARR)); goto done;} + if(NCJsort(jtmp2) != NCJ_ARRAY) {stat = (THROW(NC_ENOTZARR)); goto done;} + + /* Verify the rank */ + assert(rank != 0); + if(rank != NCJlength(jtmp2)) {stat = (THROW(NC_ENCZARR)); goto done;} + + /* Get the chunks and chunkproduct */ + if((chunks = (size64_t*)malloc(sizeof(size64_t)*rank)) == NULL) + {stat = (THROW(NC_ENOMEM)); goto done;} + if((stat = NCZ_decodeints(jtmp2, chunks))) goto done; + zvar->chunkproduct = 1; + for(k=0;kchunkproduct *= chunks[i]; + zvar->chunksize = zvar->chunkproduct * var->type_info->size; + /* Create the cache */ + if((stat = NCZ_create_chunk_cache(var,zvar->chunksize,zvar->dimension_separator,&zvar->cache))) goto done; + } else {/* zvar->scalar */ + zvar->chunkproduct = 1; + zvar->chunksize = zvar->chunkproduct * var->type_info->size; + var->chunksizes = NULL; + zvar->chunksize = zvar->chunkproduct * var->type_info->size; + /* Create the cache */ + if((stat = NCZ_create_chunk_cache(var,zvar->chunksize,zvar->dimension_separator,&zvar->cache))) goto done; + } + if((stat = NCZ_adjust_var_cache(var))) goto done; } - /* filters key */ + + /* codecs key */ /* From V2 Spec: A list of JSON objects providing codec configurations, or null if no filters are to be applied. Each codec configuration object MUST contain a "id" key identifying the codec to be used. */ - /* Do filters key before compressor key so final filter chain is in correct order */ { + NCJdictget(jvar,"codecs",&jcodecs); + assert(jcodecs != NULL); + /* Process the first codec which must be bytes, to get the endianness */ + jcodec = NCJith(jcodecs,0); + if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_ENOTZARR; goto done;} + if((stat = NCJdictget(jcodec,"name",&jvalue))) goto done; + if(jvalue == NULL || strcmp(ZBYTES3,NCJstring(jvalue))!=0) {stat = NC_ENCZARR; goto done;} + if((stat = NCJdictget(jcodec,"configuration",&jvalue))) goto done; + if(jvalue == NULL || NCJsort(jvalue) != NCJ_DICT) {stat = NC_ENCZARR; goto done;} + if((stat = NCJdictget(jvalue,"endian",&jtmp))) goto done; + if(strcasecmp("big",NCJstring(jtmp))==0) + var->endianness = NC_ENDIAN_BIG; + else if(strcasecmp("little",NCJstring(jtmp))==0) + var->endianness = NC_ENDIAN_LITTLE; + else {stat = NC_EINVAL; goto done;} + #ifdef ENABLE_NCZARR_FILTERS if(var->filters == NULL) var->filters = (void*)nclistnew(); if(zvar->incompletefilters == NULL) zvar->incompletefilters = (void*)nclistnew(); chainindex = 0; /* track location of filter in the chain */ if((stat = NCZ_filter_initialize())) goto done; - if((stat = NCJdictget(jvar,"filters",&jvalue))) goto done; - if(jvalue != NULL && NCJsort(jvalue) != NCJ_NULL) { - int k; - if(NCJsort(jvalue) != NCJ_ARRAY) {stat = NC_EFILTER; goto done;} - for(k=0;;k++) { - jfilter = NULL; - jfilter = NCJith(jvalue,k); - if(jfilter == NULL) break; /* done */ - if(NCJsort(jfilter) != NCJ_DICT) {stat = NC_EFILTER; goto done;} - if((stat = NCZ_filter_build(file,var,jfilter,chainindex++))) goto done; - } + if((stat = NCJdictget(jvar,"codecs",&jcodecs))) goto done; + if(jcodecs == NULL || NCJsort(jcodecs) == NCJ_NULL) + {stat = NC_ENOTZARR; goto done;} + if(NCJsort(jcodecs) != NCJ_ARRAY) {stat = NC_ENOTZARR; goto done;} + if(NCJlength(jcodecs) == 0) {stat = NC_ENOTZARR; goto done;} + + /* Process remaining filters */ + for(k=1;;k++) { + jcodec = NULL; + jcodec = NCJith(jcodecs,k); + if(jcodec == NULL) break; /* done */ + if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EFILTER; goto done;} + if((stat = NCZ_filter_build(file,var,jcodec,chainindex++))) goto done; } + /* Suppress variable if there are filters and var is not fixed-size */ + if(varsized && nclistlength((NClist*)var->filters) > 0) suppress = 1; #endif } - /* compressor key */ - /* From V2 Spec: A JSON object identifying the primary compression codec and providing - configuration parameters, or ``null`` if no compressor is to be used. */ -#ifdef ENABLE_NCZARR_FILTERS - { - if(var->filters == NULL) var->filters = (void*)nclistnew(); - if((stat = NCZ_filter_initialize())) goto done; - if((stat = NCJdictget(jvar,"compressor",&jfilter))) goto done; - if(jfilter != NULL && NCJsort(jfilter) != NCJ_NULL) { - if(NCJsort(jfilter) != NCJ_DICT) {stat = NC_EFILTER; goto done;} - if((stat = NCZ_filter_build(file,var,jfilter,chainindex++))) goto done; - } - } - /* Suppress variable if there are filters and var is not fixed-size */ - if(varsized && nclistlength((NClist*)var->filters) > 0) - suppress = 1; -#endif - if(zarr_rank > 0) { - if((stat = NCZ_computedimrefs(file, zfile, map, var, rank, dimnames, shapes, var->dim))) - goto done; + if(rank > 0) { + if((stat = NCZ_computedimrefs(file, var, jvar, dimnames, shapes, var->dim))) goto done; if(!zvar->scalar) { /* Extract the dimids */ for(j=0;jcontroller->path,grp->hdr.name,nclistlength(subgrpnames)); - /* Load each subgroup name in turn */ - for(i = 0; i < nclistlength(subgrpnames); i++) { - NC_GRP_INFO_T* g = NULL; - const char* gname = nclistget(subgrpnames,i); - char norm_name[NC_MAX_NAME]; - /* Check and normalize the name. */ - if((stat = nc4_check_name(gname, norm_name))) - goto done; - if((stat = nc4_grp_list_add(file, grp, norm_name, &g))) - goto done; - if(!(g->format_grp_info = calloc(1, sizeof(NCZ_GRP_INFO_T)))) - {stat = NC_ENOMEM; goto done;} - ((NCZ_GRP_INFO_T*)g->format_grp_info)->common.file = file; - } - /* Recurse to fill in subgroups */ for(i=0;ichildren);i++) { NC_GRP_INFO_T* g = (NC_GRP_INFO_T*)ncindexith(grp->children,i); - if((stat = read_grp(file,zfile,map,g))) + if((stat = read_grp(file,g))) goto done; } @@ -1373,145 +1444,177 @@ NCZ_load_jatts(NCjson* jblock, NCjson** jattsp, NClist** atypesp) /* Potentially shared functions */ /** - * @internal Synchronize dimension data from memory to map. + * @internal JSONize dims in a group * * @param file pointer to file struct - * + * @param parent group + * @param jdimsp return the array of jsonized dimensions * @return ::NC_NOERR No error. * @author Dennis Heimbigner */ static int -NCZ_collect_dims(NC_FILE_INFO_T* file, NCjson** jdimsp) +NCZ_collect_dims(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, NCjson** jdimsp) { int i, stat=NC_NOERR; NCjson* jdims = NULL; + NCjson* jdim = NULL; + NCjson* jdimname = NULL; NCjson* jdimsize = NULL; - NCjson* jdimargs = NULL; - NCbytes* fqn = ncbytesnew(); + NCjson* jdimunlimited = NULL; + NCjson* jcfg = NULL; char slen[64]; - ZTRACE(3,"file=%s grp=%s",file->controller->path,grp->hdr.name); - - NCJnew(NCJ_DICT,&jdims); + ZTRACE(3,"file=%s parent=%s",file->controller->path,parent->hdr.name); - for(i=0; ialldims); i++) { - NC_DIM_INFO_T* dim = (NC_DIM_INFO_T*)nclistget(file->alldims,i); - - /* Compute FQN for dimension */ - ncbytesclear(fqn); - if((stat = NCZ_makeFQN(dim->container,(NC_OBJ*)dim,fqn))) goto done; + NCJnew(NCJ_ARRAY,&jdims); + for(i=0; idim); i++) { + NC_DIM_INFO_T* dim = (NC_DIM_INFO_T*)ncindexith(parent->dim,i); + jdimsize = NULL; + jdimname = NULL; + jdimunlimited = NULL; + jdim = NULL; + jcfg = NULL; + + /* Collect the dimension info */ + + /* jsonize dim name */ + NCJnewstring(NCJ_STRING,dim->hdr.name,&jdimname); + /* Jsonize the size */ snprintf(slen,sizeof(slen),"%llu",(unsigned long long)dim->len); - if((stat = NCJnewstring(NCJ_INT,slen,&jdimsize))) goto done; + NCJnewstring(NCJ_INT,slen,&jdimsize); + /* And the unlimited flag*/ + if(dim->unlimited) { + NCJnewstring(NCJ_INT,"1",&jdimunlimited); + } - if((stat = NCJnew(NCJ_DICT,&jdimargs))) goto done; - if((stat = NCJinsert(jdimargs,"size",jdimsize))) goto done; - jdimsize = NULL; - if(dim->unlimited) - {if((stat = NCJinsertstring(jdimargs,"unlimited","1"))) goto done;} - if((stat = NCJinsert(jdims,ncbytescontents(fqn),jdimargs))) goto done; + /* Assemble the dimension dict */ + + /* Create configuration dict */ + NCJnew(NCJ_DICT,&jcfg); + NCJinsert(jcfg,"size",jdimsize); + jdimsize = NULL; + if(jdimunlimited != NULL) NCJinsert(jcfg,"unlimited",jdimunlimited); + jdimunlimited = NULL; + + /* Create dim dict */ + NCJnew(NCJ_DICT,&jdim); + + /* Add the dimension name*/ + NCJinsert(jdim,"name",jdimname); + jdimname = NULL; + + /* Add the configuration */ + NCJinsert(jdim,"configuration",jcfg); + jcfg = NULL; + + /* Add to jdims */ + NCJappend(jdims,jdim); + jdim = NULL; } if(jdimsp) {*jdimsp = jdims; jdims = NULL;} -done: - ncbytesfree(fqn); + NCJreclaim(jdims); + NCJreclaim(jdim); + NCJreclaim(jdimsize); + NCJreclaim(jdimname); + NCJreclaim(jdimunlimited); + NCJreclaim(jcfg); return ZUNTRACE(THROW(stat)); } + +/** + * @internal JSONize arrays in a group + * + * @param file pointer to file struct + * @param parent group + * @param jarrayp return the array of jsonized arrays + * @return ::NC_NOERR No error. + * + * @author Dennis Heimbigner + */ static int -NCZ_parse_group_content(NCjson* jcontent, NClist* dimdefs, NClist* varnames, NClist* subgrps) +NCZ_collect_arrays(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, NCjson** jarraysp) { - int i,stat = NC_NOERR; - NCjson* jvalue = NULL; - NCjson* jgrp = NULL; + int i, stat=NC_NOERR; + NCjson* jarrays = NULL; + NCjson* jarray = NULL; + NCjson* jarrayname = NULL; + ZTRACE(3,"file=%s parent=%s",file->controller->path,parent->hdr.name); - ZTRACE(3,"jcontent=|%s| |dimdefs|=%u |varnames|=%u |subgrps|=%u",NCJtotext(jcontent),(unsigned)nclistlength(dimdefs),(unsigned)nclistlength(varnames),(unsigned)nclistlength(subgrps)); + NCJnew(NCJ_ARRAY,&jarrays); - /* Get the _nczarr_group key */ - if((stat=NCJdictget(jcontent,NCZ_V3_GROUP,&jgrp))) goto done; - if(jgrp == NULL) {stat = NC_ENCZARR; goto done;} - - /* Now get nczarr specific keys */ - if((stat=NCJdictget(jgrp,"dims",&jvalue))) goto done; - if(jvalue != NULL) { - if(NCJsort(jvalue) != NCJ_DICT) {stat = (THROW(NC_ENCZARR)); goto done;} - /* Extract the dimensions defined in this group */ - for(i=0;ivars); i++) { + NC_VAR_INFO_T* var = (NC_VAR_INFO_T*)ncindexith(parent->vars,i); + jarrayname = NULL; + + /* jsonize array name */ + NCJnewstring(NCJ_STRING,var->hdr.name,&jarrayname); - if((stat=NCJdictget(jgrp,"vars",&jvalue))) goto done; - if(jvalue != NULL) { - /* Extract the variable names in this group */ - for(i=0;icommon.file->controller->path,grp->hdr.name,(unsigned)nclistlength(varnames),(unsigned)nclistlength(subgrps)); + ZTRACE(3,"file=%s grp=%s",file->controller->path,grp->hdr.name); - nclistclear(varnames); - if((stat = NCZ_searchvars(zfile,grp,varnames))) goto done; - nclistclear(subgrps); - if((stat = NCZ_searchsubgrps(zfile,grp,subgrps))) goto done; + /* Array of subgroup dicts */ + NCJnew(NCJ_ARRAY, &jsubgrps); + /* For each subgroup, build the subgroup metadata and insert into + the json for the group here. This is recursive. + */ + for(i=0; ichildren); i++) { + NC_GRP_INFO_T* g = (NC_GRP_INFO_T*)ncindexith(grp->children,i); + /* Recurse to build the subgroup dict for g*/ + if((stat = build_group_json(file,g,&jsubgrp))) goto done; + NCJappend(jsubgrps,jsubgrp); + jsubgrp = NULL; + } + + if(jsubgrpsp) {*jsubgrpsp = jsubgrps; jsubgrps = NULL;} done: + NCJreclaim(jsubgrp); + NCJreclaim(jsubgrps); + NCJreclaim(jgrpname); return ZUNTRACE(THROW(stat)); } + /** @internal Read attributes from a group or var and create a list of annotated NC_ATT_INFO_T* objects. This will process @@ -1524,10 +1627,11 @@ _NCProperties attribute specially. @author Dennis Heimbigner */ static int -NCZ_read_atts(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NC_OBJ* container, NCjson* jblock) +NCZ_read_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCjson* jblock) { int stat = NC_NOERR; int i; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; NC_VAR_INFO_T* var = NULL; NCZ_VAR_INFO_T* zvar = NULL; NC_GRP_INFO_T* grp = NULL; @@ -1565,7 +1669,7 @@ NCZ_read_atts(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NC_OBJ* container, N if(jattrs != NULL) { /* Iterate over the attributes to create the in-memory attributes */ - /* Watch for special cases: _FillValue and _ARRAY_DIMENSIONS (xarray), etc. */ + /* Watch for special cases: _FillValue etc. */ for(i=0;iroot_grp == grp) { /* Setup provenance */ - if(NCJsort(value) != NCJ_STRING) + if(!NCJisatomic(value)) {stat = (THROW(NC_ENCZARR)); goto done;} /*malformed*/ if((stat = NCZ_read_provenance(file,aname,NCJstring(value)))) goto done; } - /* case 2: name = _ARRAY_DIMENSIONS, sort==NCVAR, flags & HIDDENATTRFLAG */ - if(strcmp(aname,NC_XARRAY_DIMS)==0 && var != NULL && (ra->flags & HIDDENATTRFLAG)) { - /* store for later */ - int i; - assert(NCJsort(value) == NCJ_ARRAY); - if((zvar->xarray = nclistnew())==NULL) - {stat = NC_ENOMEM; goto done;} - for(i=0;ixarray,strdup(NCJstring(k))); - } - } /* case other: if attribute is hidden */ if(ra->flags & HIDDENATTRFLAG) continue; /* ignore it */ } @@ -1650,71 +1741,84 @@ NCZ_read_atts(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NC_OBJ* container, N } static int -NCZ_searchvars(NCZ_FILE_INFO_T* zfile, NC_GRP_INFO_T* grp, NClist* varnames) +subobjects_pure(NCZ_FILE_INFO_T* zfile, NC_GRP_INFO_T* grp, NClist* varnames, NClist* grpnames) { int i,stat = NC_NOERR; char* grpkey = NULL; - char* varkey = NULL; - char* zarray = NULL; + char* objkey = NULL; + char* zjkey = NULL; + char* content = NULL; NClist* matches = nclistnew(); + NCjson* json = NULL; + NCjson* jnodetype = NULL; /* do not reclaim */ /* Compute the key for the grp */ if((stat = NCZ_grpkey(grp,&grpkey))) goto done; /* Get the map and search group */ if((stat = nczmap_search(zfile->map,grpkey,matches))) goto done; for(i=0;imap,zarray)) == NC_NOERR) - nclistpush(varnames,strdup(name)); + nullfree(content); content = NULL; + NCJreclaim(json); json = NULL; + if(strcmp(name,Z3GROUP)==0) continue; /* only want next level */ + /* See if name/zarr.json exists */ + if((stat = nczm_concat(grpkey,name,&objkey))) goto done; + if((stat = nczm_concat(objkey,Z3ARRAY,&zjkey))) goto done; + if((stat = nczmap_len(zfile->map,zjkey,&size)) == NC_NOERR) { + /* Read contents of the zarr.json */ + if((content = (char*)malloc(size))==NULL) {stat = NC_ENOMEM; goto done;} + if((stat = nczmap_read(zfile->map,zjkey,0,size,content))) goto done; + /* Parse */ + if((NCJparse(content,0,&json))) {stat = NC_ENOTZARR; goto done;} + if(NCJsort(json) != NCJ_DICT) {stat = NC_ENOTZARR; goto done;} + if((NCJdictget(json,"node_type",&jnodetype))) {stat = NC_ENOTZARR; goto done;} + if(strcasecmp("group",NCJstring(jnodetype))==0) + nclistpush(grpnames,strdup(NCJstring(jnodetype))); + else if(strcasecmp("array",NCJstring(jnodetype))==0) + nclistpush(varnames,strdup(NCJstring(jnodetype))); + /* else ignore */ + } stat = NC_NOERR; - nullfree(varkey); varkey = NULL; - nullfree(zarray); zarray = NULL; } done: - nullfree(grpkey); - nullfree(varkey); - nullfree(zarray); + nullfree(objkey); + nullfree(zjkey); + nullfree(content); nclistfreeall(matches); + NCJreclaim(json); return stat; } - static int -NCZ_searchsubgrps(NCZ_FILE_INFO_T* zfile, NC_GRP_INFO_T* grp, NClist* subgrpnames) +subobjects(NCZ_FILE_INFO_T* zfile, NC_GRP_INFO_T* grp, NClist* varnames, NClist* grpnames) { - int i,stat = NC_NOERR; - char* grpkey = NULL; - char* subkey = NULL; - char* zgroup = NULL; - NClist* matches = nclistnew(); + int i; + NCZ_GRP_INFO_T* zgrp = (NCZ_GRP_INFO_T*)grp->format_grp_info; + NCjson* jgrp = zgrp->grpsuper; + NCjson* jarrays = NULL; + NCjson* jsubgrps = NULL; - /* Compute the key for the grp */ - if((stat = NCZ_grpkey(grp,&grpkey))) goto done; - /* Get the map and search group */ - if((stat = nczmap_search(zfile->map,grpkey,matches))) goto done; - for(i=0;imap,zgroup)) == NC_NOERR) - nclistpush(subgrpnames,strdup(name)); - stat = NC_NOERR; - nullfree(subkey); subkey = NULL; - nullfree(zgroup); zgroup = NULL; - } + NCJdictget(jgrp,"arrays",&jarrays); + NCJdictget(jgrp,"children",&jsubgrps); -done: - nullfree(grpkey); - nullfree(subkey); - nullfree(zgroup); - nclistfreeall(matches); - return stat; + for(i=0;i we need to create the dims in root if they do not already exist */ int purezarr = 0; - int xarray = 0; - NCZ_VAR_INFO_T* zvar = NULL; - - ZTRACE(3,"file=%s var=%s purezarr=%d xarray=%d ndims=%d shape=%s", - file->controller->path,var->hdr.name,purezarr,xarray,(int)ndims,nczprint_vector(ndims,shapes)); + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + NCjson* jdimnames = NULL; + NCjson* jshape = NULL; + NCjson* jdimfqns = NULL; + NCjson* jnczarray = NULL; + int ndims; + NC_DIM_INFO_T* vardims[NC_MAX_VAR_DIMS]; - zvar = (NCZ_VAR_INFO_T*)(var->format_var_info); - assert(zvar != NULL); + ZTRACE(3,"file=%s var=%s purezarr=%d ndims=%d shape=%s", + file->controller->path,var->hdr.name,purezarr,(int)ndims,nczprint_vector(ndims,shapes)); if(zfile->controls.flags & FLAG_PUREZARR) purezarr = 1; - if(zfile->controls.flags & FLAG_XARRAYDIMS) xarray = 1; assert(var->atts_read); - if(purezarr && xarray) {/* process xarray dimdef attribute; Note that it might not exist */ - /* Note that if xarray && !purezarr, then xarray will be superceded by the nczarr dimensions key */ - char zdimname[4096]; - if(zvar->xarray == NULL) { - assert(nclistlength(dimnames) == 0); - } else { /* (zvar->xarray != NULL) */ - /* convert xarray to the dimnames */ - for(i=0;ixarray);i++) { - snprintf(zdimname,sizeof(zdimname),"/%s",(const char*)nclistget(zvar->xarray,i)); - nclistpush(dimnames,strdup(zdimname)); - } - } - createdims = 1; /* may need to create them */ + if(dims) *dims = NULL; + ndims = var->ndims; + if(ndims == 0) goto done; /* scalar */ + + /* We have two sources for the dimension names for this variable. + 1. the "dimension_names" inside the zarr.json dictionary. + 2. the "dimensions" key inside the _nczarr_array dictionary. + + If purezarr, then we only have #1. In that case, for each name in "dimension_names", + we need to do the following: + 1. get the corresponding size from the "shapes" vector. + 2. if the name is null, then set the name to "_zdim_", + where n is the size from the shape vector. + 3. search up the containing parent groups looking for a dimension + with that name. + 4. if found, then use that dimension. + 5. if not found, then create the dimension in the immediately containing group. + + If not purezarr, then we verify for each FQN in #2 that + 1. its simple name (the last segment of the FQN) is the same as the name from #1. + 2. we use the FQN to find the actual dimension. + */ + + /* Get the shape vector */ + NCJdictget(jvar,"shape",&jshape); + assert(jshape != NULL && NCJsort(jshape)==NCJ_ARRAY && NCJlength(jshape)==ndims); + + /* Get the dimension_names array */ + NCJdictget(jvar,"dimension_names",&jdimnames); + + if(!purezarr) { + NCJdictget(jvar,NCZ_V3_ARRAY,&jnczarray); + if(jnczarray == NULL) {stat = NC_ENCZARR; goto done;} + /* get the FQNS */ + NCJdictget(jnczarray,"dimensions",&jdimfqns); + if(jdimfqns == NULL) {stat = NC_ENCZARR; goto done;} + assert(jdimfqns != NULL && NCJlength(jdimfqns) == ndims); } - /* If pure zarr and we have no dimref names, then fake it */ - if(purezarr && nclistlength(dimnames) == 0) { - createdims = 1; - for(i=0;icontainer,dimname,size,&dim))) goto done; + if(dim == NULL) { + /* Create dim in this grp */ + if((stat = nc4_dim_list_add(var->container, dimname, (size_t)size, -1, &dim))) goto done; + } + assert(dim != NULL); + vardims[i] = dim; + } + } else { /* !purezarr */ + for(i=0;icontainer,p,NCDIM,(NC_OBJ**)&dim))) goto done; + assert(dim != NULL); + vardims[i] = dim; + } } } - - /* Now, use dimnames to get the dims; create if necessary */ - if((stat = NCZ_parsedimrefs(file,dimnames,shapes,dims,createdims))) - goto done; - + for(i=0;idim && var->dimids); + for(i=0;idim[i] = vardims[i]; + var->dimids[i] = vardims[i]->hdr.id; + } + done: return ZUNTRACE(THROW(stat)); } @@ -1972,6 +2147,7 @@ NCZ_attr_convert(NCjson* src, nc_type typeid, size_t typelen, int* countp, NCbyt return ZUNTRACE(THROW(stat)); } +#if 0 static int NCZ_parsedimrefs(NC_FILE_INFO_T* file, NClist* dimnames, size64_t* shape, NC_DIM_INFO_T** dims, int create) { @@ -2018,6 +2194,7 @@ NCZ_parsedimrefs(NC_FILE_INFO_T* file, NClist* dimnames, size64_t* shape, NC_DIM nclistfreeall(segments); return THROW(stat); } +#endif /* Convert an attribute "types list to an envv style list */ static int @@ -2027,8 +2204,8 @@ NCZ_jtypes2atypes(NCjson* jtypes, NClist* atypes) for(i=0;iroot_grp; - for(i=0;ichildren);j++) { - NC_GRP_INFO_T* subgrp = (NC_GRP_INFO_T*)ncindexith(grp->children,j); - if(strcmp(subgrp->hdr.name,norm_name)==0) { - grp = subgrp; - found = 1; - break; - } - } - if(!found) {stat = NC_ENOGRP; goto done;} - } - /* grp should be group of interest */ - if(grpp) *grpp = grp; + int stat = NC_NOERR; + NC_DIM_INFO_T* dim = NULL; + dim = (NC_DIM_INFO_T*)ncindexlookup(grp->dim,dimname); + if(dim == NULL && grp->parent != NULL) + {if((stat = finddim(file,grp->parent,dimname,size,&dim))) goto done;} + if(dimp) *dimp = dim; done: return THROW(stat); } -/* This code is a subset of NCZ_def_dim */ -static int -NCZ_createdim(NC_FILE_INFO_T* file, const char* name, size64_t dimlen, NC_DIM_INFO_T** dimp) -{ - int stat = NC_NOERR; - NC_GRP_INFO_T* root = file->root_grp; - NC_DIM_INFO_T* thed = NULL; - if((stat = nc4_dim_list_add(root, name, (size_t)dimlen, -1, &thed))) - goto done; - assert(thed != NULL); - /* Create struct for NCZ-specific dim info. */ - if (!(thed->format_dim_info = calloc(1, sizeof(NCZ_DIM_INFO_T)))) - {stat = NC_ENOMEM; goto done;} - ((NCZ_DIM_INFO_T*)thed->format_dim_info)->common.file = file; - *dimp = thed; thed = NULL; -done: - return stat; -} /**************************************************/ /* Format Filter Support Functions */ @@ -2118,7 +2262,6 @@ int ZF3_hdf2codec(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, NCZ_Filter* filter) { int stat = NC_NOERR; - NCZ_codec_env_t env = NCZ_CODEC_ENV_EMPTY_V3; /* Convert the HDF5 id + visible parameters to the codec form */ @@ -2127,7 +2270,7 @@ ZF3_hdf2codec(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, NCZ_Filter* nullfree(filter->codec.codec); filter->codec.codec = NULL; filter->codec.id = strdup(filter->plugin->codec.codec->codecid); if(filter->plugin->codec.codec->NCZ_hdf5_to_codec) { - stat = filter->plugin->codec.codec->NCZ_hdf5_to_codec(&env,filter->hdf5.visible.nparams,filter->hdf5.visible.params,&filter->codec.codec); + stat = filter->plugin->codec.codec->NCZ_hdf5_to_codec(NCplistzarrv3,filter->hdf5.id,filter->hdf5.visible.nparams,filter->hdf5.visible.params,&filter->codec.codec); #ifdef DEBUGF fprintf(stderr,">>> DEBUGF: NCZ_hdf5_to_codec: visible=%s codec=%s\n",printnczparams(filter->hdf5.visible),filter->codec.codec); #endif @@ -2145,24 +2288,23 @@ ZF3_codec2hdf(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, const NCjson { int stat = NC_NOERR; NCjson* jvalue = NULL; - NCZ_codec_env_t env = NCZ_CODEC_ENV_EMPTY_V3; assert(jfilter != NULL); assert(filter != NULL); if(filter->codec.id == NULL) { /* Get the id of this codec filter */ - if(NCJdictget(jfilter,"id",&jvalue)<0) {stat = NC_EFILTER; goto done;} - if(NCJsort(jvalue) != NCJ_STRING) {stat = THROW(NC_ENOFILTER); goto done;} + if(NCJdictget(jfilter,"name",&jvalue)) {stat = NC_EFILTER; goto done;} + if(!NCJisatomic(jvalue)) {stat = THROW(NC_ENOFILTER); goto done;} filter->codec.id = strdup(NCJstring(jvalue)); } if(plugin != NULL) { /* Save the hdf5 id */ - filter->hdf5.id = plugin->codec.codec->hdf5id; + filter->hdf5.id = plugin->hdf5.filter->id; /* Convert the codec to hdf5 form visible parameters */ if(plugin->codec.codec->NCZ_codec_to_hdf5) { - stat = plugin->codec.codec->NCZ_codec_to_hdf5(&env,filter->codec.codec,&filter->hdf5.visible.nparams,&filter->hdf5.visible.params); + stat = plugin->codec.codec->NCZ_codec_to_hdf5(NCplistzarrv3,filter->codec.codec,&filter->hdf5.id,&filter->hdf5.visible.nparams,&filter->hdf5.visible.params); #ifdef DEBUGF fprintf(stderr,">>> DEBUGF: NCZ_codec_to_hdf5: codec=%s, hdf5=%s\n",printcodec(codec),printhdf5(hdf5)); #endif @@ -2184,9 +2326,11 @@ static const NCZ_Formatter NCZ_formatter3_table = { ZF3_create, ZF3_open, + ZF3_close, ZF3_readmeta, ZF3_writemeta, ZF3_readattrs, + ZF3_buildchunkkey, #ifdef ENABLE_NCZARR_FILTERS ZF3_codec2hdf, ZF3_hdf2codec, @@ -2194,7 +2338,6 @@ static const NCZ_Formatter NCZ_formatter3_table = { NULL, NULL, #endif - ZF3_close }; const NCZ_Formatter* NCZ_formatter3 = &NCZ_formatter3_table; diff --git a/libnczarr/zincludes.h b/libnczarr/zincludes.h index 5a5b8934fb..0a7fc0ee16 100644 --- a/libnczarr/zincludes.h +++ b/libnczarr/zincludes.h @@ -46,6 +46,7 @@ extern "C" { #include "ncs3sdk.h" #include "ncindex.h" #include "ncjson.h" +#include "ncproplist.h" #include "zmap.h" #include "zinternal.h" diff --git a/libnczarr/zinternal.c b/libnczarr/zinternal.c index 0534dc37bf..0fe153affa 100644 --- a/libnczarr/zinternal.c +++ b/libnczarr/zinternal.c @@ -27,6 +27,9 @@ extern int nc_log_level; #endif /* LOGGING */ +const NCproplist* NCplistzarrv2 = NULL; +const NCproplist* NCplistzarrv3 = NULL; + #ifdef LOOK /** * @internal Provide a wrapper for H5Eset_auto @@ -73,6 +76,10 @@ NCZ_initialize_internal(void) ngs->zarr.dimension_separator = dimsep[0]; } } + /* Build some common proplists */ + NCplistzarrv2 = ncplistnew(); + ncplistadd((NCproplist*)NCplistzarrv2,"zarrformat",(uintptr_t)2); + ncplistadd((NCproplist*)NCplistzarrv3,"zarrformat",(uintptr_t)3); return stat; } @@ -92,6 +99,11 @@ NCZ_finalize_internal(void) #ifdef ENABLE_S3 NCZ_s3finalize(); #endif + + /* Cleanup common proplists */ + ncplistfree((NCproplist*)NCplistzarrv2); NCplistzarrv2 = NULL; + ncplistfree((NCproplist*)NCplistzarrv3); NCplistzarrv3 = NULL; + return NC_NOERR; } diff --git a/libnczarr/zinternal.h b/libnczarr/zinternal.h index 08121776f3..77e8f91f6e 100644 --- a/libnczarr/zinternal.h +++ b/libnczarr/zinternal.h @@ -38,7 +38,6 @@ #define NCZ_CHUNKSIZE_FACTOR (10) #define NCZ_MIN_CHUNK_SIZE (2) - /**************************************************/ /* Constants */ @@ -78,6 +77,9 @@ /* Pure Zarr pseudo names */ #define ZDIMANON "_zdim" +/* Bytes codec name */ +#define ZBYTES3 "bytes" + /* V2 Reserved Attributes */ /* Inserted into /.zgroup @@ -102,13 +104,10 @@ Inserted into any .zattrs ? or should it go into the container? /* V3 Reserved Attributes */ /* Inserted into group zarr.json: -_nczarr_superblock: {"version": "[23].0.0"} -Inserted into any group zarr.json: -"_nczarr_group": "{ -\"dimensions\": {\"d1\": \"1\", \"d2\": \"1\",...} -\"variables\": [\"v1\", \"v2\", ...] -\"groups\": [\"g1\", \"g2\", ...] -}" +_nczarr_superblock: { + +}, + Inserted into any array zarr.json: "_nczarr_array": "{ \"dimensions\": [\"/g1/g2/d1\", \"/d2\",...] @@ -161,7 +160,7 @@ In v3, the attributes are in a dictionary under the key name "attributes", #define ncidfor(var) ncidforx((var)->container->nc4_info,(var)->container->hdr.id) /**************************************************/ -/* Forward */ +/* Opaque */ struct NClist; struct NCjson; @@ -169,6 +168,7 @@ struct NCauth; struct NCZMAP; struct NCZChunkCache; struct NCZ_Formatter; +struct NCproplist; /**************************************************/ /* Define annotation data for NCZ objects */ @@ -200,6 +200,7 @@ typedef struct NCZ_FILE_INFO { # define FLAG_NCZARR_V1 16 NCZM_IMPL mapimpl; } controls; + NCjson* superblock; /* Only used by NCZarr 3.0.0 and later */ struct NCZ_Formatter* dispatcher; } NCZ_FILE_INFO_T; @@ -216,18 +217,7 @@ typedef struct NCZ_ATT_INFO { /* Struct to hold ZARR-specific info for a group. */ typedef struct NCZ_GRP_INFO { NCZcommon common; -#if 0 - /* The jcontent field stores the following: - 1. List of (name,length) for dims in the group - 2. List of (name,type) for user-defined types in the group - 3. List of var names in the group - 4. List of subgroups names in the group - */ - NClist* dims; - NClist* types; /* currently not used */ - NClist* vars; - NClist* grps; -#endif + NCjson* grpsuper; /* Corresponding info from the superblock */ } NCZ_GRP_INFO_T; /* Struct to hold ZARR-specific info for a variable. */ @@ -254,6 +244,14 @@ typedef struct NCZ_TYPE_INFO { NCZcommon common; } NCZ_TYPE_INFO_T; +/* Parsed dimension info */ +struct NCZ_DimInfo { + char* path; + size64_t dimlen; + int unlimited; +}; + + #if 0 /* Define the contents of the .nczcontent object */ /* The .nczcontent field stores the following: @@ -272,6 +270,12 @@ typedef struct NCZCONTENT{ /**************************************************/ +/* Common property lists */ +EXTERNL const struct NCproplist* NCplistzarrv2; +EXTERNL const struct NCproplist* NCplistzarrv3; + +/**************************************************/ + extern int ncz_initialized; /**< True if initialization has happened. */ /* zinternal.c */ diff --git a/libnczarr/zmap.c b/libnczarr/zmap.c index 9126223f89..1ed10a8e3c 100644 --- a/libnczarr/zmap.c +++ b/libnczarr/zmap.c @@ -182,26 +182,14 @@ nczmap_write(NCZMAP* map, const char* key, size64_t count, const void* content) return map->api->write(map, key, count, content); } -/* Define a static qsort comparator for strings for use with qsort */ -static int -cmp_strings(const void* a1, const void* a2) -{ - const char** s1 = (const char**)a1; - const char** s2 = (const char**)a2; - return strcmp(*s1,*s2); -} - int nczmap_search(NCZMAP* map, const char* prefix, NClist* matches) { int stat = NC_NOERR; if((stat = map->api->search(map, prefix, matches)) == NC_NOERR) { - /* sort the list */ - if(nclistlength(matches) > 1) { - void* base = nclistcontents(matches); - qsort(base, nclistlength(matches), sizeof(char*), cmp_strings); - } + if((stat = NCZ_sort(nclistcontents(matches),nclistlength(matches),NULL))) goto done; /* sort the list */ } +done: return stat; } diff --git a/libnczarr/zutil.c b/libnczarr/zutil.c index c68b8f815f..a609e08752 100644 --- a/libnczarr/zutil.c +++ b/libnczarr/zutil.c @@ -51,10 +51,10 @@ Note also that if we read a pure zarr file we will probably always see "|S1", so we will never see a variable of type NC_CHAR. We might however see an attribute of type string. */ -static const struct ZTYPES { - char* zarr[3]; - char* nczarr[3]; -} znames[NUM_ATOMIC_TYPES] = { +static const struct ZTYPESV2 { + const char* zarr[3]; + const char* nczarr[3]; +} znamesv2[NUM_ATOMIC_TYPES] = { /* nc_type Pure Zarr NCZarr NE LE BE NE LE BE*/ /*NC_NAT*/ {{NULL,NULL,NULL}, {NULL,NULL,NULL}}, @@ -72,24 +72,6 @@ static const struct ZTYPES { /*NC_STRING*/ {{"|S%d","|S%d","|S%d"},{"|S%d","|S%d","|S%d"}}, }; -#if 0 -static const char* zfillvalue[NUM_ATOMIC_TYPES] = { -NULL, /*NC_NAT*/ -"-127", /*NC_BYTE*/ -"0", /*NC_CHAR*/ -"-32767", /*NC_SHORT*/ -"-2147483647", /*NC_INT*/ -"9.9692099683868690e+36f", /* near 15 * 2^119 */ /*NC_FLOAT*/ -"9.9692099683868690e+36", /*NC_DOUBLE*/ -"255", /*NC_UBYTE*/ -"65535", /*NC_USHORT*/ -"4294967295", /*NC_UINT*/ -"-9223372036854775806", /*NC_INT64*/ -"18446744073709551614", /*NC_UINT64*/ -"", /*NC_STRING*/ -}; -#endif - /* map nc_type -> NCJ_SORT */ static int zjsonsort[NUM_ATOMIC_TYPES] = { NCJ_UNDEF, /*NC_NAT*/ @@ -107,9 +89,39 @@ NCJ_INT, /*NC_UINT64*/ NCJ_STRING, /*NC_STRING*/ }; +static const struct ZTYPESV3 { + const char* zarr; + const char* nczarr; +} znamesv3[NUM_ATOMIC_TYPES] = { +/* nc_type Pure Zarr NCZarr */ +/*NC_NAT*/ {NULL, NULL}, +/*NC_BYTE*/ {"int8", "int8"}, +/*NC_CHAR*/ {"uint8", "uint8"}, +/*NC_SHORT*/ {"int16", "int16"}, +/*NC_INT*/ {"int32", "int32"}, +/*NC_FLOAT*/ {"float32", "float32"}, +/*NC_DOUBLE*/ {"float64", "float64"}, +/*NC_UBYTE*/ {"uint8", "uint8"}, +/*NC_USHORT*/ {"uint16", "uint16"}, +/*NC_UINT*/ {"uint32", "uint32"}, +/*NC_INT64*/ {"int64", "int64"}, +/*NC_UINT64*/ {"uint64", "uint64"}, +/*NC_STRING*/ {"r%u", "r%u"}, +}; + +/* Map dtypes to nctype */ +/* Keep sorted for binary search */ +static struct ZDTYPEV3 { + const char* dtype; + nc_type nctype; + size_t nctypelen; +} zdtypesv3[NUM_ATOMIC_TYPES]; +static int zdtypesv3n = 0; +static int zdtypesinitialized=0; + /* Forward */ -static char* backslashescape(const char* s); -static char* deescape(const char* esc); +static void znamesv3init(void); +static int splitfqn(const char* fqn0, NClist* segments); /**************************************************/ @@ -510,20 +522,8 @@ NCZ_subobjects(NCZMAP* map, const char* prefix, const char* tag, char dimsep, NC return stat; } -#if 0 -/* Convert a netcdf-4 type integer */ -int -ncz_nctypedecode(const char* snctype, nc_type* nctypep) -{ - unsigned nctype = 0; - if(sscanf(snctype,"%u",&nctype)!=1) return NC_EINVAL; - if(nctypep) *nctypep = nctype; - return NC_NOERR; -} -#endif - /** -@internal Given an nc_type+other, produce the corresponding dtype string. +@internal Given an nc_type+other, produce the corresponding Zarr v2 dtype string. @param nctype - [in] nc_type @param endianness - [in] endianness @param purezarr - [in] 1=>pure zarr, 0 => nczarr @@ -535,23 +535,23 @@ ncz_nctypedecode(const char* snctype, nc_type* nctypep) */ int -ncz_nctype2dtype(nc_type nctype, int endianness, int purezarr, int len, char** dnamep) +ncz2_nctype2dtype(nc_type nctype, int endianness, int purezarr, int len, char** dnamep) { char dname[64]; - char* format = NULL; + const char* format = NULL; if(nctype <= NC_NAT || nctype > NC_MAX_ATOMIC_TYPE) return NC_EINVAL; if(purezarr) - format = znames[nctype].zarr[endianness]; + format = znamesv2[nctype].zarr[endianness]; else - format = znames[nctype].nczarr[endianness]; + format = znamesv2[nctype].nczarr[endianness]; snprintf(dname,sizeof(dname),format,len); if(dnamep) *dnamep = strdup(dname); return NC_NOERR; } /* -@internal Convert a numcodecs dtype spec to a corresponding nc_type. +@internal Convert a numcodecs Zarr v2 dtype spec to a corresponding nc_type. @param nctype - [in] dtype the dtype to convert @param nctype - [in] typehint help disambiguate char vs string @param purezarr - [in] 1=>pure zarr, 0 => nczarr @@ -564,7 +564,7 @@ ncz_nctype2dtype(nc_type nctype, int endianness, int purezarr, int len, char** d */ int -ncz_dtype2nctype(const char* dtype, nc_type typehint, int purezarr, nc_type* nctypep, int* endianp, int* typelenp) +ncz2_dtype2nctype(const char* dtype, nc_type typehint, int purezarr, nc_type* nctypep, int* endianp, int* typelenp) { int stat = NC_NOERR; int typelen = 0; @@ -659,6 +659,114 @@ ncz_dtype2nctype(const char* dtype, nc_type typehint, int purezarr, nc_type* nct goto done; } +/** +@internal Given an nc_type+other, produce the corresponding Zarr v3 dtype string. +@param nctype - [in] nc_type +@param purezarr - [in] 1=>pure zarr, 0 => nczarr +@param strlen - [in] max string length +@param namep - [out] pointer to hold pointer to the dtype; user frees +@return NC_NOERR +@return NC_EINVAL +@author Dennis Heimbigner +*/ + +int +ncz3_nctype2dtype(nc_type nctype, int purezarr, int strlen, char** dnamep) +{ + char dname[64]; + const char* format = NULL; + + if(nctype <= NC_NAT || nctype > NC_MAX_ATOMIC_TYPE) return NC_EINVAL; + if(purezarr) + format = znamesv3[nctype].zarr; + else + format = znamesv3[nctype].nczarr; + snprintf(dname,sizeof(dname),format,strlen*8); + if(dnamep) *dnamep = strdup(dname); + return NC_NOERR; +} + +/* +@internal Convert a Zarr v3 data_type spec to a corresponding nc_type. +@param nctype - [in] data_type the dtype to convert +@param purezarr - [in] 1=>pure zarr, 0 => nczarr +@param nctypep - [out] hold corresponding type +@param typelenp - [out] hold corresponding type size (for fixed length strings) +@return NC_NOERR +@return NC_EINVAL +@author Dennis Heimbigner +*/ + +/* Comparator for search */ +static int +srchznamesv3(const void* a, const void* b) +{ + const char* key = (const char*)a; + const struct ZDTYPEV3* elem = (struct ZDTYPEV3*)b; + return strcasecmp(key,elem->dtype); +} + +int +ncz3_dtype2nctype(const char* dtype, int purezarr, nc_type* nctypep, int* typelenp) +{ + int stat = NC_NOERR; + nc_type nctype = NC_NAT; + int typelen = 0; + void* match = NULL; + + NC_UNUSED(purezarr); + + if(nctypep) *nctypep = NC_NAT; + if(dtype == NULL) goto zerr; + if(!zdtypesinitialized) znamesv3init(); + + /* Special case for r == string */ + if(1 == sscanf(dtype,"r%d",&typelen)) { + nctype = NC_STRING; + if((typelen % 8) != 0) goto zerr; + typelen = typelen / 8; /* convert bits to bytes */ + goto done; + } + match = bsearch((void*)dtype,(void*)zdtypesv3,zdtypesv3n,sizeof(struct ZDTYPEV3),srchznamesv3); + if(match == NULL) goto zerr; + nctype = ((struct ZDTYPEV3*)match)->nctype; + typelen = ((struct ZDTYPEV3*)match)->nctypelen; + +done: + if(nctypep) *nctypep = nctype; + if(typelenp) *typelenp = typelen; + return stat; +zerr: + stat = NC_ENCZARR; + goto done; +} + +/* Comparator for sort */ +static int +cmpznamesv3(const void* a, const void* b) +{ + struct ZDTYPEV3* za = (struct ZDTYPEV3*)a; + struct ZDTYPEV3* zb = (struct ZDTYPEV3*)b; + return strcasecmp(za->dtype,zb->dtype); +} + +static void +znamesv3init(void) +{ + uintptr_t i,j; + struct ZDTYPEV3 zt; + memset((void*)zdtypesv3,0,sizeof(zdtypesv3)); + for(j=0,i=NC_BYTE;iname))==NULL) goto done; + if((escaped = NCZ_backslashescape(object->name))==NULL) goto done; nclistpush(segments,escaped); escaped = NULL; /* Collect the group prefix segments (escaped) */ for(grp=parent;grp->parent!=NULL;grp=grp->parent) { /* Add in the group name */ - if((escaped = backslashescape(grp->hdr.name))==NULL) goto done; + if((escaped = NCZ_backslashescape(grp->hdr.name))==NULL) goto done; nclistpush(segments,escaped); escaped = NULL; } @@ -1151,23 +1259,67 @@ NCZ_makeFQN(NC_GRP_INFO_T* parent, NC_OBJ* object, NCbytes* fqn) return THROW(stat); } -/* Parse an fqn into a sequence of groups + some kind of object */ +/* Find an object matching the given name and of given sort */ int -NCZ_parseFQN(NC_FILE_INFO_T* file, const char* fqn0, NClist* path) +NCZ_locateFQN(NC_GRP_INFO_T* parent, const char* fqn, NC_SORT sort, NC_OBJ** objectp) { int i,ret = NC_NOERR; - int count; - char* p; - char* start; - char* fqn = NULL; - char* descaped = NULL; NC_GRP_INFO_T* grp = NULL; + NC_OBJ* object = NULL; + NClist* segments = nclistnew(); + size_t count = 0; + + assert(fqn != NULL && fqn[0] == '/'); + /* Step 1: Break fqn into segments at occurrences of '/' */ + if((ret = splitfqn(fqn,segments))) goto done; + count = nclistlength(segments); + + /* walk to convert to groups + 1 left over for the final object*/ + grp = parent; + for(i=0;ichildren,segment); + if(object == NULL || object->sort != NCGRP) {ret = NC_ENOOBJECT; goto done;} + grp = (NC_GRP_INFO_T*)object; object = NULL; + } + /* Find an object to match the sort and last segment */ + do { + const char* segment = (const char*)nclistget(segments,count-1); /* last segment */ + object = ncindexlookup(grp->children,segment); + if(object != NULL && (sort == NCNAT || sort == NCGRP)) break; /* not this */ + object = ncindexlookup(grp->dim,segment); + if(object != NULL && (sort == NCNAT || sort == NCDIM)) break; /* not this */ + object = ncindexlookup(grp->vars,segment); + if(object != NULL && (sort == NCNAT || sort == NCVAR)) break; /* not this */ + object = ncindexlookup(grp->type,segment); + if(object != NULL && (sort == NCNAT || sort == NCTYP)) break; /* not this */ + object = ncindexlookup(grp->att,segment); + if(object != NULL && (sort == NCNAT || sort == NCATT)) break; /* not this */ + object = NULL; /* not found */ + } while(0); + if(object == NULL) {ret = NC_ENOOBJECT; goto done;} + if(objectp) *objectp = object; +done: + nclistfreeall(segments); + return THROW(ret); +} + +static int +splitfqn(const char* fqn0, NClist* segments) +{ + int i,stat = NC_NOERR; + char* fqn = NULL; + char* p = NULL; + char* start = NULL; + int count = 0; assert(fqn0 != NULL && fqn0[0] == '/'); fqn = strdup(fqn0); start = fqn+1; /* leave off the leading '/' */ count = 0; - /* Step 1: Break fqn into pieces at occurrences of '/' */ + /* Break fqn into pieces at occurrences of '/' */ for(p=start;*p;) { switch(*p) { case '\\': @@ -1183,38 +1335,17 @@ NCZ_parseFQN(NC_FILE_INFO_T* file, const char* fqn0, NClist* path) break; } } - /* Re-walk to convert to groups */ + /* collect segments */ p = fqn+1; - grp = file->root_grp; /* Starting group */ for(i=0;ichildren,descaped); - if(object == NULL) object = ncindexlookup(grp->dim,descaped); - if(object == NULL) object = ncindexlookup(grp->vars,descaped); - if(object == NULL) object = ncindexlookup(grp->type,descaped); - if(object == NULL) object = ncindexlookup(grp->att,descaped); - if(object == NULL) {ret = NC_EINVAL; goto done;} - nclistpush(path,object); - p = p + strlen(p) + 1; /* skip past the terminating nul */ - nullfree(descaped); - descaped = NULL; + char* descaped = NCZ_deescape(p); + nclistpush(segments,descaped); } - /* Verify this looks like an FQN */ - for(i=0;i<(count-1);i++) { - NC_OBJ* object = nclistget(path,i); - if(object->sort != NCGRP) {ret = NC_EINVAL; goto done;} - } -done: - nullfree(descaped); - nullfree(fqn); - return THROW(ret); + return stat; } -static char* -backslashescape(const char* s) +char* +NCZ_backslashescape(const char* s) { const char* p; char* q; @@ -1240,8 +1371,8 @@ backslashescape(const char* s) return escaped; } -static char* -deescape(const char* esc) +char* +NCZ_deescape(const char* esc) { size_t len; char* s; @@ -1263,3 +1394,23 @@ deescape(const char* esc) *q = '\0'; return s; } + +/* Define a static qsort comparator for strings for use with qsort */ +static int +cmp_strings(const void* a1, const void* a2) +{ + const char** s1 = (const char**)a1; + const char** s2 = (const char**)a2; + return strcmp(*s1,*s2); +} + +int +NCZ_sort(void* vec, size_t count, int (*compare)(const void*, const void*)) +{ + if(compare == NULL) compare = cmp_strings; /* default comparator */ + if(vec != NULL && count > 0) { + qsort(vec, count, sizeof(void*), compare); + } + return NC_NOERR; +} + diff --git a/libnczarr/zxcache.c b/libnczarr/zxcache.c index 7a469587a3..e50b87e0da 100644 --- a/libnczarr/zxcache.c +++ b/libnczarr/zxcache.c @@ -577,54 +577,6 @@ NCZ_chunk_cache_modify(NCZChunkCache* cache, const size64_t* indices) return THROW(stat); } -/**************************************************/ -/* -From Zarr V2 Specification: -"The compressed sequence of bytes for each chunk is stored under -a key formed from the index of the chunk within the grid of -chunks representing the array. To form a string key for a -chunk, the indices are converted to strings and concatenated -with the dimension_separator character ('.' or '/') separating -each index. For example, given an array with shape (10000, -10000) and chunk shape (1000, 1000) there will be 100 chunks -laid out in a 10 by 10 grid. The chunk with indices (0, 0) -provides data for rows 0-1000 and columns 0-1000 and is stored -under the key "0.0"; the chunk with indices (2, 4) provides data -for rows 2000-3000 and columns 4000-5000 and is stored under the -key "2.4"; etc." -*/ - -/** - * @param R Rank - * @param chunkindices The chunk indices - * @param dimsep the dimension separator - * @param keyp Return the chunk key string - */ -int -NCZ_buildchunkkey(size_t R, const size64_t* chunkindices, char dimsep, char** keyp) -{ - int stat = NC_NOERR; - int r; - NCbytes* key = ncbytesnew(); - - if(keyp) *keyp = NULL; - - assert(islegaldimsep(dimsep)); - - for(r=0;r 0) ncbytesappend(key,dimsep); - /* Print as decimal with no leading zeros */ - snprintf(sindex,sizeof(sindex),"%lu",(unsigned long)chunkindices[r]); - ncbytescat(key,sindex); - } - ncbytesnull(key); - if(keyp) *keyp = ncbytesextract(key); - - ncbytesfree(key); - return THROW(stat); -} - /** * @internal Push data to chunk of a file. * If chunk does not exist, create it @@ -847,7 +799,7 @@ NCZ_buildchunkpath(NCZChunkCache* cache, const size64_t* chunkindices, struct Ch assert(key != NULL); /* Get the chunk object name */ - if((stat = NCZ_buildchunkkey(cache->ndims, chunkindices, cache->dimension_separator, &chunkname))) goto done; + if((stat = NCZF_buildchunkkey(cache->var->container->nc4_info,cache->ndims, chunkindices, cache->dimension_separator, &chunkname))) goto done; /* Get the var object key */ if((stat = NCZ_varkey(cache->var,&varkey))) goto done; key->varkey = varkey; varkey = NULL; diff --git a/nczarr_test/ref_multi.cdl b/nczarr_test/ref_multi.cdl index b81c63c020..d854dac1eb 100644 --- a/nczarr_test/ref_multi.cdl +++ b/nczarr_test/ref_multi.cdl @@ -9,7 +9,7 @@ variables: var:_Storage = "chunked" ; var:_ChunkSizes = 4, 4, 4, 4 ; var:_Filter = "307,9|1,2|40000" ; - var:_Codecs = "[{\"id\": \"bz2\",\"level\": \"9\"},{\"id\": \"zlib\",\"level\": \"2\"},{\"id\": \"noop\"}]" ; + var:_Codecs = "[{\"id\": \"bz2\", \"level\": \"9\"},{\"id\": \"zlib\", \"level\": \"2\"},{\"id\": 40000, \"nparams\": 0}]" ; var:_NoFill = "true" ; // global attributes: diff --git a/nczarr_test/run_interop.sh b/nczarr_test/run_interop.sh index ed22197112..52bbffee69 100755 --- a/nczarr_test/run_interop.sh +++ b/nczarr_test/run_interop.sh @@ -1,7 +1,5 @@ #!/bin/sh -set -e - if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh @@ -45,7 +43,6 @@ testcasezip() { } testcases3() { - set -x echo -e "\to Running S3 Testcase:\t$1\t$2" zext=s3 base=$1 diff --git a/nczarr_test/run_notzarr.sh b/nczarr_test/run_notzarr.sh index 86bd570dab..24cc47d21b 100755 --- a/nczarr_test/run_notzarr.sh +++ b/nczarr_test/run_notzarr.sh @@ -1,11 +1,12 @@ #!/bin/sh -#set -x -#set -e + if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh . "$srcdir/test_nczarr.sh" +set -e + # Build both ISOPATH and S3ISOPATH s3isolate "testdir_notzarr" THISDIR=`pwd` @@ -30,7 +31,7 @@ cp ${srcdir}/ref_notzarr.tar.gz . gunzip ref_notzarr.tar.gz tar -xf ref_notzarr.tar if test "x$FEATURE_S3TESTS" = xyes ; then - ${execdir}/s3util -f notzarr.file/notzarr.txt -u "https://${URL}" -k "//${S3ISOPATH}/notzarr.s3/notzarr.txt" upload + ${execdir}/s3util -f notzarr.file/notzarr.txt -u "https://${URL}" -k "/${S3ISOPATH}/notzarr.s3/notzarr.txt" upload fi echo "Test empty file" diff --git a/nczarr_test/testfilter_misc.c b/nczarr_test/testfilter_misc.c index 85fdf3df9d..52e6a6a5ea 100644 --- a/nczarr_test/testfilter_misc.c +++ b/nczarr_test/testfilter_misc.c @@ -111,7 +111,7 @@ report(const char* msg, int lineno) static int verifychunks(void) { - int i; + size_t i; int store = -1; size_t localchunks[MAXDIMS]; memset(localchunks,0,sizeof(localchunks)); @@ -122,7 +122,7 @@ verifychunks(void) } for(i=0;i=0;i--) { odom[i] += 1; if(odom[i] < dimsize[i]) break; @@ -510,7 +510,7 @@ odom_next(void) static int odom_offset(void) { - int i; + size_t i; int offset = 0; for(i=0;i 1) diff --git a/nczarr_test/zmapio.c b/nczarr_test/zmapio.c index 065a2ba9dc..9c267120ed 100644 --- a/nczarr_test/zmapio.c +++ b/nczarr_test/zmapio.c @@ -85,6 +85,7 @@ struct Dumpptions { int xflags; # define XNOZMETADATA 1 int strlen; + int zarrformat; } dumpoptions; /* Forward */ @@ -110,7 +111,7 @@ static void nccheck(int stat, int line) static void zmapusage(void) { - fprintf(stderr,"usage: zmapio [-t ][-d][-v][-h][-x] \n"); + fprintf(stderr,"usage: zmapio [-2|-3][-t ][-d][-v][-h][-x] \n"); exit(1); } @@ -152,8 +153,14 @@ main(int argc, char** argv) /* Init options */ memset((void*)&dumpoptions,0,sizeof(dumpoptions)); - while ((c = getopt(argc, argv, "dhvx:t:F:T:X:")) != EOF) { + while ((c = getopt(argc, argv, "23dhvx:t:F:T:X:")) != EOF) { switch(c) { + case '2': + dumpoptions.zarrformat = 2; + break; + case '3': + dumpoptions.zarrformat = 3; + break; case 'd': dumpoptions.debug = 1; break; @@ -497,7 +504,7 @@ printcontent(size64_t len, const char* content, OBJKIND kind) } } -static char chunkchars[] = ".0123456789"; +static char chunkchars[] = "./c0123456789"; static OBJKIND keykind(const char* key) @@ -513,8 +520,13 @@ keykind(const char* key) kind = OK_IGNORE; else kind = OK_META; + if(!dumpoptions.zarrformat) dumpoptions.zarrformat = 2; + } else if(strcasecmp(&suffix[1],"zarr.json")==0) { + kind = OK_META; + if(!dumpoptions.zarrformat) dumpoptions.zarrformat = 3; } else if(strcasecmp(&suffix[1],"zarr.json")==0) { kind = OK_META; + if(!dumpoptions.zarrformat) dumpoptions.zarrformat = 3; } else if(suffix[strlen(suffix)-1] == '/') kind = OK_GROUP; else { diff --git a/plugins/H5Znoop.c b/plugins/H5Znoop.c index 9ba1dc5862..5fae7d0f32 100644 --- a/plugins/H5Znoop.c +++ b/plugins/H5Znoop.c @@ -5,8 +5,6 @@ #include #include -#include - #include "netcdf_filter_build.h" #ifndef NOOP_INSTANCE @@ -131,23 +129,29 @@ H5Z_filter_noop(unsigned int flags, size_t cd_nelmts, return *buf_size; } +#if 0 /**************************************************/ /* NCZarr Codec API */ -/* Codec Format +/* This codec test various things: +test=1 + +It also tests the use of hdf5raw plugin. +Codec Format (V2) { "id": "test", -"p0": "", -"p1": "", -"p2": "", +"nparams": "n", +"0": "", +"1": "", +"2": "", ... -"pn": "", +"n": "", } */ /* Forward */ -static int NCZ_noop_codec_to_hdf5(void* env, const char* codec, size_t* nparamsp, unsigned** paramsp); -static int NCZ_noop_hdf5_to_codec(void* env, size_t nparams, const unsigned* params, char** codecp); +static int NCZ_noop_codec_to_hdf5(const NCproplist* env, const char* codec, int* idp, size_t* nparamsp, unsigned** paramsp); +static int NCZ_noop_hdf5_to_codec(const NCproplist* env, int id, size_t nparams, const unsigned* params, char** codecp); /* Structure for NCZ_PLUGIN_CODEC */ static NCZ_codec_t NCZ_noop_codec = {/* NCZ_codec_t codec fields */ @@ -178,7 +182,7 @@ NCZ_get_codec_info(void) /* NCZarr Interface Functions */ static int -NCZ_noop_codec_to_hdf5(void* env, const char* codec_json, size_t* nparamsp, unsigned** paramsp) +NCZ_noop_codec_to_hdf5(const NCproplist* env, const char* codec_json, int* idp, size_t* nparamsp, unsigned** paramsp) { int stat = NC_NOERR; NCjson* jcodec = NULL; @@ -187,8 +191,10 @@ NCZ_noop_codec_to_hdf5(void* env, const char* codec_json, size_t* nparamsp, unsi int i,nparams = 0; unsigned* params = NULL; char field[1024]; - NCZ_codec_env_t* codec_env = (NCZ_codec_env_t*)env; + uintptr_t zarrformat = 0; + ncplistget(env,"zarrformat",&zarrformat,NULL); + /* parse the JSON */ if(NCJparse(codec_json,0,&jcodec)) {stat = NC_EFILTER; goto done;} @@ -196,18 +202,18 @@ NCZ_noop_codec_to_hdf5(void* env, const char* codec_json, size_t* nparamsp, unsi /* Get and Verify the codec ID */ - if(codec_env->zarrformat == 2) { - if(NCJdictget(jcodec,"id",&jtmp)) {stat = NC_EFILTER; goto done;} - if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EINVAL; goto done;} - if(strcmp(NCJstring(jtmp),NCZ_noop_codec.codecid)!=0) {stat = NC_EINVAL; goto done;} - nparams = (NCJlength(jcodec) - 1) / 2; /* -1 for id each param is key+value */ - jdict = jcodec; - } else { + if(zarrformat == 3) { if(NCJdictget(jcodec,"name",&jtmp)) {stat = NC_EFILTER; goto done;} if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EINVAL; goto done;} if(strcmp(NCJstring(jtmp),NCZ_noop_codec.codecid)!=0) {stat = NC_EINVAL; goto done;} if(NCJdictget(jcodec,"configuration",&jdict)) {stat = NC_EFILTER; goto done;} nparams = (NCJlength(jdict) / 2); /* each param is key+value */ + } else { + if(NCJdictget(jcodec,"id",&jtmp)) {stat = NC_EFILTER; goto done;} + if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EINVAL; goto done;} + if(strcmp(NCJstring(jtmp),NCZ_noop_codec.codecid)!=0) {stat = NC_EINVAL; goto done;} + nparams = (NCJlength(jcodec) - 1) / 2; /* -1 for id each param is key+value */ + jdict = jcodec; } if((params = (unsigned*)calloc(nparams,sizeof(unsigned)))== NULL) {stat = NC_ENOMEM; goto done;} /* This filter has an arbitrary number of parameters named p0...pn */ @@ -222,6 +228,7 @@ NCZ_noop_codec_to_hdf5(void* env, const char* codec_json, size_t* nparamsp, unsi if(nparamsp) *nparamsp = nparams; if(paramsp) {*paramsp = params; params = NULL;} + if(idp) *idp = H5Z_FILTER_NOOP; done: if(params) free(params); @@ -230,17 +237,21 @@ NCZ_noop_codec_to_hdf5(void* env, const char* codec_json, size_t* nparamsp, unsi } static int -NCZ_noop_hdf5_to_codec(void* env, size_t nparams, const unsigned* params, char** codecp) +NCZ_noop_hdf5_to_codec(const NCproplist* env, int id, size_t nparams, const unsigned* params, char** codecp) { int i,stat = NC_NOERR; char json[8192]; char value[8192]; - NCZ_codec_env_t* codec_env = (NCZ_codec_env_t*)env; + uintptr_t zarrformat = 0; + + NC_UNUSED(id); if(nparams != 0 && params == NULL) {stat = NC_EINVAL; goto done;} - if(codec_env->zarrformat == 2) { + ncplistget(env,"zarrformat",&zarrformat,NULL); + + if(zarrformat == 2) { snprintf(json,sizeof(json),"{\"id\": \"%s\"",NCZ_noop_codec.codecid); /* Accept an arbitrary number of params */ for(i=0;izarrformat == 2) { - snprintf(json,sizeof(json),"{\"id\": \"%s\"}",NCZ_unknown_codec.codecid); - } else { + if(zarrformat == 3) { snprintf(json,sizeof(json),"{\"name\": \"%s\"}",NCZ_unknown_codec.codecid); + } else { + snprintf(json,sizeof(json),"{\"id\": \"%s\"}",NCZ_unknown_codec.codecid); } + if(codecp) { if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} } diff --git a/plugins/NCZhdf5filters.c b/plugins/NCZhdf5filters.c index a892ad7a55..6598401a24 100644 --- a/plugins/NCZhdf5filters.c +++ b/plugins/NCZhdf5filters.c @@ -20,6 +20,8 @@ Author: Dennis Heimbigner #include "netcdf_filter.h" #include "netcdf_filter_build.h" #include "netcdf_json.h" +#include "netcdf_proplist.h" +#include "netcdf_vutils.h" #ifdef HAVE_SZ #include @@ -34,23 +36,163 @@ Author: Dennis Heimbigner /* NCZarr Filter Objects */ /* Forward */ -static int NCZ_shuffle_codec_to_hdf5(void*, const char* codec, size_t* nparamsp, unsigned** paramsp); -static int NCZ_shuffle_hdf5_to_codec(void*, size_t nparams, const unsigned* params, char** codecp); -static int NCZ_shuffle_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); -static int NCZ_fletcher32_codec_to_hdf5(void*, const char* codec, size_t* nparamsp, unsigned** paramsp); -static int NCZ_fletcher32_hdf5_to_codec(void*, size_t nparams, const unsigned* params, char** codecp); -static int NCZ_fletcher32_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); +static int NCZ_raw_codec_to_hdf5(const NCproplist*, const char* codec, int* idp, size_t* nparamsp, unsigned** paramsp); +static int NCZ_raw_hdf5_to_codec(const NCproplist*, int id, size_t nparams, const unsigned* params, char** codecp); -static int NCZ_deflate_codec_to_hdf5(void*, const char* codec, size_t* nparamsp, unsigned** paramsp); -static int NCZ_deflate_hdf5_to_codec(void*, size_t nparams, const unsigned* params, char** codecp); +static int NCZ_shuffle_codec_to_hdf5(const NCproplist*, const char* codec, int* idp, size_t* nparamsp, unsigned** paramsp); +static int NCZ_shuffle_hdf5_to_codec(const NCproplist*, int id, size_t nparams, const unsigned* params, char** codecp); +static int NCZ_shuffle_modify_parameters(const NCproplist*, int* idp, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); + +static int NCZ_fletcher32_codec_to_hdf5(const NCproplist*, const char* codec, int* idp, size_t* nparamsp, unsigned** paramsp); +static int NCZ_fletcher32_hdf5_to_codec(const NCproplist*, int id, size_t nparams, const unsigned* params, char** codecp); +static int NCZ_fletcher32_modify_parameters(const NCproplist*, int* idp, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); + +static int NCZ_deflate_codec_to_hdf5(const NCproplist*, const char* codec, int* idp, size_t* nparamsp, unsigned** paramsp); +static int NCZ_deflate_hdf5_to_codec(const NCproplist*, int id, size_t nparams, const unsigned* params, char** codecp); #ifdef HAVE_SZ -static int NCZ_szip_codec_to_hdf5(void*, const char* codec, size_t* nparamsp, unsigned** paramsp); -static int NCZ_szip_hdf5_to_codec(void*, size_t nparams, const unsigned* params, char** codecp); -static int NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); +static int NCZ_szip_codec_to_hdf5(const NCproplist*, const char* codec, int* idp, size_t* nparamsp, unsigned** paramsp); +static int NCZ_szip_hdf5_to_codec(const NCproplist*, int id, size_t nparams, const unsigned* params, char** codecp); +static int NCZ_szip_modify_parameters(const NCproplist*, int* idp, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); #endif +/**************************************************/ +/* Provide a default codec for HDF5 filters. + This encodes the parameters of the HDF5 filter + into one of these two codec forms: + zarr v2: {"id": "hdf5raw", "hdf5id": ", "nparams": , "0": ...,"": } + zarr v3: {"name": "hdf5raw, "configuration": {"hdf5id": , "nparams": , "0": ...,"": }} + This codec is called hdf5raw. + NOTES: + 1. this cannot be used if a modify_parameters function is required. +*/ + +static NCZ_codec_t NCZ_hdf5_raw_codec = { + NCZ_CODEC_CLASS_VER, /* Struct version number */ + NCZ_CODEC_HDF5, /* Struct sort */ + "hdf5raw", /* Standard name/id of the codec */ + 0, /* HDF5 alias for raw */ + NULL, /*NCZ_raw_codec_initialize*/ + NULL, /*NCZ_raw_codec_finalize*/ + NCZ_raw_codec_to_hdf5, + NCZ_raw_hdf5_to_codec, + NULL +}; + +static int +NCZ_raw_codec_to_hdf5(const NCproplist* env, const char* codec, int* idp, size_t* nparamsp, unsigned** paramsp) +{ + int i,stat = NC_NOERR; + NCjson* jcodec = NULL; + NCjson* jname = NULL; + NCjson* jdict = NULL; + NCjson* jtmp = NULL; + const char* name = NULL; + struct NCJconst jc; + uintptr_t zarrformat = 0; + int hdf5id = 0; + size_t nparams = 0; + unsigned* params = NULL; + + if(nparamsp == NULL || paramsp == NULL) + {stat = NC_EINTERNAL; goto done;} + + if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL) + {stat = NC_ENOMEM; goto done;} + + ncplistget(env,"zarrformat",&zarrformat,NULL); + + /* parse the JSON */ + if(NCJparse(codec,0,&jcodec)) {stat = NC_EFILTER; goto done;} + if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;} + + if(zarrformat == 3) { + if(NCJdictget(jcodec,"name",&jname)) {stat = NC_EFILTER; goto done;} + if(NCJdictget(jcodec,"configuration",&jdict)) {stat = NC_EFILTER; goto done;} + } else { + if(NCJdictget(jcodec,"id",&jname)) {stat = NC_EFILTER; goto done;} + jdict = jcodec; + } + + /* Verify the codec ID */ + if(jname == NULL || !NCJisatomic(jname)) {stat = NC_EFILTER; goto done;} + name = NCJstring(jname); + if(1 != sscanf(name,"%d",&hdf5id)) {stat = NC_EINVAL; goto done;} + /* Return the id */ + if(idp) *idp = hdf5id; + + /* Get nparams */ + if(NCJdictget(jdict,"nparams",&jtmp)) {stat = NC_EFILTER; goto done;} + if(jtmp == NULL || NCJsort(jtmp) != NCJ_INT) {stat = NC_EFILTER; goto done;} + if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;} + if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;} + nparams = (unsigned)jc.ival; + + /* Validate nparams */ + if(NCJlength(jdict) != 2* (nparams + 2)) {stat = NC_EFILTER; goto done;} + + /* Get params */ + for(i=0;i<(int)nparams;i++) { + unsigned param; + char n[64]; + snprintf(n,sizeof(n),"%u",i); + if(NCJdictget(jdict,n,&jtmp)) {stat = NC_EFILTER; goto done;} /* get parameter "" */ + if(jtmp == NULL || NCJsort(jtmp) != NCJ_INT) {stat = NC_EFILTER; goto done;} + if(1 != sscanf(NCJstring(jtmp),"%u",¶m)) {stat = NC_EFILTER; goto done;} + params[i] = param; + } + + if(idp) *idp = hdf5id; + *nparamsp = nparams; + *paramsp = params; params = NULL; + +done: + if(params) free(params); + NCJreclaim(jcodec); + return stat; +} + +static int +NCZ_raw_hdf5_to_codec(const NCproplist* env, int id, size_t nparams, const unsigned* params, char** codecp) +{ + int i,stat = NC_NOERR; + uintptr_t zarrformat = 0; + VString* json = NULL; + char stmp[128]; + + if(codecp == NULL) {stat = NC_EINTERNAL; goto done;} + + ncplistget(env,"zarrformat",&zarrformat,NULL); + + json = vsnew(); + + if(zarrformat == 3) { + snprintf(stmp,sizeof(stmp),"{\"name\": %d, \"configuration\": {",id); + } else { + snprintf(stmp,sizeof(stmp),"{\"id\": %d, ",id); + } + vscat(json,stmp); + + snprintf(stmp,sizeof(stmp),"\"nparams\": %u",(unsigned)nparams); + vscat(json,stmp); + + /* add the raw parameters */ + for(i=0;i<(int)nparams;i++) { + snprintf(stmp,sizeof(stmp),", \"%u\": %u",i,params[i]); + vscat(json,stmp); + } + if(zarrformat == 3) + vscat(json,"}"); + vscat(json,"}"); + + if(codecp) *codecp = vsextract(json); + +done: + vsfree(json); + return stat; +} + /**************************************************/ static NCZ_codec_t NCZ_shuffle_codec = { @@ -66,27 +208,38 @@ static NCZ_codec_t NCZ_shuffle_codec = { }; static int -NCZ_shuffle_codec_to_hdf5(void* env, const char* codec, size_t* nparamsp, unsigned** paramsp) +NCZ_shuffle_codec_to_hdf5(const NCproplist* env, const char* codec, int* idp, size_t* nparamsp, unsigned** paramsp) { int stat = NC_NOERR; - - /* Ignore any incoming elementsize */ - if(nparamsp) *nparamsp = 0; - + NC_UNUSED(env); + NC_UNUSED(codec); + + if(nparamsp == NULL || paramsp == NULL) + {stat = NC_EINTERNAL; goto done;} + + if(idp) *idp = H5Z_FILTER_SHUFFLE; + *nparamsp = 0; + *paramsp = NULL; +done: return stat; } static int -NCZ_shuffle_hdf5_to_codec(void* env, size_t nparams, const unsigned* params, char** codecp) +NCZ_shuffle_hdf5_to_codec(const NCproplist* env, int id, size_t nparams, const unsigned* params, char** codecp) { int stat = NC_NOERR; char json[1024]; - unsigned typesize = 0; + uintptr_t zarrformat = 0; - if(nparams > 0) - typesize = params[0]; - snprintf(json,sizeof(json),"{\"id\": \"%s\", \"elementsize\": \"%u\"}",NCZ_shuffle_codec.codecid,typesize); + ncplistget(env,"zarrformat",&zarrformat,NULL); + + if(zarrformat == 3) { + snprintf(json,sizeof(json),"{\"name\": \"%s\"}",NCZ_shuffle_codec.codecid); + } else { + snprintf(json,sizeof(json),"{\"id\": \"%s\"}",NCZ_shuffle_codec.codecid); + } + if(codecp) { if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} } @@ -96,20 +249,24 @@ NCZ_shuffle_hdf5_to_codec(void* env, size_t nparams, const unsigned* params, cha } static int -NCZ_shuffle_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp) +NCZ_shuffle_modify_parameters(const NCproplist* env, int* idp, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp) { int stat = NC_NOERR; nc_type vtype; size_t typesize; char vname[NC_MAX_NAME+1]; unsigned int* params = NULL; + uintptr_t ncid, varid; /* Ignore the visible parameters */ if(!wnparamsp || !wparamsp) {stat = NC_EINTERNAL; goto done;} + ncplistget(env,"fileid",&ncid,NULL); + ncplistget(env,"varid",&varid,NULL); + /* Get variable info */ - if((stat = nc_inq_var(ncid,varid,vname,&vtype,NULL,NULL,NULL))) goto done; + if((stat = nc_inq_var((int)ncid,(int)varid,vname,&vtype,NULL,NULL,NULL))) goto done; /* Get the typesize */ if((stat = nc_inq_type(ncid,vtype,NULL,&typesize))) goto done; @@ -129,35 +286,6 @@ NCZ_shuffle_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** return stat; } -#if 0 -static int -NCZ_shuffle_visible_parameters(int ncid, int varid, size_t nparamsin, const unsigned int* paramsin, size_t* nparamsp, unsigned** paramsp) -{ - int stat = NC_NOERR; - unsigned* params = NULL; - size_t typesize = 0; - - if(!nparamsp || !paramsp) {stat = NC_EINTERNAL; goto done;} - - if(nparamsin > 0) - typesize = paramsin[0]; - - if((params=(unsigned*)malloc(sizeof(unsigned)))==NULL) - {stat = NC_ENOMEM; goto done;} - - params[0] = (unsigned)typesize; - - /* add the typesize as a new parameter */ - nullfree(*paramsp); - *paramsp = params; params = NULL; - *nparamsp = 1; - -done: - nullfree(params); - return stat; -} -#endif - /**************************************************/ static NCZ_codec_t NCZ_fletcher32_codec = {/* NCZ_codec_t codec fields */ @@ -173,7 +301,7 @@ static NCZ_codec_t NCZ_fletcher32_codec = {/* NCZ_codec_t codec fields */ }; static int -NCZ_fletcher32_codec_to_hdf5(void* env, const char* codec, size_t* nparamsp, unsigned** paramsp) +NCZ_fletcher32_codec_to_hdf5(const NCproplist* env, const char* codec, int* idp, size_t* nparamsp, unsigned** paramsp) { int stat = NC_NOERR; @@ -181,7 +309,6 @@ NCZ_fletcher32_codec_to_hdf5(void* env, const char* codec, size_t* nparamsp, uns if(!nparamsp || !paramsp) {stat = NC_EINTERNAL; goto done;} *nparamsp = 0; - nullfree(*paramsp); *paramsp = NULL; done: @@ -189,12 +316,20 @@ NCZ_fletcher32_codec_to_hdf5(void* env, const char* codec, size_t* nparamsp, uns } static int -NCZ_fletcher32_hdf5_to_codec(void* env, size_t nparams, const unsigned* params, char** codecp) +NCZ_fletcher32_hdf5_to_codec(const NCproplist* env, int id, size_t nparams, const unsigned* params, char** codecp) { int stat = NC_NOERR; char json[1024]; + uintptr_t zarrformat = 0; - snprintf(json,sizeof(json),"{\"id\": \"%s\"}",NCZ_fletcher32_codec.codecid); + ncplistget(env,"zarrformat",&zarrformat,NULL); + + if(zarrformat == 3) { + snprintf(json,sizeof(json),"{\"name\": \"%s\"}",NCZ_fletcher32_codec.codecid); + } else { + snprintf(json,sizeof(json),"{\"id\": \"%s\"}",NCZ_fletcher32_codec.codecid); + } + if(codecp) { if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} } @@ -204,10 +339,13 @@ NCZ_fletcher32_hdf5_to_codec(void* env, size_t nparams, const unsigned* params, } static int -NCZ_fletcher32_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp) +NCZ_fletcher32_modify_parameters(const NCproplist* env, int* idp, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp) { int stat = NC_NOERR; + NC_UNUSED(env); + NC_UNUSED(idp); + /* Ignore the visible parameters */ if(!wnparamsp || !wparamsp) {stat = NC_EINTERNAL; goto done;} @@ -220,22 +358,6 @@ NCZ_fletcher32_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigne return stat; } -#if 0 -static int -NCZ_fletcher32_visible_parameters(int ncid, int varid, size_t nparamsin, const unsigned int* paramsin, size_t* nparamsp, unsigned** paramsp) -{ - int stat = NC_NOERR; - - if(!nparamsp || !paramsp) {stat = NC_EINTERNAL; goto done;} - - nullfree(*paramsp); *paramsp = NULL; - *nparamsp = 0; - -done: - return stat; -} -#endif - /**************************************************/ static NCZ_codec_t NCZ_zlib_codec = {/* NCZ_codec_t codec fields */ @@ -251,13 +373,15 @@ static NCZ_codec_t NCZ_zlib_codec = {/* NCZ_codec_t codec fields */ }; static int -NCZ_deflate_codec_to_hdf5(void* env, const char* codec_json, size_t* nparamsp, unsigned** paramsp) +NCZ_deflate_codec_to_hdf5(const NCproplist* env, const char* codec_json, int* idp, size_t* nparamsp, unsigned** paramsp) { int stat = NC_NOERR; NCjson* jcodec = NULL; NCjson* jtmp = NULL; + NCjson* jdict = NULL; unsigned* params = NULL; struct NCJconst jc; + uintptr_t zarrformat = 0; if(nparamsp == NULL || paramsp == NULL) {stat = NC_EINTERNAL; goto done;} @@ -265,22 +389,32 @@ NCZ_deflate_codec_to_hdf5(void* env, const char* codec_json, size_t* nparamsp, u if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL) {stat = NC_ENOMEM; goto done;} + ncplistget(env,"zarrformat",&zarrformat,NULL); + /* parse the JSON */ if(NCJparse(codec_json,0,&jcodec)) {stat = NC_EFILTER; goto done;} if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;} + + if(zarrformat == 3) { + if(NCJdictget(jcodec,"name",&jtmp)){stat = NC_EFILTER; goto done;} + if(NCJdictget(jcodec,"configuration",&jdict)){stat = NC_EFILTER; goto done;} + } else { + if(NCJdictget(jcodec,"id",&jtmp)){stat = NC_EFILTER; goto done;} + jdict = jcodec; + } + if(jdict == NULL){stat = NC_EFILTER; goto done;} + /* Verify the codec ID */ - if(NCJdictget(jcodec,"id",&jtmp)) - {stat = NC_EFILTER; goto done;} if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EFILTER; goto done;} if(strcmp(NCJstring(jtmp),NCZ_zlib_codec.codecid)!=0) {stat = NC_EINVAL; goto done;} /* Get Level */ - if(NCJdictget(jcodec,"level",&jtmp)) - {stat = NC_EFILTER; goto done;} - if(NCJcvt(jtmp,NCJ_INT,&jc)) - {stat = NC_EFILTER; goto done;} + if(NCJdictget(jdict,"level",&jtmp)) {stat = NC_EFILTER; goto done;} + if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;} if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;} + + if(idp) *idp = H5Z_FILTER_DEFLATE; params[0] = (unsigned)jc.ival; *nparamsp = 1; *paramsp = params; params = NULL; @@ -292,17 +426,25 @@ NCZ_deflate_codec_to_hdf5(void* env, const char* codec_json, size_t* nparamsp, u } static int -NCZ_deflate_hdf5_to_codec(void* env, size_t nparams, const unsigned* params, char** codecp) +NCZ_deflate_hdf5_to_codec(const NCproplist* env, int id, size_t nparams, const unsigned* params, char** codecp) { int stat = NC_NOERR; unsigned level = 0; char json[1024]; + uintptr_t zarrformat = 0; if(nparams == 0 || params == NULL) {stat = NC_EFILTER; goto done;} + ncplistget(env,"zarrformat",&zarrformat,NULL); + level = params[0]; - snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_zlib_codec.codecid,level); + if(zarrformat == 3) { + snprintf(json,sizeof(json),"{\"name\": \"%s\", \"configuration\": {\"level\": \"%u\"}}",NCZ_zlib_codec.codecid,level); + } else { + snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_zlib_codec.codecid,level); + } + if(codecp) { if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} } @@ -328,14 +470,16 @@ static NCZ_codec_t NCZ_szip_codec = { }; static int -NCZ_szip_codec_to_hdf5(void* env, const char* codec_json, size_t* nparamsp, unsigned** paramsp) +NCZ_szip_codec_to_hdf5(const NCproplist* env, const char* codec_json, int* idp, size_t* nparamsp, unsigned** paramsp) { int stat = NC_NOERR; unsigned* params = NULL; size_t nparams = 2; /* No. of visible parameters */ NCjson* json = NULL; + NCjson* jdict = NULL; NCjson* jtmp = NULL; struct NCJconst jc = {0,0,0,NULL}; + uintptr_t zarrformat = 0; if(nparamsp == NULL || paramsp == NULL) {stat = NC_EINTERNAL; goto done;} @@ -343,22 +487,26 @@ NCZ_szip_codec_to_hdf5(void* env, const char* codec_json, size_t* nparamsp, unsi if((params = (unsigned*)calloc(nparams,sizeof(unsigned)))== NULL) {stat = NC_ENOMEM; goto done;} + ncplistget(env,"zarrformat",&zarrformat,NULL); + if(NCJparse(codec_json,0,&json)) {stat = NC_EFILTER; goto done;} - if(NCJdictget(json,"mask",&jtmp) || jtmp == NULL) - {stat = NC_EFILTER; goto done;} - if(NCJcvt(jtmp,NCJ_INT,&jc)) - {stat = NC_EFILTER; goto done;} + if(zarrformat == 3) { + if(NCJdictget(json,"configuration",&jdict)) {stat = NC_EFILTER; goto done;} + } else + jdict = json; + + if(NCJdictget(jdict,"mask",&jtmp) || jtmp == NULL) {stat = NC_EFILTER; goto done;} + if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;} params[H5Z_SZIP_PARM_MASK] = (unsigned)jc.ival; jtmp = NULL; - if(NCJdictget(json,"pixels-per-block",&jtmp) || jtmp == NULL) - {stat = NC_EFILTER; goto done;} - if(NCJcvt(jtmp,NCJ_INT,&jc)) - {stat = NC_EFILTER; goto done;} + if(NCJdictget(jdict,"pixels-per-block",&jtmp) || jtmp == NULL) {stat = NC_EFILTER; goto done;} + if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;} params[H5Z_SZIP_PARM_PPB] = (unsigned)jc.ival; + if(idp) *idp = H5Z_FILTER_SZIP; *nparamsp = nparams; *paramsp = params; params = NULL; @@ -369,15 +517,26 @@ NCZ_szip_codec_to_hdf5(void* env, const char* codec_json, size_t* nparamsp, unsi } static int -NCZ_szip_hdf5_to_codec(void* env, size_t nparams, const unsigned* params, char** codecp) +NCZ_szip_hdf5_to_codec(const NCproplist* env, int id, size_t nparams, const unsigned* params, char** codecp) { int stat = NC_NOERR; char json[2048]; + uintptr_t zarrformat = 0; - snprintf(json,sizeof(json),"{\"id\": \"%s\", \"mask\": %u, \"pixels-per-block\": %u}", + ncplistget(env,"zarrformat",&zarrformat,NULL); + + if(zarrformat == 3) { + snprintf(json,sizeof(json),"{\"name\": \"%s\", \"configuration\": {\"mask\": %u, \"pixels-per-block\": %u}}", + NCZ_szip_codec.codecid, + params[H5Z_SZIP_PARM_MASK], + params[H5Z_SZIP_PARM_PPB]); + } else { + snprintf(json,sizeof(json),"{\"id\": \"%s\", \"mask\": %u, \"pixels-per-block\": %u}", NCZ_szip_codec.codecid, params[H5Z_SZIP_PARM_MASK], params[H5Z_SZIP_PARM_PPB]); + } + if(codecp) { if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} } @@ -387,7 +546,7 @@ NCZ_szip_hdf5_to_codec(void* env, size_t nparams, const unsigned* params, char** } static int -NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp) +NCZ_szip_modify_parameters(const NCproplist* env, int* idp, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp) { int i,ret_value = NC_NOERR; nc_type vtype; @@ -399,6 +558,7 @@ NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vp unsigned* params = NULL; unsigned* vparams = NULL; size_t wnparams = 4; + uintptr_t ncid, varid; if(wnparamsp == NULL || wparamsp == NULL) {ret_value = NC_EFILTER; goto done;} @@ -409,11 +569,14 @@ NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vp vparams = *vparamsp; + ncplistget(env,"fileid",&ncid,NULL); + ncplistget(env,"varid",&varid,NULL); + /* Get variable info */ - if((ret_value = nc_inq_var(ncid,varid,vname,&vtype,&ndims,dimids,NULL))) goto done; + if((ret_value = nc_inq_var((int)ncid,(int)varid,vname,&vtype,&ndims,dimids,NULL))) goto done; /* Get the typesize */ - if((ret_value = nc_inq_type(ncid,vtype,NULL,&typesize))) goto done; + if((ret_value = nc_inq_type((int)ncid,vtype,NULL,&typesize))) goto done; /* Get datatype's precision, in case is less than full bits */ dtype_precision = typesize*8; @@ -428,18 +591,18 @@ NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vp if(ndims == 0) {ret_value = NC_EFILTER; goto done;} /* Set "local" parameter for this dataset's "pixels-per-scanline" */ - if((ret_value = nc_inq_dimlen(ncid,dimids[ndims-1],&scanline))) goto done; + if((ret_value = nc_inq_dimlen((int)ncid,dimids[ndims-1],&scanline))) goto done; /* Get number of elements for the dataspace; use total number of elements in the chunk to define the new 'scanline' size */ /* Compute chunksize */ - if((ret_value = nc_inq_var_chunking(ncid,varid,&storage,chunklens))) goto done; + if((ret_value = nc_inq_var_chunking((int)ncid,varid,&storage,chunklens))) goto done; if(storage != NC_CHUNKED) {ret_value = NC_EFILTER; goto done;} npoints = 1; for(i=0;izarrformat == 2) isv2 = 1; + ncplistget(env,"zarrformat",&zarrformat,NULL); + + if(zarrformat == 2) isv2 = 1; /* parse the JSON */ if(NCJparse(codec_json,0,&jcodec)) @@ -120,18 +121,18 @@ NCZ_misc_codec_to_hdf5(void* env0, const char* codec_json, size_t* nparamsp, uns /* Verify the codec ID */ - if(env->zarrformat == 2) { - if(NCJdictget(jcodec,"id",&jtmp)) {stat = NC_EFILTER; goto done;} - } else { + if(zarrformat == 3) { if(NCJdictget(jcodec,"name",&jtmp)) {stat = NC_EFILTER; goto done;} + } else { + if(NCJdictget(jcodec,"id",&jtmp)) {stat = NC_EFILTER; goto done;} } if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EINVAL; goto done;} if(strcmp(NCJstring(jtmp),NCZ_misc_codec.codecid)!=0) {stat = NC_EINVAL; goto done;} - if(env->zarrformat == 2) - jparams = jcodec; - else { /* V3 */ + if(zarrformat == 3) { if(NCJdictget(jcodec,"configuration",&jparams)) {stat = NC_EFILTER; goto done;} + } else { /* V2 */ + jparams = jcodec; } /* The codec will have (2*14 + 1) + 1 = 29 dict entries + id if v2*/ @@ -154,6 +155,7 @@ NCZ_misc_codec_to_hdf5(void* env0, const char* codec_json, size_t* nparamsp, uns } if(nparamsp) *nparamsp = nparams; if(paramsp) {*paramsp = params; params = NULL;} + if(idp) *idp = H5Z_FILTER_TEST; done: if(params) free(params); @@ -162,13 +164,15 @@ NCZ_misc_codec_to_hdf5(void* env0, const char* codec_json, size_t* nparamsp, uns } static int -NCZ_misc_hdf5_to_codec(void* env0, size_t nparams, const unsigned* params, char** codecp) +NCZ_misc_hdf5_to_codec(const NCproplist* env, int id, size_t nparams, const unsigned* params, char** codecp) { int i,stat = NC_NOERR; char json[4096]; char value[1024]; size_t count, jlen; - NCZ_codec_env_t* env = (NCZ_codec_env_t*)env0; + uintptr_t zarrformat = 0; + + NC_UNUSED(id); if(nparams == 0 || params == NULL) {stat = NC_EINVAL; goto done;} @@ -177,8 +181,11 @@ NCZ_misc_hdf5_to_codec(void* env0, size_t nparams, const unsigned* params, char* stat = NC_EINVAL; goto done; } + + ncplistget(env,"zarrformat",&zarrformat,NULL); + jlen = sizeof(json); - if(env->zarrformat == 2) { + if(zarrformat == 2) { count = snprintf(json,sizeof(json),"{\"id\": \"%s\"",NCZ_misc_codec.codecid); for(i=0;i<14;i++) { size_t len = snprintf(value,sizeof(value),", \"%s\": \"%u\"",fields[i],params[i]); @@ -186,7 +193,7 @@ NCZ_misc_hdf5_to_codec(void* env0, size_t nparams, const unsigned* params, char* strcat(json,value); } count += 1; assert(jlen > count); - } else if(env->zarrformat == 3) { + } else if(zarrformat == 3) { snprintf(json,sizeof(json),"{\"name\": \"%s\", \"configuration\": {",NCZ_misc_codec.codecid); for(i=0;i<14;i++) { if(i > 0) strcat(json,", "); diff --git a/plugins/NCZstdfilters.c b/plugins/NCZstdfilters.c index 84b775e96f..3436527031 100644 --- a/plugins/NCZstdfilters.c +++ b/plugins/NCZstdfilters.c @@ -20,13 +20,14 @@ Author: Dennis Heimbigner #include "netcdf_filter.h" #include "netcdf_filter_build.h" #include "netcdf_json.h" +#include "netcdf_proplist.h" /* Provide Codec information for the standard filters */ #ifndef H5Z_FILTER_BZIP2 -#define H5Z_FILTER_BZIP2 1 -#define H5Z_FILTER_ZSTD 2 -#define H5Z_FILTER_BLOSC 3 +#define H5Z_FILTER_BZIP2 307 +#define H5Z_FILTER_ZSTD 32015 +#define H5Z_FILTER_BLOSC 32001 #endif #ifdef HAVE_BLOSC @@ -36,21 +37,22 @@ Author: Dennis Heimbigner /**************************************************/ /* NCZarr Filter Objects */ + /* Forward */ -static int NCZ_bzip2_codec_to_hdf5(void* codec_env, const char* codec, size_t* nparamsp, unsigned** paramsp); -static int NCZ_bzip2_hdf5_to_codec(void* codec_env, size_t nparams, const unsigned* params, char** codecp); +static int NCZ_bzip2_codec_to_hdf5(const NCproplist* env, const char* codec, int* idp, size_t* nparamsp, unsigned** paramsp); +static int NCZ_bzip2_hdf5_to_codec(const NCproplist* env, int id, size_t nparams, const unsigned* params, char** codecp); #ifdef HAVE_ZSTD -static int NCZ_zstd_codec_to_hdf5(void* codec_env, const char* codec, size_t* nparamsp, unsigned** paramsp); -static int NCZ_zstd_hdf5_to_codec(void* codec_env, size_t nparams, const unsigned* params, char** codecp); +static int NCZ_zstd_codec_to_hdf5(const struct NCproplist* env, const char* codec, int* idp, size_t* nparamsp, unsigned** paramsp); +static int NCZ_zstd_hdf5_to_codec(const struct NCproplist* env, int id, size_t nparams, const unsigned* params, char** codecp); #endif #ifdef HAVE_BLOSC -static int NCZ_blosc_codec_to_hdf5(void* codec_env, const char* codec, size_t* nparamsp, unsigned** paramsp); -static int NCZ_blosc_hdf5_to_codec(void* codec_env, size_t nparams, const unsigned* params, char** codecp); -static int NCZ_blosc_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); -static void NCZ_blosc_codec_finalize(void); +static int NCZ_blosc_codec_to_hdf5(const NCproplist* env, const char* codec, int* idp, size_t* nparamsp, unsigned** paramsp); +static int NCZ_blosc_hdf5_to_codec(const NCproplist* env, int id, size_t nparams, const unsigned* params, char** codecp); +static int NCZ_blosc_modify_parameters(const NCproplist* env, int* idp, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); +static void NCZ_blosc_codec_finalize(const NCproplist*); #endif /**************************************************/ @@ -77,13 +79,15 @@ NCZ_get_codec_info(void) } static int -NCZ_bzip2_codec_to_hdf5(void* env, const char* codec_json, size_t* nparamsp, unsigned** paramsp) +NCZ_bzip2_codec_to_hdf5(const NCproplist* env, const char* codec_json, int* idp, size_t* nparamsp, unsigned** paramsp) { int stat = NC_NOERR; NCjson* jcodec = NULL; + NCjson* jdict = NULL; NCjson* jtmp = NULL; unsigned* params = NULL; struct NCJconst jc; + uintptr_t zarrformat = 0; if(nparamsp == NULL || paramsp == NULL) {stat = NC_EINTERNAL; goto done;} @@ -91,25 +95,31 @@ NCZ_bzip2_codec_to_hdf5(void* env, const char* codec_json, size_t* nparamsp, uns if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL) {stat = NC_ENOMEM; goto done;} + ncplistget(env,"zarrformat",&zarrformat,NULL); + /* parse the JSON */ - if(NCJparse(codec_json,0,&jcodec)) - {stat = NC_EFILTER; goto done;} + if(NCJparse(codec_json,0,&jcodec)) {stat = NC_EFILTER; goto done;} if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;} + + if(zarrformat == 3) { + NCJdictget(jcodec,"configuration",&jdict); + if(NCJdictget(jdict,"name",&jtmp)) {stat = NC_EFILTER; goto done;} + } else { + jdict = jcodec; + if(NCJdictget(jdict,"id",&jtmp)) {stat = NC_EFILTER; goto done;} + } /* Verify the codec ID */ - if(NCJdictget(jcodec,"id",&jtmp)) - {stat = NC_EFILTER; goto done;} if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EFILTER; goto done;} if(strcmp(NCJstring(jtmp),NCZ_bzip2_codec.codecid)!=0) {stat = NC_EINVAL; goto done;} /* Get Level */ - if(NCJdictget(jcodec,"level",&jtmp)) - {stat = NC_EFILTER; goto done;} - if(NCJcvt(jtmp,NCJ_INT,&jc)) - {stat = NC_EFILTER; goto done;} + if(NCJdictget(jdict,"level",&jtmp)) {stat = NC_EFILTER; goto done;} + if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;} if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;} params[0] = (unsigned)jc.ival; *nparamsp = 1; *paramsp = params; params = NULL; + if(idp) *idp = H5Z_FILTER_BZIP2; done: if(params) free(params); @@ -118,17 +128,25 @@ NCZ_bzip2_codec_to_hdf5(void* env, const char* codec_json, size_t* nparamsp, uns } static int -NCZ_bzip2_hdf5_to_codec(void* env, size_t nparams, const unsigned* params, char** codecp) +NCZ_bzip2_hdf5_to_codec(const NCproplist* env, int id, size_t nparams, const unsigned* params, char** codecp) { int stat = NC_NOERR; unsigned level = 0; char json[1024]; + uintptr_t zarrformat; + + NC_UNUSED(id); if(nparams == 0 || params == NULL) {stat = NC_EFILTER; goto done;} + ncplistget(env,"zarrformat",&zarrformat,NULL); level = params[0]; - snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_bzip2_codec.codecid,level); + + if(zarrformat == 3) + snprintf(json,sizeof(json),"{\"name\": \"%s\", \"configuration\": {\"level\": \"%u\"}}",NCZ_bzip2_codec.codecid,level); + else + snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_bzip2_codec.codecid,level); if(codecp) { if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} } @@ -155,13 +173,15 @@ static NCZ_codec_t NCZ_zstd_codec = {/* NCZ_codec_t codec fields */ }; static int -NCZ_zstd_codec_to_hdf5(void* env, const char* codec_json, size_t* nparamsp, unsigned** paramsp) +NCZ_zstd_codec_to_hdf5(const NCproplist* env, const char* codec_json, int* idp, size_t* nparamsp, unsigned** paramsp) { int stat = NC_NOERR; NCjson* jcodec = NULL; + NCjson* jdict = NULL; NCjson* jtmp = NULL; unsigned* params = NULL; struct NCJconst jc; + uintptr_t zarrformat = 0; if(nparamsp == NULL || paramsp == NULL) {stat = NC_EINTERNAL; goto done;} @@ -169,25 +189,32 @@ NCZ_zstd_codec_to_hdf5(void* env, const char* codec_json, size_t* nparamsp, unsi if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL) {stat = NC_ENOMEM; goto done;} + ncplistget(env,"zarrformat",&zarrformat,NULL); + /* parse the JSON */ - if(NCJparse(codec_json,0,&jcodec)) - {stat = NC_EFILTER; goto done;} + if(NCJparse(codec_json,0,&jcodec)) {stat = NC_EFILTER; goto done;} if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;} + + if(zarrformat == 3) { + NCJdictget(jcodec,"configuration",&jdict); + if(NCJdictget(jdict,"name",&jtmp)) {stat = NC_EFILTER; goto done;} + } else { + jdict = jcodec; + if(NCJdictget(jdict,"id",&jtmp)) {stat = NC_EFILTER; goto done;} + } + /* Verify the codec ID */ - if(NCJdictget(jcodec,"id",&jtmp)) - {stat = NC_EFILTER; goto done;} if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EFILTER; goto done;} if(strcmp(NCJstring(jtmp),NCZ_zstd_codec.codecid)!=0) {stat = NC_EINVAL; goto done;} /* Get Level */ - if(NCJdictget(jcodec,"level",&jtmp)) - {stat = NC_EFILTER; goto done;} - if(NCJcvt(jtmp,NCJ_INT,&jc)) - {stat = NC_EFILTER; goto done;} + if(NCJdictget(jdict,"level",&jtmp)) {stat = NC_EFILTER; goto done;} + if(NCJcvt(jtmp,NCJ_INT,&jc)) {stat = NC_EFILTER; goto done;} if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;} params[0] = (unsigned)jc.ival; *nparamsp = 1; *paramsp = params; params = NULL; + if(idp) *idp = H5Z_FILTER_ZSTD; done: if(params) free(params); @@ -196,17 +223,23 @@ NCZ_zstd_codec_to_hdf5(void* env, const char* codec_json, size_t* nparamsp, unsi } static int -NCZ_zstd_hdf5_to_codec(void* env, size_t nparams, const unsigned* params, char** codecp) +NCZ_zstd_hdf5_to_codec(const NCproplist* env, int id, size_t nparams, const unsigned* params, char** codecp) { int stat = NC_NOERR; unsigned level = 0; char json[1024]; - + uintptr_t zarrformat; + if(nparams == 0 || params == NULL) {stat = NC_EFILTER; goto done;} + ncplistget(env,"zarrformat",&zarrformat,NULL); level = params[0]; - snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_zstd_codec.codecid,level); + + if(zarrformat == 3) + snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_zstd_codec.codecid,level); + else + snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_zstd_codec.codecid,level); if(codecp) { if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} } @@ -261,8 +294,9 @@ void blosc_destroy(void); static int ncz_blosc_initialized = 0; static void -NCZ_blosc_codec_finalize(void) +NCZ_blosc_codec_finalize(const NCproplist* env) { + NC_UNUSED(env); if(ncz_blosc_initialized) { blosc_destroy(); ncz_blosc_initialized = 0; @@ -270,7 +304,7 @@ NCZ_blosc_codec_finalize(void) } static int -NCZ_blosc_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp) +NCZ_blosc_modify_parameters(const NCproplist* env, int* idp, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp) { int i,stat = NC_NOERR; nc_type vtype; @@ -282,7 +316,10 @@ NCZ_blosc_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** v size_t nparams; size_t vnparams = *vnparamsp; unsigned* vparams = *vparamsp; + uintptr_t ncid, varid; + NC_UNUSED(idp); + if(vnparams < 7) {stat = NC_EFILTER; goto done;} nparams = 7; @@ -293,20 +330,23 @@ NCZ_blosc_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** v if(wnparamsp == NULL || wparamsp == NULL) {stat = NC_EFILTER; goto done;} + ncplistget(env,"fileid",&ncid,NULL); + ncplistget(env,"varid",&varid,NULL); + vnparams = *vnparamsp; vparams = *vparamsp; /* Get variable info */ - if((stat = nc_inq_var(ncid,varid,vname,&vtype,&ndims,NULL,NULL))) goto done; + if((stat = nc_inq_var((int)ncid,(int)varid,vname,&vtype,&ndims,NULL,NULL))) goto done; if(ndims == 0) {stat = NC_EFILTER; goto done;} /* Get the typesize */ - if((stat = nc_inq_type(ncid,vtype,NULL,&typesize))) goto done; + if((stat = nc_inq_type((int)ncid,vtype,NULL,&typesize))) goto done; /* Compute chunksize */ if((chunklens = (size_t*)calloc(ndims,sizeof(size_t)))==NULL) goto done; - if((stat = nc_inq_var_chunking(ncid,varid,&storage,chunklens))) goto done; + if((stat = nc_inq_var_chunking((int)ncid,varid,&storage,chunklens))) goto done; if(storage != NC_CHUNKED) {stat = NC_EFILTER; goto done;} chunksize = typesize; for(i=0;izarrformat == 3) { + if(zarrformat == 3) { if(NCZdictget(json,"configuration",&jdict)) {stat = NC_EFILTER; goto done;} } else jdict = json; @@ -89,13 +91,15 @@ NCZ_szip_codec_to_hdf5(void* env0, const char* codec_json, size_t* nparamsp, uns } static int -NCZ_szip_hdf5_to_codec(void* env0, size_t nparams, const unsigned* params, char** codecp) +NCZ_szip_hdf5_to_codec(const NCproplist* env, size_t nparams, const unsigned* params, char** codecp) { int stat = NC_NOERR; char json[2048]; - NCZ_codec_env_t* env = (NCZ_codec_env_t*)env0; + uintptr_t zarrformat = 0; + + ncplistget(env,"zarrformat",&zarrformat,NULL); - if(env->zarrformat == 2) { + if(zarrformat == 2) { snprintf(json,sizeof(json),"{\"id\": \"%s\", \"mask\": %u, \"pixels-per-block\": %u}", NCZ_szip_codec.codecid, params[H5Z_SZIP_PARM_MASK], @@ -116,7 +120,7 @@ NCZ_szip_hdf5_to_codec(void* env0, size_t nparams, const unsigned* params, char* } static int -NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp) +NCZ_szip_modify_parameters(const NCZproplist* env, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp) { int i,ret_value = NC_NOERR; nc_type vtype; @@ -128,6 +132,7 @@ NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vp unsigned* params = NULL; unsigned* vparams = NULL; size_t wnparams = 4; + uintptr_t ncid, varid, zarrformat; if(wnparamsp == NULL || wparamsp == NULL) {ret_value = NC_EFILTER; goto done;} @@ -138,6 +143,10 @@ NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vp vparams = *vparamsp; + ncplistget(env,"zarrformat",&zarrformat,NULL); + ncplistget(env,"fileid",&ncid,NULL); + ncplistget(env,"varid",&varid,NULL); + /* Get variable info */ if((ret_value = nc_inq_var(ncid,varid,vname,&vtype,&ndims,dimids,NULL))) goto done; diff --git a/s3gc.in b/s3gc.in index 612e371d7a..6f2c335f27 100755 --- a/s3gc.in +++ b/s3gc.in @@ -54,14 +54,16 @@ lastdate=$((current-deltasec)) rm -f s3gc.json # Get complete set of keys in ${S3TESTSUBTREE} prefix -if ! aws s3api list-objects-v2 --bucket ${S3TESTBUCKET} --prefix "${S3TESTSUBTREE}" | grep -F '"Key":' >s3gc.keys ; then +if ! aws s3api list-objects-v2 --bucket ${S3TESTBUCKET} --prefix "/${S3TESTSUBTREE}" | grep -F '"Key":' >s3gc.keys ; then echo "No keys found" rm -f s3gc.json exit 0 fi -aws s3api list-objects-v2 --bucket ${S3TESTBUCKET} --prefix "${S3TESTSUBTREE}" | grep -F '"Key":' >s3gc.keys +aws s3api list-objects-v2 --bucket ${S3TESTBUCKET} --prefix "/${S3TESTSUBTREE}" | grep -F '"Key":' >s3gc.keys while read -r line; do - KEY=`echo "$line" | sed -e 's|[^"]*"Key":[^"]*"\([^"]*\)".*|\1|'` + KEY0=`echo "$line" | sed -e 's|[^"]*"Key":[^"]*"\([^"]*\)".*|\1|'` + # Strip off any leading '/' + KEY=`echo "$KEY0" | sed -e 's|^[/]*\(.*\)|\1|'` # Ignore keys that do not start with ${S3TESTSUBTREE} PREFIX=`echo "$KEY" | sed -e 's|\([^/]*\)/.*|\1|'` if test "x$PREFIX" = "x$S3TESTSUBTREE" ; then @@ -117,4 +119,4 @@ while test "x$REM" != x ; do aws s3api delete-objects --bucket ${S3TESTBUCKET} --delete "file://s3gc.json" fi done -rm -f s3gc.json +#rm -f s3gc.json