From 16ab2e12aa644e9fa6866184834b9bbccb58c1c2 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 24 Feb 2024 16:22:15 -0700 Subject: [PATCH] ckp --- include/ncbytes.h | 3 + include/netcdf.h | 3 +- libdispatch/ncbytes.c | 15 + libnczarr/zarr.h | 3 +- libnczarr/zattr.c | 238 +++++++----- libnczarr/zcvt.c | 4 +- libnczarr/zfile.c | 446 ++++++++++++++++++++++- libnczarr/zfilter.c | 8 +- libnczarr/zformat2.c | 251 +++++++------ libnczarr/zformat3.c | 279 +++++++------- libnczarr/zinternal.c | 13 +- libnczarr/zinternal.h | 4 +- libnczarr/zutil.c | 19 +- libnczarr/zvar.c | 8 +- libnczarr/zxcache.c | 2 +- libsrc4/nc4internal.c | 2 +- nczarr_test/ref_byte_fill_value_null.cdl | 4 +- plugins/NCZmisc.c | 2 +- 18 files changed, 947 insertions(+), 357 deletions(-) diff --git a/include/ncbytes.h b/include/ncbytes.h index b00d428c16..678d0c6bc9 100644 --- a/include/ncbytes.h +++ b/include/ncbytes.h @@ -47,6 +47,9 @@ EXTERNL int ncbytesremove(NCbytes*,unsigned long); /* Concatenate a null-terminated string to the end of the buffer */ EXTERNL int ncbytescat(NCbytes*,const char*); +/* Insert n bytes into the buffer at position pos*/ +EXTERNL int ncbytesinsert(NCbytes*,size_t pos, size_t n, const char*); + /* Set the contents of the buffer; mark the buffer as non-extendible */ EXTERNL int ncbytessetcontents(NCbytes*, void*, unsigned long); diff --git a/include/netcdf.h b/include/netcdf.h index 267c0eda48..29474412c5 100644 --- a/include/netcdf.h +++ b/include/netcdf.h @@ -118,6 +118,7 @@ extern "C" { 0x0002 All upper 16 bits are unused except 0x20000 + 0x40000 */ /* Lower 16 bits */ @@ -162,7 +163,7 @@ Use this in mode flags for both nc_create() and nc_open(). */ #define NC_INMEMORY 0x8000 /**< Read from memory. Mode flag for nc_open() or nc_create() */ /* Upper 16 bits */ -#define NC_NOATTCREORD 0x20000 /**< Disable the netcdf-4 (hdf5) attribute creation order tracking */ +#define NC_NOATTCREORD 0x20000 /**< Disable the netcdf-4 (hdf5) attribute creation order tracking */ #define NC_NODIMSCALE_ATTACH 0x40000 /**< Disable the netcdf-4 (hdf5) attaching of dimscales to variables (#2128) */ #define NC_MAX_MAGIC_NUMBER_LEN 8 /**< Max len of user-defined format magic number. */ diff --git a/libdispatch/ncbytes.c b/libdispatch/ncbytes.c index 84eb93e090..e4814f9441 100644 --- a/libdispatch/ncbytes.c +++ b/libdispatch/ncbytes.c @@ -209,3 +209,18 @@ ncbytesremove(NCbytes* bb, unsigned long pos) bb->length--; return TRUE; } + +/* Insert n bytes into the buffer at position pos*/ +int +ncbytesinsert(NCbytes* bb, size_t pos, size_t n, const char* s) +{ + if(bb == NULL) return ncbytesfail(); + if(pos > bb->length) ncbytesfail(); + if((bb->length + n) >= bb->alloc) if(!ncbytessetalloc(bb,bb->length+n+1)) return ncbytesfail(); + if(bb->length > 0) + memmove(bb->content+pos+n,bb->content+pos,(bb->length - pos)); + memcpy(bb->content+pos,s,n); + bb->length += n; + bb->content[bb->length] = '\0'; + return TRUE; +} diff --git a/libnczarr/zarr.h b/libnczarr/zarr.h index 41ee8be265..a5d513b119 100644 --- a/libnczarr/zarr.h +++ b/libnczarr/zarr.h @@ -84,10 +84,9 @@ EXTERNL int NCZ_comma_parse(const char* s, NClist* list); EXTERNL int NCZ_swapatomicdata(size_t datalen, void* data, int typesize); EXTERNL char** NCZ_clonestringvec(size_t len, const char** vec); EXTERNL void NCZ_freestringvec(size_t len, char** vec); +EXTERNL void NCZ_clearstringvec(size_t len, char** vec); EXTERNL int NCZ_ischunkname(const char* name,char dimsep); EXTERNL char* NCZ_chunkpath(struct ChunkKey key); -EXTERNL int NCZ_reclaim_fill_value(NC_VAR_INFO_T* var); -EXTERNL int NCZ_copy_fill_value(NC_VAR_INFO_T* var, void** dstp); EXTERNL size_t NCZ_get_maxstrlen(NC_OBJ* obj); EXTERNL int NCZ_fixed2char(const void* fixed, char** charp, size_t count, size_t maxstrlen); EXTERNL int NCZ_char2fixed(const char** charp, void* fixed, size_t count, size_t maxstrlen); diff --git a/libnczarr/zattr.c b/libnczarr/zattr.c index c5a1415da0..17172c5d7c 100644 --- a/libnczarr/zattr.c +++ b/libnczarr/zattr.c @@ -33,23 +33,23 @@ static int NCZ_json_convention_read(const NCjson* json, NCjson** jtextp); * [Candidate for moving to libsrc4] */ int -ncz_getattlist(NC_GRP_INFO_T *grp, int varid, NC_VAR_INFO_T **varp, NCindex **attlist) +ncz_getattlist(NC_GRP_INFO_T *grp, int varid, NC_VAR_INFO_T **varp, NCindex **attlistp) { int stat = NC_NOERR; NC_FILE_INFO_T* file = grp->nc4_info; NCZ_FILE_INFO_T* zinfo = file->format_file_info; - assert(grp && attlist && file && zinfo); + assert(grp && file && zinfo); if (varid == NC_GLOBAL) { if (varp) *varp = NULL; - *attlist = grp->att; + if(attlistp) *attlistp = grp->att; } else { NC_VAR_INFO_T *var; if (!(var = (NC_VAR_INFO_T *)ncindexith(grp->vars, (size_t)varid))) return NC_ENOTVAR; assert(var->hdr.id == varid); if (varp) *varp = var; - *attlist = var->att; + if(attlistp) *attlistp = var->att; } return stat; } @@ -423,7 +423,10 @@ ncz_put_att(int ncid, int containerid, const char *name, nc_type file_type, void* copy = NULL; NC_OBJ* obj = NULL; int isfillvalue = (strcmp(name,_FillValue)==0); - + char norm_name[NC_MAX_NAME + 1]; + const NC_reservedatt* ra = NULL; + NC_ATT_INFO_T* att = NULL; + NCindex* attlist = NULL; if(containerid == NC_GLOBAL) { if((stat= nc4_find_grp_h5(ncid, &grp, &file))) goto done; obj = (NC_OBJ*)grp; @@ -436,6 +439,12 @@ ncz_put_att(int ncid, int containerid, const char *name, nc_type file_type, nc = file->controller; assert(nc && grp && file); + if(obj->sort == NCGRP) + attlist = ((NC_GRP_INFO_T*)obj)->att; + else + attlist = ((NC_VAR_INFO_T*)obj)->att; + assert(attlist != NULL); + LOG((1, "%s: ncid 0x%x containerid %d name %s file_type %d mem_type %d len %d", __func__,ncid, containerid, name, file_type, mem_type, len)); @@ -453,6 +462,30 @@ ncz_put_att(int ncid, int containerid, const char *name, nc_type file_type, if (file->cmode & NC_CLASSIC_MODEL && file_type > NC_DOUBLE) return NC_ESTRICTNC3; + /* The length needs to be positive (cast needed for brain-dead systems with signed size_t). */ + if((unsigned long) len > X_INT_MAX) {stat = NC_EINVAL; goto done;} + + /* If len is not zero, then there must be some data. */ + if (len > 0 && data == NULL) {stat = NC_EINVAL; goto done;} + + /* If the file is read-only, return an error. */ + if (file->no_write) {stat = NC_EPERM; goto done;} + + /* Check and normalize the name. */ + if (!name || strlen(name) > NC_MAX_NAME) {stat = NC_EBADNAME; goto done;} + if ((stat = nc4_check_name(name, norm_name))) goto done; + + /* Check that a reserved att name is not being used improperly */ + ra = NC_findreserved(name); + if(ra != NULL) { + /* case 1: grp=root, containerid==NC_GLOBAL, flags & READONLYFLAG */ + if (obj->sort == NCGRP && ((NC_GRP_INFO_T*)obj)->parent == NULL && (ra->flags & READONLYFLAG)) + {stat = NC_ENAMEINUSE; goto done;} + /* case 2: grp=NA, objid!=NC_GLOBAL, flags & HIDDENATTRFLAG */ + if (obj->sort != NCGRP && (ra->flags & HIDDENATTRFLAG)) + {stat = NC_ENAMEINUSE; goto done;} + } + if(isfillvalue) file_type = var->type_info->hdr.id; @@ -472,7 +505,27 @@ ncz_put_att(int ncid, int containerid, const char *name, nc_type file_type, } else { /* no conversion */ /* Still need a copy of the input data */ copy = NULL; - if((stat = NC_copy_data_all(file->controller, mem_type, data, 1, ©))) goto done; + if((stat = NC_copy_data_all(file->controller, mem_type, data, len, ©))) goto done; + } + + /* See if there is already an attribute with this name. */ + att = (NC_ATT_INFO_T*)ncindexlookup(attlist,norm_name); + + if (!att) { + if (!(file->flags & NC_INDEF)) { /* If this is a new att, require define mode. */ + if (file->cmode & NC_CLASSIC_MODEL) {stat = NC_ENOTINDEFINE; goto done;} + file->flags |= NC_INDEF;/* Set define mode. */ + file->redef = NC_TRUE; /* For nc_abort, we need to remember if we're in define mode as a redef. */ + } + } else { + /* For an existing att, if we're not in define mode, the len + must not be greater than the existing len for classic model. */ + if(!(file->flags & NC_INDEF) + && len * (size_t)nc4typelen(att->nc_typeid) > (size_t)att->len * (size_t)nc4typelen(att->nc_typeid)) { + if (file->cmode & NC_CLASSIC_MODEL) {stat = NC_ENOTINDEFINE; goto done;} + file->flags |= NC_INDEF;/* Set define mode. */ + file->redef = NC_TRUE; /* For nc_abort, we need to remember if we're in define mode as a redef. */ + } } if((stat = ncz_makeattr(file,obj,name,file_type,len,copy,NULL))) goto done; @@ -662,21 +715,25 @@ NCZ_inq_attname(int ncid, int varid, int attnum, char *name) */ int NCZ_get_att(int ncid, int varid, const char *name, void *value, - nc_type memtype) + nc_type mem_type) { + int stat = NC_NOERR; NC_FILE_INFO_T *file; NC_GRP_INFO_T *grp; NC_VAR_INFO_T *var = NULL; + NC_ATT_INFO_T *att = NULL; char norm_name[NC_MAX_NAME + 1]; - int stat = NC_NOERR; + nc_type file_type = NC_NAT; + int range_error = 0; LOG((2, "%s: ncid 0x%x varid %d", __func__, ncid, varid)); - /* Find the file, group, and var info, and do lazy att read if - * needed. */ - if ((stat = ncz_find_grp_var_att(ncid, varid, name, 0, 1, norm_name, - &file, &grp, &var, NULL))) - return stat; + /* Find the file, group, and var info */ + if ((stat = ncz_find_file_grp_var(ncid, varid, &file, &grp, &var))) goto done; + + /* Check and normalize the name. */ + if (!name || strlen(name) > NC_MAX_NAME) {stat = NC_EBADNAME; goto done;} + if ((stat = nc4_check_name(name, norm_name))) goto done; /* If this is one of the reserved global atts, use nc_get_att_special. */ { @@ -687,13 +744,57 @@ NCZ_get_att(int ncid, int varid, const char *name, void *value, } /* See if the attribute exists */ - stat = nc4_get_att_ptrs(file, grp, var, norm_name, NULL, memtype, - NULL, NULL, value); + stat = nc4_find_grp_att(grp,varid,norm_name,0,&att); /* If asking for _FillValue and it does not exist, build it */ - if(stat == NC_ENOTATT && varid != NC_GLOBAL && strcmp(norm_name,"_FillValue")==0) { - stat = ncz_create_fillvalue(file,var); + if(stat == NC_ENOTATT && att != NULL && varid != NC_GLOBAL && strcmp(norm_name,"_FillValue")==0) { + if((stat = ncz_create_fillvalue(file,var))) goto done; + /* get fill value attribute again */ + if((stat = nc4_find_grp_att(grp,varid,norm_name,0,&att))) goto done; } + + /* stop if error */ + if(stat) goto done; + + assert(att != NULL); + file_type = att->nc_typeid; + + /* We must have two valid types to continue. */ + if (file_type == NC_NAT || mem_type == NC_NAT) return NC_EBADTYPE; + + /* No character conversions are allowed. */ + if (file_type != mem_type && + (file_type == NC_CHAR || mem_type == NC_CHAR || + file_type == NC_STRING || mem_type == NC_STRING)) + return NC_ECHAR; + + /* For classic mode file, only allow atts with classic types to be + * created. */ + if (file->cmode & NC_CLASSIC_MODEL && file_type > NC_DOUBLE) + return NC_ESTRICTNC3; + + /* copy and/or convert memory data to file format data */ + if(mem_type != file_type && mem_type < NC_STRING && mem_type < NC_STRING) { + size_t mem_type_len = 0; + size_t file_type_len = 0; + if ((stat = nc4_get_typelen_mem(file, mem_type, &mem_type_len))) return stat; + if ((stat = nc4_get_typelen_mem(file, file_type, &file_type_len))) return stat; + /* Need to convert from file_type data into output buffer */ + if ((stat = nc4_convert_type(att->data, value, file_type, mem_type, + att->len, &range_error, NULL, + (file->cmode & NC_CLASSIC_MODEL), + NC_NOQUANTIZE, 0))) + goto done; + } else { /* no conversion */ + /* Still need a copy of the input data */ + if((stat = NC_copy_data(file->controller, file_type, att->data, att->len, value))) goto done; + } + +done: + /* If there was an error return it, otherwise return any potential + range error value. If none, return NC_NOERR as usual.*/ + if (range_error) return NC_ERANGE; + if (stat) return stat; return THROW(stat); } @@ -764,42 +865,53 @@ int ncz_create_fillvalue(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var) { int stat = NC_NOERR; - size_t i; - NC_ATT_INFO_T* fv = NULL; /* Have the var's attributes been read? */ if(!var->atts_read) goto done; /* above my pay grade */ /* Is FillValue warranted? */ - if(!var->no_fill && var->fill_value != NULL && !isdfaltfillvalue(var->type_info->hdr.id,var->fill_value)) { - /* Make sure _FillValue does not exist */ +#if 0 + if(!var->no_fill && var->fill_value != NULL && !isdfaltfillvalue(vartype,var->fill_value)) { + /* Make sure _FillValue att does not exist */ for(i=0;iatt);i++) { - fv = (NC_ATT_INFO_T*)ncindexith(var->att,i); - if(strcmp(fv->hdr.name,NC_ATT_FILLVALUE)==0) break; - fv = NULL; + fvatt = (NC_ATT_INFO_T*)ncindexith(var->att,i); + if(strcmp(fvatt->hdr.name,NC_ATT_FILLVALUE)==0) break; + fvatt = NULL; } - if(fv == NULL) { - /* Create it */ - if((stat = ncz_makeattr(file, (NC_OBJ*)var,_FillValue,var->type_info->hdr.id,1,var->fill_value,&fv))) + if(fvatt == NULL) { /* Create it */ + /* Make a copy of the var->fill_value to be stored in the attribute */ + if((stat = NC_copy_data_all(file->controller, vartype, var->fill_value, 1, ©))) goto done; + if((stat = ncz_makeattr(file, (NC_OBJ*)var,_FillValue,vartype,1,copy,&fvatt))) + copy = NULL; goto done; } } +#else + if(!var->no_fill && var->fill_value != NULL) { + if((stat = NCZ_copy_var_to_fillatt(file,var))) goto done; + } +#endif + done: +#if 0 + if(copy) + NC_reclaim_data_all(file->controller,vartype,copy,1); +#endif return THROW(stat); } -/* Create an attribute; This is the core of ncz_put_att above */ +/* +Create an attribute; This is the core of ncz_put_att above +*/ int ncz_makeattr(NC_FILE_INFO_T* file, NC_OBJ* container, const char* name, nc_type typeid, size_t len, void* values, NC_ATT_INFO_T** attp) { int stat = NC_NOERR; NC_ATT_INFO_T* att = NULL; NCZ_ATT_INFO_T* zatt = NULL; - NCindex* attlist = NULL; - char norm_name[NC_MAX_NAME + 1]; - const NC_reservedatt* ra = NULL; int new_att = 0; size_t typesize = 0; + NCindex* attlist = NULL; if(container->sort == NCGRP) attlist = ((NC_GRP_INFO_T*)container)->att; @@ -807,68 +919,28 @@ ncz_makeattr(NC_FILE_INFO_T* file, NC_OBJ* container, const char* name, nc_type attlist = ((NC_VAR_INFO_T*)container)->att; assert(attlist != NULL); - /* The length needs to be positive (cast needed for brain-dead systems with signed size_t). */ - if((unsigned long) len > X_INT_MAX) {stat = NC_EINVAL; goto done;} - - /* If len is not zero, then there must be some data. */ - if (len && !values) {stat = NC_EINVAL; goto done;} - - /* If the file is read-only, return an error. */ - if (file->no_write) {stat = NC_EPERM; goto done;} - - /* Check and normalize the name. */ - if (!name || strlen(name) > NC_MAX_NAME) {stat = NC_EBADNAME; goto done;} - if ((stat = nc4_check_name(name, norm_name))) goto done; - - /* Check that a reserved att name is not being used improperly */ - ra = NC_findreserved(name); - if(ra != NULL) { - /* case 1: grp=root, containerid==NC_GLOBAL, flags & READONLYFLAG */ - if (container->sort == NCGRP && ((NC_GRP_INFO_T*)container)->parent == NULL && (ra->flags & READONLYFLAG)) - {stat = NC_ENAMEINUSE; goto done;} - /* case 2: grp=NA, containerid!=NC_GLOBAL, flags & HIDDENATTRFLAG */ - if (container->sort != NCGRP && (ra->flags & HIDDENATTRFLAG)) - {stat = NC_ENAMEINUSE; goto done;} - } - - /* See if there is already an attribute with this name. */ - att = (NC_ATT_INFO_T*)ncindexlookup(attlist,norm_name); - - if (!att) { - if (!(file->flags & NC_INDEF)) { /* If this is a new att, require define mode. */ - if (file->cmode & NC_CLASSIC_MODEL) {stat = NC_ENOTINDEFINE; goto done;} - file->flags |= NC_INDEF;/* Set define mode. */ - file->redef = NC_TRUE; /* For nc_abort, we need to remember if we're in define mode as a redef. */ - } - new_att = NC_TRUE; - } else { - /* For an existing att, if we're not in define mode, the len - must not be greater than the existing len for classic model. */ - if(!(file->flags & NC_INDEF) - && len * (size_t)nc4typelen(typeid) > (size_t)att->len * (size_t)nc4typelen(att->nc_typeid)) { - if (file->cmode & NC_CLASSIC_MODEL) {stat = NC_ENOTINDEFINE; goto done;} - file->flags |= NC_INDEF;/* Set define mode. */ - file->redef = NC_TRUE; /* For nc_abort, we need to remember if we're in define mode as a redef. */ - } - } - if ((stat = nc4_get_typelen_mem(file, typeid, &typesize))) goto done; + /* See if there is already an attribute with this name. */ + att = (NC_ATT_INFO_T*)ncindexlookup(attlist,name); + new_att = (att == NULL); + if(new_att) { if((stat=nc4_att_list_add(attlist,name,&att))) goto done; if((zatt = calloc(1,sizeof(NCZ_ATT_INFO_T))) == NULL) {stat = NC_ENOMEM; goto done;} zatt->common.file = file; att->container = container; att->format_att_info = zatt; - } else { + } else if(att->data) { + /* remove old att data */ (void)NC_reclaim_data_all(file->controller,att->nc_typeid,att->data,att->len); att->data = NULL; } - + /* Fill in the attribute's type and value */ att->nc_typeid = typeid; att->len = (int)len; - att->data = values; values = NULL; + if((stat = NC_copy_data_all(file->controller,typeid,values,len,&att->data))) goto done; att->dirty = NC_TRUE; if(attp) {*attp = att; att = NULL;} @@ -876,7 +948,6 @@ ncz_makeattr(NC_FILE_INFO_T* file, NC_OBJ* container, const char* name, nc_type ((NC_VAR_INFO_T*)container)->attr_dirty = NC_TRUE; done: - if(values) (void)NC_reclaim_data_all(file->controller,typeid,values,len); if(stat) { if(new_att && att) { nc4_att_list_del(attlist,att); @@ -906,6 +977,10 @@ NCZ_computeattrdata(nc_type* typeidp, const NCjson* values, size_t* typelenp, si typeid = *typeidp; /* initial assumption */ + + if((stat = NC4_inq_atomic_type(typeid, NULL, &typelen))) + goto done; + /* Handle special types */ if(typeid == NC_JSON) { /* Apply the JSON attribute convention and convert to JSON string */ @@ -926,9 +1001,6 @@ NCZ_computeattrdata(nc_type* typeidp, const NCjson* values, size_t* typelenp, si } assert(typeid != NC_NAT && typeid != NC_JSON); - if((stat = NC4_inq_atomic_type(typeid, NULL, &typelen))) - goto done; - if(typelenp) *typelenp = typelen; if(typeidp) *typeidp = typeid; /* return possibly inferred type */ if(countp) *countp = count; diff --git a/libnczarr/zcvt.c b/libnczarr/zcvt.c index 82e884c059..991989c5b6 100644 --- a/libnczarr/zcvt.c +++ b/libnczarr/zcvt.c @@ -431,7 +431,6 @@ NCZ_stringconvert1(nc_type srctype, char* src, NCjson* jvalue) struct ZCVT zcvt; nc_type dsttype = NC_NAT; char s[1024]; - char sq[1024+2+1]; char* p = NULL; int isnanorinf = 0; @@ -519,12 +518,15 @@ NCZ_stringconvert1(nc_type srctype, char* src, NCjson* jvalue) #endif /* Quote the nan/inf constant */ if(isnanorinf) { +#if 0 + char sq[1024+2+1]; size_t l = strlen(s); memcpy(sq,s,l+1); s[0] = '"'; memcpy(s+1,sq,l); s[l+1] = '"'; s[l+2] = '\0'; +#endif /*0*/ } } break; case NC_STRING: { diff --git a/libnczarr/zfile.c b/libnczarr/zfile.c index e264f7144a..b2a1101ac8 100644 --- a/libnczarr/zfile.c +++ b/libnczarr/zfile.c @@ -9,26 +9,458 @@ * This file is part of netcdf-4, a netCDF-like interface for NCZ, or * a ZARR backend for netCDF, depending on your point of view. * - * @author Dennis Heimbigner + * @author Dennis Heimbigner, Ed Hartnett */ #include "zincludes.h" #include "zfilter.h" +#include + +/* Forward */ +static int NCZ_enddef(NC_FILE_INFO_T* h5); +static int ncz_sync_netcdf4_file(NC_FILE_INFO_T* file, int isclose); + +/** + * @internal Put the file back in redef mode. This is done + * automatically for netcdf-4 files, if the user forgets. + * + * @param ncid File and group ID. + * + * @return ::NC_NOERR No error. + * @author Dennis Heimbigner, Ed Hartnett + */ +int +NCZ_redef(int ncid) +{ + NC_FILE_INFO_T* file = NULL; + int stat = NC_NOERR; + + ZTRACE(0,"NCZ_redef(ncid)"); + + /* Find this file's metadata. */ + if ((stat = nc4_find_grp_h5(ncid, NULL, &file))) + goto done; + assert(file); + + /* If we're already in define mode, return an error. */ + if (file->flags & NC_INDEF) + {stat = NC_EINDEFINE; goto done;} + + /* If the file is read-only, return an error. */ + if (file->no_write) + {stat = NC_EPERM; goto done;} + + /* Set define mode. */ + file->flags |= NC_INDEF; + + /* For nc_abort, we need to remember if we're in define mode as a + redef. */ + file->redef = NC_TRUE; + +done: + return ZUNTRACE(stat); +} + +/** + * @internal For netcdf-4 files, this just calls nc_enddef, ignoring + * the extra parameters. + * + * @param ncid File and group ID. + * @param h_minfree Ignored for netCDF-4 files. + * @param v_align Ignored for netCDF-4 files. + * @param v_minfree Ignored for netCDF-4 files. + * @param r_align Ignored for netCDF-4 files. + * + * @return ::NC_NOERR No error. + * @author Dennis Heimbigner, Ed Hartnett + */ +int +NCZ__enddef(int ncid, size_t h_minfree, size_t v_align, + size_t v_minfree, size_t r_align) +{ + int stat = NC_NOERR; + NC_FILE_INFO_T* h5 = NULL; + NC_GRP_INFO_T* grp = NULL; + ZTRACE(0,"ncid=%d",ncid); + if ((stat = nc4_find_grp_h5(ncid, &grp, &h5))) + goto done; + stat = NCZ_enddef(h5); +done: + return ZUNTRACE(stat); +} + +/** + * @internal Take the file out of define mode. This is called + * automatically for netcdf-4 files, if the user forgets. + * + * @param h5 File object + * + * @return ::NC_NOERR No error. + * @return ::NC_EBADID Bad ncid. + * @return ::NC_EBADGRPID Bad group ID. + * @author Dennis Heimbigner, Ed Hartnett + */ +static int +NCZ_enddef(NC_FILE_INFO_T* h5) +{ + NC_VAR_INFO_T *var; + size_t i,j; + int stat = NC_NOERR; + + ZTRACE(1,"h5=%s",h5->hdr.name); -/* Sync from NC_VAR_INFO_T.fill_value to attribute _FillValue */ + /* When exiting define mode, process all variables */ + for (i = 0; i < nclistlength(h5->allgroups); i++) { + NC_GRP_INFO_T* g = nclistget(h5->allgroups,i); + for (j = 0; j < ncindexsize(g->vars); j++) { + var = (NC_VAR_INFO_T *)ncindexith(g->vars, j); + assert(var); + var->written_to = NC_TRUE; /* mark it written */ + var->created = 1; + } + } + if((stat = ncz_enddef_netcdf4_file(h5))) goto done; +done: + return ZUNTRACE(stat); +} + +/** + * @internal Flushes all buffers associated with the file, after + * writing all changed metadata. This may only be called in data mode. + * + * @param ncid File and group ID. + * + * @return ::NC_NOERR No error. + * @return ::NC_EBADID Bad ncid. + * @return ::NC_EINDEFINE Classic model file is in define mode. + * @author Dennis Heimbigner, Ed Hartnett + */ int -NCZ_copy_var_to_att(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, NC_ATT_INFO_T* att) +NCZ_sync(int ncid) { int stat = NC_NOERR; - return THROW(stat); + NC_FILE_INFO_T* file = NULL; + + ZTRACE(0,"ncid=%d",ncid); + + LOG((2, "%s: ncid 0x%x", __func__, ncid)); + + if ((stat = nc4_find_grp_h5(ncid, NULL, &file))) + return stat; + assert(file); + + /* If we're in define mode, we can't sync. */ + if (file->flags & NC_INDEF) + { + if (file->cmode & NC_CLASSIC_MODEL) + return NC_EINDEFINE; + if ((stat = NCZ_enddef(file))) + return stat; + } + + /* do not do this if file is writeonce */ + stat = ncz_sync_netcdf4_file(file,!ZCLOSE); + return stat; } -/* Sync from Attribute_FillValue to NC_VAR_INFO_T.fill_value */ +/** + * @internal From the netcdf-3 docs: The function nc_abort just closes + * the netCDF dataset, if not in define mode. If the dataset is being + * created and is still in define mode, the dataset is deleted. If + * define mode was entered by a call to nc_redef, the netCDF dataset + * is restored to its state before definition mode was entered and the + * dataset is closed. + * + * @param ncid File and group ID. + * + * @return ::NC_NOERR No error. + * @author Dennis Heimbigner, Ed Hartnett + */ int -NCZ_copy_att_to_var(NC_FILE_INFO_T* file, NC_ATT_INFO_T* att, NC_VAR_INFO_T* var) +NCZ_abort(int ncid) { int stat = NC_NOERR; - return THROW(stat); + NC_FILE_INFO_T* h5 = NULL; + ZTRACE(0,"ncid=%d",ncid); + LOG((2, "%s: ncid 0x%x", __func__, ncid)); + /* Find metadata for this file. */ + if ((stat = nc4_find_grp_h5(ncid, NULL, &h5))) + return stat; + assert(h5); + stat = ncz_closeorabort(h5, NULL, 1); + return ZUNTRACE(stat); } +/** + * @internal Close the netcdf file, writing any changes first. + * + * @param ncid File and group ID. + * @param params any extra parameters in/out of close + * + * @return ::NC_NOERR No error. + * @author Dennis Heimbigner, Ed Hartnett + */ +int +NCZ_close(int ncid, void* params) +{ + int stat = NC_NOERR; + NC_FILE_INFO_T* h5 = NULL; + + ZTRACE(0,"ncid=%d",ncid); + LOG((1, "%s: ncid 0x%x", __func__, ncid)); + /* Find metadata for this file. */ + if ((stat = nc4_find_grp_h5(ncid, NULL, &h5))) + return stat; + assert(h5); + return ncz_closeorabort(h5, params, 0); +} + +/** + * @internal From the netcdf-3 docs: The function nc_abort just closes + * the netCDF dataset, if not in define mode. If the dataset is being + * created and is still in define mode, the dataset is deleted. If + * define mode was entered by a call to nc_redef, the netCDF dataset + * is restored to its state before definition mode was entered and the + * dataset is closed. + * + * @param ncid File and group ID. + * + * @return ::NC_NOERR No error. + * @author Dennis Heimbigner, Ed Hartnett + */ +int +ncz_closeorabort(NC_FILE_INFO_T* h5, void* params, int abort) +{ + int stat = NC_NOERR; + + assert(h5); + + NC_UNUSED(params); + + ZTRACE(3,"file=%s abort=%d",h5->hdr.name,abort); + LOG((2, "%s: file: %p", __func__, h5)); + + /* If we're in define mode, but not redefing the file, delete it. */ + if(!abort) { + /* Invoke enddef if needed, which includes sync first */ + if(h5->flags & NC_INDEF) h5->flags ^= NC_INDEF; + /* Sync the file unless this is a read-only file. */ + if(!h5->no_write) { + if((stat = ncz_sync_netcdf4_file(h5,ZCLOSE))) + goto done; + } + } + + /* Reclaim memory */ + + /* Free any zarr-related data, including the map */ + if ((stat = ncz_close_file(h5, abort))) + goto done; + + /* Reclaim provenance info */ + NCZ_clear_provenance(&h5->provenance); + + /* Free the NC_FILE_INFO_T struct. */ + if ((stat = nc4_nc4f_list_del(h5))) + return stat; + +done: + return ZUNTRACE(stat); +} + +/**************************************************/ +/** + * @internal Learn number of dimensions, variables, global attributes, + * and the ID of the first unlimited dimension (if any). + * + * @note It's possible for any of these pointers to be NULL, in which + * case don't try to figure out that value. + * + * @param ncid File and group ID. + * @param ndimsp Pointer that gets number of dimensions. + * @param nvarsp Pointer that gets number of variables. + * @param nattsp Pointer that gets number of global attributes. + * @param unlimdimidp Pointer that gets first unlimited dimension ID, + * or -1 if there are no unlimied dimensions. + * + * @return ::NC_NOERR No error. + * @author Dennis Heimbigner, Ed Hartnett + */ +int +NCZ_inq(int ncid, int *ndimsp, int *nvarsp, int *nattsp, int *unlimdimidp) +{ + NC *nc; + NC_FILE_INFO_T* file; + NC_GRP_INFO_T *grp; + int stat = NC_NOERR; + size_t i; + + LOG((2, "%s: ncid 0x%x", __func__, ncid)); + + /* Find file metadata. */ + if ((stat = nc4_find_nc_grp_h5(ncid, &nc, &grp, &file))) + return stat; + + assert(file && grp && nc); + + /* Count the number of dims, vars, and global atts; need to iterate + * because of possible nulls. */ + if (ndimsp) + { + *ndimsp = ncindexcount(grp->dim); + } + if (nvarsp) + { + *nvarsp = ncindexcount(grp->vars); + } + if (nattsp) + { + *nattsp = ncindexcount(grp->att); + } + + if (unlimdimidp) + { + /* Default, no unlimited dimension */ + *unlimdimidp = -1; + + /* If there's more than one unlimited dim, which was not possible + with netcdf-3, then only the last unlimited one will be reported + back in xtendimp. */ + /* Note that this code is inconsistent with nc_inq_unlimid() */ + for(i=0;idim);i++) { + NC_DIM_INFO_T* d = (NC_DIM_INFO_T*)ncindexith(grp->dim,i); + if(d == NULL) continue; + if(d->unlimited) { + *unlimdimidp = d->hdr.id; + break; + } + } + } + + return NC_NOERR; +} + +/** + * @internal This function will write all changed metadata and flush + * ZARR file to disk. + * + * @param file Pointer to file info struct. + * + * @return ::NC_NOERR No error. + * @return ::NC_EINDEFINE Classic model file in define mode. + * @return ::NC_EHDFERR ZARR error. + * @author Dennis Heimbigner, Ed Hartnett + */ + +static int +ncz_sync_netcdf4_file(NC_FILE_INFO_T* file, int isclose) +{ + int stat = NC_NOERR; + + assert(file && file->format_file_info); + + LOG((3, "%s", __func__)); + ZTRACE(2,"file=%s",file->hdr.name); + + /* End depend mode if needed. (Error checking for classic mode has + * already happened). */ + if (file->flags & NC_INDEF) + { + /* Turn define mode off. */ + file->flags ^= NC_INDEF; + + /* Redef mode needs to be tracked separately for nc_abort. */ + file->redef = NC_FALSE; + } + +#ifdef LOGGING + /* This will print out the names, types, lens, etc of the vars and + atts in the file, if the logging level is 2 or greater. */ + log_metadata_nc(file); +#endif + + /* Write any metadata that has changed. */ + if (!file->no_write) + { + /* Write out provenance; will create _NCProperties */ + if((stat = NCZ_write_provenance(file))) + goto done; + + /* Write out meta-data if we are closing as opposed to enddef() */ + if(isclose) + {if((stat = NCZF_writemeta(file))) goto done;} + } +done: + return ZUNTRACE(stat); +} + +/** + * @internal This function will do the enddef stuff for an nczarr file. + * + * @param file Pointer to ZARR file info struct. + * + * @return ::NC_NOERR No error. + * @return ::NC_ENOTINDEFINE Not in define mode. + * @author Dennis Heimbigner, Ed Hartnett + */ +int +ncz_enddef_netcdf4_file(NC_FILE_INFO_T* file) +{ + assert(file); + LOG((3, "%s", __func__)); + + /* If we're not in define mode, return an error. */ + if (!(file->flags & NC_INDEF)) + return NC_ENOTINDEFINE; + + /* Turn define mode off. */ + file->flags ^= NC_INDEF; + + /* Redef mode needs to be tracked separately for nc_abort. */ + file->redef = NC_FALSE; + + return ncz_sync_netcdf4_file(file,!ZCLOSE); +} + +/** + * @internal IN netcdf, you first create + * the variable and then (optionally) specify the fill value. + * + * @param ncid File and group ID. + * @param fillmode File mode. + * @param old_modep Pointer that gets old mode. Ignored if NULL. + * + * @return ::NC_NOERR No error. + * @author Dennis Heimbigner + */ +int +NCZ_set_fill(int ncid, int fillmode, int *old_modep) +{ + NC_FILE_INFO_T* h5 = NULL; + int stat = NC_NOERR; + + ZTRACE(0,"NCZ_set_fill(ncid,fillmode,old)"); + + /* Get pointer to file info. */ + if ((stat = nc4_find_grp_h5(ncid, NULL, &h5))) + goto done; + assert(h5); + + /* Trying to set fill on a read-only file? You sicken me! */ + if (h5->no_write) + {stat = NC_EPERM; goto done;} + + /* Did you pass me some weird fillmode? */ + if (fillmode != NC_FILL && fillmode != NC_NOFILL) + {stat = NC_EINVAL; goto done;} + + /* If the user wants to know, tell him what the old mode was. */ + if (old_modep) + *old_modep = h5->fill_mode; + + h5->fill_mode = fillmode; + +done: + return ZUNTRACE(stat); +} diff --git a/libnczarr/zfilter.c b/libnczarr/zfilter.c index 5b20a54e21..0173cdff5f 100644 --- a/libnczarr/zfilter.c +++ b/libnczarr/zfilter.c @@ -405,7 +405,7 @@ nc_var_filter_remove(int ncid, int varid, unsigned int filterid) int stat; /* Get pointer to the var. */ - if ((stat = ncz_find_grp_file_var(ncid, varid, NULL, NULL, &var))) + if ((stat = ncz_find_file_grp_var(ncid, varid, NULL, NULL, &var))) return stat; assert(var); @@ -438,7 +438,7 @@ NCZ_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, assert(nc); /* Find info for this file and group and var, and set pointer to each. */ - if ((stat = ncz_find_grp_file_var(ncid, varid, &h5, &grp, &var))) + if ((stat = ncz_find_file_grp_var(ncid, varid, &h5, &grp, &var))) {stat = THROW(stat); goto done;} assert(h5 && var && var->hdr.id == varid); @@ -652,7 +652,7 @@ NCZ_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int* ids assert(nc); /* Find info for this file and group and var, and set pointer to each. */ - if ((stat = ncz_find_grp_file_var(ncid, varid, &h5, &grp, &var))) + if ((stat = ncz_find_file_grp_var(ncid, varid, &h5, &grp, &var))) {stat = THROW(stat); goto done;} assert(h5 && var && var->hdr.id == varid); @@ -692,7 +692,7 @@ NCZ_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparamsp, assert(nc); /* Find info for this file and group and var, and set pointer to each. */ - if ((stat = ncz_find_grp_file_var(ncid, varid, &h5, &grp, &var))) + if ((stat = ncz_find_file_grp_var(ncid, varid, &h5, &grp, &var))) {stat = THROW(stat); goto done;} assert(h5 && var && var->hdr.id == varid); diff --git a/libnczarr/zformat2.c b/libnczarr/zformat2.c index cabebab350..1cf4da2dfa 100644 --- a/libnczarr/zformat2.c +++ b/libnczarr/zformat2.c @@ -53,6 +53,7 @@ static int build_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCindex* attlist, static int read_superblock(NC_FILE_INFO_T* file, int* nczarrvp); static int read_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, const char* name, NC_GRP_INFO_T** grpp); +static int read_grp_contents(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp); static int read_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, NClist* varnames); static int read_subgrps(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, NClist* subgrpnames); static int read_attrs(NC_FILE_INFO_T* file, NC_OBJ* container); @@ -344,16 +345,13 @@ write_var_meta(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_VAR NC_DIM_INFO_T* dim = var->dim[i]; shape[i] = dim->len; } - /* but might be scalar */ - if(var->ndims == 0) - shape[0] = 1; /* shape key */ /* Integer list defining the length of each dimension of the array.*/ /* Create the list */ NCJcheck(NCJnew(NCJ_ARRAY,&jtmp)); - if(zvar->scalar) { - NCJaddstring(jtmp,NCJ_INT,"1"); + if(var->ndims == 0) { /* scalar */ + NCJaddstring(jtmp,NCJ_INT,"1"); /* pretend scalar x is x[1] */ } else for(i=0;indims;i++) { snprintf(number,sizeof(number),"%llu",shape[i]); NCJaddstring(jtmp,NCJ_INT,number); @@ -374,21 +372,16 @@ write_var_meta(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_VAR } /* chunks key */ - /* The zarr format does not support the concept - of contiguous (or compact), so it will never appear in the read case. - */ - /* list of chunk sizes */ - NCJcheck(NCJaddstring(jvar,NCJ_STRING,"chunks")); /* Create the list */ NCJcheck(NCJnew(NCJ_ARRAY,&jtmp)); - if(zvar->scalar) { + if(var->ndims == 0) {/* treat scalar like chunks of [1] */ NCJaddstring(jtmp,NCJ_INT,"1"); /* one chunk of size 1 */ } else for(i=0;indims;i++) { size64_t len = var->chunksizes[i]; snprintf(number,sizeof(number),"%lld",len); NCJaddstring(jtmp,NCJ_INT,number); } - NCJcheck(NCJappend(jvar,jtmp)); + NCJcheck(NCJinsert(jvar,"chunks",jtmp)); jtmp = NULL; /* fill_value key */ @@ -495,17 +488,19 @@ write_var_meta(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_VAR NCJinsert(jncvar,"dimrefs",jdimrefs); jdimrefs = NULL; /* Avoid memory problems */ - /* Add the _Storage flag */ /* Record if this is a scalar */ if(var->ndims == 0) { NCJnewstring(NCJ_INT,"1",&jtmp); NCJinsert(jncvar,"scalar",jtmp); jtmp = NULL; } + +#if 0 /* everything looks like it is chunked */ NCJnewstring(NCJ_STRING,"chunked",&jtmp); NCJinsert(jncvar,"storage",jtmp); jtmp = NULL; +#endif if(!purezarr) { NCJinsert(jvar,NCZ_V2_ARRAY,jncvar); @@ -646,7 +641,7 @@ build_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCindex* attlist, NCjson** j else {if((stat = NC4_inq_atomic_type(a->nc_typeid,NULL,&typesize))) goto done;} - /* Track complex json representation*/ + /* Track complex json representation*/ if(internaltype == NC_CHAR && NCZ_iscomplexjsontext((size_t)a->len,(char*)a->data,&jdata)) { internaltype = NC_JSON; /* hack to remember this case */ typesize = 0; @@ -806,7 +801,6 @@ ZF2_readmeta(NC_FILE_INFO_T* file) int purezarr = 0; int nczarr_format = 0; NCZ_FILE_INFO_T* zfile = NULL; - NC_GRP_INFO_T* root = NULL; ZTRACE(3,"file=%s",file->controller->path); @@ -825,8 +819,8 @@ ZF2_readmeta(NC_FILE_INFO_T* file) } /* Now load the groups starting with root */ - if((stat = read_grp(file,NULL,"/",&root))) goto done; - file->root_grp = root; + assert(file->root_grp != NULL); + if((stat = read_grp_contents(file,file->root_grp))) goto done; done: return ZUNTRACE(THROW(stat)); @@ -845,7 +839,7 @@ static int read_attrs(NC_FILE_INFO_T* file, NC_OBJ* container) { int stat = NC_NOERR; - char* fullpath = NULL; + char* objpath = NULL; char* key = NULL; NCjson* jattrs = NULL; const NCjson* jncattr = NULL; @@ -868,18 +862,15 @@ read_attrs(NC_FILE_INFO_T* file, NC_OBJ* container) if(container->sort == NCGRP) { grp = (NC_GRP_INFO_T*)container; /* Construct grp key */ - if((stat = NCZ_grpkey(grp,&fullpath))) goto done; + if((stat = NCZ_grpkey(grp,&objpath))) goto done; } else { var = (NC_VAR_INFO_T*)container; zvar = (NCZ_VAR_INFO_T*)var->format_var_info; /* Construct var key */ - if((stat = NCZ_varkey(var,&fullpath))) goto done; + if((stat = NCZ_varkey(var,&objpath))) goto done; } - /* Construct the path to the .zattrs object */ - if((stat = nczm_concat(fullpath,Z2ATTRS,&key))) - goto done; - + if((stat = nczm_concat(objpath,Z2ATTRS,&key))) goto done; /* Download the .zattrs object: may not exist */ switch ((stat=NCZ_downloadjson(zfile->map,key,&jattrs))) { case NC_NOERR: break; @@ -891,16 +882,17 @@ read_attrs(NC_FILE_INFO_T* file, NC_OBJ* container) if(jattrs != NULL) { if(NCJsort(jattrs) != NCJ_DICT) {stat = THROW(NC_ENCZARR); goto done;} natts = NCJdictlength(jattrs); - /* Get _nczarr_attrs from .zattrs (may be null)*/ - NCJcheck(NCJdictget(jattrs,NCZ_V2_ATTR,&jncattr)); - nullfree(key); key = NULL; - if(jncattr != NULL) { - /* jncattr attribute should be a dict */ - if(NCJsort(jncattr) != NCJ_DICT) {stat = (THROW(NC_ENCZARR)); goto done;} - /* Extract "types; may not exist if purezarr or only hidden attributes are defined */ - NCJcheck(NCJdictget(jncattr,"types",&jtypes)); - } - /* Convert to a vector of nc_types */ + if(!purezarr) { + /* Get _nczarr_attrs from .zattrs (may be null)*/ + NCJcheck(NCJdictget(jattrs,NCZ_V2_ATTR,&jncattr)); + if(jncattr != NULL) { + /* jncattr attribute should be a dict */ + if(NCJsort(jncattr) != NCJ_DICT) {stat = (THROW(NC_ENCZARR)); goto done;} + /* Extract "types; may not exist if purezarr or only hidden attributes are defined */ + NCJcheck(NCJdictget(jncattr,"types",&jtypes)); + } + } + /* Compute the attribute types */ if((stat = jtypes2atypes(purezarr, jattrs, jtypes, &atypes))) goto done; /* Fill in the attributes */ @@ -922,12 +914,15 @@ read_attrs(NC_FILE_INFO_T* file, NC_OBJ* container) if(var != NULL && NCJarraylength(jvalues)==1 && atypes[i] == NC_INT) sscanf(NCJstring(jkey),"%zu",&zvar->maxstrlen); continue; /* _FillValue: ignore */ - } + } else if(memcmp(NCJstring(jkey),NCZ_V2_PREFIX,strlen(NCZ_V2_PREFIX))==0) + continue; /* do not materialize any "_nczarr..." attributes */ /* Convert jvalues to void* data */ typeid = atypes[i]; if((stat = NCZ_computeattrdata(&typeid, jvalues, NULL, &len, &data))) goto done; - /* Create the attribute; will take control of data */ - if((stat = ncz4_create_attr(file,container,NCJstring(jkey),typeid,len,data,NULL))) goto done; + /* Create the attribute */ + stat = ncz4_create_attr(file,container,NCJstring(jkey),typeid,len,data,NULL); + (void)NC_reclaim_data_all(file->controller,typeid,data,len); + if(stat) goto done; data = NULL; } } @@ -939,6 +934,7 @@ read_attrs(NC_FILE_INFO_T* file, NC_OBJ* container) done: NCJreclaim(jattrs); nullfree(atypes); + nullfree(objpath); nullfree(key); return ZUNTRACE(THROW(stat)); } @@ -1016,44 +1012,66 @@ static int read_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, const char* name, NC_GRP_INFO_T** grpp) { int stat = NC_NOERR; - char* parentpath = NULL; + NC_GRP_INFO_T* grp = NULL; + + ZTRACE(3,"file=%s grp=%s",file->controller->path,grp->hdr.name); + + /* Build group */ + if((stat = ncz4_create_grp(file,parent,name,&grp))) goto done; + + /* Fill in the group */ + if((stat = read_grp_contents(file,grp))) goto done; + + if(grpp) {*grpp = grp; grp = NULL;} + +done: + return ZUNTRACE(THROW(stat)); +} + +/** + * @internal Read group contents from map to memory + * + * @param file Pointer to file struct + * @param grp Pointer to grp + * + * @return ::NC_NOERR No error. + * @author Dennis Heimbigner + */ +static int +read_grp_contents(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp) +{ + int stat = NC_NOERR; char* grppath = NULL; char* key = NULL; int purezarr = 0; - NCjson* jdict = NULL; NClist* dimdefs = nclistnew(); NClist* varnames = nclistnew(); NClist* subgrps = nclistnew(); NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; - NC_GRP_INFO_T* grp = NULL; int* dimids = NULL; + NCjson* jgroup = NULL; ZTRACE(3,"file=%s grp=%s",file->controller->path,grp->hdr.name); purezarr = (zfile->flags & FLAG_PUREZARR); /* Construct grp path */ - if((stat = NCZ_grpkey(parent,&parentpath))) goto done; - if((stat = nczm_concat(parentpath,name,&grppath))) goto done; + if((stat = NCZ_grpkey(grp,&grppath))) goto done; if(purezarr) { - if((stat = parse_group_content_pure(file,parent,varnames,subgrps))) goto done; + if((stat = parse_group_content_pure(file,grp,varnames,subgrps))) goto done; } else { /*!purezarr*/ - /* build Z2METAROOT path */ - if((stat = nczm_concat(grppath,Z2METAROOT,&key))) goto done; + /* build Z2GROUP path */ + if((stat = nczm_concat(grppath,Z2GROUP,&key))) goto done; /* Read */ - jdict = NULL; - stat=NCZ_downloadjson(zfile->map,key,&jdict); + stat=NCZ_downloadjson(zfile->map,key,&jgroup); nullfree(key); key = NULL; - if(!jdict) {stat = NC_ENOTZARR; goto done;} + if(!jgroup) {stat = NC_ENOTZARR; goto done;} /* Pull out lists about group content */ - if((stat = parse_group_content(jdict,dimdefs,varnames,subgrps))) goto done; + if((stat = parse_group_content(jgroup,dimdefs,varnames,subgrps))) goto done; } - /* Build group */ - if((stat = ncz4_create_grp(file,parent,name,&grp))) goto done; - - /* Read the attributes */ + /* Read and build the attributes */ if((stat = read_attrs(file,(NC_OBJ*)grp))) goto done; /* Fill in the group recursively */ @@ -1069,13 +1087,13 @@ read_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, const char* name, NC_GRP_I if((stat = read_subgrps(file,grp,subgrps))) goto done; done: - NCJreclaim(jdict); nclistfreeall(dimdefs); nclistfreeall(varnames); nclistfreeall(subgrps); - nullfree(parentpath); + nullfree(dimids); nullfree(grppath); nullfree(key); + NCJreclaim(jgroup); return ZUNTRACE(THROW(stat)); } @@ -1153,17 +1171,18 @@ read_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) const NCjson* jncvar = NULL; const NCjson* jdimrefs = NULL; const NCjson* jvalue = NULL; + char* grppath = NULL; char* varpath = NULL; char* key = NULL; - char* dimnames[NC_MAX_VAR_DIMS]; int suppress = 0; /* Abort processing of this variable */ + size_t isscalar = 0; size_t vtypelen = 0; - size_t zarr_rank = 0; /* Need to watch out for scalars */ + char* dimnames[NC_MAX_VAR_DIMS]; #ifdef ENABLE_NCZARR_FILTERS int varsized = 0; - const NCjson* jfilter = NULL; int chainindex = 0; NClist* codecs = nclistnew(); /* NClist */ + const NCjson* jfilter = NULL; #endif /* Capture arguments for ncz4_create_var */ struct { @@ -1192,11 +1211,13 @@ read_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) #ifdef ENABLE_NCZARR_FILTERS cvargs.filterlist = nclistnew(); #endif + memset(dimnames,0,sizeof(dimnames)); purezarr = (zfile->flags & FLAG_PUREZARR)?1:0; /* Construct var path */ - if((stat = NCZ_varkey(var,&varpath))) goto done; + if((stat = NCZ_grpkey(grp,&grppath))) goto done; + if((stat = nczm_concat(grppath,varname,&varpath))) goto done; /* Construct the path to the zarray object */ if((stat = nczm_concat(varpath,Z2ARRAY,&key))) goto done; /* Download the zarray object */ @@ -1215,6 +1236,28 @@ read_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) {stat = (THROW(NC_ENCZARR)); goto done;} } + /* shape */ + { + NCJcheck(NCJdictget(jvar,"shape",&jvalue)); + if(NCJsort(jvalue) != NCJ_ARRAY) {stat = (THROW(NC_ENCZARR)); goto done;} + cvargs.rank = NCJarraylength(jvalue); + /* extract the shapes */ + if((stat = NCZ_decodesizet64vec(jvalue, cvargs.shape))) goto done; + cvargs.storage = NC_CHUNKED; + /* Note: we detect and adjust for scalar later */ + } + + /* chunks */ + { + NCJcheck(NCJdictget(jvar,"chunks",&jvalue)); + if(jvalue != NULL && NCJsort(jvalue) != NCJ_ARRAY) + {stat = (THROW(NC_ENCZARR)); goto done;} + /* Verify the rank */ + if(cvargs.rank != NCJarraylength(jvalue)) {stat = NC_ENCZARR; goto done;} + /* get chunk sizes */ + if((stat = NCZ_decodesizet64vec(jvalue, cvargs.chunks))) goto done; + } + /* Set the type and endianness of the variable */ { NCJcheck(NCJdictget(jvar,"dtype",&jvalue)); @@ -1230,33 +1273,29 @@ read_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) if(!purezarr) { /* Extract the _NCZARR_ARRAY values */ - /* Do this first so we know about storage esp. scalar */ /* Extract the NCZ_V2_ARRAY dict */ NCJcheck(NCJdictget(jvar,NCZ_V2_ARRAY,&jncvar)); if(!stat && jncvar == NULL) NCJcheck(NCJdictget(jvar,NCZ_V2_ARRAY,&jncvar)); if(jncvar == NULL) {stat = NC_ENCZARR; goto done;} assert((NCJsort(jncvar) == NCJ_DICT)); - /* Extract scalar flag */ - NCJcheck(NCJdictget(jncvar,"scalar",&jvalue)); - if(jvalue != NULL) { - cvargs.storage = NC_CHUNKED; - cvargs.rank = 0; - } - /* Extract storage flag */ - NCJcheck(NCJdictget(jncvar,"storage",&jvalue)); - if(jvalue != NULL) { - cvargs.storage = NC_CHUNKED; - } + /* Extract scalar flag and adjust accordingly */ + { + NCJcheck(NCJdictget(jncvar,"scalar",&jvalue)); + if(jvalue != NULL) { + if(NCJsort(jvalue) != NCJ_INT) {stat = NC_ENCZARR; goto done;} + sscanf(NCJstring(jvalue),"%zu",&isscalar); + } + } /* Extract dimrefs list */ NCJcheck(NCJdictget(jncvar,"dimrefs",&jdimrefs)); if(jdimrefs != NULL) { /* Extract the dimref names */ assert((NCJsort(jdimrefs) == NCJ_ARRAY)); - if(cvargs.rank == 0) { + if(isscalar) { assert(NCJarraylength(jdimrefs) == 0); } else { - cvargs.rank = NCJarraylength(jdimrefs); + if(cvargs.rank != NCJarraylength(jdimrefs)) {stat = NC_ENCZARR; goto done;} for(j=0;jzarr.dimension_separator; /* use global value */ NCJcheck(NCJdictget(jvar,"dimension_separator",&jvalue)); if(jvalue != NULL) { /* Verify its value */ @@ -1281,49 +1320,10 @@ read_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) } /* If value is invalid, then use global default */ if(!islegaldimsep(cvargs.dimension_separator)) - cvargs.dimension_separator = ngs->zarr.dimension_separator; /* use global value */ + cvargs.dimension_separator = DFALT_DIM_SEPARATOR_V2; assert(islegaldimsep(cvargs.dimension_separator)); /* we are hosed */ } - /* shape */ - { - NCJcheck(NCJdictget(jvar,"shape",&jvalue)); - if(NCJsort(jvalue) != NCJ_ARRAY) {stat = (THROW(NC_ENCZARR)); goto done;} - - /* Process the rank */ - zarr_rank = NCJarraylength(jvalue); - if(zarr_rank == 0) { - /* suppress variable */ - ZLOG(NCLOGWARN,"Empty shape for variable %s suppressed",varname); - suppress = 1; - goto suppressvar; - } - - if(cvargs.rank == 0) { - zarr_rank = 1; /* Zarr does not support scalars */ - } else - cvargs.rank = (zarr_rank = NCJarraylength(jvalue)); - - if(zarr_rank > 0) { - /* Save the rank of the variable */ - if((stat = nc4_var_set_ndims(var, (int)cvargs.rank))) goto done; - /* extract the shapes */ - if((stat = NCZ_decodesizet64vec(jvalue, cvargs.shape))) goto done; - } - } - - /* chunks */ - { - NCJcheck(NCJdictget(jvar,"chunks",&jvalue)); - if(jvalue != NULL && NCJsort(jvalue) != NCJ_ARRAY) - {stat = (THROW(NC_ENCZARR)); goto done;} - /* Verify the rank */ - if(cvargs.rank > 0) { - if(zarr_rank == 0) {stat = NC_ENCZARR; goto done;} - if((stat = NCZ_decodesizet64vec(jvalue, cvargs.chunks))) goto done; - } - } - /* Capture row vs column major; currently, column major not used*/ { char* sorder = "C"; @@ -1363,9 +1363,10 @@ read_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) { if((stat = NCZ_filter_initialize())) goto done; NCJcheck(NCJdictget(jvar,"compressor",&jfilter)); - if(NCJsort(jfilter) != NCJ_DICT) {stat = NC_EFILTER; goto done;} - nclistpush(codecs,jfilter); - + if(jfilter != NULL && NCJsort(jfilter) != NCJ_NULL) { + if(NCJsort(jfilter) != NCJ_DICT) {stat = NC_EFILTER; goto done;} + nclistpush(codecs,jfilter); + } } { @@ -1385,7 +1386,7 @@ read_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) #endif /*ENABLE_NCZARR_FILTERS*/ /* Compute the set of dim references for this variable */ - if(zarr_rank > 0) { + if(!isscalar) { if((stat = NCZ_computedimrefs(file, grp, cvargs.rank, cvargs.shape, dimnames, cvargs.dimfqns))) goto done; } @@ -1403,7 +1404,12 @@ read_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) } } -suppressvar: + /* Make final scalar adjustments */ + if(isscalar) { + cvargs.rank = 0; + cvargs.storage = NC_CONTIGUOUS; + } + if(!suppress) { /* Create the variable */ if((stat = ncz4_create_var(file,grp, @@ -1437,11 +1443,14 @@ read_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) /* Clean up cvargs */ nclistfree(cvargs.filterlist); nullfree(cvargs.fill_value); + NCZ_clearstringvec(cvargs.rank,cvargs.dimfqns); /* Clean up */ + NCZ_clearstringvec(cvargs.rank,dimnames); + nclistfreeall(codecs); + nullfree(grppath); nullfree(varpath); nullfree(key); NCJreclaim(jvar); - return THROW(stat); } @@ -1894,7 +1903,7 @@ dtype2nctype(const char* dtype, nc_type* nctypep, size_t* typelenp, int* endianp nctype = NC_JSON; typelen = 1; goto exit; - } else if(strcmp(dtype,"S1")==0) { nctype = NC_CHAR; typelen = 1; goto exit; diff --git a/libnczarr/zformat3.c b/libnczarr/zformat3.c index 5b85830947..ae7da99ca1 100644 --- a/libnczarr/zformat3.c +++ b/libnczarr/zformat3.c @@ -91,6 +91,7 @@ static int build_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCindex* attlist, static int build_superblock(NC_FILE_INFO_T* file, NCjson** jsuperp); static int read_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, const char* name, NC_GRP_INFO_T** grpp); +static int read_grp_contents(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp); static int read_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames); static int read_subgrps(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* subgrpnames); static int verify_superblock(NC_FILE_INFO_T* file, const NCjson* jsuper); @@ -891,9 +892,8 @@ ZF3_readmeta(NC_FILE_INFO_T* file) } /* Now load the groups contents starting with root */ - assert(file->root_grp == NULL); - if((stat = read_grp(file,NULL,"/",&file->root_grp))) - goto done; + assert(file->root_grp != NULL); + if((stat = read_grp_contents(file,file->root_grp))) goto done; done: NCJreclaim(jrootgrp); @@ -983,7 +983,8 @@ read_attrs(NC_FILE_INFO_T* file, NC_OBJ* container, const NCjson* jatts, const N if(var != NULL && NCJarraylength(jvalues)==1 && atypes[i] == NC_INT) sscanf(NCJstring(jkey),"%zu",&zvar->maxstrlen); continue; /* _FillValue: ignore */ - } + } else if(memcmp(NCJstring(jkey),NCZ_V3_PREFIX,strlen(NCZ_V3_PREFIX))==0) + continue; /* do not materialize any "_nczarr..." attributes */ /* Convert jvalues to void* data */ typeid = atypes[i]; if((stat = NCZ_computeattrdata(&typeid, jvalues, NULL, &len, &data))) goto done; @@ -1094,71 +1095,94 @@ verify_superblock(NC_FILE_INFO_T* file, const NCjson* jsuper) * @internal Read group data from map to memory * * @param file Pointer to file struct - * @param grp Pointer to grp struct + * @param parent Pointer to parent group + * @param name of the group to create + * @param grpp created group * * @return ::NC_NOERR No error. * @author Dennis Heimbigner */ static int read_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, const char* name, NC_GRP_INFO_T** grpp) +{ + int stat = NC_NOERR; + NC_GRP_INFO_T* grp = NULL; + + ZTRACE(3,"file=%s grp=%s",file->controller->path,grp->hdr.name); + + /* Build group */ + if((stat = ncz4_create_grp(file,parent,name,&grp))) goto done; + + /* Fill in the group recursively */ + if((stat = read_grp_contents(file,grp))) goto done; + + if(grpp) {*grpp = grp; grp = NULL;} + +done: + return ZUNTRACE(THROW(stat)); +} + +/** + * @internal fill in group content from map to memory + * + * @param file Pointer to file struct + * @param grp Pointer to grp struct + * + * @return ::NC_NOERR No error. + * @author Dennis Heimbigner + */ +static int +read_grp_contents(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp) { int stat = NC_NOERR; NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; NCZMAP* map = zfile->map; - char* fullpath = NULL; + char* grppath = NULL; char* key = NULL; int purezarr = 0; - NCjson* jgroup = NULL; NClist* subvars = nclistnew(); NClist* subgrps = nclistnew(); + NCjson* jgroup = NULL; const NCjson* jatts = NULL; - const NCjson* jtypes = NULL; const NCjson* jnczgrp = NULL; const NCjson* jdims = NULL; - NC_GRP_INFO_T* grp = NULL; ZTRACE(3,"file=%s grp=%s",file->controller->path,grp->hdr.name); purezarr = (zfile->flags & FLAG_PUREZARR); - /* Construct grp path */ - if((stat = NCZ_grpkey(grp,&fullpath))) - goto done; - /* Download the grp meta-data; might not exist for virtual groups */ /* build Z3GROUP path */ - if((stat = nczm_concat(fullpath,Z3GROUP,&key))) goto done; + /* Construct grp path */ + if((stat = NCZ_grpkey(grp,&grppath))) goto done; + /* and the grp's zarr.json */ + if((stat = nczm_concat(grppath,Z3GROUP,&key))) goto done; - /* Read */ + /* Read zarr.json */ stat=NCZ_downloadjson(map,key,&jgroup); nullfree(key); key = NULL; if(stat) goto done; - /* Build group */ - if((stat = ncz4_create_grp(file,parent,name,&grp))) goto done; - if(purezarr || jgroup == NULL) { /* Apparently this is a virtual group; treat like purezarr to find the vars and groups */ if((stat = subobjects_pure(zfile,grp,subvars,subgrps))) goto done; } else { + /* Get attributes */ + NCJcheck(NCJdictget(jgroup,"attributes",&jatts)); /* Get _nczarr_group */ NCJcheck(NCJdictget(jatts,NCZ_V3_GROUP,&jnczgrp)); - if(jnczgrp != NULL) { - /* Define dimensions */ - NCJcheck(NCJdictget(jnczgrp,"dimensions",&jdims)); - if(jdims != NULL) { - if((stat = parse_dims(file,grp,jdims))) goto done; - } - /* Get lists of subvars and subgrps */ - if((stat = subobjects(zfile,grp,jnczgrp,subvars,subgrps))) goto done; - + if(jnczgrp == NULL) {stat = NC_ENCZARR; goto done;} + + /* Define dimensions */ + NCJcheck(NCJdictget(jnczgrp,"dimensions",&jdims)); + if(jdims != NULL) { + if((stat = parse_dims(file,grp,jdims))) goto done; } + /* Get lists of subvars and subgrps */ + if((stat = subobjects(zfile,grp,jnczgrp,subvars,subgrps))) goto done; } - /* Read the attributes */ - if((stat = read_attrs(file,(NC_OBJ*)grp,jatts,jtypes))) goto done; - /* Fill in the group recursively */ /* Define vars */ @@ -1171,7 +1195,7 @@ read_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, const char* name, NC_GRP_I NCJreclaim(jgroup); nclistfreeall(subvars); nclistfreeall(subgrps); - nullfree(fullpath); + nullfree(grppath); nullfree(key); return ZUNTRACE(THROW(stat)); } @@ -1329,40 +1353,56 @@ read_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) const NCjson* jcodecs = NULL; const NCjson* jcodec = NULL; const NCjson* jhint = NULL; + char* grppath = NULL; char* varpath = NULL; char* key = NULL; int suppress = 0; /* Abort processing of this variable */ - nc_type vtype = NC_NAT; size_t vtypelen = 0; - size_t rank = 0; - size64_t shape[NC_MAX_VAR_DIMS]; - size64_t chunks[NC_MAX_VAR_DIMS]; char* dimnames[NC_MAX_VAR_DIMS]; - char* dimfqns[NC_MAX_VAR_DIMS]; - int endianness = NC_ENDIAN_NATIVE; - size_t maxstrlen = 0; - int scalar = 0; - int storage = NC_CHUNKED; - char dimension_separator = '\0'; NC_TYPE_INFO_T* type_info = NULL; - int no_fill = 0; - void* fillvalue = NULL; - int order = 'C'; #ifdef ENABLE_NCZARR_FILTERS int varsized = 0; - NClist* filterchain = nclistnew(); /*NClist*/ int chainindex = 0; #endif + /* Capture arguments for ncz4_create_var */ + struct { + const char* varname; + nc_type vtype; + int endianness; + size_t maxstrlen; + int storage; + char dimension_separator; + char order; + size_t rank; + size64_t shape[NC_MAX_VAR_DIMS]; + size64_t chunks[NC_MAX_VAR_DIMS]; + char* dimfqns[NC_MAX_VAR_DIMS]; + NClist* filterlist; + int no_fill; + void* fill_value; + } cvargs; + + /* initialize cvargs defaults */ + memset(&cvargs,0,sizeof(cvargs)); + cvargs.varname = varname; + cvargs.endianness = (NC_isLittleEndian()?NC_ENDIAN_LITTLE:NC_ENDIAN_BIG); + cvargs.storage = NC_CHUNKED; + cvargs.no_fill = 1; +#ifdef ENABLE_NCZARR_FILTERS + cvargs.filterlist = nclistnew(); +#endif + memset(dimnames,0,sizeof(dimnames)); if(zfile->flags & FLAG_PUREZARR) purezarr = 1; - endianness = (NC_isLittleEndian()?NC_ENDIAN_LITTLE:NC_ENDIAN_BIG); + cvargs.endianness = (NC_isLittleEndian()?NC_ENDIAN_LITTLE:NC_ENDIAN_BIG); - /* Construct var path */ - if((stat = NCZ_varkey(var,&varpath))) goto done; - + /* Construct parent path */ + if((stat = NCZ_grpkey(grp,&grppath))) goto done; + /* Construct the path to the var */ + if((stat = nczm_concat(grppath,varname,&varpath))) goto done; /* Construct the path to the zarray object */ - if((stat = nczm_concat(varpath,Z2ARRAY,&key))) goto done; + if((stat = nczm_concat(varpath,Z3ARRAY,&key))) goto done; /* Download the zarray object */ if((stat=NCZ_readdict(zfile->map,key,&jvar))) goto done; @@ -1386,22 +1426,14 @@ read_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) { NCJcheck(NCJdictget(jvar,"shape",&jvalue)); if(NCJsort(jvalue) != NCJ_ARRAY) {stat = (THROW(NC_ENCZARR)); goto done;} + cvargs.rank = NCJarraylength(jvalue); - if(NCJarraylength(jvalue) == 0) { - scalar = 1; - rank = 0; - } else { - scalar = 0; - rank = NCJarraylength(jvalue); - } - - if(rank > 0) { + if(cvargs.rank > 0) { /* extract the shape */ - if((stat = NCZ_decodesizet64vec(jvalue, shape))) goto done; + if((stat = NCZ_decodesizet64vec(jvalue, cvargs.shape))) goto done; } /* Set storage flag */ - storage = (scalar?NC_CONTIGUOUS:NC_CHUNKED); - + cvargs.storage = (cvargs.rank==0?NC_CONTIGUOUS:NC_CHUNKED); } /* Get dimension_names */ @@ -1410,8 +1442,7 @@ read_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) /* get the "dimension_names" */ NCJcheck(NCJdictget(jvar,"dimension_names",&jdimnames)); if(jdimnames != NULL) { - rank = NCJarraylength(jdimnames); - for(i=0;i NC_NAT && vtype <= NC_MAX_ATOMIC_TYPE) { /* Disallows NC_JSON */ + if((stat = dtype2nctype(NCJstring(jvalue),&cvargs.vtype,&vtypelen,NULL))) goto done; + if(cvargs.vtype > NC_NAT && cvargs.vtype <= NC_MAX_ATOMIC_TYPE) { /* Disallows NC_JSON */ /* Locate the NC_TYPE_INFO_T object */ - if((stat = ncz_gettype(file,grp,vtype,&type_info))) goto done; + if((stat = ncz_gettype(file,grp,cvargs.vtype,&type_info))) goto done; } else {stat = NC_EBADTYPE; goto done;} - if(vtype == NC_STRING) { - maxstrlen = vtypelen; + if(cvargs.vtype == NC_STRING) { + cvargs.maxstrlen = vtypelen; vtypelen = sizeof(char*); /* in-memory len */ - if(maxstrlen <= 0) maxstrlen = NCZ_get_maxstrlen((NC_OBJ*)var); + if(cvargs.maxstrlen <= 0) cvargs.maxstrlen = NCZ_get_maxstrlen((NC_OBJ*)var); } } @@ -1475,12 +1505,12 @@ read_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) { NCglobalstate* ngs = NC_getglobalstate(); assert(ngs != NULL); - dimension_separator = '\0'; + cvargs.dimension_separator = '\0'; NCJcheck(NCJdictget(jvar,"chunk_key_encoding",&jvalue)); if(jvalue == NULL) { /* If value is invalid, then use global default */ - if(!islegaldimsep(dimension_separator)) - dimension_separator = ngs->zarr.dimension_separator; /* use global value */ + if(!islegaldimsep(cvargs.dimension_separator)) + cvargs.dimension_separator = ngs->zarr.dimension_separator; /* use global value */ /* Verify its value */ } else { if(NCJsort(jvalue) != NCJ_DICT) {stat = NC_ENOTZARR; goto done;} @@ -1491,27 +1521,27 @@ read_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) NCJcheck(NCJdictget(jtmp,"separator",&jsep)); if(jsep != NULL) { if(NCJisatomic(jsep) && NCJstring(jsep) != NULL && strlen(NCJstring(jsep)) == 1) - dimension_separator = NCJstring(jsep)[0]; + cvargs.dimension_separator = NCJstring(jsep)[0]; } else - dimension_separator = '/'; + cvargs.dimension_separator = DFALT_DIM_SEPARATOR_V3; } else - dimension_separator = '/'; + cvargs.dimension_separator = DFALT_DIM_SEPARATOR_V3; } else if(strcasecmp("v2",NCJstring(jtmp))==0) { NCJcheck(NCJdictget(jvalue,"separator",&jsep)); if(jsep != NULL) { if(NCJsort(jsep) == NCJ_STRING && NCJstring(jsep) != NULL && strlen(NCJstring(jsep)) == 1) - dimension_separator = NCJstring(jsep)[0]; + cvargs.dimension_separator = NCJstring(jsep)[0]; else {stat = NC_ENOTZARR; goto done;} } else - dimension_separator = '.'; + cvargs.dimension_separator = '.'; } else {stat = NC_ENOTZARR; goto done;} } - assert(islegaldimsep(dimension_separator)); /* we are hosed */ + assert(islegaldimsep(cvargs.dimension_separator)); /* we are hosed */ } /* chunks */ { - if(!scalar) { + if(cvargs.rank > 0) { NCJcheck(NCJdictget(jvar,"chunk_grid",&jvalue)); if(jvalue == NULL) {stat = (THROW(NC_ENOTBUILT)); goto done;} NCJcheck(NCJdictget(jvalue,"name",&jtmp)); @@ -1523,12 +1553,10 @@ read_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) if(jtmp2 == NULL) {stat = (THROW(NC_ENOTZARR)); goto done;} if(NCJsort(jtmp2) != NCJ_ARRAY) {stat = (THROW(NC_ENOTZARR)); goto done;} - /* Verify the rank */ - assert(rank != 0); - if(rank != NCJarraylength(jtmp2)) {stat = (THROW(NC_ENCZARR)); goto done;} + if(cvargs.rank != NCJarraylength(jtmp2)) {stat = (THROW(NC_ENCZARR)); goto done;} /* Get the chunks and chunkproduct */ - if((stat = NCZ_decodesizet64vec(jtmp2, chunks))) goto done; + if((stat = NCZ_decodesizet64vec(jtmp2, cvargs.chunks))) goto done; } } @@ -1548,9 +1576,9 @@ read_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) if(jvalue == NULL || NCJsort(jvalue) != NCJ_DICT) {stat = NC_ENCZARR; goto done;} NCJcheck(NCJdictget(jvalue,"endian",&jtmp)); if(strcasecmp("big",NCJstring(jtmp))==0) - endianness = NC_ENDIAN_BIG; + cvargs.endianness = NC_ENDIAN_BIG; else if(strcasecmp("little",NCJstring(jtmp))==0) - endianness = NC_ENDIAN_LITTLE; + cvargs.endianness = NC_ENDIAN_LITTLE; else {stat = NC_EINVAL; goto done;} #ifdef ENABLE_NCZARR_FILTERS @@ -1567,51 +1595,51 @@ read_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) jcodec = NCJith(jcodecs,k); if(jcodec == NULL) break; /* done */ if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EFILTER; goto done;} - if((stat = json2filter(file,jcodec,chainindex++,filterchain,NULL))) goto done; + if((stat = json2filter(file,jcodec,chainindex++,cvargs.filterlist,NULL))) goto done; } /* Suppress variable if there are filters and var is not fixed-size */ - if(varsized && nclistlength((NClist*)filterchain) > 0) suppress = 1; + if(varsized && nclistlength(cvargs.filterlist) > 0) suppress = 1; #endif } -#ifdef ENABLE_NCZARR_FILTERS - if(!suppress) { - /* At this point, we can finalize the filters */ - if((stat = NCZ_filter_setup(var))) goto done; - } -#endif - - /* fill in var dimids corresponding to the dim references; create dimensions as necessary */ - if((stat = NCZ_computedimrefs(file,grp,rank,shape,dimnames,dimfqns))) goto done; - /* fill_value; must precede calls to adjust cache */ { NCJcheck(NCJdictget(jvar,"fill_value",&jvalue)); if(jvalue == NULL || NCJsort(jvalue) == NCJ_NULL) - no_fill = 1; + cvargs.no_fill = 1; else { size_t fvlen; - nc_type atypeid = vtype; - if((stat = NCZ_computeattrdata(&atypeid, jvalue, NULL, &fvlen, &fillvalue))) goto done; - assert(atypeid == vtype); - no_fill = 0; + nc_type atypeid = cvargs.vtype; + if((stat = NCZ_computeattrdata(&atypeid, jvalue, NULL, &fvlen, &cvargs.fill_value))) goto done; + assert(atypeid == cvargs.vtype); + cvargs.no_fill = 0; } } if(!suppress) { /* Create the variable */ - if((stat = ncz4_create_var(file,grp,varname,vtype, - endianness, - maxstrlen, - storage, - dimension_separator, - order, - rank,shape,chunks,dimfqns, - filterchain, - no_fill, - fillvalue, + if((stat = ncz4_create_var(file,grp, + cvargs.varname, + cvargs.vtype, + cvargs.endianness, + cvargs.maxstrlen, + cvargs.storage, + cvargs.dimension_separator, + cvargs.order, + cvargs.rank, + cvargs.shape, + cvargs.chunks, + cvargs.dimfqns, + cvargs.filterlist, + cvargs.no_fill, + cvargs.fill_value, &var))) goto done; +#ifdef ENABLE_NCZARR_FILTERS + /* At this point, we can finalize the filters */ + if((stat = NCZ_filter_setup(var))) goto done; +#endif + if((stat = NCZ_adjust_var_cache(var))) goto done; /* Now that we have the variable, create its attributes */ @@ -1633,7 +1661,12 @@ read_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) } done: + /* Clean up cvargs */ + nclistfree(cvargs.filterlist); + nullfree(cvargs.fill_value); + NCZ_clearstringvec(cvargs.rank,cvargs.dimfqns); /* Clean up */ + nullfree(grppath); nullfree(varpath); nullfree(key); NCJreclaim(jvar); diff --git a/libnczarr/zinternal.c b/libnczarr/zinternal.c index 53b405195a..5331535052 100644 --- a/libnczarr/zinternal.c +++ b/libnczarr/zinternal.c @@ -349,9 +349,10 @@ close_vars(NC_GRP_INFO_T *grp) if (var->type_info) { int stat = NC_NOERR; - if((stat = NC_reclaim_data(grp->nc4_info,var->type_info->hdr.id,var->fill_value,1))) - return stat; - nullfree(var->fill_value); + if((stat = NCZ_free_fillvalue(var))) return stat; +// if((stat = NC_reclaim_data(grp->nc4_info,var->type_info->hdr.id,var->fill_value,1))) return stat; +// nullfree(var->fill_value); + var->fill_value = NULL; } } } @@ -532,7 +533,7 @@ ncz_rec_grp_NCZ_del(NC_GRP_INFO_T *grp) * @author Dennis Heimbigner, Ed Hartnett */ int -ncz_find_grp_file_var(int ncid, int varid, NC_FILE_INFO_T **h5, +ncz_find_file_grp_var(int ncid, int varid, NC_FILE_INFO_T **h5, NC_GRP_INFO_T **grp, NC_VAR_INFO_T **var) { NC_FILE_INFO_T *my_h5; @@ -663,7 +664,9 @@ NCZ_ensure_fill_value(NC_VAR_INFO_T *var) return NC_NOERR; /* If the user has set a fill_value for this var, use, otherwise find the default fill value. */ - if((stat = NCZ_set_fill_value(var->container->nc4_info,var,var->no_fill,var->fill_value))) goto done; + if(var->fill_value == NULL) { + if((stat = NCZ_set_fill_value(var->container->nc4_info,var,var->no_fill,var->fill_value))) goto done; + } assert(var->fill_value != NULL); LOG((4, "Found a fill value for var %s", var->hdr.name)); diff --git a/libnczarr/zinternal.h b/libnczarr/zinternal.h index f0436a4f37..35d049e81a 100644 --- a/libnczarr/zinternal.h +++ b/libnczarr/zinternal.h @@ -153,12 +153,14 @@ Optionally Inserted into any group zarr.json or array zarr.json is the extra att */ +#define NCZ_V2_PREFIX "_nczarr" #define NCZ_V2_SUPERBLOCK "_nczarr_superblock" /* Must match values in include/nc4internal.h */ #define NCZ_V2_GROUP "_nczarr_group" #define NCZ_V2_ARRAY "_nczarr_array" #define NCZ_V2_ATTR "_nczarr_attrs" +#define NCZ_V3_PREFIX NCZ_V2_PREFIX #define NCZ_V3_SUPERBLOCK "_nczarr_superblock" /* Must match values in include/nc4internal.h */ #define NCZ_V3_GROUP "_nczarr_group" @@ -365,7 +367,7 @@ int NCZ_write_var_data(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var); /* Undefined */ /* Find var, doing lazy var metadata read if needed. */ -int ncz_find_grp_file_var(int ncid, int varid, NC_FILE_INFO_T** file, +int ncz_find_file_grp_var(int ncid, int varid, NC_FILE_INFO_T** file, NC_GRP_INFO_T** grp, NC_VAR_INFO_T** var); #endif /* ZINTERNAL_H */ diff --git a/libnczarr/zutil.c b/libnczarr/zutil.c index 83de266b38..6500f23722 100644 --- a/libnczarr/zutil.c +++ b/libnczarr/zutil.c @@ -787,7 +787,7 @@ NCZ_clonestringvec(size_t len, const char** vec) } void -NCZ_freestringvec(size_t len, char** vec) +NCZ_clearstringvec(size_t len, char** vec) { size_t i; if(vec == NULL) return; @@ -798,6 +798,12 @@ NCZ_freestringvec(size_t len, char** vec) for(i=0;i=0;i--) { ncbytescat(fqn,"/"); ncbytescat(fqn,nclistget(segments,i)); } +#endif done: nclistfreeall(segments); @@ -1399,7 +1412,7 @@ NCZ_computedimrefs(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, if(purezarr) { assert(dimfqns != NULL); for(i=0;ihdr.name)); - /* (over-) write the NC_VAR_INFO_T.fill_value */ + /* Reclaim existing fill value */ + if(var->fill_value) { + if((stat = NCZ_reclaim_fill_value(var))) goto done; + } + assert(var->fill_value == NULL); + + /* set the NC_VAR_INFO_T.fill_value */ if((stat = NCZ_set_fill_value(h5,var,*no_fill,fill_value))) goto done; /* synchronize to Attribute */ diff --git a/libnczarr/zxcache.c b/libnczarr/zxcache.c index eb23b02d0c..d416d1775e 100644 --- a/libnczarr/zxcache.c +++ b/libnczarr/zxcache.c @@ -541,7 +541,7 @@ NCZ_ensure_fill_chunk(NCZChunkCache* cache) default: { unsigned char* p; for(p=cache->fillchunk,i=0;ichunksize;i+=typesize,p+=typesize) - memcpy(p,var->fill_value,typesize); + memcpy(p,var->fill_value,typesize); /* Warning: only works for non-pointer values */ } break; } done: diff --git a/libsrc4/nc4internal.c b/libsrc4/nc4internal.c index 357279ce17..be2ae48222 100644 --- a/libsrc4/nc4internal.c +++ b/libsrc4/nc4internal.c @@ -42,7 +42,7 @@ static NC_reservedatt NC_reserved[] = { {NC_ATT_DIMENSION_LIST, READONLYFLAG|HIDDENATTRFLAG}, /*DIMENSION_LIST*/ {NC_ATT_NAME, READONLYFLAG|HIDDENATTRFLAG}, /*NAME*/ {NC_ATT_REFERENCE_LIST, READONLYFLAG|HIDDENATTRFLAG}, /*REFERENCE_LIST*/ - {NC_XARRAY_DIMS, READONLYFLAG|NAMEONLYFLAG|HIDDENATTRFLAG}, /*_ARRAY_DIMENSIONS*/ + {NC_XARRAY_DIMS, READONLYFLAG|HIDDENATTRFLAG}, /*_ARRAY_DIMENSIONS*/ {NC_ATT_CODECS, VARFLAG|READONLYFLAG|NAMEONLYFLAG}, /*_Codecs*/ {NC_ATT_FORMAT, READONLYFLAG}, /*_Format*/ {ISNETCDF4ATT, READONLYFLAG|NAMEONLYFLAG}, /*_IsNetcdf4*/ diff --git a/nczarr_test/ref_byte_fill_value_null.cdl b/nczarr_test/ref_byte_fill_value_null.cdl index 6afd2ef373..93bcad2988 100644 --- a/nczarr_test/ref_byte_fill_value_null.cdl +++ b/nczarr_test/ref_byte_fill_value_null.cdl @@ -1,8 +1,8 @@ netcdf ref_byte_fill_value_null { dimensions: - _zdim_20 = 20 ; + _Anonymous_Dim_20 = 20 ; variables: - ubyte byt(_zdim_20, _zdim_20) ; + ubyte byt(_Anonymous_Dim_20, _Anonymous_Dim_20) ; byt:_Storage = "chunked" ; byt:_ChunkSizes = 20, 20 ; byt:_NoFill = "true" ; diff --git a/plugins/NCZmisc.c b/plugins/NCZmisc.c index 9b8dd5c363..8ce72e2311 100644 --- a/plugins/NCZmisc.c +++ b/plugins/NCZmisc.c @@ -142,7 +142,7 @@ NCZ_misc_codec_to_hdf5(const NCproplist* env, const char* codec_json, int* idp, /* verify the dict size */ if(isv3) dictpairs = (14); else dictpairs = (14+1); if(NCJdictlength(jparams) != dictpairs) { - fprintf(stderr,"(1) Incorrect no. of codec parameters: need=%d sent=%d\n",(int)dictpairs,NCJdictlength(jparams)); + fprintf(stderr,"(1) Incorrect no. of codec parameters: need=%d sent=%zu\n",(int)dictpairs,NCJdictlength(jparams)); fprintf(stderr,"jparams=%s\n",NCJtotext(jparams,0)); stat = NC_EINVAL; goto done;