diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 8d5eca680a..8ec6c93b7f 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,10 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.3 - TBD +* Improve the speed and data quantity for DAP4 queries. See [Github #2765](https://github.com/Unidata/netcdf-c/pull/2765). +* Remove the use of execinfo to programmatically dump the stack; it never worked. See [Github #2789](https://github.com/Unidata/netcdf-c/pull/2789). +* Update the internal copy of tinyxml2 to latest code. See [Github #2771](https://github.com/Unidata/netcdf-c/pull/2771). +* Mitigate the problem of remote/nczarr-related test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). * Fix DAP2 proxy problems. See [Github #2764](https://github.com/Unidata/netcdf-c/pull/2764). * Cleanup a number of misc issues. See [Github #2763](https://github.com/Unidata/netcdf-c/pull/2763). * Mitigate the problem of test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). diff --git a/config.h.cmake.in b/config.h.cmake.in index 274ec63328..de1bc2f281 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -468,9 +468,6 @@ with zip */ /* if true, HDF5 is at least version 1.10.5 and supports UTF8 paths */ #cmakedefine HDF5_UTF8_PATHS 1 -/* if true, backtrace support will be used. */ -#cmakedefine HAVE_EXECINFO_H 1 - /* if true, include JNA bug fix */ #cmakedefine JNA 1 diff --git a/configure.ac b/configure.ac index a357b3e4dd..d5e00a9d5f 100644 --- a/configure.ac +++ b/configure.ac @@ -872,129 +872,136 @@ AC_MSG_RESULT([${have_sz}]) ########## -# See if we have libzip for NCZarr -AC_SEARCH_LIBS([zip_open],[zip zip.dll cygzip.dll],[have_zip=yes],[have_zip=no]) -AC_MSG_CHECKING([whether libzip library is available]) -AC_MSG_RESULT([${have_zip}]) -enable_nczarr_zip=${have_zip} # alias +## +# Check to see if we're using NCZarr. If not, we don't need to check for dependencies and such. +## if test "x$enable_nczarr" = xno ; then enable_nczarr_zip=no -fi +else -AC_MSG_CHECKING([whether nczarr zip support is enabled]) -AC_MSG_RESULT([${enable_nczarr_zip}]) + # See if we have libzip for NCZarr + AC_SEARCH_LIBS([zip_open],[zip zip.dll cygzip.dll],[have_zip=yes],[have_zip=no]) + AC_MSG_CHECKING([whether libzip library is available]) + AC_MSG_RESULT([${have_zip}]) + enable_nczarr_zip=${have_zip} # alias -if test "x$enable_nczarr_zip" = xyes ; then -AC_DEFINE([ENABLE_NCZARR_ZIP], [1], [If true, then libzip found]) -fi -# Check for enabling of S3 support -AC_MSG_CHECKING([whether netcdf S3 support should be enabled]) -AC_ARG_ENABLE([s3], - [AS_HELP_STRING([--enable-s3], - [enable netcdf S3 support])]) -test "x$enable_s3" = xyes || enable_s3=no -AC_MSG_RESULT($enable_s3) -if test "x$enable_remote_functionality" = xno ; then - AC_MSG_WARN([--disable-remote-functionality => --disable-s3]) - enable_s3=no -fi - -# --enable-nczarr-s3 is synonym for --enable-s3 (but...) -AC_MSG_CHECKING([whether netcdf NCZarr S3 support should be enabled]) -AC_ARG_ENABLE([nczarr-s3], - [AS_HELP_STRING([--enable-nczarr-s3], - [(Deprecated) enable netcdf NCZarr S3 support; Deprecated in favor of --enable-s3])]) -AC_MSG_RESULT([$enable_nczarr_s3 (Deprecated) Please use --enable-s3)]) - -# Set enable_s3 instead of enable_nczarr_s3 -if test "x$enable_s3" = xno && test "x$enable_nczarr_s3" = xyes && test "x$enable_remote_functionality" = xyes; then -enable_s3=yes # back compatibility -fi -unset enable_nczarr_s3 - -# Note we check for the library after checking for enable_s3 -# because for some reason this fails if we unconditionally test for sdk -# and it is not available. Fix someday -S3LIBS="" -if test "x$enable_s3" = xyes ; then -# See if we have the s3 aws library -# Check for the AWS S3 SDK library - AC_LANG_PUSH([C++]) - AC_CHECK_LIB([aws-c-common], [aws_string_destroy], [enable_s3_aws=yes],[enable_s3_aws=no]) - if test "x$enable_s3_aws" = "xyes" ; then - S3LIBS="-laws-cpp-sdk-core -laws-cpp-sdk-s3" + AC_MSG_CHECKING([whether nczarr zip support is enabled]) + AC_MSG_RESULT([${enable_nczarr_zip}]) + + if test "x$enable_nczarr_zip" = xyes ; then + AC_DEFINE([ENABLE_NCZARR_ZIP], [1], [If true, then libzip found]) fi - AC_LANG_POP -else - enable_s3_aws=no -fi -AC_MSG_CHECKING([whether AWS S3 SDK library is available]) -AC_MSG_RESULT([$enable_s3_aws]) + # Check for enabling of S3 support + AC_MSG_CHECKING([whether netcdf S3 support should be enabled]) + AC_ARG_ENABLE([s3], + [AS_HELP_STRING([--enable-s3], + [enable netcdf S3 support])]) + test "x$enable_s3" = xyes || enable_s3=no + AC_MSG_RESULT($enable_s3) + + if test "x$enable_remote_functionality" = xno ; then + AC_MSG_WARN([--disable-remote-functionality => --disable-s3]) + enable_s3=no + fi -# Check for enabling forced use of Internal S3 library -AC_MSG_CHECKING([whether internal S3 support should be used]) -AC_ARG_ENABLE([s3-internal], - [AS_HELP_STRING([--enable-s3-internal], - [enable internal S3 support])]) -test "x$enable_s3_internal" = xyes || enable_s3_internal=no -AC_MSG_RESULT($enable_s3_internal) + # --enable-nczarr-s3 is synonym for --enable-s3 (but...) + AC_MSG_CHECKING([whether netcdf NCZarr S3 support should be enabled]) + AC_ARG_ENABLE([nczarr-s3], + [AS_HELP_STRING([--enable-nczarr-s3], + [(Deprecated) enable netcdf NCZarr S3 support; Deprecated in favor of --enable-s3])]) + AC_MSG_RESULT([$enable_nczarr_s3 (Deprecated) Please use --enable-s3)]) -if test "x$enable_s3_aws" = xno && test "x$enable_s3_internal" = xno ; then -AC_MSG_WARN([No S3 library available => S3 support disabled]) -enable_s3=no -fi + # Set enable_s3 instead of enable_nczarr_s3 + if test "x$enable_s3" = xno && test "x$enable_nczarr_s3" = xyes && test "x$enable_remote_functionality" = xyes; then + enable_s3=yes # back compatibility + fi + unset enable_nczarr_s3 + + # Note we check for the library after checking for enable_s3 + # because for some reason this fails if we unconditionally test for sdk + # and it is not available. Fix someday + S3LIBS="" + if test "x$enable_s3" = xyes ; then + # See if we have the s3 aws library + # Check for the AWS S3 SDK library + AC_LANG_PUSH([C++]) + AC_CHECK_LIB([aws-c-common], [aws_string_destroy], [enable_s3_aws=yes],[enable_s3_aws=no]) + if test "x$enable_s3_aws" = "xyes" ; then + S3LIBS="-laws-cpp-sdk-core -laws-cpp-sdk-s3" + fi + AC_LANG_POP + else + enable_s3_aws=no + fi -if test "x$enable_s3_aws" = xyes && test "x$enable_s3_internal" = xyes ; then -AC_MSG_WARN([Both aws-sdk-cpp and s3-internal enabled => use s3-internal.]) -enable_s3_aws=no -fi + AC_MSG_CHECKING([whether AWS S3 SDK library is available]) + AC_MSG_RESULT([$enable_s3_aws]) -if test "x$enable_s3_internal" = xyes ; then -if test "x$ISOSX" != xyes && test "x$ISMINGW" != xyes && test "x$ISMSVC" != xyes ; then -# Find crypto libraries if using ssl -AC_CHECK_LIB([ssl],[ssl_create_cipher_list]) -AC_CHECK_LIB([crypto],[SHA256]) -fi -fi + # Check for enabling forced use of Internal S3 library + AC_MSG_CHECKING([whether internal S3 support should be used]) + AC_ARG_ENABLE([s3-internal], + [AS_HELP_STRING([--enable-s3-internal], + [enable internal S3 support])]) + test "x$enable_s3_internal" = xyes || enable_s3_internal=no + AC_MSG_RESULT($enable_s3_internal) + + if test "x$enable_s3_aws" = xno && test "x$enable_s3_internal" = xno ; then + AC_MSG_WARN([No S3 library available => S3 support disabled]) + enable_s3=no + fi -# Check for enabling S3 testing -AC_MSG_CHECKING([what level of netcdf S3 testing should be enabled]) -AC_ARG_WITH([s3-testing], - [AS_HELP_STRING([--with-s3-testing=yes|no|public], - [control netcdf S3 testing])], - [], [with_s3_testing=public]) -AC_MSG_RESULT($with_s3_testing) + if test "x$enable_s3_aws" = xyes && test "x$enable_s3_internal" = xyes ; then + AC_MSG_WARN([Both aws-sdk-cpp and s3-internal enabled => use s3-internal.]) + enable_s3_aws=no + fi -# Disable S3 tests if S3 support is disabled -if test "x$enable_s3" = xno ; then - if test "x$with_s3_testing" != xno ; then - AC_MSG_WARN([S3 support is disabled => no testing]) - with_s3_testing=no - fi -fi + if test "x$enable_s3_internal" = xyes ; then + if test "x$ISOSX" != xyes && test "x$ISMINGW" != xyes && test "x$ISMSVC" != xyes ; then + # Find crypto libraries if using ssl + AC_CHECK_LIB([ssl],[ssl_create_cipher_list]) + AC_CHECK_LIB([crypto],[SHA256]) + fi + fi -if test "x$enable_s3" = xyes ; then - AC_DEFINE([ENABLE_S3], [1], [if true, build netcdf-c with S3 support enabled]) -fi + # Check for enabling S3 testing + AC_MSG_CHECKING([what level of netcdf S3 testing should be enabled]) + AC_ARG_WITH([s3-testing], + [AS_HELP_STRING([--with-s3-testing=yes|no|public], + [control netcdf S3 testing])], + [], [with_s3_testing=public]) + AC_MSG_RESULT($with_s3_testing) + + # Disable S3 tests if S3 support is disabled + if test "x$enable_s3" = xno ; then + if test "x$with_s3_testing" != xno ; then + AC_MSG_WARN([S3 support is disabled => no testing]) + with_s3_testing=no + fi + fi -if test "x$enable_s3_aws" = xyes ; then - LIBS="$LIBS$S3LIBS" - AC_DEFINE([ENABLE_S3_AWS], [1], [If true, then use aws S3 library]) -fi + if test "x$enable_s3" = xyes ; then + AC_DEFINE([ENABLE_S3], [1], [if true, build netcdf-c with S3 support enabled]) + fi -if test "x$enable_s3_internal" = xyes ; then - AC_DEFINE([ENABLE_S3_INTERNAL], [1], [If true, then use internal S3 library]) -fi + if test "x$enable_s3_aws" = xyes ; then + LIBS="$LIBS$S3LIBS" + AC_DEFINE([ENABLE_S3_AWS], [1], [If true, then use aws S3 library]) + fi -AC_DEFINE_UNQUOTED([WITH_S3_TESTING], [$with_s3_testing], [control S3 testing.]) + if test "x$enable_s3_internal" = xyes ; then + AC_DEFINE([ENABLE_S3_INTERNAL], [1], [If true, then use internal S3 library]) + fi + + AC_DEFINE_UNQUOTED([WITH_S3_TESTING], [$with_s3_testing], [control S3 testing.]) -if test "x$with_s3_testing" = xyes ; then - AC_MSG_WARN([*** DO NOT SPECIFY WITH_S3_TESTING=YES UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***]) - AC_DEFINE([ENABLE_S3_TESTALL], [yes], [control S3 testing.]) + if test "x$with_s3_testing" = xyes ; then + AC_MSG_WARN([*** DO NOT SPECIFY WITH_S3_TESTING=YES UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***]) + AC_DEFINE([ENABLE_S3_TESTALL], [yes], [control S3 testing.]) + fi fi # Check whether we want to enable strict null byte header padding. @@ -1320,9 +1327,6 @@ AC_CHECK_HEADERS([sys/resource.h]) # See if we have ftw.h to walk directory trees AC_CHECK_HEADERS([ftw.h]) -# See if we can do stack tracing programmatically -AC_CHECK_HEADERS([execinfo.h]) - # Check for these functions... AC_CHECK_FUNCS([strlcat snprintf strcasecmp fileno \ strdup strtoll strtoull \ diff --git a/dap4_test/test_common.h b/dap4_test/test_common.h index a005d9c9cd..d7e096e2da 100644 --- a/dap4_test/test_common.h +++ b/dap4_test/test_common.h @@ -21,6 +21,7 @@ typedef int TDMR; static NCbytes* input = NULL; static NCbytes* output = NULL; static NCD4meta* metadata = NULL; +static NCD4response* resp = NULL; static char* infile = NULL; static char* outfile = NULL; static int ncid = 0; @@ -85,16 +86,21 @@ setup(int tdmr, int argc, char** argv) if(translatenc4) controller->controls.translation = NCD4_TRANSNC4; NCD4_applyclientfragmentcontrols(controller); - if((metadata=NCD4_newmeta(controller))==NULL) - fail(NC_ENOMEM); - metadata->mode = mode; - NCD4_attachraw(metadata, ncbyteslength(input),ncbytescontents(input)); - if((ret=NCD4_dechunk(metadata))) /* ok for mode == DMR or mode == DAP */ + if((ret=NCD4_newMeta(controller,&metadata))) + fail(ret); + + if((ret=NCD4_newResponse(controller,&resp))) + fail(ret); + resp->raw.size = ncbyteslength(input); + resp->raw.memory = ncbytescontents(input); + resp->mode = mode; + + if((ret=NCD4_dechunk(resp))) /* ok for mode == DMR or mode == DAP */ fail(ret); #ifdef DEBUG { - int swap = (metadata->serial.hostbigendian != metadata->serial.remotebigendian); + int swap = (controller->platform.hostlittleendian != resp->remotelittleendian); void* d = metadata->serial.dap; size_t sz = metadata->serial.dapsize; fprintf(stderr,"====================\n"); diff --git a/dap4_test/test_meta.c b/dap4_test/test_meta.c index f0bf2273b4..478fcfb62d 100644 --- a/dap4_test/test_meta.c +++ b/dap4_test/test_meta.c @@ -20,7 +20,7 @@ main(int argc, char** argv) fprintf(stderr,"t_dmrmeta %s -> %s\n",infile,outfile); #endif - if((ret = NCD4_parse(metadata))) goto done; + if((ret = NCD4_parse(metadata,resp,0))) goto done; if((ret = NCD4_metabuild(metadata,ncid))) goto done; done: diff --git a/dap4_test/test_parse.c b/dap4_test/test_parse.c index 44075ffaa8..59afb74d5e 100644 --- a/dap4_test/test_parse.c +++ b/dap4_test/test_parse.c @@ -17,7 +17,7 @@ main(int argc, char** argv) setup(TDMR_PARSE,argc,argv); - if((ret = NCD4_parse(metadata))) goto done; + if((ret = NCD4_parse(metadata,resp,0))) goto done; ret = NCD4_print(metadata,output); ncbytesnull(output); if(ret == NC_NOERR) { diff --git a/examples/C/format.c b/examples/C/format.c index 291aad65b6..40413dbe3f 100644 --- a/examples/C/format.c +++ b/examples/C/format.c @@ -45,7 +45,7 @@ main() /* Create a bunch of phoney data so we have something to write in the example file. */ for (fp=(float *)temp, i=0; ierrdata); - nullfree(serial->dmr); - nullfree(serial->dap); - nullfree(serial->rawdata); - /* clear all fields */ - memset(serial,0,sizeof(NCD4serial)); - /* Reset fields */ - serial->hostlittleendian = NCD4_isLittleEndian(); - serial->rawsize = rawsize; - serial->rawdata = rawdata; -} - int -NCD4_dechunk(NCD4meta* metadata) +NCD4_dechunk(NCD4response* resp) { - unsigned char *praw, *phdr, *pdap; + unsigned char *praw, *pdmr, *phdr, *pdap, *pappend, *pchunk; NCD4HDR hdr; + int firstchunk; #ifdef D4DUMPRAW - NCD4_tagdump(metadata->serial.rawsize,metadata->serial.rawdata,0,"RAW"); + NCD4_tagdump(resp->serial.raw.size,resp->serial.raw.data,0,"RAW"); #endif /* Access the returned raw data */ - praw = metadata->serial.rawdata; + praw = (unsigned char*)resp->raw.memory; - if(metadata->mode == NCD4_DSR) { + if(resp->mode == NCD4_DSR) { return THROW(NC_EDMR); - } else if(metadata->mode == NCD4_DMR) { + } else if(resp->mode == NCD4_DMR) { /* Verify the mode; assume that the is optional */ if(memcmp(praw,"serial.rawsize; - if((metadata->serial.dmr = malloc(len+1)) == NULL) + len = resp->raw.size; + if((resp->serial.dmr = malloc(len+1)) == NULL) return THROW(NC_ENOMEM); - memcpy(metadata->serial.dmr,praw,len); - metadata->serial.dmr[len] = '\0'; + memcpy(resp->serial.dmr,praw,len); + resp->serial.dmr[len] = '\0'; /* Suppress nuls */ - (void)NCD4_elidenuls(metadata->serial.dmr,len); + (void)NCD4_elidenuls(resp->serial.dmr,len); return THROW(NC_NOERR); } - } else if(metadata->mode != NCD4_DAP) + } else if(resp->mode != NCD4_DAP) return THROW(NC_EDAP); /* We must be processing a DAP mode packet */ - praw = (metadata->serial.dap = metadata->serial.rawdata); - metadata->serial.rawdata = NULL; + praw = resp->raw.memory; /* If the raw data looks like xml, then we almost certainly have an error */ if(memcmp(praw,"serial.rawsize, metadata->serial.rawdata); + int stat = NCD4_seterrormessage(resp, resp->raw.size, resp->raw.memory); return THROW(stat); /* slight lie */ } - /* Get the DMR chunk header*/ - phdr = NCD4_getheader(praw,&hdr,metadata->serial.hostlittleendian); + /* Get the first header to get dmr content and endian flags*/ + pdmr = NCD4_getheader(praw,&hdr,resp->controller->platform.hostlittleendian); if(hdr.count == 0) return THROW(NC_EDMR); - if(hdr.flags & NCD4_ERR_CHUNK) { - return processerrchunk(metadata, (void*)phdr, hdr.count); - } + if(hdr.flags & NCD4_ERR_CHUNK) + return processerrchunk(resp, (void*)pdmr, hdr.count); + resp->remotelittleendian = ((hdr.flags & NCD4_LITTLE_ENDIAN_CHUNK) ? 1 : 0); - metadata->serial.remotelittleendian = ((hdr.flags & NCD4_LITTLE_ENDIAN_CHUNK) ? 1 : 0); - /* Again, avoid strxxx operations on dmr */ - if((metadata->serial.dmr = malloc(hdr.count+1)) == NULL) + /* avoid strxxx operations on dmr */ + if((resp->serial.dmr = malloc(hdr.count+1)) == NULL) return THROW(NC_ENOMEM); - memcpy(metadata->serial.dmr,phdr,hdr.count); - metadata->serial.dmr[hdr.count-1] = '\0'; + memcpy(resp->serial.dmr,pdmr,hdr.count); + resp->serial.dmr[hdr.count-1] = '\0'; /* Suppress nuls */ - (void)NCD4_elidenuls(metadata->serial.dmr,hdr.count); + (void)NCD4_elidenuls(resp->serial.dmr,hdr.count); + /* See if there is any data after the DMR */ if(hdr.flags & NCD4_LAST_CHUNK) return THROW(NC_ENODATA); /* Read and concat together the data chunks */ - phdr = phdr + hdr.count; /* point to data chunk header */ + phdr = pdmr + hdr.count; /* point to data chunk header */ /* Do a sanity check in case the server has shorted us with no data */ - if((hdr.count + CHUNKHDRSIZE) >= metadata->serial.rawsize) { + if((hdr.count + CHUNKHDRSIZE) >= resp->raw.size) { /* Server only sent the DMR part */ - metadata->serial.dapsize = 0; + resp->serial.dapsize = 0; return THROW(NC_EDATADDS); } - pdap = metadata->serial.dap; - for(;;) { - phdr = NCD4_getheader(phdr,&hdr,metadata->serial.hostlittleendian); - if(hdr.flags & NCD4_ERR_CHUNK) { - return processerrchunk(metadata, (void*)phdr, hdr.count); + /* walk all the data chunks */ + /* invariants: + praw -- beginning of the raw response + pdmr -- beginning of the dmr in the raw data + pdap -- beginning of the dechunked dap data + phdr -- pointer to the hdr of the current chunk + pchunk -- pointer to the data part of the current chunk + pappend -- where to append next chunk to the growing dechunked data + */ + for(firstchunk=1;;firstchunk=0) { + pchunk = NCD4_getheader(phdr,&hdr,resp->controller->platform.hostlittleendian); /* Process first data chunk header */ + if(firstchunk) { + pdap = phdr; /* remember start point of the dechunked data */ + pappend = phdr; /* start appending here */ } + if(hdr.flags & NCD4_ERR_CHUNK) + return processerrchunk(resp, (void*)pchunk, hdr.count); /* data chunk; possibly last; possibly empty */ - if(hdr.count > 0) { - d4memmove(pdap,phdr,hdr.count); /* will overwrite the header */ - phdr += hdr.count; - pdap += hdr.count; - } + if(hdr.count > 0) + d4memmove(pappend,pchunk,hdr.count); /* overwrite the header; this the heart of dechunking */ + pappend += hdr.count; /* next append point */ + phdr = pchunk + hdr.count; /* point to header of next chunk */ if(hdr.flags & NCD4_LAST_CHUNK) break; } - metadata->serial.dapsize = (size_t)DELTA(pdap,metadata->serial.dap); + resp->serial.dap = pdap; /* start of dechunked data */ + resp->serial.dapsize = (size_t)DELTA(pappend,pdap); #ifdef D4DUMPDMR - fprintf(stderr,"%s\n",metadata->serial.dmr); + fprintf(stderr,"%s\n",resp->serial.dmr); fflush(stderr); #endif #ifdef D4DUMPDAP - NCD4_tagdump(metadata->serial.dapsize,metadata->serial.dap,0,"DAP"); + NCD4_tagdump(resp->serial.dapsize,resp->serial.dap,0,"DAP"); #endif return THROW(NC_NOERR); } static int -processerrchunk(NCD4meta* metadata, void* errchunk, unsigned int count) +processerrchunk(NCD4response* resp, void* errchunk, unsigned int count) { - metadata->serial.errdata = (char*)d4alloc(count+1); - if(metadata->serial.errdata == NULL) + resp->serial.errdata = (char*)d4alloc(count+1); + if(resp->serial.errdata == NULL) return THROW(NC_ENOMEM); - memcpy(metadata->serial.errdata,errchunk,count); - metadata->serial.errdata[count] = '\0'; + memcpy(resp->serial.errdata,errchunk,count); + resp->serial.errdata[count] = '\0'; return THROW(NC_ENODATA); /* slight lie */ } @@ -157,26 +152,26 @@ Given a raw response, attempt to infer the mode: DMR, DAP, DSR. Since DSR is not standardizes, it becomes the default. */ int -NCD4_infermode(NCD4meta* meta) +NCD4_infermode(NCD4response* resp) { - d4size_t size = meta->serial.rawsize; - char* raw = meta->serial.rawdata; + d4size_t size = resp->raw.size; + char* raw = resp->raw.memory; if(size < 16) return THROW(NC_EDAP); /* must have at least this to hold a hdr + partial dmr*/ if(memcmp(raw,"mode = NCD4_DMR; + resp->mode = NCD4_DMR; goto done; } raw += 4; /* Pretend we have a DAP hdr */ if(memcmp(raw,"mode = NCD4_DAP; + resp->mode = NCD4_DAP; goto done; } /* Default to DSR */ - meta->mode = NCD4_DSR; + resp->mode = NCD4_DSR; done: return NC_NOERR; diff --git a/libdap4/d4curlfunctions.c b/libdap4/d4curlfunctions.c index ee06e4cacd..b1f949b9f4 100644 --- a/libdap4/d4curlfunctions.c +++ b/libdap4/d4curlfunctions.c @@ -340,7 +340,7 @@ NCD4_get_rcproperties(NCD4INFO* state) ncerror err = NC_NOERR; char* option = NULL; #ifdef HAVE_CURLOPT_BUFFERSIZE - option = NC_rclookup(D4BUFFERSIZE,state->uri->uri,NULL); + option = NC_rclookup(D4BUFFERSIZE,state->dmruri->uri,NULL); if(option != NULL && strlen(option) != 0) { long bufsize; if(strcasecmp(option,"max")==0) @@ -351,7 +351,7 @@ NCD4_get_rcproperties(NCD4INFO* state) } #endif #ifdef HAVE_CURLOPT_KEEPALIVE - option = NC_rclookup(D4KEEPALIVE,state->uri->uri,NULL); + option = NC_rclookup(D4KEEPALIVE,state->dmruri->uri,NULL); if(option != NULL && strlen(option) != 0) { /* The keepalive value is of the form 0 or n/m, where n is the idle time and m is the interval time; diff --git a/libdap4/d4data.c b/libdap4/d4data.c index 5d61b81dc4..eae5f32ed8 100644 --- a/libdap4/d4data.c +++ b/libdap4/d4data.c @@ -16,7 +16,7 @@ This code serves two purposes (NCD4_processdata) 2. Walk a specified variable instance to convert to netcdf4 memory representation. - (NCD4_fillinstance) + (NCD4_movetoinstance) */ @@ -29,7 +29,6 @@ static int fillopfixed(NCD4meta*, d4size_t opaquesize, NCD4offset* offset, void* static int fillopvar(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs); static int fillstruct(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs); static int fillseq(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs); -static int NCD4_inferChecksums(NCD4meta* meta, NClist* toplevel); static unsigned NCD4_computeChecksum(NCD4meta* meta, NCD4node* topvar); /***************************************************/ @@ -54,8 +53,9 @@ static unsigned int debugcrc32(unsigned int crc, const void *buf, size_t size) /***************************************************/ /* API */ +/* Parcel out the dechunked data to the corresponding vars */ int -NCD4_processdata(NCD4meta* meta) +NCD4_parcelvars(NCD4meta* meta, NCD4response* resp) { int ret = NC_NOERR; int i; @@ -68,35 +68,57 @@ NCD4_processdata(NCD4meta* meta) toplevel = nclistnew(); NCD4_getToplevelVars(meta,root,toplevel); - /* Otherwise */ - NCD4_inferChecksums(meta,toplevel); + /* Compute the offset and size of the toplevel vars in the raw dap data. */ + offset = BUILDOFFSET(resp->serial.dap,resp->serial.dapsize); + for(i=0;iinferredchecksumming))) { + FAIL(ret,"delimit failure"); + } + var->data.response = resp; /* cross link */ + } +done: + nclistfree(toplevel); + nullfree(offset); + return THROW(ret); +} + +/* Process top level vars wrt checksums and swapping */ +int +NCD4_processdata(NCD4meta* meta, NCD4response* resp) +{ + int ret = NC_NOERR; + int i; + NClist* toplevel = NULL; + NCD4node* root = meta->root; + NCD4offset* offset = NULL; - /* If necessary, byte swap the serialized data */ /* Do we need to swap the dap4 data? */ - meta->swap = (meta->serial.hostlittleendian != meta->serial.remotelittleendian); + meta->swap = (meta->controller->platform.hostlittleendian != resp->remotelittleendian); - /* Compute the offset and size of the toplevel vars in the raw dap data. */ - /* Also extract remote checksums */ - offset = BUILDOFFSET(meta->serial.dap,meta->serial.dapsize); + /* Recursively walk the tree in prefix order + to get the top-level variables; also mark as unvisited */ + toplevel = nclistnew(); + NCD4_getToplevelVars(meta,root,toplevel); + + /* Extract remote checksums */ for(i=0;icontroller->data.inferredchecksumming) { - /* Compute remote checksum: must occur before any byte swapping */ + if(resp->inferredchecksumming) { + /* Compute checksum of response data: must occur before any byte swapping and after delimiting */ var->data.localchecksum = NCD4_computeChecksum(meta,var); #ifdef DUMPCHECKSUM fprintf(stderr,"var %s: remote-checksum = 0x%x\n",var->name,var->data.remotechecksum); #endif /* verify checksums */ - if(!meta->controller->data.checksumignore) { + if(!resp->checksumignore) { if(var->data.localchecksum != var->data.remotechecksum) { nclog(NCLOGERR,"Checksum mismatch: %s\n",var->name); ret = NC_EDAP; goto done; } /* Also verify checksum attribute */ - if(meta->controller->data.attrchecksumming) { + if(resp->attrchecksumming) { if(var->data.attrchecksum != var->data.remotechecksum) { nclog(NCLOGERR,"Attribute Checksum mismatch: %s\n",var->name); ret = NC_EDAP; @@ -105,13 +127,11 @@ NCD4_processdata(NCD4meta* meta) } } } - } - - /* Swap the data for each top level variable, - */ - if(meta->swap) { - if((ret=NCD4_swapdata(meta,toplevel))) - FAIL(ret,"byte swapping failed"); + if(meta->swap) { + if((ret=NCD4_swapdata(resp,var,meta->swap))) + FAIL(ret,"byte swapping failed"); + } + var->data.valid = 1; /* Everything should be in place */ } done: @@ -133,7 +153,7 @@ Assumes that NCD4_processdata has been called. */ int -NCD4_fillinstance(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs) +NCD4_movetoinstance(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs) { int ret = NC_NOERR; void* dst = *dstp; @@ -149,30 +169,30 @@ NCD4_fillinstance(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dst } else switch(type->subsort) { case NC_STRING: /* oob strings */ if((ret=fillstring(meta,offset,&dst,blobs))) - FAIL(ret,"fillinstance"); + FAIL(ret,"movetoinstance"); break; case NC_OPAQUE: if(type->opaque.size > 0) { /* We know the size and its the same for all instances */ if((ret=fillopfixed(meta,type->opaque.size,offset,&dst))) - FAIL(ret,"fillinstance"); + FAIL(ret,"movetoinstance"); } else { /* Size differs per instance, so we need to convert each opaque to a vlen */ if((ret=fillopvar(meta,type,offset,&dst,blobs))) - FAIL(ret,"fillinstance"); + FAIL(ret,"movetoinstance"); } break; case NC_STRUCT: if((ret=fillstruct(meta,type,offset,&dst,blobs))) - FAIL(ret,"fillinstance"); + FAIL(ret,"movetoinstance"); break; case NC_SEQ: if((ret=fillseq(meta,type,offset,&dst,blobs))) - FAIL(ret,"fillinstance"); + FAIL(ret,"movetoinstance"); break; default: ret = NC_EINVAL; - FAIL(ret,"fillinstance"); + FAIL(ret,"movetoinstance"); } *dstp = dst; @@ -196,7 +216,7 @@ fillstruct(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NCli NCD4node* field = nclistget(type->vars,i); NCD4node* ftype = field->basetype; void* fdst = (((char*)dst) + field->meta.offset); - if((ret=NCD4_fillinstance(meta,ftype,offset,&fdst,blobs))) + if((ret=NCD4_movetoinstance(meta,ftype,offset,&fdst,blobs))) FAIL(ret,"fillstruct"); } dst = ((char*)dst) + type->meta.memsize; @@ -231,7 +251,7 @@ fillseq(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NClist* for(i=0;ip))+(recordsize * i); - if((ret=NCD4_fillinstance(meta,vlentype,offset,&recdst,blobs))) + if((ret=NCD4_movetoinstance(meta,vlentype,offset,&recdst,blobs))) FAIL(ret,"fillseq"); } dst++; @@ -373,12 +393,16 @@ fprintf(stderr,"toplevel: var=%s\n",node->name); return THROW(ret); } -static int -NCD4_inferChecksums(NCD4meta* meta, NClist* toplevel) +int +NCD4_inferChecksums(NCD4meta* meta, NCD4response* resp) { int ret = NC_NOERR; int i, attrfound; - NCD4INFO* info = meta->controller; + NClist* toplevel = NULL; + + /* Get the toplevel vars */ + toplevel = nclistnew(); + NCD4_getToplevelVars(meta,meta->root,toplevel); /* First, look thru the DMR to see if there is a checksum attribute */ attrfound = 0; @@ -399,9 +423,10 @@ NCD4_inferChecksums(NCD4meta* meta, NClist* toplevel) } } } - info->data.attrchecksumming = (attrfound ? 1 : 0); + nclistfree(toplevel); + resp->attrchecksumming = (attrfound ? 1 : 0); /* Infer checksums */ - info->data.inferredchecksumming = ((info->data.attrchecksumming || info->data.querychecksumming) ? 1 : 0); + resp->inferredchecksumming = ((resp->attrchecksumming || resp->querychecksumming) ? 1 : 0); return THROW(ret); } diff --git a/libdap4/d4debug.c b/libdap4/d4debug.c index d2d40b44a6..caed80027a 100644 --- a/libdap4/d4debug.c +++ b/libdap4/d4debug.c @@ -97,7 +97,7 @@ int NCD4_debugcopy(NCD4INFO* info) { int i,ret=NC_NOERR; - NCD4meta* meta = info->substrate.metadata; + NCD4meta* meta = info->dmrmetadata; NClist* topvars = nclistnew(); NC* ncp = info->controller; void* memory = NULL; diff --git a/libdap4/d4file.c b/libdap4/d4file.c index aeccc7423f..46c1a0fdbb 100644 --- a/libdap4/d4file.c +++ b/libdap4/d4file.c @@ -6,7 +6,6 @@ #include "ncdispatch.h" #include "ncd4dispatch.h" #include "d4includes.h" -#include "d4read.h" #include "d4curlfunctions.h" #ifdef _MSC_VER @@ -23,19 +22,20 @@ static int constrainable(NCURI*); static void freeCurl(NCD4curl*); -static void freeInfo(NCD4INFO*); static int fragmentcheck(NCD4INFO*, const char* key, const char* subkey); static const char* getfragment(NCD4INFO* info, const char* key); static const char* getquery(NCD4INFO* info, const char* key); static int set_curl_properties(NCD4INFO*); static int makesubstrate(NCD4INFO* d4info); -static void resetInfoforRead(NCD4INFO* d4info); /**************************************************/ /* Constants */ static const char* checkseps = "+,:;"; +/*Define the set of protocols known to be constrainable */ +static const char* constrainableprotocols[] = {"http", "https",NULL}; + /**************************************************/ int NCD4_open(const char * path, int mode, @@ -46,10 +46,10 @@ NCD4_open(const char * path, int mode, NCD4INFO* d4info = NULL; const char* value; NC* nc; - NCD4meta* meta = NULL; size_t len = 0; void* contents = NULL; - + NCD4response* dmrresp = NULL; + if(path == NULL) return THROW(NC_EDAPURL); @@ -61,29 +61,27 @@ NCD4_open(const char * path, int mode, /* Setup our NC and NCDAPCOMMON state*/ - d4info = (NCD4INFO*)calloc(1,sizeof(NCD4INFO)); - if(d4info == NULL) {ret = NC_ENOMEM; goto done;} - + if((ret=NCD4_newInfo(&d4info))) goto done; nc->dispatchdata = d4info; nc->int_ncid = nc__pseudofd(); /* create a unique id */ d4info->controller = (NC*)nc; /* Parse url and params */ - if(ncuriparse(nc->path,&d4info->uri)) + if(ncuriparse(nc->path,&d4info->dmruri)) {ret = NC_EDAPURL; goto done;} /* Load auth info from rc file */ - if((ret = NC_authsetup(&d4info->auth, d4info->uri))) + if((ret = NC_authsetup(&d4info->auth, d4info->dmruri))) goto done; NCD4_curl_protocols(d4info); - if(!constrainable(d4info->uri)) + if(!constrainable(d4info->dmruri)) SETFLAG(d4info->controls.flags,NCF_UNCONSTRAINABLE); /* fail if we are unconstrainable but have constraints */ if(FLAGSET(d4info->controls.flags,NCF_UNCONSTRAINABLE)) { - if(d4info->uri != NULL) { - const char* ce = ncuriquerylookup(d4info->uri,DAP4CE); /* Look for dap4.ce */ + if(d4info->dmruri != NULL) { + const char* ce = ncuriquerylookup(d4info->dmruri,DAP4CE); /* Look for dap4.ce */ if(ce != NULL) { nclog(NCLOGWARN,"Attempt to constrain an unconstrainable data source: %s=%s", DAP4CE,ce); @@ -115,7 +113,7 @@ NCD4_open(const char * path, int mode, } /* Turn on logging; only do this after oc_open*/ - if((value = ncurifragmentlookup(d4info->uri,"log")) != NULL) { + if((value = ncurifragmentlookup(d4info->dmruri,"log")) != NULL) { ncloginit(); ncsetloglevel(NCLOGNOTE); } @@ -150,30 +148,34 @@ NCD4_open(const char * path, int mode, /* Reset the substrate */ if((ret=makesubstrate(d4info))) goto done; - /* Always start by reading the DMR only */ + /* Always start by reading the whole DMR only */ /* reclaim substrate.metadata */ - resetInfoforRead(d4info); + NCD4_resetInfoForRead(d4info); /* Rebuild metadata */ - if((d4info->substrate.metadata=NCD4_newmeta(d4info))==NULL) - {ret = NC_ENOMEM; goto done;} + if((ret = NCD4_newMeta(d4info,&d4info->dmrmetadata))) goto done; + + /* Capture response */ + if((dmrresp = (NCD4response*)calloc(1,sizeof(NCD4response)))==NULL) + {ret = NC_ENOMEM; goto done;} + dmrresp->controller = d4info; - if((ret=NCD4_readDMR(d4info, d4info->controls.flags.flags))) goto done; + if((ret=NCD4_readDMR(d4info, d4info->controls.flags.flags, d4info->dmruri, dmrresp))) goto done; /* set serial.rawdata */ len = ncbyteslength(d4info->curl->packet); contents = ncbytesextract(d4info->curl->packet); - NCD4_attachraw(d4info->substrate.metadata, len, contents); - - /* process query parameters */ - NCD4_applyclientquerycontrols(d4info); + assert(dmrresp != NULL); + dmrresp->raw.size = len; + dmrresp->raw.memory = contents; - meta = d4info->substrate.metadata; + /* process checksum parameters */ + NCD4_applychecksumcontrols(d4info,dmrresp); /* Infer the mode */ - if((ret=NCD4_infermode(meta))) goto done; + if((ret=NCD4_infermode(dmrresp))) goto done; /* Process the dmr part */ - if((ret=NCD4_dechunk(meta))) goto done; + if((ret=NCD4_dechunk(dmrresp))) goto done; #ifdef D4DUMPDMR { @@ -184,13 +186,14 @@ NCD4_open(const char * path, int mode, } #endif - if((ret = NCD4_parse(d4info->substrate.metadata))) goto done; + if((ret = NCD4_parse(d4info->dmrmetadata,dmrresp,0))) goto done; #ifdef D4DEBUGMETA { + meta = d4info->dmrmetadata; fprintf(stderr,"\n/////////////\n"); NCbytes* buf = ncbytesnew(); - NCD4_print(d4info->substrate.metadata,buf); + NCD4_print(meta,buf); ncbytesnull(buf); fputs(ncbytescontents(buf),stderr); ncbytesfree(buf); @@ -200,12 +203,20 @@ NCD4_open(const char * path, int mode, #endif /* Build the substrate metadata */ - ret = NCD4_metabuild(d4info->substrate.metadata,d4info->substrate.metadata->ncid); + ret = NCD4_metabuild(d4info->dmrmetadata,d4info->dmrmetadata->ncid); if(ret != NC_NOERR && ret != NC_EVARSIZE) goto done; + /* Remember the response */ + nclistpush(d4info->responses,dmrresp); + + /* Avoid duplicate reclaims */ + dmrresp = NULL; + d4info = NULL; + done: + NCD4_reclaimResponse(dmrresp); + NCD4_reclaimInfo(d4info); if(ret) { - freeInfo(d4info); nc->dispatchdata = NULL; } return THROW(ret); @@ -236,7 +247,7 @@ NCD4_close(int ncid, void* ignore) ret = nc_abort(substrateid); } - freeInfo(d4info); + NCD4_reclaimInfo(d4info); done: return THROW(ret); @@ -248,82 +259,6 @@ NCD4_abort(int ncid) return NCD4_close(ncid,NULL); } -/**************************************************/ - -/* Reclaim an NCD4INFO instance */ -static void -freeInfo(NCD4INFO* d4info) -{ - if(d4info == NULL) return; - d4info->controller = NULL; /* break link */ - nullfree(d4info->rawurltext); - nullfree(d4info->urltext); - ncurifree(d4info->uri); - freeCurl(d4info->curl); - nullfree(d4info->data.memory); - nullfree(d4info->data.ondiskfilename); - if(d4info->data.ondiskfile != NULL) - fclose(d4info->data.ondiskfile); - nullfree(d4info->fileproto.filename); - if(d4info->substrate.realfile - && !FLAGSET(d4info->controls.debugflags,NCF_DEBUG_COPY)) { - /* We used real file, so we need to delete the temp file - unless we are debugging. - Assume caller has done nc_close|nc_abort on the ncid. - Note that in theory, this should not be necessary since - AFAIK the substrate file is still in def mode, and - when aborted, it should be deleted. But that is not working - for some reason, so we delete it ourselves. - */ - if(d4info->substrate.filename != NULL) { - unlink(d4info->substrate.filename); - } - } - nullfree(d4info->substrate.filename); /* always reclaim */ - NCD4_reclaimMeta(d4info->substrate.metadata); - NC_authfree(d4info->auth); - nclistfree(d4info->blobs); - free(d4info); -} - -/* Reset NCD4INFO instance for new read request */ -static void -resetInfoforRead(NCD4INFO* d4info) -{ - if(d4info == NULL) return; - if(d4info->substrate.realfile - && !FLAGSET(d4info->controls.debugflags,NCF_DEBUG_COPY)) { - /* We used real file, so we need to delete the temp file - unless we are debugging. - Assume caller has done nc_close|nc_abort on the ncid. - Note that in theory, this should not be necessary since - AFAIK the substrate file is still in def mode, and - when aborted, it should be deleted. But that is not working - for some reason, so we delete it ourselves. - */ - if(d4info->substrate.filename != NULL) { - unlink(d4info->substrate.filename); - } - } - NCD4_resetMeta(d4info->substrate.metadata); - nullfree(d4info->substrate.metadata); - d4info->substrate.metadata = NULL; -} - -static void -freeCurl(NCD4curl* curl) -{ - if(curl == NULL) return; - NCD4_curlclose(curl->curl); - ncbytesfree(curl->packet); - nullfree(curl->errdata.code); - nullfree(curl->errdata.message); - free(curl); -} - -/* Define the set of protocols known to be constrainable */ -static const char* constrainableprotocols[] = {"http", "https",NULL}; - static int constrainable(NCURI* durl) { @@ -449,11 +384,6 @@ NCD4_applyclientfragmentcontrols(NCD4INFO* info) if(value != NULL) strncpy(info->controls.substratename,value,(NC_MAX_NAME-1)); - value = getfragment(info,"hyrax"); - if(value != NULL) { - info->data.checksumignore = 1; /* Assume checksum, but ignore */ - } - info->controls.opaquesize = DFALTOPAQUESIZE; value = getfragment(info,"opaquesize"); if(value != NULL) { @@ -476,22 +406,29 @@ NCD4_applyclientfragmentcontrols(NCD4INFO* info) } } +/* Checksum controls are found both in the query and fragment + parts of a URL. +*/ void -NCD4_applyclientquerycontrols(NCD4INFO* info) +NCD4_applychecksumcontrols(NCD4INFO* info, NCD4response* resp) { const char* value = getquery(info,DAP4CSUM); if(value == NULL) { - info->data.querychecksumming = DEFAULT_CHECKSUM_STATE; + resp->querychecksumming = DEFAULT_CHECKSUM_STATE; } else { if(strcasecmp(value,"false")==0) { - info->data.querychecksumming = 0; + resp->querychecksumming = 0; } else if(strcasecmp(value,"true")==0) { - info->data.querychecksumming = 1; + resp->querychecksumming = 1; } else { nclog(NCLOGWARN,"Unknown checksum mode: %s ; using default",value); - info->data.querychecksumming = DEFAULT_CHECKSUM_STATE; + resp->querychecksumming = DEFAULT_CHECKSUM_STATE; } } + value = getfragment(info,"hyrax"); + if(value != NULL) { + resp->checksumignore = 1; /* Assume checksum, but ignore */ + } } /* Search for substring in value of param. If substring == NULL; then just @@ -523,7 +460,7 @@ getfragment(NCD4INFO* info, const char* key) const char* value; if(info == NULL || key == NULL) return NULL; - if((value=ncurifragmentlookup(info->uri,key)) == NULL) + if((value=ncurifragmentlookup(info->dmruri,key)) == NULL) return NULL; return value; } @@ -537,7 +474,7 @@ getquery(NCD4INFO* info, const char* key) const char* value; if(info == NULL || key == NULL) return NULL; - if((value=ncuriquerylookup(info->uri,key)) == NULL) + if((value=ncuriquerylookup(info->dmruri,key)) == NULL) return NULL; return value; } @@ -596,3 +533,175 @@ NCD4_get_substrate(NC* nc) } else subnc = nc; return subnc; } + +/**************************************************/ +/* Allocate/Free for various structures */ + +int +NCD4_newInfo(NCD4INFO** d4infop) +{ + int ret = NC_NOERR; + NCD4INFO* info = NULL; + if((info = calloc(1,sizeof(NCD4INFO)))==NULL) + {ret = NC_ENOMEM; goto done;} + info->platform.hostlittleendian = NCD4_isLittleEndian(); + info->responses = nclistnew(); + if(d4infop) {*d4infop = info; info = NULL;} +done: + if(info) NCD4_reclaimInfo(info); + return THROW(ret); +} + +/* Reclaim an NCD4INFO instance */ +void +NCD4_reclaimInfo(NCD4INFO* d4info) +{ + size_t i; + if(d4info == NULL) return; + d4info->controller = NULL; /* break link */ + nullfree(d4info->rawdmrurltext); + nullfree(d4info->dmrurltext); + ncurifree(d4info->dmruri); + freeCurl(d4info->curl); + nullfree(d4info->fileproto.filename); + NCD4_resetInfoForRead(d4info); + nullfree(d4info->substrate.filename); /* always reclaim */ + NC_authfree(d4info->auth); + nclistfree(d4info->blobs); + /* Reclaim dmr node tree */ + NCD4_reclaimMeta(d4info->dmrmetadata); + /* Reclaim all responses */ + for(i=0;iresponses);i++) { + NCD4response* resp = nclistget(d4info->responses,i); + NCD4_reclaimResponse(resp); + } + nclistfree(d4info->responses); + free(d4info); +} + +/* Reset NCD4INFO instance for new read request */ +void +NCD4_resetInfoForRead(NCD4INFO* d4info) +{ + if(d4info == NULL) return; + if(d4info->substrate.realfile + && !FLAGSET(d4info->controls.debugflags,NCF_DEBUG_COPY)) { + /* We used real file, so we need to delete the temp file + unless we are debugging. + Assume caller has done nc_close|nc_abort on the ncid. + Note that in theory, this should not be necessary since + AFAIK the substrate file is still in def mode, and + when aborted, it should be deleted. But that is not working + for some reason, so we delete it ourselves. + */ + if(d4info->substrate.filename != NULL) { + unlink(d4info->substrate.filename); + } + } + NCD4_reclaimMeta(d4info->dmrmetadata); + d4info->dmrmetadata = NULL; +} + +static void +freeCurl(NCD4curl* curl) +{ + if(curl == NULL) return; + NCD4_curlclose(curl->curl); + ncbytesfree(curl->packet); + nullfree(curl->errdata.code); + nullfree(curl->errdata.message); + free(curl); +} + +int +NCD4_newResponse(NCD4INFO* info, NCD4response** respp) +{ + int ret = NC_NOERR; + NCD4response* resp = NULL; + NC_UNUSED(info); + if((resp = calloc(1,sizeof(NCD4response)))==NULL) + {ret = NC_ENOMEM; goto done;} + resp->controller = info; + if(respp) {*respp = resp; resp = NULL;} +done: + if(resp) NCD4_reclaimResponse(resp); + return THROW(ret); +} + + +/* Reclaim an NCD4response instance */ +void +NCD4_reclaimResponse(NCD4response* d4resp) +{ + struct NCD4serial* serial = NULL; + if(d4resp == NULL) return; + serial = &d4resp->serial; + d4resp->controller = NULL; /* break link */ + + nullfree(d4resp->raw.memory); + nullfree(serial->dmr); + nullfree(serial->errdata); + + /* clear all fields */ + memset(serial,0,sizeof(struct NCD4serial)); + + nullfree(d4resp->error.parseerror); + nullfree(d4resp->error.message); + nullfree(d4resp->error.context); + nullfree(d4resp->error.otherinfo); + memset(&d4resp->error,0,sizeof(d4resp->error)); + + free(d4resp); +} + +/* Create an empty NCD4meta object for + use in subsequent calls + (is the the right src file to hold this?) +*/ + +int +NCD4_newMeta(NCD4INFO* info, NCD4meta** metap) +{ + int ret = NC_NOERR; + NCD4meta* meta = (NCD4meta*)calloc(1,sizeof(NCD4meta)); + if(meta == NULL) return NC_ENOMEM; + meta->allnodes = nclistnew(); +#ifdef D4DEBUG + meta->debuglevel = 1; +#endif + meta->controller = info; + meta->ncid = info->substrate.nc4id; /* Transfer netcdf ncid */ + if(metap) {*metap = meta; meta = NULL;} + return THROW(ret); +} + +void +NCD4_reclaimMeta(NCD4meta* dataset) +{ + int i; + if(dataset == NULL) return; + + for(i=0;iallnodes);i++) { + NCD4node* node = (NCD4node*)nclistget(dataset->allnodes,i); + reclaimNode(node); + } + nclistfree(dataset->allnodes); + nclistfree(dataset->groupbyid); + nclistfree(dataset->atomictypes); + free(dataset); +} + +#if 0 +void +NCD4_resetMeta(NCD4meta* dataset) +{ + if(dataset == NULL) return; +#if 0 + for(i=0;iblobs);i++) { + void* p = nclistget(dataset->blobs,i); + nullfree(p); + } + nclistfree(dataset->blobs); +#endif +} +#endif diff --git a/libdap4/d4fix.c b/libdap4/d4fix.c index 82049dd90f..b76a52d28b 100644 --- a/libdap4/d4fix.c +++ b/libdap4/d4fix.c @@ -190,7 +190,7 @@ walk(NCD4node* node, NClist* sorted) */ int -NCD4_delimit(NCD4meta* compiler, NCD4node* topvar, NCD4offset* offset) +NCD4_delimit(NCD4meta* compiler, NCD4node* topvar, NCD4offset* offset, int inferredchecksumming) { int ret = NC_NOERR; NCD4mark mark = 0; @@ -214,7 +214,7 @@ NCD4_delimit(NCD4meta* compiler, NCD4node* topvar, NCD4offset* offset) topvar->data.dap4data.memory = mark; topvar->data.dap4data.size = OFFSETSIZE(offset,mark); /* extract the dap4 data checksum, if present */ - if(compiler->controller->data.inferredchecksumming) { + if(inferredchecksumming) { union ATOMICS csum; TRANSFER(csum.u8,offset,CHECKSUMSIZE); topvar->data.remotechecksum = csum.u32[0]; diff --git a/libdap4/d4http.c b/libdap4/d4http.c index 8e370d8793..13cb82a193 100644 --- a/libdap4/d4http.c +++ b/libdap4/d4http.c @@ -6,7 +6,6 @@ #include "d4includes.h" #include "d4curlfunctions.h" -static size_t WriteFileCallback(void*, size_t, size_t, void*); static size_t WriteMemoryCallback(void*, size_t, size_t, void*); static int curlerrtoncerr(CURLcode cstat); @@ -33,59 +32,6 @@ NCD4_fetchhttpcode(CURL* curl) return httpcode; } -int -NCD4_fetchurl_file(CURL* curl, const char* url, FILE* stream, - d4size_t* sizep, long* filetime) -{ - int ret = NC_NOERR; - CURLcode cstat = CURLE_OK; - struct Fetchdata fetchdata; - - /* Set the URL */ - cstat = curl_easy_setopt(curl, CURLOPT_URL, (void*)url); - if (cstat != CURLE_OK) goto fail; - - /* send all data to this function */ - cstat = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteFileCallback); - if (cstat != CURLE_OK) goto fail; - - /* we pass our file to the callback function */ - cstat = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&fetchdata); - if(cstat != CURLE_OK) goto fail; - - /* One last thing; always try to get the last modified time */ - cstat = curl_easy_setopt(curl, CURLOPT_FILETIME, (long)1); - if (cstat != CURLE_OK) goto fail; - - fetchdata.stream = stream; - fetchdata.size = 0; - cstat = curl_easy_perform(curl); - if (cstat != CURLE_OK) - {ret = NC_EDAPSVC; goto fail;} - - if (ret == NC_NOERR) { - /* return the file size*/ -#ifdef D4DEBUG - nclog(NCLOGNOTE,"filesize: %lu bytes",fetchdata.size); -#endif - if (sizep != NULL) - *sizep = fetchdata.size; - /* Get the last modified time */ - if(filetime != NULL) - cstat = curl_easy_getinfo(curl,CURLINFO_FILETIME,filetime); - if(cstat != CURLE_OK) - {ret = NC_ECURL; goto fail;} - } - return THROW(ret); - -fail: - if(cstat != CURLE_OK) { - nclog(NCLOGERR, "curl error: %s", curl_easy_strerror(cstat)); - ret = curlerrtoncerr(cstat); - } - return THROW(ret); -} - int NCD4_fetchurl(CURL* curl, const char* url, NCbytes* buf, long* filetime, int* httpcodep) { @@ -155,27 +101,6 @@ NCD4_fetchurl(CURL* curl, const char* url, NCbytes* buf, long* filetime, int* ht return THROW(ret); } -static size_t -WriteFileCallback(void* ptr, size_t size, size_t nmemb, void* data) -{ - size_t realsize = size * nmemb; - size_t count; - struct Fetchdata* fetchdata; - fetchdata = (struct Fetchdata*) data; - if(realsize == 0) - nclog(NCLOGWARN,"WriteFileCallback: zero sized chunk"); - count = fwrite(ptr, size, nmemb, fetchdata->stream); - if (count > 0) { - fetchdata->size += (count * size); - } else { - nclog(NCLOGWARN,"WriteFileCallback: zero sized write"); - } -#ifdef PROGRESS - nclog(NCLOGNOTE,"callback: %lu bytes",(d4size_t)realsize); -#endif - return count; -} - static size_t WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *data) { diff --git a/libdap4/d4meta.c b/libdap4/d4meta.c index 4293b3727c..1cea62f613 100644 --- a/libdap4/d4meta.c +++ b/libdap4/d4meta.c @@ -89,75 +89,6 @@ NCD4_metabuild(NCD4meta* metadata, int ncid) return THROW(ret); } - -/* Create an empty NCD4meta object for - use in subsequent calls - (is the the right src file to hold this?) -*/ - -NCD4meta* -NCD4_newmeta(NCD4INFO* info) -{ - NCD4meta* meta = (NCD4meta*)calloc(1,sizeof(NCD4meta)); - if(meta == NULL) return NULL; - meta->allnodes = nclistnew(); -#ifdef D4DEBUG - meta->debuglevel = 1; -#endif - meta->controller = info; - meta->ncid = info->substrate.nc4id; /* Transfer netcdf ncid */ - return meta; -} - -/* Attach raw data to metadata */ -void -NCD4_attachraw(NCD4meta* meta, size_t rawsize, void* rawdata) -{ - assert(meta != NULL); - NCD4_resetSerial(&meta->serial,rawsize,rawdata); -} - -void -NCD4_setdebuglevel(NCD4meta* meta, int debuglevel) -{ - meta->debuglevel = debuglevel; -} - -void -NCD4_reclaimMeta(NCD4meta* dataset) -{ - int i; - if(dataset == NULL) return; - NCD4_resetMeta(dataset); - - for(i=0;iallnodes);i++) { - NCD4node* node = (NCD4node*)nclistget(dataset->allnodes,i); - reclaimNode(node); - } - nclistfree(dataset->allnodes); - nclistfree(dataset->groupbyid); - nclistfree(dataset->atomictypes); - free(dataset); -} - -void -NCD4_resetMeta(NCD4meta* dataset) -{ - if(dataset == NULL) return; - nullfree(dataset->error.parseerror); dataset->error.parseerror = NULL; - nullfree(dataset->error.message); dataset->error.message = NULL; - nullfree(dataset->error.context); dataset->error.context = NULL; - nullfree(dataset->error.otherinfo); dataset->error.otherinfo = NULL; - NCD4_resetSerial(&dataset->serial,0,NULL); -#if 0 - for(i=0;iblobs);i++) { - void* p = nclistget(dataset->blobs,i); - nullfree(p); - } - nclistfree(dataset->blobs); -#endif -} - void reclaimNode(NCD4node* node) { @@ -676,6 +607,39 @@ savevarbyid(NCD4node* group, NCD4node* var) nclistinsert(group->group.varbyid,var->meta.id,var); } +/* Collect FQN path from var node up to and including + the root group and create an name from it +*/ +char* +NCD4_getVarFQN(NCD4node* var, const char* tail) +{ + int i; + NCD4node* x = NULL; + NClist* path = NULL; + NCbytes* fqn = NULL; + char* result; + + path = nclistnew(); + for(x=var->container;ISGROUP(x->sort);x=x->container) { + nclistinsert(path,0,x); + } + fqn = ncbytesnew(); + for(i=0;iname); + if(escaped == NULL) return NULL; + if(i > 0) ncbytesappend(fqn,'/'); + ncbytescat(fqn,escaped); + free(escaped); + } + nclistfree(path); + if(tail != NULL) + ncbytescat(fqn,tail); + result = ncbytesextract(fqn); + ncbytesfree(fqn); + return result; +} + /* Collect FQN path from node up to (but not including) the first enclosing group and create an name from it */ @@ -1180,7 +1144,7 @@ markdapsize(NCD4meta* meta) } int -NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node** grpp) +NCD4_findvar(NC* ncp, int gid, int varid, NCD4node** varp, NCD4node** grpp) { int ret = NC_NOERR; NCD4INFO* info = NULL; @@ -1192,11 +1156,11 @@ NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node** grpp) info = getdap(ncp); if(info == NULL) return THROW(NC_EBADID); - meta = info->substrate.metadata; + meta = info->dmrmetadata; if(meta == NULL) return THROW(NC_EBADID); /* Locate var node via (grpid,varid) */ - grp_id = GROUPIDPART(ncid); + grp_id = GROUPIDPART(gid); group = nclistget(meta->groupbyid,grp_id); if(group == NULL) return THROW(NC_EBADID); @@ -1205,7 +1169,7 @@ NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node** grpp) return THROW(NC_EBADID); if(varp) *varp = var; if(grpp) *grpp = group; - return ret; + return THROW(ret); } static int diff --git a/libdap4/d4parser.c b/libdap4/d4parser.c index 013e320928..2948f80672 100644 --- a/libdap4/d4parser.c +++ b/libdap4/d4parser.c @@ -154,7 +154,7 @@ static int defineBytestringType(NCD4parser*); /* API */ int -NCD4_parse(NCD4meta* metadata) +NCD4_parse(NCD4meta* metadata, NCD4response* resp, int dapparse) { int ret = NC_NOERR; NCD4parser* parser = NULL; @@ -168,8 +168,10 @@ NCD4_parse(NCD4meta* metadata) /* Create and fill in the parser state */ parser = (NCD4parser*)calloc(1,sizeof(NCD4parser)); if(parser == NULL) {ret=NC_ENOMEM; goto done;} + parser->controller = metadata->controller; parser->metadata = metadata; - doc = ncxml_parse(parser->metadata->serial.dmr,strlen(parser->metadata->serial.dmr)); + parser->response = resp; + doc = ncxml_parse(parser->response->serial.dmr,strlen(parser->response->serial.dmr)); if(doc == NULL) {ret=NC_ENOMEM; goto done;} dom = ncxml_root(doc); parser->types = nclistnew(); @@ -178,6 +180,7 @@ NCD4_parse(NCD4meta* metadata) #ifdef D4DEBUG parser->debuglevel = 1; #endif + parser->dapparse = dapparse; /*Walk the DOM tree to build the DAP4 node tree*/ ret = traverse(parser,dom); @@ -214,9 +217,9 @@ traverse(NCD4parser* parser, ncxml_t dom) ret=parseError(parser,dom); /* Report the error */ fprintf(stderr,"DAP4 Error: http-code=%d message=\"%s\" context=\"%s\"\n", - parser->metadata->error.httpcode, - parser->metadata->error.message, - parser->metadata->error.context); + parser->response->error.httpcode, + parser->response->error.message, + parser->response->error.context); fflush(stderr); ret=NC_EDMR; goto done; @@ -225,7 +228,8 @@ traverse(NCD4parser* parser, ncxml_t dom) if((ret=makeNode(parser,NULL,NULL,NCD4_GROUP,NC_NULL,&parser->metadata->root))) goto done; parser->metadata->root->group.isdataset = 1; parser->metadata->root->meta.id = parser->metadata->ncid; - parser->metadata->groupbyid = nclistnew(); + if(parser->metadata->groupbyid == NULL) + parser->metadata->groupbyid = nclistnew(); SETNAME(parser->metadata->root,"/"); xattr = ncxml_attr(dom,"name"); if(xattr != NULL) parser->metadata->root->group.datasetname = xattr; @@ -847,23 +851,23 @@ parseError(NCD4parser* parser, ncxml_t errxml) char* shttpcode = ncxml_attr(errxml,"httpcode"); ncxml_t x; if(shttpcode == NULL) shttpcode = strdup("400"); - if(sscanf(shttpcode,"%d",&parser->metadata->error.httpcode) != 1) + if(sscanf(shttpcode,"%d",&parser->response->error.httpcode) != 1) nclog(NCLOGERR,"Malformed response"); nullfree(shttpcode); x=ncxml_child(errxml, "Message"); if(x != NULL) { char* txt = ncxml_text(x); - parser->metadata->error.message = (txt == NULL ? NULL : txt); + parser->response->error.message = (txt == NULL ? NULL : txt); } x = ncxml_child(errxml, "Context"); if(x != NULL) { const char* txt = ncxml_text(x); - parser->metadata->error.context = (txt == NULL ? NULL : strdup(txt)); + parser->response->error.context = (txt == NULL ? NULL : strdup(txt)); } x=ncxml_child(errxml, "OtherInformation"); if(x != NULL) { const char* txt = ncxml_text(x); - parser->metadata->error.otherinfo = (txt == NULL ? NULL : strdup(txt)); + parser->response->error.otherinfo = (txt == NULL ? NULL : strdup(txt)); } return THROW(NC_NOERR); } @@ -1321,7 +1325,7 @@ makeNode(NCD4parser* parser, NCD4node* parent, ncxml_t xml, NCD4sort sort, nc_ty record(parser,node); if(nodep) *nodep = node; done: - return ret; + return THROW(ret); } static int @@ -1652,12 +1656,19 @@ parseForwards(NCD4parser* parser, NCD4node* root) const char* mapname = (const char*)nclistget(var->mapnames,j); /* Find the corresponding variable */ NCD4node* mapref = lookupFQN(parser,mapname,NCD4_VAR); - if(mapref == NULL) + if(mapref != NULL) + PUSH(var->maps,mapref); + else if(!parser->dapparse) FAIL(NC_ENOTVAR," name does not refer to a variable: %s",mapname); - PUSH(var->maps,mapref); } } done: return THROW(ret); } + +void +NCD4_setdebuglevel(NCD4parser* parser, int debuglevel) +{ + parser->debuglevel = debuglevel; +} diff --git a/libdap4/d4read.c b/libdap4/d4read.c index 3a0e95a1d1..3a24b08ff0 100644 --- a/libdap4/d4read.c +++ b/libdap4/d4read.c @@ -19,9 +19,8 @@ See \ref copyright file for more info. /* Do conversion if this code was compiled via Vis. Studio or Mingw */ /*Forward*/ -static int readpacket(NCD4INFO* state, NCURI*, NCbytes*, NCD4mode, NCD4format, long*); +static int readpacket(NCD4INFO* state, NCURI*, NCbytes*, NCD4mode, NCD4format, int*, long*); static int readfile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, NCbytes* packet); -static int readfiletofile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, FILE* stream, d4size_t* sizep); static int readfileDAPDMR(NCD4INFO* state, const NCURI* uri, NCbytes* packet); #ifdef HAVE_GETTIMEOFDAY @@ -38,82 +37,27 @@ deltatime(struct timeval time0,struct timeval time1) #endif int -NCD4_readDMR(NCD4INFO* state, int flags) +NCD4_readDMR(NCD4INFO* state, int flags, NCURI* url, NCD4response* resp) { int stat = NC_NOERR; - long lastmod = -1; - - if((flags & NCF_ONDISK) == 0) { - ncbytesclear(state->curl->packet); - stat = readpacket(state,state->uri,state->curl->packet,NCD4_DMR,NCD4_FORMAT_XML,&lastmod); - if(stat == NC_NOERR) - state->data.dmrlastmodified = lastmod; - } else { /*((flags & NCF_ONDISK) != 0) */ - NCURI* url = state->uri; - int fileprotocol = (strcmp(url->protocol,"file")==0); - if(fileprotocol) { - stat = readfiletofile(state, url, NCD4_DMR, NCD4_FORMAT_XML, state->data.ondiskfile, &state->data.datasize); - } else { - char* readurl = NULL; - int flags = 0; - if(!fileprotocol) flags |= NCURIQUERY; - flags |= NCURIENCODE; - flags |= NCURIPWD; -#ifdef FIX - ncurisetconstraints(url,state->constraint); -#endif - readurl = ncuribuild(url,NULL,".dmr.xml",NCURISVC); - if(readurl == NULL) - return THROW(NC_ENOMEM); - stat = NCD4_fetchurl_file(state->curl, readurl, state->data.ondiskfile, - &state->data.datasize, &lastmod); - nullfree(readurl); - if(stat == NC_NOERR) - state->data.dmrlastmodified = lastmod; - } - } + ncbytesclear(state->curl->packet); + stat = readpacket(state,url,state->curl->packet,NCD4_DMR,NCD4_FORMAT_XML,&resp->serial.httpcode,NULL); return THROW(stat); } int -NCD4_readDAP(NCD4INFO* state, int flags) +NCD4_readDAP(NCD4INFO* state, int flags, NCURI* url, NCD4response* resp) { int stat = NC_NOERR; - long lastmod = -1; - - if((flags & NCF_ONDISK) == 0) { - ncbytesclear(state->curl->packet); - stat = readpacket(state,state->uri,state->curl->packet,NCD4_DAP,NCD4_FORMAT_NONE,&lastmod); - if(stat) { - NCD4_seterrormessage(state->substrate.metadata, nclistlength(state->curl->packet), nclistcontents(state->curl->packet)); - goto done; - } else - state->data.daplastmodified = lastmod; - } else { /*((flags & NCF_ONDISK) != 0) */ - NCURI* url = state->uri; - int fileprotocol = (strcmp(url->protocol,"file")==0); - if(fileprotocol) { - stat = readfiletofile(state, url, NCD4_DAP, NCD4_FORMAT_NONE, state->data.ondiskfile, &state->data.datasize); - } else { - char* readurl = NULL; - int flags = 0; - if(!fileprotocol) flags |= NCURIQUERY; - flags |= NCURIENCODE; - flags |= NCURIPWD; -#ifdef FIX - ncurisetconstraints(url,state->constraint); -#endif - readurl = ncuribuild(url,NULL,".dap",NCURISVC); - if(readurl == NULL) - return THROW(NC_ENOMEM); - stat = NCD4_fetchurl_file(state->curl, readurl, state->data.ondiskfile, - &state->data.datasize, &lastmod); - nullfree(readurl); - if(stat == NC_NOERR) - state->data.daplastmodified = lastmod; - } + + ncbytesclear(state->curl->packet); + stat = readpacket(state,url,state->curl->packet,NCD4_DAP,NCD4_FORMAT_NONE,&resp->serial.httpcode,NULL); + if(stat) { + NCD4_seterrormessage(resp, nclistlength(state->curl->packet), nclistcontents(state->curl->packet)); + } else { + resp->raw.size = ncbyteslength(state->curl->packet); + resp->raw.memory = ncbytesextract(state->curl->packet); } -done: return THROW(stat); } @@ -150,7 +94,7 @@ dxxformat(int fxx, int dxx) } static int -readpacket(NCD4INFO* state, NCURI* url, NCbytes* packet, NCD4mode dxx, NCD4format fxx, long* lastmodified) +readpacket(NCD4INFO* state, NCURI* url, NCbytes* packet, NCD4mode dxx, NCD4format fxx, int* httpcodep, long* lastmodified) { int stat = NC_NOERR; int fileprotocol = 0; @@ -185,7 +129,7 @@ readpacket(NCD4INFO* state, NCURI* url, NCbytes* packet, NCD4mode dxx, NCD4forma gettimeofday(&time0,NULL); #endif } - stat = NCD4_fetchurl(curl,fetchurl,packet,lastmodified,&state->substrate.metadata->error.httpcode); + stat = NCD4_fetchurl(curl,fetchurl,packet,lastmodified,httpcodep); nullfree(fetchurl); if(stat) goto fail; if(FLAGSET(state->controls.flags,NCF_SHOWFETCH)) { @@ -207,37 +151,26 @@ fprintf(stderr,"readpacket: packet.size=%lu\n", return THROW(stat); } +#if 0 static int -readfiletofile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, FILE* stream, d4size_t* sizep) +readfromfile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, d4size_t* sizep) { int stat = NC_NOERR; - NCbytes* packet = ncbytesnew(); size_t len; - stat = readfile(state, uri, dxx, fxx, packet); + ncbytesclear(state->curl->packet); + stat = readfile(state, uri, dxx, fxx, state->curl->packet); #ifdef D4DEBUG fprintf(stderr,"readfiletofile: packet.size=%lu\n", - (unsigned long)ncbyteslength(packet)); + (unsigned long)ncbyteslength(state->curl->packet)); #endif if(stat != NC_NOERR) goto unwind; - len = nclistlength(packet); - if(stat == NC_NOERR) { - size_t written; - fseek(stream,0,SEEK_SET); - written = fwrite(ncbytescontents(packet),1,len,stream); - if(written != len) { -#ifdef D4DEBUG -fprintf(stderr,"readfiletofile: written!=length: %lu :: %lu\n", - (unsigned long)written,(unsigned long)len); -#endif - stat = NC_EIO; - } - } + len = nclistlength(state->curl->packet); if(sizep != NULL) *sizep = len; unwind: - ncbytesfree(packet); return THROW(stat); } +#endif static int readfile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, NCbytes* packet) @@ -365,12 +298,12 @@ readfileDAPDMR(NCD4INFO* state, const NCURI* uri, NCbytes* packet) /* Extract packet as error message; assume httpcode set */ int -NCD4_seterrormessage(NCD4meta* metadata, size_t len, char* msg) +NCD4_seterrormessage(NCD4response* resp, size_t len, char* msg) { - metadata->error.message = (char*)d4alloc(len+1); - if(metadata->error.message == NULL) + resp->error.message = (char*)d4alloc(len+1); + if(resp->error.message == NULL) return THROW(NC_ENOMEM); - memcpy(metadata->error.message,msg,len); - metadata->error.message[len] = '\0'; + memcpy(resp->error.message,msg,len); + resp->error.message[len] = '\0'; return THROW(NC_ENODATA); /* slight lie */ } diff --git a/libdap4/d4read.h b/libdap4/d4read.h index 826a2217cc..39ad8d97ce 100644 --- a/libdap4/d4read.h +++ b/libdap4/d4read.h @@ -7,6 +7,6 @@ #define D4READ_H extern int NCD4_readDMR(NCD4INFO*, int flags); -extern int NCD4_readDAP(NCD4INFO*, int flags); +extern int NCD4_readDAP(NCD4INFO*, int flags, NCURI* uri); #endif /*READ_H*/ diff --git a/libdap4/d4swap.c b/libdap4/d4swap.c index ed4c8dc1d8..8568fd328c 100644 --- a/libdap4/d4swap.c +++ b/libdap4/d4swap.c @@ -14,12 +14,12 @@ the incoming data to get the endianness correct. /* Forward */ -static int walkAtomicVar(NCD4meta*, NCD4node*, NCD4node*, NCD4offset* offset); -static int walkOpaqueVar(NCD4meta*,NCD4node*, NCD4node*, NCD4offset* offset); -static int walkStructArray(NCD4meta*,NCD4node*, NCD4node*, NCD4offset* offset); -static int walkStruct(NCD4meta*, NCD4node*, NCD4node*, NCD4offset* offset); -static int walkSeqArray(NCD4meta*, NCD4node*, NCD4node*, NCD4offset* offset); -static int walkSeq(NCD4meta*,NCD4node*, NCD4node*, NCD4offset* offset); +static int walkAtomicVar(NCD4response*, NCD4node*, NCD4node*, NCD4offset* offset,int doswap); +static int walkOpaqueVar(NCD4response*,NCD4node*, NCD4node*, NCD4offset* offset,int doswap); +static int walkStructArray(NCD4response*,NCD4node*, NCD4node*, NCD4offset* offset,int doswap); +static int walkStruct(NCD4response*, NCD4node*, NCD4node*, NCD4offset* offset,int doswap); +static int walkSeqArray(NCD4response*, NCD4node*, NCD4node*, NCD4offset* offset,int doswap); +static int walkSeq(NCD4response*,NCD4node*, NCD4node*, NCD4offset* offset,int doswap); /**************************************************/ @@ -28,43 +28,39 @@ Assumes that compiler->swap is true; does necessary byte swapping. */ int -NCD4_swapdata(NCD4meta* compiler, NClist* topvars) +NCD4_swapdata(NCD4response* resp, NCD4node* var, int doswap) { int ret = NC_NOERR; - int i; NCD4offset* offset = NULL; - offset = BUILDOFFSET(compiler->serial.dap,compiler->serial.dapsize); - for(i=0;iserial.dap,resp->serial.dapsize); OFFSET2BLOB(var->data.dap4data,offset); switch (var->subsort) { default: - if((ret=walkAtomicVar(compiler,var,var,offset))) goto done; + if((ret=walkAtomicVar(resp,var,var,offset,doswap))) goto done; break; case NC_OPAQUE: /* The only thing we need to do is swap the counts */ - if((ret=walkOpaqueVar(compiler,var,var,offset))) goto done; + if((ret=walkOpaqueVar(resp,var,var,offset,doswap))) goto done; break; case NC_STRUCT: - if((ret=walkStructArray(compiler,var,var,offset))) goto done; + if((ret=walkStructArray(resp,var,var,offset,doswap))) goto done; break; case NC_SEQ: - if((ret=walkSeqArray(compiler,var,var,offset))) goto done; + if((ret=walkSeqArray(resp,var,var,offset,doswap))) goto done; break; } var->data.dap4data.size = DELTA(offset,var->data.dap4data.memory); /* skip checksum, if there is one */ - if(compiler->controller->data.inferredchecksumming) + if(resp->inferredchecksumming) INCR(offset,CHECKSUMSIZE); - } done: if(offset) free(offset); return THROW(ret); } static int -walkAtomicVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* offset) +walkAtomicVar(NCD4response* resp, NCD4node* topvar, NCD4node* var, NCD4offset* offset, int doswap) { int ret = NC_NOERR; d4size_t i; @@ -87,7 +83,7 @@ walkAtomicVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o } else { /*(typesize > 1)*/ for(i=0;ioffset; - if(compiler->swap) { + if(doswap) { switch (typesize) { case 2: swapinline16(sp); break; case 4: swapinline32(sp); break; @@ -102,7 +98,7 @@ walkAtomicVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o COUNTERTYPE count; for(i=0;iswap) + if(doswap) swapinline64(offset); count = GETCOUNTER(offset); SKIPCOUNTER(offset); @@ -114,7 +110,7 @@ walkAtomicVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o } static int -walkOpaqueVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* offset) +walkOpaqueVar(NCD4response* resp, NCD4node* topvar, NCD4node* var, NCD4offset* offset, int doswap) { int ret = NC_NOERR; d4size_t i; @@ -125,7 +121,7 @@ walkOpaqueVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o for(i=0;iswap) + if(doswap) swapinline64(offset); count = GETCOUNTER(offset); SKIPCOUNTER(offset); @@ -135,7 +131,7 @@ walkOpaqueVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o } static int -walkStructArray(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* offset) +walkStructArray(NCD4response* resp, NCD4node* topvar, NCD4node* var, NCD4offset* offset, int doswap) { int ret = NC_NOERR; d4size_t i; @@ -144,7 +140,7 @@ walkStructArray(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset for(i=0;ibasetype; switch (fieldbase->subsort) { default: - if((ret=walkAtomicVar(compiler,topvar,field,offset))) goto done; + if((ret=walkAtomicVar(resp,topvar,field,offset,doswap))) goto done; break; case NC_OPAQUE: /* The only thing we need to do is swap the counts */ - if((ret=walkOpaqueVar(compiler,topvar,field,offset))) goto done; + if((ret=walkOpaqueVar(resp,topvar,field,offset,doswap))) goto done; break; case NC_STRUCT: - if((ret=walkStructArray(compiler,topvar,field,offset))) goto done; + if((ret=walkStructArray(resp,topvar,field,offset,doswap))) goto done; break; case NC_SEQ: - if((ret=walkSeqArray(compiler,topvar,field,offset))) goto done; + if((ret=walkSeqArray(resp,topvar,field,offset,doswap))) goto done; break; } } @@ -182,7 +178,7 @@ walkStruct(NCD4meta* compiler, NCD4node* topvar, NCD4node* structtype, NCD4offse } static int -walkSeqArray(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* offset) +walkSeqArray(NCD4response* resp, NCD4node* topvar, NCD4node* var, NCD4offset* offset, int doswap) { int ret = NC_NOERR; d4size_t i; @@ -195,7 +191,7 @@ walkSeqArray(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* of for(i=0;iswap) + if(doswap) swapinline64(offset); recordcount = GETCOUNTER(offset); SKIPCOUNTER(offset); @@ -225,17 +221,17 @@ walkSeq(NCD4meta* compiler, NCD4node* topvar, NCD4node* vlentype, NCD4offset* of for(i=0;isubsort) { default: /* atomic basetype */ - if((ret=walkAtomicVar(compiler,topvar,basetype,offset))) goto done; + if((ret=walkAtomicVar(resp,topvar,basetype,offset,doswap))) goto done; break; case NC_OPAQUE: - if((ret=walkOpaqueVar(compiler,topvar,basetype,offset))) goto done; + if((ret=walkOpaqueVar(resp,topvar,basetype,offset,doswap))) goto done; break; case NC_STRUCT: /* We can treat each record like a structure instance */ - if((ret=walkStruct(compiler,topvar,basetype,offset))) goto done; + if((ret=walkStruct(resp,topvar,basetype,offset,doswap))) goto done; break; case NC_SEQ: - if((ret=walkSeq(compiler,topvar,basetype,offset))) goto done; + if((ret=walkSeq(resp,topvar,basetype,offset,doswap))) goto done; break; } } diff --git a/libdap4/d4util.c b/libdap4/d4util.c index 4156519305..d1ba7f5828 100644 --- a/libdap4/d4util.c +++ b/libdap4/d4util.c @@ -107,44 +107,45 @@ NCD4_makeFQN(NCD4node* node) { char* fqn = NULL; char* escaped; - int i; - NCD4node* g = node; - NClist* path = nclistnew(); - size_t estimate; - - for(estimate=0;g != NULL;g=g->container) { - estimate += strlen(g->name); - nclistinsert(path,0,g); + NCbytes* buf = ncbytesnew(); + NClist* grps = nclistnew(); + NClist* parts = nclistnew(); + NCD4node* n; + size_t i; + + /* collect all the non-groups */ + for(n=node;n;n=n->container) { + if(ISGROUP(n->sort)) + nclistinsert(grps,0,n); /* keep the correct order of groups */ + else + nclistinsert(parts,0,n); } - estimate = (estimate*2) + 2*nclistlength(path); - estimate++; /*strlcat nul*/ - fqn = (char*)malloc(estimate+1); - if(fqn == NULL) goto done; - fqn[0] = '\0'; - /* Create the group-based fqn prefix */ - /* start at 1 to avoid dataset */ - for(i=1;isort != NCD4_GROUP) break; + + /* Build grp prefix of the fqn */ + for(i=1;iname); - if(escaped == NULL) {free(fqn); fqn = NULL; goto done;} - strlcat(fqn,"/",estimate); - strlcat(fqn,escaped,estimate); + escaped = backslashEscape(n->name); + if(escaped == NULL) goto done; + ncbytescat(buf,"/"); + ncbytescat(buf,escaped); free(escaped); } /* Add in the final name part (if not group) */ - if(i < nclistlength(path)) { - int last = nclistlength(path)-1; - NCD4node* n = (NCD4node*)nclistget(path,last); - char* name = NCD4_makeName(n,"."); - strlcat(fqn,"/",estimate); - strlcat(fqn,name,estimate); - nullfree(name); + for(i=0;iname); + if(escaped == NULL) goto done; + ncbytescat(buf,(i==0?"/":".")); + ncbytescat(buf,escaped); + free(escaped); } + fqn = ncbytesextract(buf); done: - nclistfree(path); + ncbytesfree(buf); + nclistfree(grps); + nclistfree(parts); return fqn; } @@ -446,11 +447,9 @@ NCD4_getheader(void* p, NCD4HDR* hdr, int hostlittleendian) } void -NCD4_reporterror(NCD4INFO* state) +NCD4_reporterror(NCD4response* resp, NCURI* uri) { - NCD4meta* meta = state->substrate.metadata; char* u = NULL; - if(meta == NULL) return; - u = ncuribuild(state->uri,NULL,NULL,NCURIALL); - fprintf(stderr,"***FAIL: url=%s httpcode=%d errmsg->\n%s\n",u,meta->error.httpcode,meta->error.message); + u = ncuribuild(uri,NULL,NULL,NCURIALL); + fprintf(stderr,"***FAIL: url=%s httpcode=%d errmsg->\n%s\n",u,resp->serial.httpcode,resp->error.message); } diff --git a/libdap4/d4util.h b/libdap4/d4util.h index b6cbde8f07..38cda6a179 100644 --- a/libdap4/d4util.h +++ b/libdap4/d4util.h @@ -17,9 +17,12 @@ */ typedef unsigned long long d4size_t; -/* Define a counted memory marker */ +/* Define a (size, memory) pair */ typedef struct D4blob {d4size_t size; void* memory;} D4blob; +/* Empty blob constant */ +#define NULLBLOB(blob) {blob.size = 0; blob.memory = NULL;} + #define OFFSET2BLOB(blob,offset) do{(blob).size = ((offset)->limit - (offset)->base); (blob).memory = (offset)->base; }while(0) #define BLOB2OFFSET(offset,blob) do{\ (offset)->base = (blob).memory; \ diff --git a/libdap4/d4varx.c b/libdap4/d4varx.c index 9a708a5d5f..b098855f83 100644 --- a/libdap4/d4varx.c +++ b/libdap4/d4varx.c @@ -11,7 +11,8 @@ #include "d4odom.h" /* Forward */ -static int getvarx(int ncid, int varid, NCD4INFO**, NCD4node** varp, nc_type* xtypep, size_t*, nc_type* nc4typep, size_t*); +static int getvarx(int gid, int varid, NCD4INFO**, NCD4node** varp, nc_type* xtypep, size_t*, nc_type* nc4typep, size_t*); +static int mapvars(NCD4meta* dapmeta, NCD4meta* dmrmeta, int inferredchecksumming); int NCD4_get_vara(int ncid, int varid, @@ -22,11 +23,11 @@ NCD4_get_vara(int ncid, int varid, int ret; /* TODO: optimize since we know stride is 1 */ ret = NCD4_get_vars(ncid,varid,start,edges,NC_stride_one,value,memtype); - return ret; + return THROW(ret); } int -NCD4_get_vars(int ncid, int varid, +NCD4_get_vars(int gid, int varid, const size_t *start, const size_t *edges, const ptrdiff_t* stride, void *memoryin, nc_type xtype) { @@ -43,14 +44,14 @@ NCD4_get_vars(int ncid, int varid, int rank; size_t dimsizes[NC_MAX_VAR_DIMS]; d4size_t dimproduct; - size_t dstcount; + size_t dstpos; NCD4offset* offset = NULL; - /* Get netcdf type info */ - if((ret=getvarx(ncid, varid, &info, &ncvar, &xtype, &xsize, &nc4type, &nc4size))) + /* Get netcdf var metadata and data */ + if((ret=getvarx(gid, varid, &info, &ncvar, &xtype, &xsize, &nc4type, &nc4size))) {goto done;} - meta = info->substrate.metadata; + meta = info->dmrmetadata; nctype = ncvar->basetype; rank = nclistlength(ncvar->dims); blobs = nclistnew(); @@ -74,18 +75,18 @@ NCD4_get_vars(int ncid, int varid, odom = d4odom_new(rank,start,edges,stride,dimsizes); else odom = d4scalarodom_new(); - dstcount = 0; /* We always write into dst starting at position 0*/ - for(;d4odom_more(odom);dstcount++) { + dstpos = 0; /* We always write into dst starting at position 0*/ + for(;d4odom_more(odom);dstpos++) { void* xpos; void* dst; - d4size_t count; + d4size_t pos; - count = d4odom_next(odom); - if(count >= dimproduct) { + pos = d4odom_next(odom); + if(pos >= dimproduct) { ret = THROW(NC_EINVALCOORDS); goto done; } - xpos = ((char*)memoryin)+(xsize * dstcount); /* ultimate destination */ + xpos = ((char*)memoryin)+(xsize * dstpos); /* ultimate destination */ /* We need to compute the offset in the dap4 data of this instance; for fixed size types, this is easy, otherwise we have to walk the variable size type @@ -95,16 +96,16 @@ NCD4_get_vars(int ncid, int varid, offset = NULL; offset = BUILDOFFSET(NULL,0); BLOB2OFFSET(offset,ncvar->data.dap4data); - /* Move offset to the count'th element of the array */ + /* Move offset to the pos'th element of the array */ if(nctype->meta.isfixedsize) { - INCR(offset,(dapsize*count)); + INCR(offset,(dapsize*pos)); } else { - /* We have to walk to the count'th location in the data */ - if((ret=NCD4_moveto(meta,ncvar,count,offset))) + /* We have to walk to the pos'th location in the data */ + if((ret=NCD4_moveto(meta,ncvar,pos,offset))) {goto done;} } dst = instance; - if((ret=NCD4_fillinstance(meta,nctype,offset,&dst,blobs))) + if((ret=NCD4_movetoinstance(meta,nctype,offset,&dst,blobs))) {goto done;} if(xtype == nc4type) { /* We can just copy out the data */ @@ -132,45 +133,29 @@ NCD4_get_vars(int ncid, int varid, } static int -getvarx(int ncid, int varid, NCD4INFO** infop, NCD4node** varp, +getvarx(int gid, int varid, NCD4INFO** infop, NCD4node** varp, nc_type* xtypep, size_t* xsizep, nc_type* nc4typep, size_t* nc4sizep) { int ret = NC_NOERR; - NC* ncp; - NCD4INFO* info; - NCD4meta* meta; - NCD4node* group; - NCD4node* var; - NCD4node* type; + NC* ncp = NULL; + NCD4INFO* info = NULL; + NCD4meta* dmrmeta = NULL; + NCD4node* group = NULL; + NCD4node* var = NULL; + NCD4node* type = NULL; nc_type xtype, actualtype; size_t instancesize, xsize; + NCURI* ceuri = NULL; /* Constrained uri */ + NCD4meta* dapmeta = NULL; + NCD4response* dapresp = NULL; - if((ret = NC_check_id(ncid, (NC**)&ncp)) != NC_NOERR) + if((ret = NC_check_id(gid, (NC**)&ncp)) != NC_NOERR) goto done; info = getdap(ncp); - meta = info->substrate.metadata; - - /* If the data has not already been read and processed, then do so. */ - if(meta->serial.dap == NULL) { - size_t len = 0; - void* content = NULL; - /* (Re)Build the meta data; sets serial.rawdata */ - NCD4_resetMeta(info->substrate.metadata); - meta->controller = info; - meta->ncid = info->substrate.nc4id; /* Transfer netcdf ncid */ - - if((ret=NCD4_readDAP(info, info->controls.flags.flags))) goto done; - len = ncbyteslength(info->curl->packet); - content = ncbytesextract(info->curl->packet); - NCD4_resetSerial(&meta->serial, len, content); - /* Process the data part */ - if((ret=NCD4_dechunk(meta))) goto done; - if((ret = NCD4_processdata(info->substrate.metadata))) goto done; - } - - if((ret = NCD4_findvar(ncp,ncid,varid,&var,&group))) goto done; + dmrmeta = info->dmrmetadata; + if((ret = NCD4_findvar(ncp,gid,varid,&var,&group))) goto done; type = var->basetype; actualtype = type->meta.id; instancesize = type->meta.memsize; @@ -189,6 +174,46 @@ getvarx(int ncid, int varid, NCD4INFO** infop, NCD4node** varp, else xsize = instancesize; + /* If we already have valid data, then just return */ + if(var->data.valid) goto validated; + + /* Ok, we need to read from the server */ + + /* Add the variable to the URI, unless the URI is already constrained or is unconstrainable */ + ceuri = ncuriclone(info->dmruri); + /* append the request for a specific variable */ + if(ncuriquerylookup(ceuri,DAP4CE) == NULL && !FLAGSET(info->controls.flags,NCF_UNCONSTRAINABLE)) { + ncurisetquerykey(ceuri,strdup("dap4.ce"),NCD4_makeFQN(var)); + } + + /* Read and process the data */ + + /* Setup the meta-data for the DAP */ + if((ret=NCD4_newMeta(info,&dapmeta))) goto done; + if((ret=NCD4_newResponse(info,&dapresp))) goto done; + dapresp->mode = NCD4_DAP; + nclistpush(info->responses,dapresp); + if((ret=NCD4_readDAP(info, info->controls.flags.flags, ceuri, dapresp))) goto done; + + /* Extract DMR and dechunk the data part */ + if((ret=NCD4_dechunk(dapresp))) goto done; + + /* Process the dmr part */ + if((ret=NCD4_parse(dapmeta,dapresp,1))) goto done; + + /* See if we are checksumming */ + if((ret=NCD4_inferChecksums(dapmeta,dapresp))) goto done; + + /* connect variables and corresponding dap data */ + if((ret = NCD4_parcelvars(dapmeta,dapresp))) goto done; + + /* Process checksums and byte-order swapping */ + if((ret = NCD4_processdata(dapmeta,dapresp))) goto done; + + /* Transfer and process the data */ + if((ret = mapvars(dapmeta,dmrmeta,dapresp->inferredchecksumming))) goto done; + +validated: /* Return relevant info */ if(infop) *infop = info; if(xtypep) *xtypep = xtype; @@ -197,8 +222,96 @@ getvarx(int ncid, int varid, NCD4INFO** infop, NCD4node** varp, if(nc4sizep) *nc4sizep = instancesize; if(varp) *varp = var; done: - if(meta->error.message != NULL) - NCD4_reporterror(info); /* Make sure the user sees this */ + if(dapmeta) NCD4_reclaimMeta(dapmeta); + ncurifree(ceuri); + if(dapresp != NULL && dapresp->error.message != NULL) + NCD4_reporterror(dapresp,ceuri); /* Make sure the user sees this */ return THROW(ret); } +#if 0 +static NCD4node* +findbyname(const char* name, NClist* nodes) +{ + int i; + for(i=0;iname)==0) + return node; + } + return NULL; +} +#endif + +static int +matchvar(NCD4meta* dmrmeta, NCD4node* dapvar, NCD4node** dmrvarp) +{ + int i,ret = NC_NOERR; + NCD4node* x = NULL; + NClist* dappath = nclistnew(); + NClist* dmrpath = nclistnew(); /* compute path for this dmr var */ + int found = 0; + NCD4node* match = NULL; + + /* Capture the dap path starting at root and ending at the dapvar (assumed to be topvar) */ + for(x=dapvar;x != NULL;x=x->container) nclistinsert(dappath,0,x); + /* Iterate over all variable nodes to find matching one */ + for(i=0;iallnodes);i++) { + NCD4node* node = (NCD4node*)nclistget(dmrmeta->allnodes,i); + if(ISVAR(node->sort) && strcmp(node->name,dapvar->name)==0) { /* possible candidate */ + int j; + found = 0; + nclistclear(dmrpath); + for(x=node;x != NULL;x=x->container) nclistinsert(dmrpath,0,x); + if(nclistlength(dmrpath) == nclistlength(dappath)) { /* same length paths */ + /* compare paths: name and sort */ + for(found=1,j=0;jsort != pdap->sort || strcmp(pdmr->name,pdap->name) != 0) + {found = 0; break;} + } + if(found) {match = node; break;} + } + } + } + if(!found) {ret = NC_EINVAL; goto done;} + if(dmrvarp) *dmrvarp = match; + +done: + nclistfree(dappath); + nclistfree(dmrpath); + return THROW(ret); +} + +/* +Map each toplevel dap var to the corresponding +toplevel dmr var and transfer necessary info; +*/ + +static int +mapvars(NCD4meta* dapmeta, NCD4meta* dmrmeta, int inferredchecksumming) +{ + int i, ret = NC_NOERR; + NCD4node* daproot = dapmeta->root; + NClist* daptop = NULL; /* top variables in dap tree */ + + /* Get top level variables from the dap node tree */ + daptop = nclistnew(); + NCD4_getToplevelVars(dapmeta,daproot,daptop); + + /* Match up the dap top variables with the dmr top variables */ + for(i=0;idata = dapvar->data; + memset(&dapvar->data,0,sizeof(NCD4vardata)); + dmrvar->data.valid = 1; + } + +done: + nclistfree(daptop); + return THROW(ret); +} diff --git a/libdap4/ncd4.h b/libdap4/ncd4.h index 058e40ff7c..286406d771 100644 --- a/libdap4/ncd4.h +++ b/libdap4/ncd4.h @@ -77,7 +77,6 @@ EXTERNL int dsp_open(const char* path, ND4dsp** dspp); /* From d4http.c */ EXTERNL long NCD4_fetchhttpcode(CURL* curl); -EXTERNL int NCD4_fetchurl_file(CURL* curl, const char* url, FILE* stream, d4size_t* sizep, long* filetime); EXTERNL int NCD4_fetchurl(CURL* curl, const char* url, NCbytes* buf, long* filetime, int* httpcode); EXTERNL int NCD4_curlopen(CURL** curlp); EXTERNL void NCD4_curlclose(CURL* curl); @@ -85,15 +84,16 @@ EXTERNL int NCD4_fetchlastmodified(CURL* curl, char* url, long* filetime); EXTERNL int NCD4_ping(const char* url); /* From d4read.c */ -EXTERNL int NCD4_readDMR(NCD4INFO* state, int flags); -EXTERNL int NCD4_readDAP(NCD4INFO* state, int flags); -EXTERNL int NCD4_seterrormessage(NCD4meta* metadata, size_t len, char* msg); +EXTERNL int NCD4_readDMR(NCD4INFO* state, int flags, NCURI* url, NCD4response*); +EXTERNL int NCD4_readDAP(NCD4INFO* state, int flags, NCURI* ceuri, NCD4response*); +EXTERNL int NCD4_seterrormessage(NCD4response*, size_t len, char* msg); /* From d4parser.c */ -EXTERNL int NCD4_parse(NCD4meta*); +EXTERNL int NCD4_parse(NCD4meta*, NCD4response*, int dapparse); EXTERNL NCD4node* NCD4_findAttr(NCD4node* container, const char* attrname); EXTERNL NCD4node* NCD4_groupFor(NCD4node* node); EXTERNL int NCD4_defineattr(NCD4meta* meta, NCD4node* parent, const char* aname, const char* typename, NCD4node** attrp); +EXTERNL void NCD4_setdebuglevel(NCD4parser*,int); /* From d4printer.c */ EXTERNL int NCD4_print(NCD4meta*, NCbytes* output); @@ -104,29 +104,33 @@ EXTERNL void NCD4_attachraw(NCD4meta*, size_t size, void* rawdata); EXTERNL void NCD4_reclaimMeta(NCD4meta*); EXTERNL void NCD4_resetMeta(NCD4meta*); EXTERNL void reclaimNode(NCD4node* node); -EXTERNL void NCD4_setdebuglevel(NCD4meta*,int); EXTERNL int NCD4_metabuild(NCD4meta*, int ncid); EXTERNL size_t NCD4_computeTypeSize(NCD4meta*, NCD4node* type); EXTERNL int NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node** grpp); +EXTERNL char* NCD4_getVarFQN(NCD4node* var, const char* tail); + /* From d4chunk.c */ -EXTERNL int NCD4_dechunk(NCD4meta*); -EXTERNL int NCD4_infermode(NCD4meta* meta); +EXTERNL int NCD4_dechunk(NCD4response*); +EXTERNL int NCD4_infermode(NCD4response*); struct NCD4serial; EXTERNL void NCD4_resetSerial(struct NCD4serial* serial, size_t rawsize, void* rawdata); +EXTERNL void NCD4_moveSerial(struct NCD4serial* serial, struct NCD4serial* dst); /* From d4swap.c */ -EXTERNL int NCD4_swapdata(NCD4meta*, NClist* topvars); +EXTERNL int NCD4_swapdata(NCD4response*, NCD4node* topvar, int doswap); /* From d4fix.c */ -EXTERNL int NCD4_delimit(NCD4meta*, NCD4node* var, NCD4offset* offset); +EXTERNL int NCD4_delimit(NCD4meta*, NCD4node* var, NCD4offset* offset, int inferredchecksumming); EXTERNL int NCD4_moveto(NCD4meta*, NCD4node* var, d4size_t count, NCD4offset* offset); EXTERNL int NCD4_toposort(NCD4meta*); /* From d4data.c */ -EXTERNL int NCD4_processdata(NCD4meta*); -EXTERNL int NCD4_fillinstance(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs); +EXTERNL int NCD4_parcelvars(NCD4meta* meta, NCD4response* resp); +EXTERNL int NCD4_processdata(NCD4meta*,NCD4response*); +EXTERNL int NCD4_movetoinstance(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs); EXTERNL int NCD4_getToplevelVars(NCD4meta* meta, NCD4node* group, NClist* toplevel); +EXTERNL int NCD4_inferChecksums(NCD4meta* meta, NCD4response* resp); /* From d4util.c */ EXTERNL d4size_t NCD4_dimproduct(NCD4node* node); @@ -141,7 +145,7 @@ EXTERNL char* NCD4_deescape(const char* esc); EXTERNL char* NCD4_entityescape(const char* s); EXTERNL size_t NCD4_elidenuls(char* s, size_t slen); EXTERNL void* NCD4_getheader(void* p, NCD4HDR* hdr, int hostlittleendian); -EXTERNL void NCD4_reporterror(NCD4INFO* state); +EXTERNL void NCD4_reporterror(NCD4response*, NCURI* uri); /* From d4dump.c */ EXTERNL void NCD4_dumpbytes(size_t size, const void* data0, int swap); @@ -163,7 +167,15 @@ EXTERNL int NCD4_convert(nc_type srctype, nc_type dsttype, char* memory0, char* /* d4file.c */ EXTERNL int NCD4_readDMRorDAP(NCD4INFO* d4info, NCD4mode mode); EXTERNL void NCD4_applyclientfragmentcontrols(NCD4INFO* d4info); -EXTERNL void NCD4_applyclientquerycontrols(NCD4INFO* d4info); +EXTERNL void NCD4_applychecksumcontrols(NCD4INFO* d4info, NCD4response*); +EXTERNL int NCD4_newInfo(NCD4INFO** d4infop); +EXTERNL void NCD4_reclaimInfo(NCD4INFO* d4info); +EXTERNL void NCD4_resetInfoforRead(NCD4INFO* d4info); +EXTERNL int NCD4_newResponse(NCD4INFO*,NCD4response** respp); +EXTERNL void NCD4_reclaimResponse(NCD4response* d4resp); +EXTERNL void NCD4_resetInfoForRead(NCD4INFO* d4info); +EXTERNL int NCD4_newMeta(NCD4INFO*,NCD4meta**); +EXTERNL void NCD4_reclaimMeta(NCD4meta*); /* ncd4dispatch.c */ struct NC_reservedatt; /*forward*/ diff --git a/libdap4/ncd4dispatch.c b/libdap4/ncd4dispatch.c index 9d9cf08a1c..e7eefb40a4 100644 --- a/libdap4/ncd4dispatch.c +++ b/libdap4/ncd4dispatch.c @@ -388,11 +388,11 @@ NCD4_inq_attname(int ncid, int varid, int attnum, char* name) const NC_reservedatt* rsvp = NULL; if((ret = NC_check_id(ncid, (NC**)&ncp)) != NC_NOERR) return (ret); + substrateid = makenc4id(ncp,ncid); + ret = nc_inq_attname(substrateid, varid, attnum, name); /* Is this a reserved attribute name? */ if(name && (rsvp = NCD4_lookupreserved(name))) return NC_EATTMETA; - substrateid = makenc4id(ncp,ncid); - ret = nc_inq_attname(substrateid, varid, attnum, name); return (ret); } @@ -847,7 +847,7 @@ NCD4_inq_dim(int ncid, int dimid, char* name, size_t* lenp) if((ret = NC_check_id(ncid, (NC**)&ncp)) != NC_NOERR) goto done; info = (NCD4INFO*)ncp->dispatchdata; - meta = info->substrate.metadata; + meta = info->dmrmetadata; /* Locate the dimension specified by dimid */ for(i=0;iallnodes);i++) { @@ -871,16 +871,15 @@ static int ncd4_get_att_reserved(NC* ncp, int ncid, int varid, const char* name, void* value, nc_type t, const NC_reservedatt* rsvp) { int ret = NC_NOERR; - NCD4INFO* info = (NCD4INFO*)(ncp->dispatchdata); - NCD4meta* meta = info->substrate.metadata; NCD4node* var = NULL; + if((ret=NCD4_findvar(ncp,ncid,varid,&var,NULL))) goto done; + if(strcmp(rsvp->name,D4CHECKSUMATTR)==0) { unsigned int* ip = (unsigned int*)value; if(varid == NC_GLOBAL) {ret = NC_EBADID; goto done;} if(t != NC_UINT) {ret = NC_EBADTYPE; goto done;} - if((ret=NCD4_findvar(ncp,ncid,varid,&var,NULL))) goto done; if(var->data.checksumattr == 0) {ret = NC_ENOTATT; goto done;} *ip = (var->data.remotechecksum); @@ -889,7 +888,7 @@ ncd4_get_att_reserved(NC* ncp, int ncid, int varid, const char* name, void* valu if(varid != NC_GLOBAL) {ret = NC_EBADID; goto done;} if(t != NC_INT) {ret = NC_EBADTYPE; goto done;} - *ip = (meta->serial.remotelittleendian?1:0); + *ip = (var->data.response->remotelittleendian?1:0); } done: return THROW(ret); @@ -925,7 +924,7 @@ static int globalinit(void) { int stat = NC_NOERR; - return stat; + return THROW(stat); } /**************************************************/ diff --git a/libdap4/ncd4types.h b/libdap4/ncd4types.h index 1f84a0f569..7bed38a845 100644 --- a/libdap4/ncd4types.h +++ b/libdap4/ncd4types.h @@ -50,6 +50,8 @@ typedef struct NCD4node NCD4node; typedef struct NCD4params NCD4params; typedef struct NCD4HDR NCD4HDR; typedef struct NCD4offset NCD4offset; +typedef struct NCD4vardata NCD4vardata; +typedef struct NCD4response NCD4response; /* Define the NCD4HDR flags */ /* Header flags */ @@ -228,19 +230,20 @@ struct NCD4node { int isfixedsize; /* sort == NCD4_TYPE; Is this a fixed size (recursively) type? */ d4size_t dapsize; /* size of the type as stored in the dap data; will, as a rule, be same as memsize only for types <= NC_UINT64 */ - nc_type cmpdid; /*netcdf id for the compound type created for seq type */ + nc_type cmpdid; /* netcdf id for the compound type created for seq type */ size_t memsize; /* size of a memory instance without taking dimproduct into account, but taking compound alignment into account */ d4size_t offset; /* computed structure field offset in memory */ size_t alignment; /* computed structure field alignment in memory */ } meta; - struct { /* Data compilation info */ - int flags; /* See d4data for actual flags */ + struct NCD4vardata { /* Data compilation info */ + int valid; /* 1 => this contains valid data */ D4blob dap4data; /* offset and start pos for this var's data in serialization */ - unsigned int remotechecksum; /* checksum from data as sent by server */ - unsigned int localchecksum; /* toplevel variable checksum as computed by client */ - int checksumattr; /* 1=> _DAP4_Checksum_CRC32 is defined */ - int attrchecksum; /* _DAP4_Checksum_CRC32 value */ + unsigned remotechecksum; /* toplevel per-variable checksum contained in the data */ + unsigned localchecksum; /* toplevel variable checksum as computed by client */ + int checksumattr; /* 1 => _DAP4_Checksum_CRC32 is defined */ + unsigned attrchecksum; /* _DAP4_Checksum_CRC32 value; this is the checksum computed by server */ + NCD4response* response; /* Response from which this data is taken */ } data; struct { /* Track netcdf-4 conversion info */ int isvlen; /* _edu.ucar.isvlen */ @@ -253,36 +256,13 @@ struct NCD4node { } nc4; }; -/* Tracking info about the serialized input before and after de-chunking */ -typedef struct NCD4serial { - size_t rawsize; /* |rawdata| */ - void* rawdata; - size_t dapsize; /* |dap|; this is transient */ - void* dap; /* pointer into rawdata where dap data starts */ - char* dmr;/* copy of dmr */ - char* errdata; /* null || error chunk (null terminated) */ - int httpcode; /* returned from last request */ - int hostlittleendian; /* 1 if the host is little endian */ - int remotelittleendian; /* 1 if the packet says data is little endian */ -} NCD4serial; - -/* This will be passed out of the parse */ +/* DMR information from a response; this will be passed out of the parse */ struct NCD4meta { NCD4INFO* controller; int ncid; /* root ncid of the substrate netcdf-4 file; warning: copy of NCD4Info.substrate.nc4id */ NCD4node* root; - NCD4mode mode; /* Are we reading DMR (only) or DAP (includes DMR) */ NClist* allnodes; /*list*/ - struct Error { /* Content of any error response */ - char* parseerror; - int httpcode; - char* message; - char* context; - char* otherinfo; - } error; - int debuglevel; - NCD4serial serial; int swap; /* 1 => swap data */ /* Define some "global" (to a DMR) data */ NClist* groupbyid; /* NClist indexed by groupid >> 16; this is global */ @@ -292,9 +272,12 @@ struct NCD4meta { }; typedef struct NCD4parser { + NCD4INFO* controller; char* input; int debuglevel; + int dapparse; /* 1 => we are parsing the DAP DMR */ NCD4meta* metadata; + NCD4response* response; /* Capture useful subsets of dataset->allnodes */ NClist* types; /*list; user-defined types only*/ NClist* dims; /*list*/ @@ -303,6 +286,32 @@ typedef struct NCD4parser { NCD4node* dapopaque; /* Single non-fixed-size opaque type */ } NCD4parser; +/* Capture all the relevant info about the response to a server request */ +struct NCD4response { /* possibly processed response from a query */ + NCD4INFO* controller; /* controlling connection */ + D4blob raw; /* complete response in memory */ + int querychecksumming; /* 1 => user specified dap4.ce value */ + int attrchecksumming; /* 1=> _DAP4_Checksum_CRC32 is defined for at least one variable */ + int inferredchecksumming; /* 1 => either query checksum || att checksum */ + int checksumignore; /* 1 => assume checksum, but do not validate */ + int remotelittleendian; /* 1 if the packet says data is little endian */ + NCD4mode mode; /* Are we reading DMR (only) or DAP (includes DMR) */ + struct NCD4serial { + size_t dapsize; /* |dap|; this is transient */ + void* dap; /* pointer into raw where dap data starts */ + char* dmr;/* copy of dmr */ + char* errdata; /* null || error chunk (null terminated) */ + int httpcode; /* returned from last request */ + } serial; /* Dechunked and processed DAP part of the response */ + struct Error { /* Content of any error response */ + char* parseerror; + int httpcode; + char* message; + char* context; + char* otherinfo; + } error; +}; + /**************************************************/ /* Curl info */ @@ -328,28 +337,20 @@ struct NCD4curl { struct NCD4INFO { NC* controller; /* Parent instance of NCD4INFO */ - char* rawurltext; /* as given to ncd4_open */ - char* urltext; /* as modified by ncd4_open */ - NCURI* uri; /* parse of rawuritext */ + char* rawdmrurltext; /* as given to ncd4_open */ + char* dmrurltext; /* as modified by ncd4_open */ + NCURI* dmruri; /* parse of rawuritext */ NCD4curl* curl; int inmemory; /* store fetched data in memory? */ - struct { - char* memory; /* allocated memory if ONDISK is not set */ - char* ondiskfilename; /* If ONDISK is set */ - FILE* ondiskfile; /* ditto */ - d4size_t datasize; /* size on disk or in memory */ - long dmrlastmodified; - long daplastmodified; - int querychecksumming; /* 1 => user specified dap4.ce value */ - int attrchecksumming; /* 1=> _DAP4_Checksum_CRC32 is defined for at least one variable */ - int inferredchecksumming; /* 1 => either query checksum || att checksum */ - int checksumignore; /* 1 => assume checksum, but do not validate */ - } data; + NCD4meta* dmrmetadata; /* Independent of responses */ + NClist* responses; /* NClist all responses from this curl handle */ + struct { /* Properties that are per-platform */ + int hostlittleendian; /* 1 if the host is little endian */ + } platform; struct { int realfile; /* 1 => we created actual temp file */ char* filename; /* of the substrate file */ int nc4id; /* substrate nc4 file ncid used to hold metadata; not same as external id */ - NCD4meta* metadata; } substrate; struct { NCCONTROLS flags; @@ -362,6 +363,7 @@ struct NCD4INFO { struct { char* filename; } fileproto; + int debuglevel; NClist* blobs; }; diff --git a/libdispatch/dfile.c b/libdispatch/dfile.c index 7dba910f30..7447db3511 100644 --- a/libdispatch/dfile.c +++ b/libdispatch/dfile.c @@ -1770,7 +1770,7 @@ static int check_create_mode(int mode) { int mode_format; - int mmap = 0; + int use_mmap = 0; int inmemory = 0; int diskless = 0; @@ -1781,17 +1781,17 @@ check_create_mode(int mode) if (mode_format && (mode_format & (mode_format - 1))) return NC_EINVAL; - mmap = ((mode & NC_MMAP) == NC_MMAP); + use_mmap = ((mode & NC_MMAP) == NC_MMAP); inmemory = ((mode & NC_INMEMORY) == NC_INMEMORY); diskless = ((mode & NC_DISKLESS) == NC_DISKLESS); /* NC_INMEMORY and NC_DISKLESS and NC_MMAP are all mutually exclusive */ if(diskless && inmemory) return NC_EDISKLESS; - if(diskless && mmap) return NC_EDISKLESS; - if(inmemory && mmap) return NC_EINMEMORY; + if(diskless && use_mmap) return NC_EDISKLESS; + if(inmemory && use_mmap) return NC_EINMEMORY; /* mmap is not allowed for netcdf-4 */ - if(mmap && (mode & NC_NETCDF4)) return NC_EINVAL; + if(use_mmap && (mode & NC_NETCDF4)) return NC_EINVAL; #ifndef USE_NETCDF4 /* If the user asks for a netCDF-4 file, and the library was built @@ -1973,7 +1973,7 @@ NC_open(const char *path0, int omode, int basepe, size_t *chunksizehintp, const NC_Dispatch* dispatcher = NULL; int inmemory = 0; int diskless = 0; - int mmap = 0; + int use_mmap = 0; char* path = NULL; NCmodel model; char* newpath = NULL; @@ -1989,17 +1989,17 @@ NC_open(const char *path0, int omode, int basepe, size_t *chunksizehintp, {stat = NC_EINVAL; goto done;} /* Capture the inmemory related flags */ - mmap = ((omode & NC_MMAP) == NC_MMAP); + use_mmap = ((omode & NC_MMAP) == NC_MMAP); diskless = ((omode & NC_DISKLESS) == NC_DISKLESS); inmemory = ((omode & NC_INMEMORY) == NC_INMEMORY); /* NC_INMEMORY and NC_DISKLESS and NC_MMAP are all mutually exclusive */ if(diskless && inmemory) {stat = NC_EDISKLESS; goto done;} - if(diskless && mmap) {stat = NC_EDISKLESS; goto done;} - if(inmemory && mmap) {stat = NC_EINMEMORY; goto done;} + if(diskless && use_mmap) {stat = NC_EDISKLESS; goto done;} + if(inmemory && use_mmap) {stat = NC_EINMEMORY; goto done;} /* mmap is not allowed for netcdf-4 */ - if(mmap && (omode & NC_NETCDF4)) {stat = NC_EINVAL; goto done;} + if(use_mmap && (omode & NC_NETCDF4)) {stat = NC_EINVAL; goto done;} /* Attempt to do file path conversion: note that this will do nothing if path is a 'file:...' url, so it will need to be diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index 02833693e7..0245b6700e 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -201,7 +201,7 @@ static struct NCPROTOCOLLIST { static int NC_omodeinfer(int useparallel, int omode, NCmodel*); static int check_file_type(const char *path, int omode, int use_parallel, void *parameters, NCmodel* model, NCURI* uri); static int processuri(const char* path, NCURI** urip, NClist* fraglist); -static int processmacros(NClist** fraglistp); +static int processmacros(NClist* fraglistp, NClist* expanded); static char* envvlist2string(NClist* pairs, const char*); static void set_default_mode(int* cmodep); static int parseonchar(const char* s, int ch, NClist* segments); @@ -238,8 +238,8 @@ processuri(const char* path, NCURI** urip, NClist* fraglenv) NCURI* uri = NULL; size_t pathlen = strlen(path); char* str = NULL; - const char** ufrags; - const char** p; + const NClist* ufrags; + size_t i; if(path == NULL || pathlen == 0) {stat = NC_EURL; goto done;} @@ -261,7 +261,6 @@ processuri(const char* path, NCURI** urip, NClist* fraglenv) /* process the corresponding fragments for that protocol */ if(protolist->fragments != NULL) { - int i; tmp = nclistnew(); if((stat = parseonchar(protolist->fragments,'&',tmp))) goto done; for(i=0;isubstitute) ncurisetprotocol(uri,protolist->substitute); /* capture the fragments of the url */ - ufrags = ncurifragmentparams(uri); - if(ufrags != NULL) { - for(p=ufrags;*p;p+=2) { - const char* key = p[0]; - const char* value = p[1]; - nclistpush(fraglenv,nulldup(key)); - value = (value==NULL?"":value); - nclistpush(fraglenv,strdup(value)); - } + ufrags = (const NClist*)ncurifragmentparams(uri); + for(i=0;i 0) { + for(i=0;iname;macros++) { if(strcmp(macros->name,key)==0) { @@ -470,14 +460,8 @@ processmacros(NClist** fraglenvp) nclistpush(expanded,strdup(key)); nclistpush(expanded,strdup(value)); } - nullfree(key); - nullfree(value); } - *fraglenvp = expanded; expanded = NULL; -done: - nclistfreeall(expanded); - nclistfreeall(fraglenv); return check(stat); } @@ -493,8 +477,6 @@ processinferences(NClist* fraglenv) int i; char* newmodeval = NULL; - if(fraglenv == NULL || nclistlength(fraglenv) == 0) goto done; - /* Get "mode" entry */ if((modeval = getmodekey(fraglenv))==NULL) goto done; @@ -532,7 +514,7 @@ printlist(newmodes,"processinferences: new mode list"); nextmodes = tmp; tmp = NULL; } - /* cleanup any unused elements in currenmodes */ + /* cleanup any unused elements in currentmodes */ nclistclearall(currentmodes); /* Ensure no duplicates */ @@ -687,21 +669,15 @@ collectallkeys(NClist* fraglenv, NClist* allkeys) /* Given a fragment envv list, coalesce duplicate keys and remove duplicate values*/ static int -cleanfragments(NClist** fraglenvp) +cleanfragments(NClist* fraglenv, NClist* newlist) { int i,stat = NC_NOERR; - NClist* fraglenv = NULL; NClist* tmp = NULL; NClist* allkeys = NULL; - NClist* newlist = NULL; NCbytes* buf = NULL; char* key = NULL; char* value = NULL; - if(fraglenvp == NULL || nclistlength(*fraglenvp) == 0) return NC_NOERR; - fraglenv = *fraglenvp; /* take control of this list */ - *fraglenvp = NULL; - newlist = nclistnew(); buf = ncbytesnew(); allkeys = nclistnew(); tmp = nclistnew(); @@ -721,13 +697,10 @@ cleanfragments(NClist** fraglenvp) nclistpush(newlist,value); nclistclear(tmp); } - *fraglenvp = newlist; newlist = NULL; done: nclistfree(allkeys); nclistfree(tmp); ncbytesfree(buf); - nclistfreeall(fraglenv); - nclistfreeall(newlist); return check(stat); } @@ -867,7 +840,8 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void char* sfrag = NULL; const char* modeval = NULL; char* abspath = NULL; - + NClist* tmp = NULL; + /* Phase 1: 1. convert special protocols to http|https 2. begin collecting fragments @@ -880,13 +854,20 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void #endif /* Phase 2: Expand macros and add to fraglenv */ - if((stat = processmacros(&fraglenv))) goto done; + nclistfreeall(tmp); + tmp = nclistnew(); + if((stat = processmacros(fraglenv,tmp))) goto done; + nclistfreeall(fraglenv); + fraglenv = tmp; tmp = NULL; #ifdef DEBUG printlist(fraglenv,"processmacros"); #endif - /* Cleanup the fragment list */ - if((stat = cleanfragments(&fraglenv))) goto done; + nclistfreeall(tmp); + tmp = nclistnew(); + if((stat = cleanfragments(fraglenv,tmp))) goto done; + nclistfreeall(fraglenv); + fraglenv = tmp; tmp = NULL; /* Phase 2a: Expand mode inferences and add to fraglenv */ if((stat = processinferences(fraglenv))) goto done; @@ -895,7 +876,11 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void #endif /* Phase 3: coalesce duplicate fragment keys and remove duplicate values */ - if((stat = cleanfragments(&fraglenv))) goto done; + nclistfreeall(tmp); + tmp = nclistnew(); + if((stat = cleanfragments(fraglenv,tmp))) goto done; + nclistfreeall(fraglenv); + fraglenv = tmp; tmp = NULL; #ifdef DEBUG printlist(fraglenv,"cleanfragments"); #endif @@ -945,13 +930,12 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void /* Phase 6: Process the non-mode keys to see if we can tell the formatx */ if(!modelcomplete(model)) { - const char** p = ncurifragmentparams(uri); /* envv format */ - if(p != NULL) { - for(;*p;p+=2) { - const char* key = p[0]; - const char* value = p[1];; - if((stat=processfragmentkeys(key,value,model))) goto done; - } + size_t i; + NClist* p = (NClist*)ncurifragmentparams(uri); /* envv format */ + for(i=0;ileaflen, leafavg); fprintf(stderr," load=%g",leafload); fprintf(stderr,"]\n"); - dirsize = (1<<(map->depth))*((unsigned long long)sizeof(void*)); + dirsize = (1ULL<<(map->depth))*((unsigned long long)sizeof(void*)); leafsize = (nleaves)*((unsigned long long)sizeof(NCexleaf)); total = dirsize + leafsize; fprintf(stderr,"\tsizeof(directory)=%llu sizeof(leaves)=%lld total=%lld\n", diff --git a/libdispatch/nclistmgr.c b/libdispatch/nclistmgr.c index 48b4eeb584..4069e87698 100644 --- a/libdispatch/nclistmgr.c +++ b/libdispatch/nclistmgr.c @@ -81,8 +81,8 @@ free_NCList(void) int add_to_NCList(NC* ncp) { - int i; - int new_id; + unsigned int i; + unsigned int new_id; if(nc_filelist == NULL) { if (!(nc_filelist = calloc(1, sizeof(NC*)*NCFILELISTLENGTH))) return NC_ENOMEM; @@ -96,7 +96,7 @@ add_to_NCList(NC* ncp) if(new_id == 0) return NC_ENOMEM; /* no more slots */ nc_filelist[new_id] = ncp; numfiles++; - ncp->ext_ncid = (new_id << ID_SHIFT); + ncp->ext_ncid = (int)(new_id << ID_SHIFT); return NC_NOERR; } diff --git a/libdispatch/nclog.c b/libdispatch/nclog.c index 6f477c18d8..bfc4b9f9fa 100644 --- a/libdispatch/nclog.c +++ b/libdispatch/nclog.c @@ -21,10 +21,6 @@ #include #endif -#ifdef HAVE_EXECINFO_H -#include -#endif - #include "netcdf.h" #include "nclog.h" @@ -276,10 +272,6 @@ ncuntrace(const char* fcn, int err, const char* fmt, ...) vfprintf(nclog_global.nclogstream, fmt, args); fprintf(nclog_global.nclogstream, "\n" ); fflush(nclog_global.nclogstream); -#ifdef HAVE_EXECINFO_H - if(err != 0) - ncbacktrace(); -#endif } done: va_end(args); @@ -302,28 +294,4 @@ ncbreakpoint(int err) return err; } -#ifdef HAVE_EXECINFO_H -#define MAXSTACKDEPTH 100 -void -ncbacktrace(void) -{ - int j, nptrs; - void* buffer[MAXSTACKDEPTH]; - char **strings; - - if(getenv("NCBACKTRACE") == NULL) return; - nptrs = backtrace(buffer, MAXSTACKDEPTH); - strings = backtrace_symbols(buffer, nptrs); - if (strings == NULL) { - perror("backtrace_symbols"); - errno = 0; - return; - } - fprintf(stderr,"Backtrace:\n"); - for(j = 0; j < nptrs; j++) - fprintf(stderr,"%s\n", strings[j]); - free(strings); -} -#endif - /**@}*/ diff --git a/libdispatch/ncuri.c b/libdispatch/ncuri.c index efd6fa4492..6f699a8591 100644 --- a/libdispatch/ncuri.c +++ b/libdispatch/ncuri.c @@ -95,16 +95,13 @@ ncstrndup(const char* s, size_t len) /* Forward */ static int collectprefixparams(char* text, char** nextp); static void freestringlist(NClist* list); -static void freestringvec(char** list); -static int ncfind(char** params, const char* key); +static int ncfind(NClist* params, const char* key); static char* nclocate(char* p, const char* charlist); static int parselist(const char* ptext, NClist* list); -static int unparselist(const char** vec, const char* prefix, int encode, char** svecp); +static int unparselist(const NClist* vec, const char* prefix, int encode, NCbytes*); static int ensurefraglist(NCURI* uri); static int ensurequerylist(NCURI* uri); -static void buildlist(const char** list, int encode, NCbytes* buf); static void removedups(NClist* list); -static int extendenvv(char*** envvp, int amount, int* oldlen); /**************************************************/ /* @@ -147,9 +144,6 @@ ncuriparse(const char* uri0, NCURI** durip) size_t len0; int pathchar; - tmp.fraglist = NULL; - tmp.querylist = NULL; - if(uri0 == NULL) {THROW(NC_EURL);} @@ -227,19 +221,15 @@ ncuriparse(const char* uri0, NCURI** durip) } /* Remove duplicates */ removedups(params); - if(nclistlength(params) > 0) { - nclistpush(params,NULL); - tmp.fraglist = nclistextract(params); - } else - tmp.fraglist = NULL; + tmp.fraglist = params; + params = NULL; + /* Parse the query */ if(tmp.query != NULL) { if(parselist(tmp.query,querylist) != NC_NOERR) {THROW(NC_EURL);} - if(nclistlength(querylist) > 0) { - nclistpush(querylist,NULL); - tmp.querylist = nclistextract(querylist); - } + tmp.querylist = querylist; + querylist = NULL; } /* Now parse the core of the url */ @@ -312,7 +302,7 @@ ncuriparse(const char* uri0, NCURI** durip) pathchar = EOFCHAR; } else { /* assume there should be a host section */ /* We already extracted the query and/or fragment sections above, - splocate the end of the host section and therefore the start + so locate the end of the host section and therefore the start of the path. */ tmp.host = p; @@ -430,9 +420,9 @@ ncuriparse(const char* uri0, NCURI** durip) freestringlist(params); freestringlist(querylist); if(tmp.fraglist) - freestringvec(tmp.fraglist); + nclistfreeall(tmp.fraglist); if(tmp.querylist) - freestringvec(tmp.querylist); + nclistfreeall(tmp.querylist); return ret; } @@ -450,6 +440,7 @@ freestringlist(NClist* list) } } +#if 0 static void freestringvec(char** list) { @@ -459,6 +450,7 @@ freestringvec(char** list) nullfree(list); } } +#endif void ncurifree(NCURI* duri) @@ -473,8 +465,8 @@ ncurifree(NCURI* duri) nullfree(duri->path); nullfree(duri->query); nullfree(duri->fragment); - freestringvec(duri->querylist); - freestringvec(duri->fraglist); + nclistfreeall(duri->querylist); + nclistfreeall(duri->fraglist); free(duri); } @@ -510,21 +502,14 @@ int ncurisetquery(NCURI* duri,const char* query) { int ret = NC_NOERR; - freestringvec(duri->querylist); + nclistfreeall((NClist*)duri->querylist); nullfree(duri->query); duri->query = NULL; duri->querylist = NULL; if(query != NULL && strlen(query) > 0) { - NClist* params = nclistnew(); - duri->query = strdup(query); - ret = parselist(duri->query,params); - if(ret != NC_NOERR) - {THROW(NC_EURL);} - nclistpush(params,NULL); - duri->querylist = nclistextract(params); - nclistfree(params); + duri->query = strdup(query); + ensurequerylist(duri); } -done: return ret; } @@ -533,12 +518,13 @@ int ncurisetfragments(NCURI* duri,const char* fragments) { int ret = NC_NOERR; - freestringvec(duri->fraglist); + nclistfreeall((NClist*)duri->fraglist); nullfree(duri->fragment); duri->fragment = NULL; duri->fraglist = NULL; if(fragments != NULL && strlen(fragments) > 0) { duri->fragment = strdup(fragments); + ensurefraglist(duri); } return ret; } @@ -559,17 +545,20 @@ ncurisetfragmentkey(NCURI* duri,const char* key, const char* value) { int ret = NC_NOERR; int pos = -1; - char* newlist = NULL; ensurefraglist(duri); pos = ncfind(duri->fraglist, key); - if(pos < 0) return NC_EINVAL; /* does not exist */ - nullfree(duri->fraglist[pos+1]); - duri->fraglist[pos+1] = strdup(value); + if(pos < 0) { /* does not exist */ + if(duri->fraglist == NULL) duri->fraglist = nclistnew(); + nclistpush(duri->fraglist,key); + nclistpush(duri->fraglist,value); + } else { + nullfree(nclistget(duri->fraglist,pos+1)); + nclistset(duri->fraglist,pos+1,strdup(value)); + } /* Rebuild the fragment */ - if((ret = unparselist((const char**)duri->fraglist,"#",0,&newlist))) goto done; - nullfree(duri->fragment); - duri->fragment = newlist; newlist = NULL; + nullfree(duri->fragment); duri->fragment = NULL; + if((ret = ensurefraglist(duri))) goto done; done: return ret; } @@ -579,25 +568,67 @@ int ncuriappendfragmentkey(NCURI* duri,const char* key, const char* value) { int ret = NC_NOERR; - int len; int pos = -1; - char* newlist = NULL; ensurefraglist(duri); pos = ncfind(duri->fraglist, key); if(pos < 0) { /* does not exist */ - if((ret = extendenvv(&duri->fraglist,2,&len))) goto done; - duri->fraglist[len] = strdup(key); - duri->fraglist[len+1] = nulldup(value); - duri->fraglist[len+2] = NULL; + nclistpush((NClist*)duri->fraglist,strdup(key)); + nclistpush((NClist*)duri->fraglist,nulldup(value)); } else { - nullfree(duri->fraglist[pos+1]); - duri->fraglist[pos+1] = strdup(value); + nullfree(nclistget(duri->fraglist,pos+1)); + nclistset(duri->fraglist,pos+1,nulldup(value)); } /* Rebuild the fragment */ - if((ret = unparselist((const char**)duri->fraglist,"#",0,&newlist))) goto done; - nullfree(duri->fragment); - duri->fragment = newlist; newlist = NULL; + nullfree(duri->fraglist); duri->fraglist = NULL; + if((ret=ensurefraglist(duri))) goto done; +done: + return ret; +} + +/* Replace a specific query key*/ +int +ncurisetquerykey(NCURI* duri,const char* key, const char* value) +{ + int ret = NC_NOERR; + int pos = -1; + + ensurequerylist(duri); + pos = ncfind(duri->querylist, key); + if(pos < 0) { /* does not exist */ + if(duri->querylist == NULL) duri->querylist = nclistnew(); + nclistpush(duri->querylist,key); + nclistpush(duri->querylist,value); + } else { + nullfree(nclistget(duri->querylist,pos+1)); + nclistset(duri->querylist,pos+1,strdup(value)); + } + /* Rebuild the query */ + nullfree(duri->query); duri->query = NULL; + if((ret = ensurequerylist(duri))) goto done; +done: + return ret; +} + +/* Replace or add a specific query key*/ +int +ncuriappendquerykey(NCURI* duri,const char* key, const char* value) +{ + int ret = NC_NOERR; + int pos = -1; + + ensurequerylist(duri); + pos = ncfind(duri->querylist, key); + if(pos < 0) { /* does not exist */ + nclistpush((NClist*)duri->querylist,strdup(key)); + nclistpush((NClist*)duri->querylist,nulldup(value)); + } else { + nullfree(nclistget(duri->querylist,pos+1)); + nclistset(duri->querylist,pos+1,nulldup(value)); + } + /* Rebuild the query */ + nullfree(duri->querylist); duri->querylist = NULL; + if((ret=ensurequerylist(duri))) goto done; done: return ret; } @@ -734,11 +765,10 @@ ncurifragmentlookup(NCURI* uri, const char* key) int i; char* value = NULL; if(uri == NULL || key == NULL) return NULL; - ensurefraglist(uri); + if(ensurefraglist(uri)) return NULL; i = ncfind(uri->fraglist,key); - if(i < 0) - return NULL; - value = uri->fraglist[(2*i)+1]; + if(i < 0) return NULL; + value = nclistget(uri->fraglist,i+1); return value; } @@ -747,20 +777,21 @@ ncuriquerylookup(NCURI* uri, const char* key) { int i; char* value = NULL; - if(uri == NULL || key == NULL || uri->querylist == NULL) return NULL; + if(uri == NULL || key == NULL) return NULL; + if(ensurequerylist(uri)) return NULL; i = ncfind(uri->querylist,key); - if(i < 0) - return NULL; - value = uri->querylist[(2*i)+1]; + if(i < 0) return NULL; + value = nclistget(uri->querylist,i+1); return value; } +#if 0 /* Obtain the complete list of fragment pairs in envv format */ const char** ncurifragmentparams(NCURI* uri) { ensurefraglist(uri); - return (const char**)uri->fraglist; + return (const char**)nclistcontents(uri->fraglist; } /* Obtain the complete list of query pairs in envv format */ @@ -771,7 +802,6 @@ ncuriqueryparams(NCURI* uri) return (const char**)uri->querylist; } -#if 0 int ncuriremoveparam(NCURI* uri, const char* key) { @@ -796,19 +826,18 @@ ncuriremoveparam(NCURI* uri, const char* key) case insensitive */ static int -ncfind(char** params, const char* key) +ncfind(NClist* params, const char* key) { int i; - char** p; if(key == NULL) return -1; if(params == NULL) return -1; - for(i=0,p=params;*p;p+=2,i++) { - if(strcasecmp(key,*p)==0) return i; + for(i=0;i 0) { + ncbytescat(buf,p0); + if(p1 != NULL && strlen(p1) > 0) { ncbytescat(buf,"="); if(encode) { - char* encoded = ncuriencodeonly(p[1],queryallow); + char* encoded = ncuriencodeonly(p1,queryallow); ncbytescat(buf,encoded); nullfree(encoded); } else - ncbytescat(buf,p[1]); + ncbytescat(buf,p1); } } - if(svecp) {*svecp = ncbytesextract(buf);} done: - ncbytesfree(buf); return stat; } @@ -1171,24 +1199,32 @@ static int ensurefraglist(NCURI* uri) { int stat = NC_NOERR; - int nofrag = 0; - int nolist = 0; - NClist* fraglist = NULL; + int hastext = 0; + int haslist = 0; + NClist* fraglist = nclistnew(); NCbytes* frag = NULL; - if(uri->fragment == NULL || strlen(uri->fragment) == 0) - {nullfree(uri->fragment); uri->fragment = NULL; nofrag=1;} - if(uri->fraglist == NULL) nolist = 1; - if(nolist && !nofrag) { - fraglist = nclistnew(); + if(nulllen(uri->fragment) == 0) + {nullfree(uri->fragment); uri->fragment = NULL; hastext=0;} + else hastext = 1; + if(nclistlength((NClist*)uri->fraglist) == 0) + {nclistfree((NClist*)uri->fraglist); uri->fraglist = NULL; haslist=0;} + else haslist = 1; + + /* Four cases: */ + if(!haslist && !hastext) { + /* do nothing */ + } else if(!haslist && hastext) { if((stat = parselist(uri->fragment,fraglist))) goto done; removedups(fraglist); - uri->fraglist = nclistextract(fraglist); - } else if(!nolist && nofrag) { + uri->fraglist = fraglist; fraglist = NULL; + } else if(haslist && !hastext) { /* Create the fragment string from fraglist */ frag = ncbytesnew(); - buildlist((const char**)uri->fraglist,0,frag); /* do not encode */ + if((stat=unparselist((const NClist*)uri->fraglist,NULL,0,frag))) goto done; /* do not encode */ uri->fragment = ncbytesextract(frag); + } else if(haslist && hastext) { + /* assume already consistent */ } done: @@ -1201,24 +1237,32 @@ static int ensurequerylist(NCURI* uri) { int stat = NC_NOERR; - int noquery = 0; - int nolist = 0; - NClist* querylist = NULL; + int hastext = 0; + int haslist = 0; + NClist* querylist = nclistnew(); NCbytes* query = NULL; - if(uri->query == NULL || strlen(uri->query) == 0) - {nullfree(uri->query); uri->query = NULL; noquery=1;} - if(uri->querylist == NULL) nolist = 1; - if(nolist && !noquery) { - querylist = nclistnew(); + if(nulllen(uri->query) == 0) + {nullfree(uri->query); uri->query = NULL; hastext=0;} + else hastext = 1; + if(nclistlength((NClist*)uri->querylist) == 0) + {nclistfree((NClist*)uri->querylist); uri->querylist = NULL; haslist=0;} + else haslist = 1; + + /* Four cases: */ + if(!haslist && !hastext) { + /* do nothing */ + } else if(!haslist && hastext) { if((stat = parselist(uri->query,querylist))) goto done; removedups(querylist); - uri->querylist = nclistextract(querylist); - } else if(!nolist && noquery) { + uri->querylist = querylist; querylist = NULL; + } else if(haslist && !hastext) { /* Create the query string from querylist */ query = ncbytesnew(); - buildlist((const char**)uri->querylist,0,query); /* do not encode */ + if((stat=unparselist((const NClist*)uri->querylist,NULL,0,query))) goto done; /* do not encode */ uri->query = ncbytesextract(query); + } else if(haslist && hastext) { + /* assume consistent */ } done: @@ -1242,30 +1286,9 @@ removedups(NClist* list) } } } - /* NULL terminate the list */ - nclistpush(list,NULL); -} - -static void -buildlist(const char** list, int encode, NCbytes* buf) -{ - const char** p; - int first = 1; - for(p=list;*p;p+=2,first=0) { - if(!first) ncbytescat(buf,"&"); - ncbytescat(buf,p[0]); - if(p[1] != NULL && strlen(p[1]) > 0) { - ncbytescat(buf,"="); - if(encode) { - char* encoded = ncuriencodeonly(p[1],queryallow); - ncbytescat(buf,encoded); - nullfree(encoded); - } else - ncbytescat(buf,p[1]); - } - } } +#if 0 static int extendenvv(char*** envvp, int amount, int* oldlenp) { @@ -1281,6 +1304,7 @@ extendenvv(char*** envvp, int amount, int* oldlenp) *envvp = envv; envv = NULL; return NC_NOERR; } +#endif /* Use for gdb debug */ char* @@ -1288,3 +1312,20 @@ ncuriunescape(const char* s) { return ncuridecodepartial(s,ascii); } + +/* Get the actual list of queryies */ +void* +ncuriqueryparams(NCURI* uri) +{ + ensurequerylist(uri); + return uri->querylist; +} + +/* Get the actual list of frags */ +void* +ncurifragmentparams(NCURI* uri) +{ + ensurefraglist(uri); + return uri->fraglist; +} + diff --git a/libdispatch/utf8proc.c b/libdispatch/utf8proc.c index 3a05dd3b34..68d8dd4a20 100644 --- a/libdispatch/utf8proc.c +++ b/libdispatch/utf8proc.c @@ -355,7 +355,8 @@ static nc_utf8proc_ssize_t nc_seqindex_write_char_decomposed(nc_utf8proc_uint16_ for (; len >= 0; entry++, len--) { nc_utf8proc_int32_t entry_cp = nc_seqindex_decode_entry(&entry); - written += nc_utf8proc_decompose_char(entry_cp, dst+written, + nc_utf8proc_int32_t *dest = dst ? (dst+written) : NULL; + written += nc_utf8proc_decompose_char(entry_cp, dest, (bufsize > written) ? (bufsize - written) : 0, options, last_boundclass); if (written < 0) return UTF8PROC_ERROR_OVERFLOW; @@ -525,8 +526,10 @@ static nc_utf8proc_ssize_t nc_seqindex_write_char_decomposed(nc_utf8proc_uint16_ if (custom_func != NULL) { uc = custom_func(uc, custom_data); /* user-specified custom mapping */ } + nc_utf8proc_int32_t *dest = NULL; + if (buffer) dest = buffer + wpos; decomp_result = nc_utf8proc_decompose_char( - uc, buffer + wpos, (bufsize > wpos) ? (bufsize - wpos) : 0, options, + uc, dest, (bufsize > wpos) ? (bufsize - wpos) : 0, options, &boundclass ); if (decomp_result < 0) return decomp_result; diff --git a/libhdf5/hdf5debug.c b/libhdf5/hdf5debug.c index 40871c148a..b4dace8d59 100644 --- a/libhdf5/hdf5debug.c +++ b/libhdf5/hdf5debug.c @@ -14,37 +14,9 @@ #ifdef H5CATCH -#define STSIZE 1000 - -#ifdef HAVE_EXECINFO_H -#ifdef H5BACKTRACE -# if !defined _WIN32 && !defined __CYGWIN__ -static void* stacktrace[STSIZE]; -# endif -#endif -#endif - int nch5breakpoint(int err) { -#ifdef HAVE_EXECINFO_H -#ifdef H5BACKTRACE -# if !defined _WIN32 && !defined __CYGWIN__ - int count = 0; - char** trace = NULL; - int i; - - count = backtrace(stacktrace,STSIZE); - trace = backtrace_symbols(stacktrace, STSIZE); - fprintf(stderr,"backtrace:\n"); - for(i=0;ichunkcache.nelems), &(var->chunkcache.size), &rdcc_w0)) < 0) BAIL(NC_EHDFERR); - var->chunkcache.preemption = rdcc_w0; + var->chunkcache.preemption = (float)rdcc_w0; /* Get the dataset creation properties. */ if ((propid = H5Dget_create_plist(hdf5_var->hdf_datasetid)) < 0) diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index 0a1273250b..6d37fe9099 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -2391,7 +2391,7 @@ nc_set_var_chunk_cache_ints(int ncid, int varid, int size, int nelems, real_nelems = nelems; if (preemption >= 0) - real_preemption = preemption / 100.; + real_preemption = (float)(preemption / 100.); return NC4_HDF5_set_var_chunk_cache(ncid, varid, real_size, real_nelems, real_preemption); diff --git a/libncxml/Makefile.am b/libncxml/Makefile.am index 3866eae5ba..f5fab3b101 100644 --- a/libncxml/Makefile.am +++ b/libncxml/Makefile.am @@ -35,7 +35,20 @@ tinyxml2:: rm -fr ./tinyxml2 ./license.txt git clone --depth=1 ${REPO} cat tinyxml2/LICENSE.txt > ./license.txt - cat tinyxml2/tinyxml2.h > ./tinyxml2.h - sed -e 's/__BORLANDC__/__APPLE__/' < tinyxml2/tinyxml2.cpp \ - | sed -e 's/ptrdiff_t/long/g' > ./tinyxml2.cpp + tr -d '\r' < tinyxml2/tinyxml2.h > tinyxml2.h + cat tinyxml2/tinyxml2.cpp \ + | sed -e 's/__BORLANDC__/__APPLE__/' \ + | sed -e 's/ptrdiff_t/long/g' \ + | sed -e '/^static[ ]*FILE[*][ ]*callfopen(/i\ +\#if 0' \ + | sed -e '/^void[ ]*XMLDocument::DeleteNode(/i\ +\#endif /*0*/\ +' \ + | sed -e '/^XMLError[ ]*XMLDocument::LoadFile([ ]*const[ ]*char[*]/i\ +\#if 0' \ + | sed -e '/^XMLError[ ]*XMLDocument::Parse(/i\ +\#endif /*0*/\ +' \ + | tr -d '\r' \ + | cat > ./tinyxml2.cpp rm -fr tinyxml2 diff --git a/libncxml/tinyxml2.cpp b/libncxml/tinyxml2.cpp index 4f761ad319..c0d16eda0f 100644 --- a/libncxml/tinyxml2.cpp +++ b/libncxml/tinyxml2.cpp @@ -2283,6 +2283,7 @@ XMLUnknown* XMLDocument::NewUnknown( const char* str ) return unk; } +#if 0 static FILE* callfopen( const char* filepath, const char* mode ) { TIXMLASSERT( filepath ); @@ -2299,6 +2300,8 @@ static FILE* callfopen( const char* filepath, const char* mode ) return fp; } +#endif /*0*/ + void XMLDocument::DeleteNode( XMLNode* node ) { TIXMLASSERT( node ); TIXMLASSERT(node->_document == this ); @@ -2317,6 +2320,7 @@ void XMLDocument::DeleteNode( XMLNode* node ) { } +#if 0 XMLError XMLDocument::LoadFile( const char* filename ) { if ( !filename ) { @@ -2420,6 +2424,8 @@ XMLError XMLDocument::SaveFile( FILE* fp, bool compact ) } +#endif /*0*/ + XMLError XMLDocument::Parse( const char* xml, size_t nBytes ) { Clear(); diff --git a/libnczarr/zarr.c b/libnczarr/zarr.c index a5649b2b10..7862aa81bd 100644 --- a/libnczarr/zarr.c +++ b/libnczarr/zarr.c @@ -22,7 +22,7 @@ static int applycontrols(NCZ_FILE_INFO_T* zinfo); */ int -ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** controls) +ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, NClist* urlcontrols) { int stat = NC_NOERR; NCZ_FILE_INFO_T* zfile = NULL; @@ -32,7 +32,7 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** contr NCjson* json = NULL; char* key = NULL; - ZTRACE(3,"file=%s root=%s controls=%s",file->hdr.name,root->hdr.name,(controls?nczprint_envv(controls):"null")); + ZTRACE(3,"file=%s root=%s urlcontrols=%s",file->hdr.name,root->hdr.name,(controls?nczprint_env(urlcontrols):"null")); nc = (NC*)file->controller; @@ -52,7 +52,7 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** contr zfile->creating = 1; zfile->common.file = file; zfile->native_endianness = (NCZ_isLittleEndian() ? NC_ENDIAN_LITTLE : NC_ENDIAN_BIG); - if((zfile->envv_controls=NCZ_clonestringvec(0,controls)) == NULL) + if((zfile->urlcontrols=nclistclone(urlcontrols,1)) == NULL) {stat = NC_ENOMEM; goto done;} zfile->default_maxstrlen = NCZ_MAXSTR_DEFAULT; @@ -71,7 +71,7 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** contr zfile->zarr.zarr_format = DFALTZARRFORMAT; /* initialize map handle*/ - if((stat = NCZ_get_map(file,uri,nc->mode,zfile->controls.flags,NULL,&zfile->map))) goto done; + if((stat = NCZ_get_map(file,uri,nc->mode,zfile->flags,NULL,&zfile->map))) goto done; /* And get the format dispatcher */ if((stat = NCZ_get_formatter(file, (const NCZ_Formatter**)&zfile->dispatcher))) goto done; @@ -92,7 +92,7 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** contr */ int -ncz_open_dataset(NC_FILE_INFO_T* file, const char** controls) +ncz_open_dataset(NC_FILE_INFO_T* file, NClist* urlcontrols) { int stat = NC_NOERR; NC* nc = NULL; @@ -121,7 +121,7 @@ ncz_open_dataset(NC_FILE_INFO_T* file, const char** controls) zfile->creating = 0; zfile->common.file = file; zfile->native_endianness = (NCZ_isLittleEndian() ? NC_ENDIAN_LITTLE : NC_ENDIAN_BIG); - if((zfile->envv_controls = NCZ_clonestringvec(0,controls))==NULL) /*0=>envv style*/ + if((zfile->urlcontrols = nclistclone(urlcontrols,1))==NULL) /*0=>envv style*/ {stat = NC_ENOMEM; goto done;} zfile->default_maxstrlen = NCZ_MAXSTR_DEFAULT; @@ -141,7 +141,7 @@ ncz_open_dataset(NC_FILE_INFO_T* file, const char** controls) } /* initialize map handle*/ - if((stat = NCZ_get_map(file,uri,nc->mode,zfile->controls.flags,NULL,&zfile->map))) goto done; + if((stat = NCZ_get_map(file,uri,nc->mode,zfile->flags,NULL,&zfile->map))) goto done; /* And get the format dispatcher */ if((stat = NCZ_get_formatter(file, (const NCZ_Formatter**)&zfile->dispatcher))) goto done; @@ -281,12 +281,13 @@ ncz_open_rootgroup(NC_FILE_INFO_T* dataset) static const char* -controllookup(const char** envv_controls, const char* key) +controllookup(NClist* controls, const char* key) { - const char** p; - for(p=envv_controls;*p;p+=2) { - if(strcasecmp(key,*p)==0) { - return p[1]; + int i; + for(i=0;ienvv_controls,"mode")) != NULL) { + if((value = controllookup(zinfo->urlcontrols,"mode")) != NULL) { if((stat = NCZ_comma_parse(value,modelist))) goto done; } /* Process the modelist */ - zinfo->controls.mapimpl = NCZM_DEFAULT; - zinfo->controls.flags |= FLAG_XARRAYDIMS; /* Always support XArray convention where possible */ + zinfo->mapimpl = NCZM_DEFAULT; + zinfo->flags |= FLAG_XARRAYDIMS; /* Always support XArray convention where possible */ for(i=0;icontrols.flags |= (FLAG_PUREZARR); + zinfo->flags |= (FLAG_PUREZARR); else if(strcasecmp(p,XARRAYCONTROL)==0) - zinfo->controls.flags |= FLAG_PUREZARR; + zinfo->flags |= FLAG_PUREZARR; else if(strcasecmp(p,NOXARRAYCONTROL)==0) noflags |= FLAG_XARRAYDIMS; else if(strcasecmp(p,ZARRFORMAT2_STRING)==0) @@ -323,16 +324,16 @@ applycontrols(NCZ_FILE_INFO_T* zinfo) /* Apply negative controls by turning off negative flags */ /* This is necessary to avoid order dependence of mode flags when both positive and negative flags are defined */ - zinfo->controls.flags &= (~noflags); + zinfo->flags &= (~noflags); /* Process other controls */ - if((value = controllookup((const char**)zinfo->envv_controls,"log")) != NULL) { - zinfo->controls.flags |= FLAG_LOGGING; + if((value = controllookup(zinfo->urlcontrols,"log")) != NULL) { + zinfo->flags |= FLAG_LOGGING; ncsetloglevel(NCLOGNOTE); } - if((value = controllookup((const char**)zinfo->envv_controls,"show")) != NULL) { + if((value = controllookup(zinfo->urlcontrols,"show")) != NULL) { if(strcasecmp(value,"fetch")==0) - zinfo->controls.flags |= FLAG_SHOWFETCH; + zinfo->flags |= FLAG_SHOWFETCH; } done: nclistfreeall(modelist); @@ -388,7 +389,7 @@ ncz_unload_jatts(NCZ_FILE_INFO_T* zinfo, NC_OBJ* container, NCjson* jattrs, NCjs } } - if(!(zinfo->controls.flags & FLAG_PUREZARR)) { + if(!(zinfo->flags & FLAG_PUREZARR)) { /* Insert the jtypes into the set of attributes */ if((stat = NCJinsert(jattrs,NCZ_V2_ATTRS,jtypes))) goto done; } diff --git a/libnczarr/zarr.h b/libnczarr/zarr.h index d250aea388..eb155bb5e5 100644 --- a/libnczarr/zarr.h +++ b/libnczarr/zarr.h @@ -26,8 +26,8 @@ struct ZCVT { #define zcvt_empty {0,0,0.0,NULL} /* zarr.c */ -EXTERNL int ncz_create_dataset(NC_FILE_INFO_T*, NC_GRP_INFO_T*, const char** controls); -EXTERNL int ncz_open_dataset(NC_FILE_INFO_T*, const char** controls); +EXTERNL int ncz_create_dataset(NC_FILE_INFO_T*, NC_GRP_INFO_T*, NClist* controls); +EXTERNL int ncz_open_dataset(NC_FILE_INFO_T*, NClist* controls); EXTERNL int ncz_del_attr(NC_FILE_INFO_T* file, NC_OBJ* container, const char* name); /* HDF5 Mimics */ diff --git a/libnczarr/zclose.c b/libnczarr/zclose.c index 1554d97e0f..4f341ae7f5 100644 --- a/libnczarr/zclose.c +++ b/libnczarr/zclose.c @@ -49,7 +49,7 @@ ncz_close_file(NC_FILE_INFO_T* file, int abort) if((stat = nczmap_close(zinfo->map,(abort && zinfo->creating)?1:0))) goto done; - NCZ_freestringvec(0,zinfo->envv_controls); + nclistfreeall(zinfo->urlcontrols); NC_authfree(zinfo->auth); nullfree(zinfo); diff --git a/libnczarr/zcreate.c b/libnczarr/zcreate.c index 3243b6f8c7..bf7c1db3e5 100644 --- a/libnczarr/zcreate.c +++ b/libnczarr/zcreate.c @@ -30,7 +30,7 @@ static const int ILLEGAL_CREATE_FLAGS = (NC_NOWRITE|NC_MMAP|NC_DISKLESS|NC_64BIT * @author Dennis Heimbigner, Ed Hartnett */ static int -ncz_create_file(const char *path, int cmode, size_t initialsz, const char** controls, int ncid) +ncz_create_file(const char *path, int cmode, size_t initialsz, NClist* controls, int ncid) { int retval = NC_NOERR; NC_FILE_INFO_T* h5 = NULL; diff --git a/libnczarr/zformat1.c b/libnczarr/zformat1.c index ec8b9d25c7..442a36c453 100644 --- a/libnczarr/zformat1.c +++ b/libnczarr/zformat1.c @@ -129,7 +129,7 @@ ZF1_readmeta(NC_FILE_INFO_T* file) zinfo = file->format_file_info; map = zinfo->map; - purezarr = (zinfo->controls.flags & FLAG_PUREZARR); + purezarr = (zinfo->flags & FLAG_PUREZARR); /* Ok, try to read superblock */ switch(stat = read_superblock(file,&nczarr_format)) { @@ -171,7 +171,7 @@ read_superblock(NC_FILE_INFO_T* file, int* nczarrvp) switch(stat = NCZ_downloadjson(zinfo->map, NCZMETAROOT, &jblock)) { case NC_EEMPTY: /* not there */ nczarr_format = NCZARRFORMAT0; /* apparently pure zarr */ - zinfo->controls.flags |= FLAG_PUREZARR; + zinfo->flags |= FLAG_PUREZARR; goto done; case NC_NOERR: if((stat = NCJdictget(jblock,"nczarr_format",&jtmp))) goto done; @@ -211,7 +211,7 @@ read_grp(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zinfo, NCZMAP* map, NC_GRP_INFO_ ZTRACE(3,"file=%s grp=%s",file->controller->path,grp->hdr.name); - purezarr = (zinfo->controls.flags & FLAG_PUREZARR); + purezarr = (zinfo->flags & FLAG_PUREZARR); /* Construct grp path */ if((stat = NCZ_grpkey(grp,&fullpath))) @@ -316,7 +316,7 @@ read_vars(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO ZTRACE(3,"file=%s grp=%s |varnames|=%u",file->controller->path,grp->hdr.name,nclistlength(varnames)); - if(zfile->controls.flags & FLAG_PUREZARR) purezarr = 1; + if(zfile->flags & FLAG_PUREZARR) purezarr = 1; /* Load each var in turn */ for(i = 0; i < nclistlength(varnames); i++) { @@ -731,7 +731,7 @@ ZF1_readattrs(NC_FILE_INFO_T* file, NC_OBJ* container) zinfo = file->format_file_info; map = zinfo->map; - purezarr = (zinfo->controls.flags & FLAG_PUREZARR)?1:0; + purezarr = (zinfo->flags & FLAG_PUREZARR)?1:0; if(container->sort == NCGRP) { grp = ((NC_GRP_INFO_T*)container); @@ -971,7 +971,7 @@ NCZ_read_atts(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zinfo, NCZMAP* map, NC_OBJ* zinfo = file->format_file_info; map = zinfo->map; - purezarr = (zinfo->controls.flags & FLAG_PUREZARR)?1:0; + purezarr = (zinfo->flags & FLAG_PUREZARR)?1:0; if(container->sort == NCGRP) { grp = ((NC_GRP_INFO_T*)container); @@ -1282,8 +1282,8 @@ NCZ_computedimrefs(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zinfo, NCZMAP* map, NC ZTRACE(3,"file=%s var=%s purezarr=%d xarray=%d ndims=%d shape=%s", file->controller->path,var->hdr.name,purezarr,xarray,(int)ndims,nczprint_vector(ndims,shapes)); - if(zinfo->controls.flags & FLAG_PUREZARR) purezarr = 1; - if(zinfo->controls.flags & FLAG_XARRAYDIMS) xarray = 1; + if(zinfo->flags & FLAG_PUREZARR) purezarr = 1; + if(zinfo->flags & FLAG_XARRAYDIMS) xarray = 1; if(purezarr && xarray) {/* Read in the attributes to get xarray dimdef attribute; Note that it might not exist */ /* Note that if xarray && !purezarr, then xarray will be superceded by the nczarr dimensions key */ diff --git a/libnczarr/zformat2.c b/libnczarr/zformat2.c index 286049f814..9b9242c580 100644 --- a/libnczarr/zformat2.c +++ b/libnczarr/zformat2.c @@ -152,7 +152,7 @@ write_grp(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO ZTRACE(3,"file=%s grp=%s isclose=%d",file->controller->path,grp->hdr.name,isclose); - purezarr = (zfile->controls.flags & FLAG_PUREZARR)?1:0; + purezarr = (zfile->flags & FLAG_PUREZARR)?1:0; /* Construct grp key */ if((stat = NCZ_grpkey(grp,&fullpath))) @@ -293,7 +293,7 @@ write_var_meta(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_VAR zfile = file->format_file_info; map = zfile->map; - purezarr = (zfile->controls.flags & FLAG_PUREZARR)?1:0; + purezarr = (zfile->flags & FLAG_PUREZARR)?1:0; /* Make sure that everything is established */ /* ensure the fill value */ @@ -600,9 +600,9 @@ write_atts(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_OBJ* co grp = (NC_GRP_INFO_T*)container; } - purezarr = (zfile->controls.flags & FLAG_PUREZARR)?1:0; + purezarr = (zfile->flags & FLAG_PUREZARR)?1:0; - if(zfile->controls.flags & FLAG_XARRAYDIMS) isxarray = 1; + if(zfile->flags & FLAG_XARRAYDIMS) isxarray = 1; /* Create the attribute dictionary */ if((stat = NCJnew(NCJ_DICT,&jatts))) goto done; @@ -775,7 +775,7 @@ ZF2_readmeta(NC_FILE_INFO_T* file) zfile = file->format_file_info; map = zfile->map; - purezarr = (zfile->controls.flags & FLAG_PUREZARR); + purezarr = (zfile->flags & FLAG_PUREZARR); /* Ok, try to read superblock */ switch(stat = read_superblock(file,&nczarr_format)) { @@ -834,7 +834,7 @@ ZF2_readattrs(NC_FILE_INFO_T* file, NC_OBJ* container) zinfo = file->format_file_info; map = zinfo->map; - purezarr = (zinfo->controls.flags & FLAG_PUREZARR)?1:0; + purezarr = (zinfo->flags & FLAG_PUREZARR)?1:0; if(container->sort == NCGRP) { grp = ((NC_GRP_INFO_T*)container); @@ -974,7 +974,7 @@ read_superblock(NC_FILE_INFO_T* file, int* nczarrvp) switch(stat = NCZ_downloadjson(zfile->map, Z2METAROOT, &jblock)) { case NC_EEMPTY: /* not there */ nczarr_format = NCZARRFORMAT0; /* apparently pure zarr */ - zfile->controls.flags |= FLAG_PUREZARR; + zfile->flags |= FLAG_PUREZARR; stat = NC_NOERR; /* reset */ goto done; case NC_NOERR: @@ -1024,7 +1024,7 @@ read_grp(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO_ ZTRACE(3,"file=%s grp=%s",file->controller->path,grp->hdr.name); - purezarr = (zfile->controls.flags & FLAG_PUREZARR); + purezarr = (zfile->flags & FLAG_PUREZARR); /* Construct grp path */ if((stat = NCZ_grpkey(grp,&fullpath))) @@ -1129,7 +1129,7 @@ read_vars(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO ZTRACE(3,"file=%s grp=%s |varnames|=%u",file->controller->path,grp->hdr.name,nclistlength(varnames)); - if(zfile->controls.flags & FLAG_PUREZARR) purezarr = 1; + if(zfile->flags & FLAG_PUREZARR) purezarr = 1; /* Load each var in turn */ for(i = 0; i < nclistlength(varnames); i++) { @@ -1769,7 +1769,7 @@ NCZ_read_atts(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_OBJ* zfile = file->format_file_info; map = zfile->map; - purezarr = (zfile->controls.flags & FLAG_PUREZARR)?1:0; + purezarr = (zfile->flags & FLAG_PUREZARR)?1:0; if(container->sort == NCGRP) { grp = ((NC_GRP_INFO_T*)container); @@ -2080,8 +2080,8 @@ NCZ_computedimrefs(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC ZTRACE(3,"file=%s var=%s purezarr=%d xarray=%d ndims=%d shape=%s", file->controller->path,var->hdr.name,purezarr,xarray,(int)ndims,nczprint_vector(ndims,shapes)); - if(zfile->controls.flags & FLAG_PUREZARR) purezarr = 1; - if(zfile->controls.flags & FLAG_XARRAYDIMS) xarray = 1; + if(zfile->flags & FLAG_PUREZARR) purezarr = 1; + if(zfile->flags & FLAG_XARRAYDIMS) xarray = 1; if(purezarr && xarray) {/* Read in the attributes to get xarray dimdef attribute; Note that it might not exist */ /* Note that if xarray && !purezarr, then xarray will be superceded by the nczarr dimensions key */ diff --git a/libnczarr/zformat3.c b/libnczarr/zformat3.c index 8bab2ea5b6..9b05725dee 100644 --- a/libnczarr/zformat3.c +++ b/libnczarr/zformat3.c @@ -160,7 +160,7 @@ write_grp(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO ZTRACE(3,"file=%s grp=%s isclose=%d",file->controller->path,grp->hdr.name,isclose); - purezarr = (zfile->controls.flags & FLAG_PUREZARR)?1:0; + purezarr = (zfile->flags & FLAG_PUREZARR)?1:0; rootgrp = (grp->parent == NULL); /* Construct grp key */ @@ -272,7 +272,7 @@ write_var_meta(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_VAR zfile = file->format_file_info; map = zfile->map; - purezarr = (zfile->controls.flags & FLAG_PUREZARR)?1:0; + purezarr = (zfile->flags & FLAG_PUREZARR)?1:0; /* Make sure that everything is established */ /* ensure the fill value */ @@ -569,7 +569,7 @@ build_atts(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NC_OBJ* container, NCin if(container->sort == NCVAR) var = (NC_VAR_INFO_T*)container; - purezarr = (zfile->controls.flags & FLAG_PUREZARR)?1:0; + purezarr = (zfile->flags & FLAG_PUREZARR)?1:0; /* Create the attribute dictionary */ if((stat = NCJnew(NCJ_DICT,&jatts))) goto done; @@ -774,7 +774,7 @@ ZF3_readmeta(NC_FILE_INFO_T* file) /* Read the root group's metadata */ switch(stat = NCZ_downloadjson(zfile->map, Z3METAROOT, &jblock)) { case NC_EEMPTY: /* not there */ - zfile->controls.flags |= FLAG_PUREZARR; + zfile->flags |= FLAG_PUREZARR; stat = NC_NOERR; /* reset */ goto done; case NC_NOERR: @@ -906,7 +906,7 @@ read_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp) ZTRACE(3,"file=%s grp=%s",file->controller->path,grp->hdr.name); - purezarr = (zfile->controls.flags & FLAG_PUREZARR); + purezarr = (zfile->flags & FLAG_PUREZARR); /* Construct grp path */ if((stat = NCZ_grpkey(grp,&fullpath))) @@ -1036,7 +1036,7 @@ read_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) ZTRACE(3,"file=%s grp=%s |varnames|=%u",file->controller->path,grp->hdr.name,nclistlength(varnames)); - if(zfile->controls.flags & FLAG_PUREZARR) purezarr = 1; + if(zfile->flags & FLAG_PUREZARR) purezarr = 1; if(nclistlength(varnames) == 0) goto done; /* Nothing to create */ @@ -1648,7 +1648,7 @@ NCZ_read_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCjson* jblock) ZTRACE(3,"file=%s container=%s",file->controller->path,container->name); - purezarr = (zfile->controls.flags & FLAG_PUREZARR)?1:0; + purezarr = (zfile->flags & FLAG_PUREZARR)?1:0; if(container->sort == NCGRP) { grp = ((NC_GRP_INFO_T*)container); @@ -1961,7 +1961,7 @@ NCZ_computedimrefs(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, NCjson* jvar, NClis ZTRACE(3,"file=%s var=%s purezarr=%d ndims=%d shape=%s", file->controller->path,var->hdr.name,purezarr,(int)ndims,nczprint_vector(ndims,shapes)); - if(zfile->controls.flags & FLAG_PUREZARR) purezarr = 1; + if(zfile->flags & FLAG_PUREZARR) purezarr = 1; assert(var->atts_read); diff --git a/libnczarr/zinternal.h b/libnczarr/zinternal.h index 77e8f91f6e..2a5d3c297c 100644 --- a/libnczarr/zinternal.h +++ b/libnczarr/zinternal.h @@ -190,16 +190,14 @@ typedef struct NCZ_FILE_INFO { int creating; /* 1=> created 0=>open */ int native_endianness; /* NC_ENDIAN_LITTLE | NC_ENDIAN_BIG */ int default_maxstrlen; /* default max str size for variables of type string */ - char** envv_controls; /* Envv format */ - struct Controls { - size64_t flags; + NClist* urlcontrols; /* controls specified by the file url fragment */ + size64_t flags; # define FLAG_PUREZARR 1 # define FLAG_SHOWFETCH 2 # define FLAG_LOGGING 4 # define FLAG_XARRAYDIMS 8 # define FLAG_NCZARR_V1 16 - NCZM_IMPL mapimpl; - } controls; + NCZM_IMPL mapimpl; NCjson* superblock; /* Only used by NCZarr 3.0.0 and later */ struct NCZ_Formatter* dispatcher; } NCZ_FILE_INFO_T; diff --git a/libnczarr/zopen.c b/libnczarr/zopen.c index 198108fe02..2e8353eb95 100644 --- a/libnczarr/zopen.c +++ b/libnczarr/zopen.c @@ -64,7 +64,7 @@ check_for_classic_model(NC_GRP_INFO_T *root_grp, int *is_classic) * @author Dennis Heimbigner, Ed Hartnett */ static int -ncz_open_file(const char *path, int mode, const char** controls, int ncid) +ncz_open_file(const char *path, int mode, NClist* controls, int ncid) { int stat = NC_NOERR; NC_FILE_INFO_T *h5 = NULL; diff --git a/libsrc/ncx.m4 b/libsrc/ncx.m4 index c8ea6df416..a9fb11f41b 100644 --- a/libsrc/ncx.m4 +++ b/libsrc/ncx.m4 @@ -307,88 +307,24 @@ swapn2b(void *dst, const void *src, IntType nn) IntType i; uint16_t *op = (uint16_t*) dst; uint16_t *ip = (uint16_t*) src; + uint16_t tmp; for (i=0; i 0) - * { - * *op++ = *(++ip); - * *op++ = *(ip++ -1); - * } - */ - while (nn > 3) - { - *op++ = *(++ip); - *op++ = *(ip++ -1); - *op++ = *(++ip); - *op++ = *(ip++ -1); - *op++ = *(++ip); - *op++ = *(ip++ -1); - *op++ = *(++ip); - *op++ = *(ip++ -1); - nn -= 4; - } - while (nn-- > 0) - { - *op++ = *(++ip); - *op++ = *(ip++ -1); - } -#endif } # ifndef vax inline static void swap4b(void *dst, const void *src) { - /* copy over, make the below swap in-place */ uint32_t tmp; - /* use memcpy to avoid type punning */ + /* memcpy is used to handle the case of unaligned memory */ memcpy(&tmp, src, sizeof(tmp)); tmp = SWAP4(tmp); memcpy(dst, &tmp, 4); - - /* Codes below will cause "break strict-aliasing rules" in gcc - uint32_t *op = (uint32_t*)dst; - *op = *(uint32_t*)src; - *op = SWAP4(*op); - */ - - /* Below are copied from netCDF-4. - * See https://bugtracking.unidata.ucar.edu/browse/NCF-338 - * Quote "One issue we are wrestling with is how compilers optimize this - * code. For some reason, we are actually needing to add an artificial - * move to a 4 byte space to get it to work. I think what is happening is - * that the optimizer is bit shifting within a double, which is incorrect. - * The following code actually does work correctly. - * This is in Linux land, gcc. - * - * However, the above in-place byte-swap does not appear affected by this. - */ -#if 0 - uint32_t *ip = (uint32_t*)src; - uint32_t tempOut; /* cannot use pointer when gcc O2 optimizer is used */ - tempOut = SWAP4(*ip); - - *(float *)dst = *(float *)(&tempOut); -#endif - - /* OLD implementation that results in four load and four store CPU - instructions - char *op = dst; - const char *ip = src; - op[0] = ip[3]; - op[1] = ip[2]; - op[2] = ip[1]; - op[3] = ip[0]; - */ - } # endif /* !vax */ @@ -398,110 +334,24 @@ swapn4b(void *dst, const void *src, IntType nn) IntType i; uint32_t *op = (uint32_t*) dst; uint32_t *ip = (uint32_t*) src; + uint32_t tmp; for (i=0; i 0) - * { - * op[0] = ip[3]; - * op[1] = ip[2]; - * op[2] = ip[1]; - * op[3] = ip[0]; - * op += 4; - * ip += 4; - * } - */ - while (nn > 3) - { - op[0] = ip[3]; - op[1] = ip[2]; - op[2] = ip[1]; - op[3] = ip[0]; - op[4] = ip[7]; - op[5] = ip[6]; - op[6] = ip[5]; - op[7] = ip[4]; - op[8] = ip[11]; - op[9] = ip[10]; - op[10] = ip[9]; - op[11] = ip[8]; - op[12] = ip[15]; - op[13] = ip[14]; - op[14] = ip[13]; - op[15] = ip[12]; - op += 16; - ip += 16; - nn -= 4; - } - while (nn-- > 0) - { - op[0] = ip[3]; - op[1] = ip[2]; - op[2] = ip[1]; - op[3] = ip[0]; - op += 4; - ip += 4; - } -#endif } # ifndef vax inline static void swap8b(void *dst, const void *src) { -#ifdef FLOAT_WORDS_BIGENDIAN - /* copy over, make the below swap in-place */ - *(uint64_t*)dst = *(uint64_t*)src; - - uint32_t *op = (uint32_t*)dst; - *op = SWAP4(*op); - op = (uint32_t*)((char*)dst+4); - *op = SWAP4(*op); -#else uint64_t tmp; - /* use memcpy to avoid type punning */ + /* memcpy is used to handle the case of unaligned memory */ memcpy(&tmp, src, sizeof(tmp)); tmp = SWAP8(tmp); memcpy(dst, &tmp, 8); - - /* Codes below will cause "break strict-aliasing rules" in gcc - uint64_t *op = (uint64_t*)dst; - *op = *(uint64_t*)src; - *op = SWAP8(*op); - */ -#endif - -#if 0 - char *op = dst; - const char *ip = src; -# ifndef FLOAT_WORDS_BIGENDIAN - op[0] = ip[7]; - op[1] = ip[6]; - op[2] = ip[5]; - op[3] = ip[4]; - op[4] = ip[3]; - op[5] = ip[2]; - op[6] = ip[1]; - op[7] = ip[0]; -# else - op[0] = ip[3]; - op[1] = ip[2]; - op[2] = ip[1]; - op[3] = ip[0]; - op[4] = ip[7]; - op[5] = ip[6]; - op[6] = ip[5]; - op[7] = ip[4]; -#endif -#endif } # endif /* !vax */ @@ -509,100 +359,16 @@ swap8b(void *dst, const void *src) inline static void swapn8b(void *dst, const void *src, IntType nn) { -#ifdef FLOAT_WORDS_BIGENDIAN - IntType i; - uint64_t *dst_p = (uint64_t*) dst; - uint64_t *src_p = (uint64_t*) src; - for (i=0; i 0) - * { - * op[0] = ip[7]; - * op[1] = ip[6]; - * op[2] = ip[5]; - * op[3] = ip[4]; - * op[4] = ip[3]; - * op[5] = ip[2]; - * op[6] = ip[1]; - * op[7] = ip[0]; - * op += 8; - * ip += 8; - * } - */ -# ifndef FLOAT_WORDS_BIGENDIAN - while (nn > 1) - { - op[0] = ip[7]; - op[1] = ip[6]; - op[2] = ip[5]; - op[3] = ip[4]; - op[4] = ip[3]; - op[5] = ip[2]; - op[6] = ip[1]; - op[7] = ip[0]; - op[8] = ip[15]; - op[9] = ip[14]; - op[10] = ip[13]; - op[11] = ip[12]; - op[12] = ip[11]; - op[13] = ip[10]; - op[14] = ip[9]; - op[15] = ip[8]; - op += 16; - ip += 16; - nn -= 2; - } - while (nn-- > 0) - { - op[0] = ip[7]; - op[1] = ip[6]; - op[2] = ip[5]; - op[3] = ip[4]; - op[4] = ip[3]; - op[5] = ip[2]; - op[6] = ip[1]; - op[7] = ip[0]; - op += 8; - ip += 8; - } -# else - while (nn-- > 0) - { - op[0] = ip[3]; - op[1] = ip[2]; - op[2] = ip[1]; - op[3] = ip[0]; - op[4] = ip[7]; - op[5] = ip[6]; - op[6] = ip[5]; - op[7] = ip[4]; - op += 8; - ip += 8; - } -#endif -#endif } # endif /* !vax */ diff --git a/nc_test/tst_diskless.c b/nc_test/tst_diskless.c index 13ae21b354..d252de6d57 100644 --- a/nc_test/tst_diskless.c +++ b/nc_test/tst_diskless.c @@ -46,7 +46,7 @@ void fail(int line) { #endif /* Control flags */ -static int flags, persist, usenetcdf4, mmap, diskless; +static int flags, persist, usenetcdf4, use_mmap, diskless; char* smode(int mode) @@ -90,13 +90,13 @@ main(int argc, char **argv) /* Set defaults */ persist = 0; usenetcdf4 = 0; - mmap = 0; + use_mmap = 0; diskless = 0; for(i=1;i= min && value <= max)) { diff --git a/nc_test4/cdm_sea_soundings.c b/nc_test4/cdm_sea_soundings.c index f7da504ce9..b2e2f681cd 100644 --- a/nc_test4/cdm_sea_soundings.c +++ b/nc_test4/cdm_sea_soundings.c @@ -28,11 +28,11 @@ main(int argc, char **argv) /* Create phony data. */ for (i = 0; i < DIM_LEN; i++) { - if (!(data[i].temp_vl.p = malloc(sizeof(float) * (i + 1)))) + if (!(data[i].temp_vl.p = malloc(sizeof(float) * (size_t)(i + 1)))) return NC_ENOMEM; for (j = 0; j < i + 1; j++) - ((float *)(data[i].temp_vl.p))[j] = 23.5 - j; - data[i].temp_vl.len = i + 1; + ((float *)(data[i].temp_vl.p))[j] = 23.5f - (float)j; + data[i].temp_vl.len = (size_t)i + 1; } printf("\n*** Testing netcdf-4 CDM compliance: sea soundings.\n"); diff --git a/nc_test4/tst_atts3.c b/nc_test4/tst_atts3.c index 0c0bd621b2..84726014d4 100644 --- a/nc_test4/tst_atts3.c +++ b/nc_test4/tst_atts3.c @@ -1215,7 +1215,7 @@ create_file() check_err(stat,__LINE__,__FILE__); } { /* Gf */ - static const float Gf_att[5] = {-3.4028231e+38, 3.4028231e+38, -9.96921e+36, 9.96921e+36, 531} ; + static const float Gf_att[5] = {-3.4028231e+38f, 3.4028231e+38f, -9.96921e+36f, 9.96921e+36f, 531} ; stat = nc_put_att_float(ncid, NC_GLOBAL, "Gf", NC_FLOAT, 5, Gf_att); check_err(stat,__LINE__,__FILE__); } @@ -1247,7 +1247,7 @@ create_file() check_err(stat,__LINE__,__FILE__); } { /* f */ - static const float i_f_att[4] = {-3.4028231e+38, 3.4028231e+38, -9.96921e+36, 9.96921e+36} ; + static const float i_f_att[4] = {-3.4028231e+38f, 3.4028231e+38f, -9.96921e+36f, 9.96921e+36f} ; stat = nc_put_att_float(ncid, i_id, "f", NC_FLOAT, 4, i_f_att); check_err(stat,__LINE__,__FILE__); } @@ -1330,7 +1330,7 @@ create_file() } { - float fr_data[2] = {-3.4028231e+38, 3.4028231e+38} ; + float fr_data[2] = {-3.4028231e+38f, 3.4028231e+38f} ; size_t fr_startset[1] = {0} ; size_t fr_countset[1] = {2} ; stat = nc_put_vara(ncid, fr_id, fr_startset, fr_countset, fr_data); @@ -1378,7 +1378,7 @@ create_file() } { - float f1_data[1] = {-3.4028231e+38} ; + float f1_data[1] = {-3.4028231e+38f} ; size_t f1_startset[1] = {0} ; size_t f1_countset[1] = {1} ; stat = nc_put_vara(ncid, f1_id, f1_startset, f1_countset, f1_data); @@ -1426,7 +1426,7 @@ create_file() } { - float f2_data[2] = {-3.4028231e+38, 3.4028231e+38} ; + float f2_data[2] = {-3.4028231e+38f, 3.4028231e+38f} ; size_t f2_startset[1] = {0} ; size_t f2_countset[1] = {2} ; stat = nc_put_vara(ncid, f2_id, f2_startset, f2_countset, f2_data); @@ -1474,7 +1474,7 @@ create_file() } { - float f3_data[3] = {-3.4028231e+38, 3.4028231e+38, -9.96921e+36} ; + float f3_data[3] = {-3.4028231e+38f, 3.4028231e+38f, -9.96921e+36f} ; size_t f3_startset[1] = {0} ; size_t f3_countset[1] = {3} ; stat = nc_put_vara(ncid, f3_id, f3_startset, f3_countset, f3_data); @@ -1522,7 +1522,7 @@ create_file() } { - float f4_data[4] = {-3.4028231e+38, 3.4028231e+38, -9.96921e+36, 9.96921e+36} ; + float f4_data[4] = {-3.4028231e+38f, 3.4028231e+38f, -9.96921e+36f, 9.96921e+36f} ; size_t f4_startset[1] = {0} ; size_t f4_countset[1] = {4} ; stat = nc_put_vara(ncid, f4_id, f4_startset, f4_countset, f4_data); diff --git a/nc_test4/tst_chunks2.c b/nc_test4/tst_chunks2.c index f166a500eb..0499d6f392 100644 --- a/nc_test4/tst_chunks2.c +++ b/nc_test4/tst_chunks2.c @@ -50,7 +50,7 @@ calculate_waste(int ndims, size_t *dimlen, size_t *chunksize, float *waste) #endif /* Percent of the chunked file that is wasted space. */ - *waste = ((float)(chunked - unchunked) / (float)chunked) * 100.0; + *waste = ((float)(chunked - unchunked) / (float)chunked) * 100.0f; #ifdef PRINT_CHUNK_WASTE_REPORT printf("\ndimlen\tchunksize\tnum_chunks\n"); diff --git a/nc_test4/tst_compounds.c b/nc_test4/tst_compounds.c index f91c210021..731ed8bc4f 100644 --- a/nc_test4/tst_compounds.c +++ b/nc_test4/tst_compounds.c @@ -546,8 +546,8 @@ main(int argc, char **argv) hr_data_out[i].starfleet_id = i; hr_data_out[i].svc_rec = data[i]; if (sprintf(hr_data_out[i].name, "alien_%d", i) < 0) ERR; - hr_data_out[i].max_temp = 99.99; - hr_data_out[i].min_temp = -9.99; + hr_data_out[i].max_temp = 99.99f; + hr_data_out[i].min_temp = -9.99f; hr_data_out[i].percent_transporter_errosion = .030303; } @@ -659,10 +659,10 @@ main(int argc, char **argv) /* Initialize data. */ for (i = 0; i < DIM6_LEN; i++) { - obsdata[i].day = 15 * i + 1; - obsdata[i].elev = 2 * i + 1; + obsdata[i].day = 15 * (char)i + 1; + obsdata[i].elev = 2 * (short)i + 1; obsdata[i].count = 2 * i + 1; - obsdata[i].relhum = 2.0 * i + 1; + obsdata[i].relhum = 2.0f * (float)i + 1; obsdata[i].time = 2.0 * i + 1; } missing_val.day = 99; @@ -863,7 +863,7 @@ main(int argc, char **argv) for (i = 0; i < DIM1_LEN; i++) { if (sprintf(hr_data_out[i].name, "alien_%d", i) < 0) ERR; - hr_data_out[i].max_temp = 99.99; + hr_data_out[i].max_temp = 99.99f; } /* Create a file with a nested compound type attribute and variable. */ @@ -927,7 +927,7 @@ main(int argc, char **argv) /* Create some phony data. */ for (i = 0; i < DIM1_LEN; i++) { - hr_data_out[i].max_temp = 99.99; + hr_data_out[i].max_temp = 99.99f; for (j = 0; j < ARRAY_LEN; j++) hr_data_out[i].name[j] = j; } diff --git a/nc_test4/tst_compounds2.c b/nc_test4/tst_compounds2.c index 7b1af1de96..8a9fadb4db 100644 --- a/nc_test4/tst_compounds2.c +++ b/nc_test4/tst_compounds2.c @@ -49,7 +49,7 @@ main(int argc, char **argv) /* Create some phony data. */ for (i = 0; i < DIM1_LEN; i++) { - data_out[i].y = 99.99; + data_out[i].y = 99.99f; for (j = 0; j < ARRAY_LEN; j++) data_out[i].x[j] = j; } diff --git a/nc_test4/tst_compounds3.c b/nc_test4/tst_compounds3.c index c09cb0b448..d388658bbb 100644 --- a/nc_test4/tst_compounds3.c +++ b/nc_test4/tst_compounds3.c @@ -136,7 +136,7 @@ main(int argc, char **argv) for (i = 0; i < ATT_LEN; i++) { - a1_att[i].s1.x = 13.3; + a1_att[i].s1.x = 13.3f; a1_att[i].s1.y = 13.3; } /* Create a file with two groups, define a type in each group, diff --git a/nc_test4/tst_converts2.c b/nc_test4/tst_converts2.c index 0f3bd92258..da6006a163 100644 --- a/nc_test4/tst_converts2.c +++ b/nc_test4/tst_converts2.c @@ -215,7 +215,7 @@ main(int argc, char **argv) if (uint64_in != ivalue) ERR; if (nc_get_var_float(ncid, varid, &float_in)) ERR; - if(fabs( (float_in-X_INT_MAX) - ((float)ivalue - X_INT_MAX)) > 1) ERR; + if(fabs( (float_in-(float)X_INT_MAX) - ((float)ivalue - (float)X_INT_MAX)) > 1) ERR; if (nc_get_var_double(ncid, varid, &double_in)) ERR; if (double_in != (double)ivalue) ERR; diff --git a/nc_test4/tst_coords.c b/nc_test4/tst_coords.c index 000b8a632b..f7133a57f9 100644 --- a/nc_test4/tst_coords.c +++ b/nc_test4/tst_coords.c @@ -38,8 +38,8 @@ main(int argc, char **argv) #define LAT_NAME "lat" #define LON_NAME "lon" #define NVARS 2 -#define START_LAT 25.0 -#define START_LON -125.0 +#define START_LAT 25.0f +#define START_LON -125.0f { int ncid, lon_dimid, lat_dimid; int lat_varid, lon_varid; @@ -52,9 +52,9 @@ main(int argc, char **argv) /* Initialize coord data. */ for (lat = 0; lat < NLAT; lat++) - lats[lat] = START_LAT + 5. * lat; + lats[lat] = START_LAT + 5.f * (float)lat; for (lon = 0; lon < NLON; lon++) - lons[lon] = START_LON + 5. * lon; + lons[lon] = START_LON + 5.f * (float)lon; /* Create file with two dimensions. */ if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; @@ -103,7 +103,7 @@ main(int argc, char **argv) #define DIM0_NAME "d0" #define CACHE_SIZE 1000000 #define CACHE_NELEMS 1009 -#define CACHE_PREEMPTION .90 +#define CACHE_PREEMPTION .90f int ncid, dimid, varid; char name_in[NC_MAX_NAME + 1]; @@ -413,12 +413,12 @@ main(int argc, char **argv) check_err(stat,__LINE__,__FILE__); } { /* missing_value */ - static const float pr_missing_value_att[1] = {1e+20} ; + static const float pr_missing_value_att[1] = {1e+20f} ; stat = nc_put_att_float(root_grp, pr_id, "missing_value", NC_FLOAT, 1, pr_missing_value_att); check_err(stat,__LINE__,__FILE__); } { /* _FillValue */ - static const float pr_FillValue_att[1] = {1e+20} ; + static const float pr_FillValue_att[1] = {1e+20f} ; stat = nc_put_att_float(root_grp, pr_id, "_FillValue", NC_FLOAT, 1, pr_FillValue_att); check_err(stat,__LINE__,__FILE__); } diff --git a/nc_test4/tst_dims.c b/nc_test4/tst_dims.c index d821e20f73..f64ba66d59 100644 --- a/nc_test4/tst_dims.c +++ b/nc_test4/tst_dims.c @@ -849,9 +849,9 @@ main(int argc, char **argv) /* Lats and lons suitable for some South American data. */ for (lat[0] = 40.0, i = 1; i < LAT_LEN; i++) - lat[i] = lat[i - 1] + .5; + lat[i] = (float)lat[i - 1] + .5f; for (lon[0] = 20.0, i = 1; i < LON_LEN; i++) - lon[i] = lon[i - 1] + 1.5; + lon[i] = (float)lon[i - 1] + 1.5f; /* Some phoney 2D pressure data. */ for (i = 0; i < LAT_LEN; i++) @@ -1002,9 +1002,9 @@ main(int argc, char **argv) /* Some phony 1D lats and lons. */ for (i = 0; i < LAT_LEN; i++) - lat[i] = i * 5.; + lat[i] = (float)i * 5.f; for (i = 0; i < LON_LEN; i++) - lon[i] = i * 5.; + lon[i] = (float)i * 5.f; if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR; diff --git a/nc_test4/tst_files.c b/nc_test4/tst_files.c index e0f7ddf302..f2ad31bedd 100644 --- a/nc_test4/tst_files.c +++ b/nc_test4/tst_files.c @@ -209,7 +209,7 @@ main(int argc, char **argv) int dimids_var[2], var_type; size_t dim_len; char dim_name[NC_MAX_NAME+1], var_name[NC_MAX_NAME+1]; - float float_in, float_out = 99.99; + float float_in, float_out = 99.99f; int int_in, int_out = -9999; /* Create a file, this time with attributes. */ diff --git a/nc_test4/tst_put_vars.c b/nc_test4/tst_put_vars.c index 62f59ff80e..1ce011c2a3 100644 --- a/nc_test4/tst_put_vars.c +++ b/nc_test4/tst_put_vars.c @@ -42,7 +42,7 @@ main() int dimids[NDIMS]; for (i = 0; i < (X_LEN * Y_LEN * Z_LEN); i++) - mydata[i] = i; + mydata[i] = (float)i; /* create the file in NetCDF-4 format */ if ((retval = nc_create(FILE_NAME, NC_NETCDF4, &ncid))) diff --git a/nc_test4/tst_quantize.c b/nc_test4/tst_quantize.c index 47c8ce2eef..eb43c30437 100644 --- a/nc_test4/tst_quantize.c +++ b/nc_test4/tst_quantize.c @@ -277,7 +277,7 @@ main(int argc, char **argv) { int ncid, varid1, varid2; int quantize_mode_in, nsd_in; - float float_data[DIM_LEN_1] = {1.1111111}; + float float_data[DIM_LEN_1] = {1.1111111f}; double double_data[DIM_LEN_1] = {1.111111111111}; printf("\t\t**** testing quantize algorithm %d...\n", quantize_mode[q]); @@ -394,7 +394,7 @@ main(int argc, char **argv) { int ncid, dimid, varid1, varid2; int quantize_mode_in, nsd_in; - float float_data[DIM_LEN_1] = {1.1111111}; + float float_data[DIM_LEN_1] = {1.1111111f}; double double_data[DIM_LEN_1] = {1.111111111111}; printf("\t\t**** testing quantize algorithm %d...\n", quantize_mode[q]); @@ -485,7 +485,7 @@ main(int argc, char **argv) { int ncid, dimid, varid1, varid2; int quantize_mode_in, nsd_in; - float float_data[DIM_LEN_5] = {1.11111111, 1.0, 9.99999999, 12345.67, .1234567}; + float float_data[DIM_LEN_5] = {1.11111111f, 1.0f, 9.99999999f, 12345.67f, .1234567f}; double double_data[DIM_LEN_5] = {1.1111111, 1.0, 9.999999999, 1234567890.12345, 123456789012345.0}; int x; @@ -616,7 +616,7 @@ main(int argc, char **argv) { int ncid, dimid, varid1, varid2; int quantize_mode_in, nsd_in; - float float_data[DIM_LEN_1] = {1.1111111}; + float float_data[DIM_LEN_1] = {1.1111111f}; double double_data[DIM_LEN_1] = {1.111111111111}; printf("\t\t**** testing quantize algorithm %d...\n", quantize_mode[q]); @@ -733,7 +733,7 @@ main(int argc, char **argv) { int ncid, dimid, varid1, varid2; int quantize_mode_in, nsd_in; - float float_data[DIM_LEN_5] = {1.11111111, 1.0, 9.99999999, 12345.67, .1234567}; + float float_data[DIM_LEN_5] = {1.11111111f, 1.0, 9.99999999f, 12345.67f, .1234567f}; double double_data[DIM_LEN_5] = {1.1111111, 1.0, 9.999999999, 1234567890.12345, 123456789012345.0}; int x; @@ -862,7 +862,7 @@ main(int argc, char **argv) { int ncid, dimid, varid1, varid2; int quantize_mode_in, nsd_in; - float float_data[DIM_LEN_5] = {1.11111111, NC_FILL_FLOAT, 9.99999999, 12345.67, NC_FILL_FLOAT}; + float float_data[DIM_LEN_5] = {1.11111111f, NC_FILL_FLOAT, 9.99999999f, 12345.67f, NC_FILL_FLOAT}; double double_data[DIM_LEN_5] = {1.1111111, NC_FILL_DOUBLE, 9.999999999, 1234567890.12345, NC_FILL_DOUBLE}; int x; @@ -987,13 +987,13 @@ main(int argc, char **argv) SUMMARIZE_ERR; printf("\t**** testing more quantization values with custom fill values...\n"); { -#define CUSTOM_FILL_FLOAT 99.99999 +#define CUSTOM_FILL_FLOAT 99.99999f #define CUSTOM_FILL_DOUBLE -99999.99999 for (q = 0; q < NUM_QUANTIZE_MODES; q++) { int ncid, dimid, varid1, varid2; int quantize_mode_in, nsd_in; - float float_data[DIM_LEN_5] = {1.11111111, CUSTOM_FILL_FLOAT, 9.99999999, 12345.67, CUSTOM_FILL_FLOAT}; + float float_data[DIM_LEN_5] = {1.11111111f, CUSTOM_FILL_FLOAT, 9.99999999f, 12345.67f, CUSTOM_FILL_FLOAT}; double double_data[DIM_LEN_5] = {1.1111111, CUSTOM_FILL_DOUBLE, 9.999999999, 1234567890.12345, CUSTOM_FILL_DOUBLE}; float custom_fill_float = CUSTOM_FILL_FLOAT; double custom_fill_double = CUSTOM_FILL_DOUBLE; @@ -1311,8 +1311,8 @@ main(int argc, char **argv) ERR; for (i = 0; i < DIM_LEN_SIMPLE; i++) { - float_data[i] = 1.5 * i; - double_data[i] = 1.5 * i; + float_data[i] = 1.5f * (float)i; + double_data[i] = 1.5 * (double)i; } /* Create the file. */ diff --git a/nc_test4/tst_unlim_vars.c b/nc_test4/tst_unlim_vars.c index e6f157a1ce..e488f36ad6 100644 --- a/nc_test4/tst_unlim_vars.c +++ b/nc_test4/tst_unlim_vars.c @@ -51,7 +51,7 @@ main(int argc, char **argv) for (time = 0; time < NUM_TIMESTEPS; time++) for (lat = 0; lat < LAT_LEN; lat++) for (lon = 0; lon < LON_LEN; lon++) - data_out[time][lat][lon] = 25.5 + lat + lon + time; + data_out[time][lat][lon] = 25.5f + (float)(lat + lon + time); /* Create a file with a 3D surface temp variable. */ if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR; diff --git a/nc_test4/tst_vars.c b/nc_test4/tst_vars.c index a605da71ef..f94fc0dcbb 100644 --- a/nc_test4/tst_vars.c +++ b/nc_test4/tst_vars.c @@ -56,10 +56,10 @@ #define NVARS_EX 4 /* These are used to construct some example data. */ -#define SAMPLE_PRESSURE 900 -#define SAMPLE_TEMP 9.0 -#define START_LAT 25.0 -#define START_LON -125.0 +#define SAMPLE_PRESSURE 900.f +#define SAMPLE_TEMP 9.0f +#define START_LAT 25.0f +#define START_LON -125.0f /* For the units attributes. */ #define UNITS "units" @@ -97,16 +97,16 @@ create_4D_example(char *file_name, int cmode) * would have some real data to write, for example, model * output. */ for (lat = 0; lat < NLAT; lat++) - lats[lat] = START_LAT + 5.*lat; + lats[lat] = START_LAT + 5.f*(float)lat; for (lon = 0; lon < NLON; lon++) - lons[lon] = START_LON + 5.*lon; + lons[lon] = START_LON + 5.f*(float)lon; for (lvl = 0; lvl < NLVL; lvl++) for (lat = 0; lat < NLAT; lat++) for (lon = 0; lon < NLON; lon++) { - pres_out[lvl][lat][lon] = SAMPLE_PRESSURE + i; - temp_out[lvl][lat][lon] = SAMPLE_TEMP + i++; + pres_out[lvl][lat][lon] = SAMPLE_PRESSURE + (float)i; + temp_out[lvl][lat][lon] = SAMPLE_TEMP + (float)i++; } /* Create the file. */ @@ -307,7 +307,7 @@ main(int argc, char **argv) unsigned short ushort_out[DIM1_LEN][DIM2_LEN] = {{110, 128, 255},{110, 128, 255}}; short short_in[DIM1_LEN][DIM2_LEN], short_out[DIM1_LEN][DIM2_LEN] = {{-110, -128, 255},{-110, -128, 255}}; int int_in[DIM1_LEN][DIM2_LEN], int_out[DIM1_LEN][DIM2_LEN] = {{0, 128, 255},{0, 128, 255}}; - float float_in[DIM1_LEN][DIM2_LEN], float_out[DIM1_LEN][DIM2_LEN] = {{-.1, 9999.99, 100.001},{-.1, 9999.99, 100.001}}; + float float_in[DIM1_LEN][DIM2_LEN], float_out[DIM1_LEN][DIM2_LEN] = {{-.1f, 9999.99f, 100.001f},{-.1f, 9999.99f, 100.001f}}; double double_in[DIM1_LEN][DIM2_LEN], double_out[DIM1_LEN][DIM2_LEN] = {{0.02, .1128, 1090.1},{0.02, .1128, 1090.1}}; unsigned int uint_in[DIM1_LEN][DIM2_LEN], uint_out[DIM1_LEN][DIM2_LEN] = {{0, 128, 255},{0, 128, 255}}; long long int64_in[DIM1_LEN][DIM2_LEN], int64_out[DIM1_LEN][DIM2_LEN] = {{-111, 777, 100},{-111, 777, 100}}; diff --git a/nc_test4/tst_xplatform2.c b/nc_test4/tst_xplatform2.c index 02b7dc7067..6d2043d9f4 100644 --- a/nc_test4/tst_xplatform2.c +++ b/nc_test4/tst_xplatform2.c @@ -312,7 +312,7 @@ main(int argc, char **argv) return NC_ENOMEM; for (j = 0; j < vlen_of_comp_out[i].len; j++) { - ((struct s1 *)vlen_of_comp_out[i].p)[j].x = 42.42; + ((struct s1 *)vlen_of_comp_out[i].p)[j].x = 42.42f; ((struct s1 *)vlen_of_comp_out[i].p)[j].y = 2.0; } } @@ -322,7 +322,7 @@ main(int argc, char **argv) for (i = 0; i < DIM2_LEN; i++) for (j = 0; j < NUM_S1; j++) { - comp_array_of_comp_out[i].data[j].x = 42.42; + comp_array_of_comp_out[i].data[j].x = 42.42f; comp_array_of_comp_out[i].data[j].y = 2.0; } @@ -336,7 +336,7 @@ main(int argc, char **argv) return NC_ENOMEM; for (k = 0; k < comp_array_of_vlen_of_comp_out[i].data[j].len; k++) { - ((struct s1 *)comp_array_of_vlen_of_comp_out[i].data[j].p)[k].x = 42.42; + ((struct s1 *)comp_array_of_vlen_of_comp_out[i].data[j].p)[k].x = 42.42f; ((struct s1 *)comp_array_of_vlen_of_comp_out[i].data[j].p)[k].y = 2.0; } } diff --git a/ncdap_test/t_dap3a.c b/ncdap_test/t_dap3a.c index bbfd36252e..4ededc15c8 100644 --- a/ncdap_test/t_dap3a.c +++ b/ncdap_test/t_dap3a.c @@ -317,7 +317,7 @@ static float int162float32_data[DIMSIZE]={0.000,256.000,512.000,768.000,1024.000 static int int32_data[DIMSIZE]={0,2048,4096,6144,8192,10240,12288,14336,16384,18432,20480,22528,24576,26624,28672,30720,32768,34816,36864,38912,40960,43008,45056,47104,49152}; static float int32tofloat32_data[DIMSIZE]={0.000,2048.000,4096.000,6144.000,8192.000,10240.000,12288.000,14336.000,16384.000,18432.000,20480.000,22528.000,24576.000,26624.000,28672.000,30720.000,32768.000,34816.000,36864.000,38912.000,40960.000,43008.000,45056.000,47104.000,49152.000}; static long int32toilong_data[DIMSIZE]={0,2048,4096,6144,8192,10240,12288,14336,16384,18432,20480,22528,24576,26624,28672,30720,32768,34816,36864,38912,40960,43008,45056,47104,49152}; -static float float32_data[DIMSIZE]={0.000,0.010,0.020,0.030,0.040,0.050,0.060,0.070,0.080,0.090,0.100,0.110,0.120,0.130,0.140,0.149,0.159,0.169,0.179,0.189,0.199,0.208,0.218,0.228,0.238}; +static float float32_data[DIMSIZE]={0.000f,0.010f,0.020f,0.030f,0.040f,0.050f,0.060f,0.070f,0.080f,0.090f,0.100f,0.110f,0.120f,0.130f,0.140f,0.149f,0.159f,0.169f,0.179f,0.189f,0.199f,0.208f,0.218f,0.228f,0.238f}; static double float64_data[DIMSIZE]={1.000,1.000,1.000,1.000,0.999,0.999,0.998,0.998,0.997,0.996,0.995,0.994,0.993,0.992,0.990,0.989,0.987,0.986,0.984,0.982,0.980,0.978,0.976,0.974,0.971}; #ifndef USE_NETCDF4 diff --git a/ncdap_test/test_cvt.c b/ncdap_test/test_cvt.c index 46f52de2a0..3485dd4f7a 100644 --- a/ncdap_test/test_cvt.c +++ b/ncdap_test/test_cvt.c @@ -316,7 +316,7 @@ static float int162float32_data[DIMSIZE]={0.000,256.000,512.000,768.000,1024.000 static int int32_data[DIMSIZE]={0,2048,4096,6144,8192,10240,12288,14336,16384,18432,20480,22528,24576,26624,28672,30720,32768,34816,36864,38912,40960,43008,45056,47104,49152}; static float int32tofloat32_data[DIMSIZE]={0.000,2048.000,4096.000,6144.000,8192.000,10240.000,12288.000,14336.000,16384.000,18432.000,20480.000,22528.000,24576.000,26624.000,28672.000,30720.000,32768.000,34816.000,36864.000,38912.000,40960.000,43008.000,45056.000,47104.000,49152.000}; static long int32toilong_data[DIMSIZE]={0,2048,4096,6144,8192,10240,12288,14336,16384,18432,20480,22528,24576,26624,28672,30720,32768,34816,36864,38912,40960,43008,45056,47104,49152}; -static float float32_data[DIMSIZE]={0.000,0.010,0.020,0.030,0.040,0.050,0.060,0.070,0.080,0.090,0.100,0.110,0.120,0.130,0.140,0.149,0.159,0.169,0.179,0.189,0.199,0.208,0.218,0.228,0.238}; +static float float32_data[DIMSIZE]={0.000f,0.010f,0.020f,0.030f,0.040f,0.050f,0.060f,0.070f,0.080f,0.090f,0.100f,0.110f,0.120f,0.130f,0.140f,0.149f,0.159f,0.169f,0.179f,0.189f,0.199f,0.208f,0.218f,0.228f,0.238f}; static double float64_data[DIMSIZE]={1.000,1.000,1.000,1.000,0.999,0.999,0.998,0.998,0.997,0.996,0.995,0.994,0.993,0.992,0.990,0.989,0.987,0.986,0.984,0.982,0.980,0.978,0.976,0.974,0.971}; #ifndef USE_NETCDF4 diff --git a/ncdap_test/test_varm3.c b/ncdap_test/test_varm3.c index 707bf53011..740d6fa6f6 100644 --- a/ncdap_test/test_varm3.c +++ b/ncdap_test/test_varm3.c @@ -33,33 +33,33 @@ TODO: Note that this test uses thredds server #define VAR "SST" static float expected_stride1[12] = { -29.430857, -29.403780, -29.325428, -29.578333, -29.660833, -29.378437, -29.151943, -29.109715, -29.114864, -29.550285, -29.542500, -29.500286 +29.430857f, +29.403780f, +29.325428f, +29.578333f, +29.660833f, +29.378437f, +29.151943f, +29.109715f, +29.114864f, +29.550285f, +29.542500f, +29.500286f }; static float expected_stride2[6] = { -29.430857, -29.325428, -29.660833, -29.151943, -29.114864, -29.542500 +29.430857f, +29.325428f, +29.660833f, +29.151943f, +29.114864f, +29.542500f }; static float expected_stride3[3] = { -29.430857, -29.378437, -29.542500 +29.430857f, +29.378437f, +29.542500f }; void diff --git a/ncdump/nccopy.c b/ncdump/nccopy.c index 2ac4a73d9c..ae041547e4 100644 --- a/ncdump/nccopy.c +++ b/ncdump/nccopy.c @@ -7,6 +7,7 @@ *********************************************************************/ #include "config.h" /* for USE_NETCDF4 macro */ +#include #include #include #ifdef HAVE_GETOPT_H @@ -2310,7 +2311,7 @@ main(int argc, char**argv) double dval = double_with_suffix(optarg); /* "K" for kilobytes. "M" for megabytes, ... */ if(dval < 0) error("Suffix used for '-m' option value must be K, M, G, T, or P"); - option_copy_buffer_size = dval; + option_copy_buffer_size = (size_t)dval; break; } case 'h': /* non-default size of chunk cache */ @@ -2318,7 +2319,7 @@ main(int argc, char**argv) double dval = double_with_suffix(optarg); /* "K" for kilobytes. "M" for megabytes, ... */ if(dval < 0) error("Suffix used for '-h' option value must be K, M, G, T, or P"); - option_chunk_cache_size = dval; + option_chunk_cache_size = (size_t)dval; break; } case 'e': /* number of elements chunk cache can hold */ @@ -2326,7 +2327,7 @@ main(int argc, char**argv) double dval = double_with_suffix(optarg); /* "K" for kilobytes. "M" for megabytes, ... */ if(dval < 0 ) error("Suffix used for '-e' option value must be K, M, G, T, or P"); - option_chunk_cache_nelems = (long)dval; + option_chunk_cache_nelems = (size_t)dval; break; } case 'r': diff --git a/ncdump/ncdump.c b/ncdump/ncdump.c index f577079813..abaa20e387 100644 --- a/ncdump/ncdump.c +++ b/ncdump/ncdump.c @@ -699,7 +699,7 @@ pr_att_valsx( case NC_BYTE: case NC_SHORT: case NC_INT: - ii = vals[iel]; + ii = (int)vals[iel]; res = snprintf(gps, PRIM_LEN, "%d", ii); assert(res < PRIM_LEN); (void) strlcat(attvals, gps, attvalslen); @@ -708,28 +708,28 @@ pr_att_valsx( case NC_UBYTE: case NC_USHORT: case NC_UINT: - ui = vals[iel]; + ui = (unsigned int)vals[iel]; res = snprintf(gps, PRIM_LEN, "%u", ui); assert(res < PRIM_LEN); (void) strlcat(attvals, gps, attvalslen); (void) strlcat(attvals, iel < len-1 ? " " : "", attvalslen); break; case NC_INT64: - i64 = vals[iel]; + i64 = (int64_t)vals[iel]; res = snprintf(gps, PRIM_LEN, "%lld", i64); assert(res < PRIM_LEN); (void) strlcat(attvals, gps, attvalslen); (void) strlcat(attvals, iel < len-1 ? " " : "", attvalslen); break; case NC_UINT64: - ui64 = vals[iel]; + ui64 = (uint64_t)vals[iel]; res = snprintf(gps, PRIM_LEN, "%llu", ui64); assert(res < PRIM_LEN); (void) strlcat(attvals, gps, attvalslen); (void) strlcat(attvals, iel < len-1 ? " " : "", attvalslen); break; case NC_FLOAT: - ff = vals[iel]; + ff = (float)vals[iel]; res = snprintf(gps, PRIM_LEN, float_attx_fmt, ff); assert(res < PRIM_LEN); tztrim(gps); /* trim trailing 0's after '.' */ @@ -847,7 +847,7 @@ pr_att( case NC_VLEN: /* because size returned for vlen is base type size, but we * need space to read array of vlen structs into ... */ - data = emalloc((att.len + 1) * sizeof(nc_vlen_t)); + data = emalloc((att.len + 1) * sizeof(nc_vlen_t)); break; case NC_OPAQUE: data = emalloc((att.len + 1) * type_size); diff --git a/ncdump/tst_create_files.c b/ncdump/tst_create_files.c index c8122042a9..e4cee78e40 100644 --- a/ncdump/tst_create_files.c +++ b/ncdump/tst_create_files.c @@ -226,8 +226,8 @@ main(int argc, char **argv) #define ATT_NAME_CMP "my_favorite_wind_speeds" #define COMPOUND_NAME "wind_vector" #define NUM_FAVS 3 -#define U_VALUE 13.3 -#define V_VALUE 12.2 +#define U_VALUE 13.3f +#define V_VALUE 12.2f { int ncid; diff --git a/ncdump/tst_vlen_data.c b/ncdump/tst_vlen_data.c index ad5d18388e..f11b3932b7 100644 --- a/ncdump/tst_vlen_data.c +++ b/ncdump/tst_vlen_data.c @@ -91,7 +91,7 @@ main(int argc, char **argv) array[i] = (float *) malloc(ncolumns * sizeof(float)); if(array[i] == NULL) ERR; for (j = 0; j < ncolumns; j++) { - array[i][j] = 10.0 * (i + 1) + j; + array[i][j] = 10.0f * (float)(i + 1) + (float)j; } } array[4][0] = missing_value; /* overwrite last row with missing for equality test */ diff --git a/nctest/CMakeLists.txt b/nctest/CMakeLists.txt index d7de84a6cd..ada220f2ab 100644 --- a/nctest/CMakeLists.txt +++ b/nctest/CMakeLists.txt @@ -18,7 +18,9 @@ TARGET_LINK_LIBRARIES(nctest netcdf) ADD_TEST(nctest ${EXECUTABLE_OUTPUT_PATH}/nctest) add_bin_test_no_prefix(tst_rename) -add_sh_test(nctest compare_test_files) -IF(HAVE_BASH) - SET_TESTS_PROPERTIES(nctest_compare_test_files PROPERTIES DEPENDS nctest) -ENDIF(HAVE_BASH) +IF(BUILD_UTILITIES) + add_sh_test(nctest compare_test_files) + IF(HAVE_BASH) + SET_TESTS_PROPERTIES(nctest_compare_test_files PROPERTIES DEPENDS nctest) + ENDIF(HAVE_BASH) +ENDIF(BUILD_UTILITIES) diff --git a/nczarr_test/CMakeLists.txt b/nczarr_test/CMakeLists.txt index e58add623a..6483f46444 100644 --- a/nczarr_test/CMakeLists.txt +++ b/nczarr_test/CMakeLists.txt @@ -9,6 +9,7 @@ SET(abs_top_srcdir ${CMAKE_CURRENT_SOURCE_DIR}) remove_definitions(-DDLL_EXPORT) + # Create nczarr tests from nc_test4 tests macro(NCZARR_SH_TEST basename src) FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/../${src}/tst_${basename}.sh SHSOURCE) @@ -55,36 +56,67 @@ ENDIF() IF(ENABLE_TESTS) - SET(COMMONSRC ut_util.c ut_test.c ${XGETOPTSRC}) - - SET(TSTCOMMONSRC test_utils.c test_utils.h ${XGETOPTSRC}) - - INCLUDE_DIRECTORIES(../libnczarr ../plugins ${CMAKE_SOURCE_DIR}/nczarr_test) + add_library(ut_util STATIC ut_util.c ut_test.c ut_includes.h ut_test.h ${XGETOPTSRC}) + IF(ENABLE_DLL) + target_compile_definitions(ut_util PUBLIC -DDLL_EXPORT -DDLL_NETCDF) + ENDIF(ENABLE_DLL) + target_include_directories(ut_util PUBLIC ../libnczarr ../plugins ${CMAKE_CURRENT_LIST_DIR}) + target_link_libraries(ut_util PUBLIC netcdf ${ALL_TLL_LIBS}) + + add_library(test_utils STATIC test_utils.c test_utils.h ${XGETOPTSRC}) + IF(ENABLE_DLL) + target_compile_definitions(test_utils PUBLIC -DDLL_EXPORT -DDLL_NETCDF) + ENDIF(ENABLE_DLL) + target_include_directories(test_utils PUBLIC ../libnczarr ../plugins ${CMAKE_CURRENT_LIST_DIR}) + target_link_libraries(test_utils PUBLIC netcdf ${ALL_TLL_LIBS}) + + set_target_properties(ut_util test_utils PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_BINARY_DIR} + RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR} + ) + + macro(build_bin_test_with_util_lib F UTIL_LIB) + build_bin_test(${F}) + IF(ENABLE_DLL) + target_compile_definitions(${F} PUBLIC -DDLL_NETCDF) + ENDIF(ENABLE_DLL) + target_link_libraries(${F} ${UTIL_LIB} ${ALL_TLL_LIBS}) + endmacro() + + macro(add_bin_test_with_util_lib PREFIX F UTIL_LIB) + add_bin_test(${PREFIX} ${F}) + IF(ENABLE_DLL) + target_compile_definitions(${PREFIX}_${F} PUBLIC -DDLL_NETCDF) + ENDIF(ENABLE_DLL) + target_link_libraries(${PREFIX}_${F} ${UTIL_LIB} ${ALL_TLL_LIBS}) + + endmacro() # Base tests # The tests are set up as a combination of shell scripts and executables that # must be run in a particular order. It is painful but will use macros to help # keep it from being too bad. - BUILD_BIN_TEST(ut_map ${COMMONSRC}) - BUILD_BIN_TEST(ut_mapapi ${COMMONSRC}) - BUILD_BIN_TEST(ut_json ${COMMONSRC}) + build_bin_test_with_util_lib(ut_map ut_util) + build_bin_test_with_util_lib(ut_mapapi ut_util) + build_bin_test_with_util_lib(ut_json ut_util) - BUILD_BIN_TEST(test_fillonlyz ${TSTCOMMONSRC}) - BUILD_BIN_TEST(test_quantize ${TSTCOMMONSRC}) - BUILD_BIN_TEST(test_notzarr ${TSTCOMMONSRC}) + build_bin_test_with_util_lib(test_fillonlyz test_utils) + build_bin_test_with_util_lib(test_quantize test_utils) + build_bin_test_with_util_lib(test_notzarr test_utils) # ADD_BIN_TEST(nczarr_test test_endians ${TSTCOMMONSRC}) # Unlimited Tests IF(USE_HDF5) - ADD_BIN_TEST(nczarr_test test_unlim_vars ${TSTCOMMONSRC}) - ADD_BIN_TEST(nczarr_test test_put_vars_two_unlim_dim ${TSTCOMMONSRC}) - BUILD_BIN_TEST(test_zchunks ${COMMONSRC}) - BUILD_BIN_TEST(test_zchunks2 ${COMMONSRC}) - BUILD_BIN_TEST(test_zchunks3 ${COMMONSRC}) - BUILD_BIN_TEST(test_unlim_io ${TSTCOMMONSRC}) - BUILD_BIN_TEST(test_chunking ${TSTCOMMONSRC}) + add_bin_test_with_util_lib(nczarr_test test_unlim_vars test_utils) + add_bin_test_with_util_lib(nczarr_test test_put_vars_two_unlim_dim test_utils) + build_bin_test_with_util_lib(test_zchunks ut_util) + build_bin_test_with_util_lib(test_zchunks2 ut_util) + build_bin_test_with_util_lib(test_zchunks3 ut_util) + build_bin_test_with_util_lib(test_unlim_io test_utils) + build_bin_test_with_util_lib(test_chunking test_utils) ADD_SH_TEST(nczarr_test run_nccopyz) ADD_SH_TEST(nczarr_test run_unlim_io) ADD_SH_TEST(nczarr_test run_nccopy5) @@ -99,11 +131,11 @@ IF(ENABLE_TESTS) # Helper programs for testing BUILD_BIN_TEST(zhex) - BUILD_BIN_TEST(zisjson ${COMMONSRC}) + build_bin_test_with_util_lib(zisjson ut_util) TARGET_INCLUDE_DIRECTORIES(zisjson PUBLIC ../libnczarr) - BUILD_BIN_TEST(zs3parse ${COMMONSRC}) + build_bin_test_with_util_lib(zs3parse ut_util) TARGET_INCLUDE_DIRECTORIES(zs3parse PUBLIC ../libnczarr) - BUILD_BIN_TEST(zmapio ${COMMONSRC}) + build_bin_test_with_util_lib(zmapio ut_util) IF(ENABLE_S3 AND NOT WITH_S3_TESTING STREQUAL "NO") # Helper programs for testing @@ -132,9 +164,7 @@ IF(ENABLE_TESTS) endif() if(ENABLE_NCDUMPCHUNKS) - SET(ncdumpchunks_SOURCE ncdumpchunks.c) - SET(ncdumpchunks_SOURCE ${ncdumpchunks_SOURCE} ${XGETOPTSRC}) - BUILD_BIN_TEST(ncdumpchunks ${ncdumpchunks_SOURCE}) + build_bin_test_with_util_lib(ncdumpchunks ut_util) endif() IF(BUILD_UTILITIES) @@ -175,7 +205,7 @@ IF(ENABLE_TESTS) build_bin_test(test_nczfilter) build_bin_test(test_filter_vlen) build_bin_test(testfilter) - build_bin_test(testfilter_misc) + build_bin_test_with_util_lib(testfilter_misc test_utils) build_bin_test(testfilter_multi) build_bin_test(testfilter_order) build_bin_test(testfilter_repeat) diff --git a/nczarr_test/test_zchunks2.c b/nczarr_test/test_zchunks2.c index 7b471a2b9f..54963761c8 100644 --- a/nczarr_test/test_zchunks2.c +++ b/nczarr_test/test_zchunks2.c @@ -52,7 +52,7 @@ calculate_waste(int ndims, size_t *dimlen, size_t *chunksize, float *waste) #endif /* Percent of the chunked file that is wasted space. */ - *waste = ((float)(chunked - unchunked) / (float)chunked) * 100.0; + *waste = ((float)(chunked - unchunked) / (float)chunked) * 100.0f; #ifdef PRINT_CHUNK_WASTE_REPORT printf("\ndimlen\tchunksize\tnum_chunks\n"); diff --git a/nczarr_test/ut_test.h b/nczarr_test/ut_test.h index 63ecad3cba..efe26c3658 100644 --- a/nczarr_test/ut_test.h +++ b/nczarr_test/ut_test.h @@ -62,18 +62,18 @@ struct Test { int (*test)(void); }; -extern struct UTOptions utoptions; +EXTERNL struct UTOptions utoptions; #define NCCHECK(expr) nccheck((expr),__LINE__) -extern void usage(int err); -extern int ut_init(int argc, char** argv, struct UTOptions* test); -extern void ut_final(void); +EXTERNL void usage(int err); +EXTERNL int ut_init(int argc, char** argv, struct UTOptions* test); +EXTERNL void ut_final(void); -extern void nccheck(int stat, int line); -extern char* makeurl(const char* file, NCZM_IMPL, struct UTOptions*); +EXTERNL void nccheck(int stat, int line); +EXTERNL char* makeurl(const char* file, NCZM_IMPL, struct UTOptions*); //extern int setup(int argc, char** argv); -extern struct Test* findtest(const char* cmd, struct Test* tests); -extern int runtests(const char** cmds, struct Test* tests); +EXTERNL struct Test* findtest(const char* cmd, struct Test* tests); +EXTERNL int runtests(const char** cmds, struct Test* tests); #endif /*ZTEST_H*/ diff --git a/plugins/NCZhdf5filters.c b/plugins/NCZhdf5filters.c index 6598401a24..0b4cdf7628 100644 --- a/plugins/NCZhdf5filters.c +++ b/plugins/NCZhdf5filters.c @@ -98,9 +98,6 @@ NCZ_raw_codec_to_hdf5(const NCproplist* env, const char* codec, int* idp, size_t if(nparamsp == NULL || paramsp == NULL) {stat = NC_EINTERNAL; goto done;} - if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL) - {stat = NC_ENOMEM; goto done;} - ncplistget(env,"zarrformat",&zarrformat,NULL); /* parse the JSON */ @@ -132,6 +129,11 @@ NCZ_raw_codec_to_hdf5(const NCproplist* env, const char* codec, int* idp, size_t /* Validate nparams */ if(NCJlength(jdict) != 2* (nparams + 2)) {stat = NC_EFILTER; goto done;} + if(nparams == 0) goto setvalues; + + if((params = (unsigned*)calloc(nparams,sizeof(unsigned)))== NULL) + {stat = NC_ENOMEM; goto done;} + /* Get params */ for(i=0;i<(int)nparams;i++) { unsigned param; @@ -143,6 +145,7 @@ NCZ_raw_codec_to_hdf5(const NCproplist* env, const char* codec, int* idp, size_t params[i] = param; } +setvalues: if(idp) *idp = hdf5id; *nparamsp = nparams; *paramsp = params; params = NULL;