From 96467826a3513476ba729148dbaabdf1b5a44be3 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Mon, 6 Jan 2025 15:09:46 -0700 Subject: [PATCH] cleanup actions --- CMakeLists.txt | 5 +- Makefile.am | 4 +- RELEASE_NOTES.md | 2 +- config.h.cmake.in | 8 + configure.ac | 36 +- docs/nczarr.md | 44 +- include/Makefile.am | 4 +- include/ncjson.h | 16 +- include/nclog.h | 2 +- include/netcdf.h | 5 +- libdispatch/derror.c | 4 +- libdispatch/dinfermodel.c | 5 + libdispatch/ncjson.c | 41 +- libnczarr/CMakeLists.txt | 5 + libnczarr/Makefile.am | 11 +- libnczarr/zarr.c | 44 +- libnczarr/zattr.c | 4 +- libnczarr/zclose.c | 11 +- libnczarr/zdebug.c | 23 + libnczarr/zdebug.h | 3 +- libnczarr/zfile.c | 4 +- libnczarr/zfilter.c | 3 +- libnczarr/zformat.h | 7 +- libnczarr/zformat2.c | 49 +- libnczarr/zformat3.c | 83 +- libnczarr/zincludes.h | 1 + libnczarr/zinfer.c | 291 +- libnczarr/zinternal.h | 27 +- libnczarr/zmap.c | 18 +- libnczarr/zmap.h | 2 +- libnczarr/zmap_file.c | 28 +- libnczarr/zmap_s3sdk.c | 33 +- libnczarr/zmap_zip.c | 2 +- libnczarr/zmetadata.c | 411 + libnczarr/zmetadata.h | 111 + libnczarr/zmetadata0.c | 147 + libnczarr/zmetadata2.c | 273 + libnczarr/zmetadata3.c | 282 + libnczarr/zopen.c | 2 +- libnczarr/zplugins.c | 4 +- libnczarr/zsync.c | 339 +- libnczarr/zutil.c | 51 +- libnczarr/zvar.c | 3 +- libnczarr/zwalk.c | 4 +- libnetcdf.settings.in | 1 + libsrc4/nc4internal.c | 11 +- ncdump/tst_nccopy4.sh | 1 + nczarr_test/CMakeLists.txt | 7 +- nczarr_test/Makefile.am | 35 +- nczarr_test/ref_cmip6.zmap | 99 +- nczarr_test/ref_consolidated_zarr.cdl | 12 + ...f_consolidated_zarr_2.18.2_python.zarr.tgz | Bin 0 -> 1144 bytes nczarr_test/ref_consolidated_zarr_base.cdl | 7 + nczarr_test/ref_filtered.cdl | 1 - nczarr_test/ref_nulls_nczarr.baseline | 1 - nczarr_test/ref_nulls_zarr.baseline | 1 - .../ref_oldformat_only_consolidated.zip | Bin 0 -> 1103 bytes nczarr_test/ref_purezarr.cdl | 1 - nczarr_test/ref_string_nczarr.baseline | 1 - nczarr_test/ref_string_zarr.baseline | 1 - nczarr_test/ref_xarray.cdl | 1 - nczarr_test/run_cachetest.sh | 2 +- nczarr_test/run_chunkcases.sh | 2 +- nczarr_test/run_consolidated_zarr.sh | 76 + nczarr_test/run_corrupt.sh | 2 +- nczarr_test/run_external.sh | 2 +- nczarr_test/run_fillonlyz.sh | 2 +- nczarr_test/run_filter.sh | 2 +- nczarr_test/run_filter_misc.sh | 2 +- nczarr_test/run_filter_vlen.sh | 2 +- nczarr_test/run_filterinstall.sh | 2 +- nczarr_test/run_interop.sh | 2 +- nczarr_test/run_it_chunks1.sh | 2 +- nczarr_test/run_jsonconvention.sh | 2 +- nczarr_test/run_misc.sh | 2 +- nczarr_test/run_mud.sh | 2 +- nczarr_test/run_nan.sh | 7 +- nczarr_test/run_nccopy5.sh | 2 +- nczarr_test/run_ncgen4.sh | 2 +- nczarr_test/run_nczarr_fill.sh | 2 +- nczarr_test/run_nczfilter.sh | 2 +- nczarr_test/run_newformat.sh | 2 +- nczarr_test/run_notzarr.sh | 6 +- nczarr_test/run_nulls.sh | 6 +- nczarr_test/run_perf_chunks1.sh | 2 +- nczarr_test/run_purezarr.sh | 2 +- nczarr_test/run_quantize.sh | 2 +- nczarr_test/run_scalar.sh | 2 +- nczarr_test/run_specific_filters.sh | 2 +- nczarr_test/run_strings.sh | 2 +- nczarr_test/run_unknown.sh | 2 +- nczarr_test/run_unlim_io.sh | 2 +- nczarr_test/run_ut_map.sh | 4 +- nczarr_test/run_ut_mapapi.sh | 2 +- nczarr_test/run_ut_misc.sh | 18 +- nczarr_test/test_nczarr.sh | 233 + nczarr_test/test_unlim_vars.c | 6 +- test_common.in | 3 + unit_test/run_s3sdk.sh | 2 +- v3_nczarr_test/CMakeLists.txt | 126 +- v3_nczarr_test/Makefile.am | 121 +- v3_nczarr_test/bm_chunks3.c | 422 + v3_nczarr_test/ncdumpchunks.c | 607 ++ v3_nczarr_test/ref_avail1.cdl | 639 ++ v3_nczarr_test/ref_avail1.dmp | 49 + v3_nczarr_test/ref_avail1.txt | 7200 +++++++++++++++++ v3_nczarr_test/ref_byte.cdl | 35 + v3_nczarr_test/ref_byte.zarr.zip | Bin 0 -> 816 bytes v3_nczarr_test/ref_byte_fill_value_null.cdl | 35 + .../ref_byte_fill_value_null.zarr.zip | Bin 0 -> 1676 bytes v3_nczarr_test/ref_cmip6.zmap | 555 ++ v3_nczarr_test/ref_consolidated_zarr.cdl | 12 + ...f_consolidated_zarr_2.18.2_python.zarr.tar | Bin 0 -> 20480 bytes ...f_consolidated_zarr_2.18.2_python.zarr.tgz | Bin 0 -> 1144 bytes v3_nczarr_test/ref_consolidated_zarr_base.cdl | 7 + v3_nczarr_test/ref_fillonly.cdl | 20 + v3_nczarr_test/ref_filtered.cdl | 5 +- v3_nczarr_test/ref_groups.h5 | Bin 0 -> 9836 bytes v3_nczarr_test/ref_misc1.cdl | 84 + v3_nczarr_test/ref_misc1.dmp | 97 + v3_nczarr_test/ref_ndims.cdl | 525 ++ v3_nczarr_test/ref_ndims.dmp | 37 + v3_nczarr_test/ref_newformatpure.cdl | 33 + v3_nczarr_test/ref_noshape.file.zip | Bin 0 -> 1482 bytes v3_nczarr_test/ref_notzarr.tar.gz | Bin 0 -> 433 bytes v3_nczarr_test/ref_nulls.cdl | 12 + v3_nczarr_test/ref_nulls_nczarr.baseline | 12 + v3_nczarr_test/ref_nulls_zarr.baseline | 1 - v3_nczarr_test/ref_oldformat.cdl | 33 + v3_nczarr_test/ref_oldformat.zip | Bin 0 -> 3342 bytes v3_nczarr_test/ref_perdimspecs.cdl | 11 + v3_nczarr_test/ref_power_901_constants.cdl | 42 + .../ref_power_901_constants_orig.zip | Bin 0 -> 3455 bytes v3_nczarr_test/ref_purezarr.cdl | 5 +- v3_nczarr_test/ref_purezarr_base.cdl | 7 + v3_nczarr_test/ref_quotes.cdl | 20 + v3_nczarr_test/ref_quotes_orig.zip | Bin 0 -> 1402 bytes v3_nczarr_test/ref_rem.cdl | 19 + v3_nczarr_test/ref_rem.dmp | 28 + v3_nczarr_test/ref_scalar.cdl | 8 + v3_nczarr_test/ref_scalar_nczarr.cdl | 8 + v3_nczarr_test/ref_skip.cdl | 17 + v3_nczarr_test/ref_skip.txt | 36 + v3_nczarr_test/ref_skipw.cdl | 17 + v3_nczarr_test/ref_string.cdl | 21 + v3_nczarr_test/ref_string_nczarr.baseline | 7 +- v3_nczarr_test/ref_string_zarr.baseline | 4 +- v3_nczarr_test/ref_t_meta_dim1.cdl | 32 + v3_nczarr_test/ref_t_meta_var1.cdl | 47 + v3_nczarr_test/ref_tst_nans.dmp | 16 + v3_nczarr_test/ref_ut_json_build.txt | 32 + v3_nczarr_test/ref_ut_json_parse.txt | 2 + v3_nczarr_test/ref_ut_map_create.cdl | 1 + v3_nczarr_test/ref_ut_map_readmeta.txt | 4 + v3_nczarr_test/ref_ut_map_readmeta2.txt | 5 + v3_nczarr_test/ref_ut_map_search.txt | 4 + v3_nczarr_test/ref_ut_map_writedata.cdl | 11 + v3_nczarr_test/ref_ut_map_writemeta.cdl | 5 + v3_nczarr_test/ref_ut_map_writemeta2.cdl | 10 + v3_nczarr_test/ref_ut_mapapi_create.cdl | 1 + v3_nczarr_test/ref_ut_mapapi_data.cdl | 8 + v3_nczarr_test/ref_ut_mapapi_meta.cdl | 7 + v3_nczarr_test/ref_ut_mapapi_search.txt | 3 + v3_nczarr_test/ref_ut_proj.txt | 13 + v3_nczarr_test/ref_ut_testmap_create.cdl | 32 + v3_nczarr_test/ref_whole.cdl | 19 + v3_nczarr_test/ref_whole.txt | 64 + v3_nczarr_test/ref_xarray.cdl | 1 - v3_nczarr_test/ref_zarr_test_data.cdl.gz | Bin 0 -> 2716 bytes v3_nczarr_test/run_chunkcases.sh | 128 + v3_nczarr_test/run_consolidated_zarr.sh | 68 + v3_nczarr_test/run_corrupt.sh | 42 + v3_nczarr_test/run_external.sh | 45 + v3_nczarr_test/run_fillonlyz.sh | 30 + v3_nczarr_test/run_filter.sh | 176 + v3_nczarr_test/run_filter_misc.sh | 117 + v3_nczarr_test/run_filter_vlen.sh | 108 + v3_nczarr_test/run_filterinstall.sh | 124 + v3_nczarr_test/run_interop.sh | 110 + v3_nczarr_test/run_jsonconvention.sh | 38 + v3_nczarr_test/run_misc.sh | 63 + v3_nczarr_test/run_mud.sh | 2 +- v3_nczarr_test/run_nan.sh | 9 +- v3_nczarr_test/run_nccopy5.sh | 277 + v3_nczarr_test/run_nccopyz.sh | 91 + v3_nczarr_test/run_ncgen4.sh | 79 + v3_nczarr_test/run_nczarr_fill.sh | 77 + v3_nczarr_test/run_nczfilter.sh | 16 + v3_nczarr_test/run_newformat.sh | 48 + v3_nczarr_test/run_notzarr.sh | 64 + v3_nczarr_test/run_nulls.sh | 56 + v3_nczarr_test/run_perf_chunks1.sh | 40 + v3_nczarr_test/run_purezarr.sh | 48 + v3_nczarr_test/run_quantize.sh | 33 + v3_nczarr_test/run_scalar.sh | 63 + v3_nczarr_test/run_specific_filters.sh | 220 + v3_nczarr_test/run_strings.sh | 59 + v3_nczarr_test/run_unknown.sh | 117 + v3_nczarr_test/run_unlim_io.sh | 107 + v3_nczarr_test/run_ut_map.sh | 118 + v3_nczarr_test/run_ut_mapapi.sh | 105 + v3_nczarr_test/run_ut_misc.sh | 49 + v3_nczarr_test/s3util.c | 442 + v3_nczarr_test/test_chunkcases.c | 125 + v3_nczarr_test/test_chunking.c | 143 + v3_nczarr_test/test_fillonlyz.c | 76 + v3_nczarr_test/test_filter.c | 314 + v3_nczarr_test/test_filter_avail.c | 107 + v3_nczarr_test/test_filter_misc.c | 562 ++ v3_nczarr_test/test_filter_order.c | 432 + v3_nczarr_test/test_filter_repeat.c | 369 + v3_nczarr_test/test_filter_vlen.c | 289 + v3_nczarr_test/test_forwardinfer.c | 6 + v3_nczarr_test/test_grpperf.c | 57 + v3_nczarr_test/test_h5_endians.c | 340 + v3_nczarr_test/test_multifilter.c | 328 + v3_nczarr_test/test_nczarr.sh | 218 + v3_nczarr_test/test_nczarr_utils.h | 167 + v3_nczarr_test/test_nczfilter.c | 67 + v3_nczarr_test/test_notzarr.c | 31 + v3_nczarr_test/test_put_vars_two_unlim_dim.c | 93 + v3_nczarr_test/test_quantize.c | 1392 ++++ v3_nczarr_test/test_readcaching.c | 84 + v3_nczarr_test/test_unlim_io.c | 125 + v3_nczarr_test/test_unlim_vars.c | 293 + v3_nczarr_test/test_unlimited.c | 196 + v3_nczarr_test/test_utils.c | 653 ++ v3_nczarr_test/test_utils.h | 99 + v3_nczarr_test/test_writecaching.c | 99 + v3_nczarr_test/test_zchunks.c | 391 + v3_nczarr_test/test_zchunks2.c | 462 ++ v3_nczarr_test/test_zchunks3.c | 84 + v3_nczarr_test/testfilter.c | 312 + v3_nczarr_test/testfilter_misc.c | 591 ++ v3_nczarr_test/testfilter_multi.c | 331 + v3_nczarr_test/testfilter_order.c | 441 + v3_nczarr_test/testfilter_repeat.c | 378 + v3_nczarr_test/timer_utils.c | 170 + v3_nczarr_test/timer_utils.h | 30 + v3_nczarr_test/tst_multifilter.c | 328 + v3_nczarr_test/tst_pure_awssdk.cpp | 19 + v3_nczarr_test/ut_chunking.c | 93 + v3_nczarr_test/ut_includes.h | 27 + v3_nczarr_test/ut_json.c | 284 + v3_nczarr_test/ut_map.c | 349 + v3_nczarr_test/ut_mapapi.c | 429 + v3_nczarr_test/ut_projections.c | 137 + v3_nczarr_test/ut_projtest.h | 22 + v3_nczarr_test/ut_test.c | 273 + v3_nczarr_test/ut_test.h | 78 + v3_nczarr_test/ut_util.c | 537 ++ v3_nczarr_test/ut_util.h | 32 + v3_nczarr_test/v3manifest.am | 2 +- v3_nczarr_test/zhex.c | 38 + v3_nczarr_test/zisjson.c | 152 + v3_nczarr_test/zmapio.c | 557 ++ v3_nczarr_test/zs3parse.c | 169 + 257 files changed, 29981 insertions(+), 852 deletions(-) create mode 100644 libnczarr/zmetadata.c create mode 100644 libnczarr/zmetadata.h create mode 100644 libnczarr/zmetadata0.c create mode 100644 libnczarr/zmetadata2.c create mode 100644 libnczarr/zmetadata3.c create mode 100644 nczarr_test/ref_consolidated_zarr.cdl create mode 100644 nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr.tgz create mode 100644 nczarr_test/ref_consolidated_zarr_base.cdl create mode 100644 nczarr_test/ref_oldformat_only_consolidated.zip create mode 100755 nczarr_test/run_consolidated_zarr.sh create mode 100644 nczarr_test/test_nczarr.sh create mode 100644 v3_nczarr_test/bm_chunks3.c create mode 100644 v3_nczarr_test/ncdumpchunks.c create mode 100644 v3_nczarr_test/ref_avail1.cdl create mode 100644 v3_nczarr_test/ref_avail1.dmp create mode 100644 v3_nczarr_test/ref_avail1.txt create mode 100644 v3_nczarr_test/ref_byte.cdl create mode 100644 v3_nczarr_test/ref_byte.zarr.zip create mode 100644 v3_nczarr_test/ref_byte_fill_value_null.cdl create mode 100644 v3_nczarr_test/ref_byte_fill_value_null.zarr.zip create mode 100644 v3_nczarr_test/ref_cmip6.zmap create mode 100644 v3_nczarr_test/ref_consolidated_zarr.cdl create mode 100644 v3_nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr.tar create mode 100644 v3_nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr.tgz create mode 100644 v3_nczarr_test/ref_consolidated_zarr_base.cdl create mode 100644 v3_nczarr_test/ref_fillonly.cdl create mode 100644 v3_nczarr_test/ref_groups.h5 create mode 100644 v3_nczarr_test/ref_misc1.cdl create mode 100644 v3_nczarr_test/ref_misc1.dmp create mode 100644 v3_nczarr_test/ref_ndims.cdl create mode 100644 v3_nczarr_test/ref_ndims.dmp create mode 100644 v3_nczarr_test/ref_newformatpure.cdl create mode 100644 v3_nczarr_test/ref_noshape.file.zip create mode 100644 v3_nczarr_test/ref_notzarr.tar.gz create mode 100644 v3_nczarr_test/ref_nulls.cdl create mode 100644 v3_nczarr_test/ref_nulls_nczarr.baseline create mode 100644 v3_nczarr_test/ref_oldformat.cdl create mode 100644 v3_nczarr_test/ref_oldformat.zip create mode 100644 v3_nczarr_test/ref_perdimspecs.cdl create mode 100644 v3_nczarr_test/ref_power_901_constants.cdl create mode 100644 v3_nczarr_test/ref_power_901_constants_orig.zip create mode 100644 v3_nczarr_test/ref_purezarr_base.cdl create mode 100644 v3_nczarr_test/ref_quotes.cdl create mode 100644 v3_nczarr_test/ref_quotes_orig.zip create mode 100644 v3_nczarr_test/ref_rem.cdl create mode 100644 v3_nczarr_test/ref_rem.dmp create mode 100644 v3_nczarr_test/ref_scalar.cdl create mode 100644 v3_nczarr_test/ref_scalar_nczarr.cdl create mode 100644 v3_nczarr_test/ref_skip.cdl create mode 100644 v3_nczarr_test/ref_skip.txt create mode 100644 v3_nczarr_test/ref_skipw.cdl create mode 100644 v3_nczarr_test/ref_string.cdl create mode 100644 v3_nczarr_test/ref_t_meta_dim1.cdl create mode 100644 v3_nczarr_test/ref_t_meta_var1.cdl create mode 100644 v3_nczarr_test/ref_tst_nans.dmp create mode 100644 v3_nczarr_test/ref_ut_json_build.txt create mode 100644 v3_nczarr_test/ref_ut_json_parse.txt create mode 100644 v3_nczarr_test/ref_ut_map_create.cdl create mode 100644 v3_nczarr_test/ref_ut_map_readmeta.txt create mode 100644 v3_nczarr_test/ref_ut_map_readmeta2.txt create mode 100644 v3_nczarr_test/ref_ut_map_search.txt create mode 100644 v3_nczarr_test/ref_ut_map_writedata.cdl create mode 100644 v3_nczarr_test/ref_ut_map_writemeta.cdl create mode 100644 v3_nczarr_test/ref_ut_map_writemeta2.cdl create mode 100644 v3_nczarr_test/ref_ut_mapapi_create.cdl create mode 100644 v3_nczarr_test/ref_ut_mapapi_data.cdl create mode 100644 v3_nczarr_test/ref_ut_mapapi_meta.cdl create mode 100644 v3_nczarr_test/ref_ut_mapapi_search.txt create mode 100644 v3_nczarr_test/ref_ut_proj.txt create mode 100644 v3_nczarr_test/ref_ut_testmap_create.cdl create mode 100644 v3_nczarr_test/ref_whole.cdl create mode 100644 v3_nczarr_test/ref_whole.txt create mode 100644 v3_nczarr_test/ref_zarr_test_data.cdl.gz create mode 100755 v3_nczarr_test/run_chunkcases.sh create mode 100755 v3_nczarr_test/run_consolidated_zarr.sh create mode 100755 v3_nczarr_test/run_corrupt.sh create mode 100755 v3_nczarr_test/run_external.sh create mode 100755 v3_nczarr_test/run_fillonlyz.sh create mode 100755 v3_nczarr_test/run_filter.sh create mode 100755 v3_nczarr_test/run_filter_misc.sh create mode 100755 v3_nczarr_test/run_filter_vlen.sh create mode 100755 v3_nczarr_test/run_filterinstall.sh create mode 100755 v3_nczarr_test/run_interop.sh create mode 100755 v3_nczarr_test/run_jsonconvention.sh create mode 100755 v3_nczarr_test/run_misc.sh create mode 100755 v3_nczarr_test/run_nccopy5.sh create mode 100755 v3_nczarr_test/run_nccopyz.sh create mode 100755 v3_nczarr_test/run_ncgen4.sh create mode 100755 v3_nczarr_test/run_nczarr_fill.sh create mode 100755 v3_nczarr_test/run_nczfilter.sh create mode 100755 v3_nczarr_test/run_newformat.sh create mode 100755 v3_nczarr_test/run_notzarr.sh create mode 100755 v3_nczarr_test/run_nulls.sh create mode 100755 v3_nczarr_test/run_perf_chunks1.sh create mode 100755 v3_nczarr_test/run_purezarr.sh create mode 100755 v3_nczarr_test/run_quantize.sh create mode 100755 v3_nczarr_test/run_scalar.sh create mode 100755 v3_nczarr_test/run_specific_filters.sh create mode 100755 v3_nczarr_test/run_strings.sh create mode 100755 v3_nczarr_test/run_unknown.sh create mode 100755 v3_nczarr_test/run_unlim_io.sh create mode 100755 v3_nczarr_test/run_ut_map.sh create mode 100755 v3_nczarr_test/run_ut_mapapi.sh create mode 100755 v3_nczarr_test/run_ut_misc.sh create mode 100644 v3_nczarr_test/s3util.c create mode 100644 v3_nczarr_test/test_chunkcases.c create mode 100644 v3_nczarr_test/test_chunking.c create mode 100644 v3_nczarr_test/test_fillonlyz.c create mode 100644 v3_nczarr_test/test_filter.c create mode 100644 v3_nczarr_test/test_filter_avail.c create mode 100644 v3_nczarr_test/test_filter_misc.c create mode 100644 v3_nczarr_test/test_filter_order.c create mode 100644 v3_nczarr_test/test_filter_repeat.c create mode 100644 v3_nczarr_test/test_filter_vlen.c create mode 100644 v3_nczarr_test/test_forwardinfer.c create mode 100644 v3_nczarr_test/test_grpperf.c create mode 100644 v3_nczarr_test/test_h5_endians.c create mode 100644 v3_nczarr_test/test_multifilter.c create mode 100644 v3_nczarr_test/test_nczarr.sh create mode 100644 v3_nczarr_test/test_nczarr_utils.h create mode 100644 v3_nczarr_test/test_nczfilter.c create mode 100644 v3_nczarr_test/test_notzarr.c create mode 100644 v3_nczarr_test/test_put_vars_two_unlim_dim.c create mode 100644 v3_nczarr_test/test_quantize.c create mode 100644 v3_nczarr_test/test_readcaching.c create mode 100644 v3_nczarr_test/test_unlim_io.c create mode 100644 v3_nczarr_test/test_unlim_vars.c create mode 100644 v3_nczarr_test/test_unlimited.c create mode 100644 v3_nczarr_test/test_utils.c create mode 100644 v3_nczarr_test/test_utils.h create mode 100644 v3_nczarr_test/test_writecaching.c create mode 100644 v3_nczarr_test/test_zchunks.c create mode 100644 v3_nczarr_test/test_zchunks2.c create mode 100644 v3_nczarr_test/test_zchunks3.c create mode 100644 v3_nczarr_test/testfilter.c create mode 100644 v3_nczarr_test/testfilter_misc.c create mode 100644 v3_nczarr_test/testfilter_multi.c create mode 100644 v3_nczarr_test/testfilter_order.c create mode 100644 v3_nczarr_test/testfilter_repeat.c create mode 100644 v3_nczarr_test/timer_utils.c create mode 100644 v3_nczarr_test/timer_utils.h create mode 100644 v3_nczarr_test/tst_multifilter.c create mode 100644 v3_nczarr_test/tst_pure_awssdk.cpp create mode 100644 v3_nczarr_test/ut_chunking.c create mode 100644 v3_nczarr_test/ut_includes.h create mode 100644 v3_nczarr_test/ut_json.c create mode 100644 v3_nczarr_test/ut_map.c create mode 100644 v3_nczarr_test/ut_mapapi.c create mode 100644 v3_nczarr_test/ut_projections.c create mode 100644 v3_nczarr_test/ut_projtest.h create mode 100644 v3_nczarr_test/ut_test.c create mode 100644 v3_nczarr_test/ut_test.h create mode 100644 v3_nczarr_test/ut_util.c create mode 100644 v3_nczarr_test/ut_util.h create mode 100644 v3_nczarr_test/zhex.c create mode 100644 v3_nczarr_test/zisjson.c create mode 100644 v3_nczarr_test/zmapio.c create mode 100644 v3_nczarr_test/zs3parse.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 21249ff14d..20483ca546 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,7 +45,6 @@ set(PACKAGE "netCDF" CACHE STRING "") include(netcdf_functions_macros) include(deprecated) - # Backport of built-in `PROJECT_IS_TOP_LEVEL` from CMake 3.21 if (NOT DEFINED NETCDF_IS_TOP_LEVEL) set(NETCDF_IS_TOP_LEVEL OFF) @@ -1763,6 +1762,10 @@ endif() # WARNING: this must match the value in configure.ac set(S3TESTBUCKET "unidata-zarr-test-data" CACHE STRING "S3 test bucket") +# Additional S3 Test Endpoing +set(S3ENDPOINT "s3.us-east-1.amazonaws.com" CACHE STRING "S3 endpoint") + + # The working S3 path tree within the Unidata bucket. # WARNING: this must match the value in configure.ac set(S3TESTSUBTREE "netcdf-c" CACHE STRING "Working S3 path.") diff --git a/Makefile.am b/Makefile.am index f99ea6c58f..6e02782330 100644 --- a/Makefile.am +++ b/Makefile.am @@ -110,9 +110,11 @@ endif # Build Cloud Storage if desired. if NETCDF_ENABLE_NCZARR +ZARR = libnczarr ZARR_TEST_DIRS = nczarr_test +if NETCDF_ENABLE_NCZARR_V3 ZARR_TEST_DIRS += v3_nczarr_test -ZARR = libnczarr +endif endif # Optionally build test plugins diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index b488c0456e..b2220bf4bc 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -6,7 +6,7 @@ Release Notes {#RELEASE_NOTES} This file contains a high-level description of this package's evolution. Releases are in reverse chronological order (most recent first). Note that, as of netcdf 4.2, the `netcdf-c++` and `netcdf-fortran` libraries have been separated into their own libraries. ## 4.9.4 - TBD -* Add experimental support for the Zarr Version 3 storage format. This code willl change as the Zarr Version 3 Specification evolves. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). +* Add experimental support for the Zarr Version 3 storage format. This code willl change as the Zarr Version 3 Specification evolves. See [Github #3068](https://github.com/Unidata/netcdf-c/pull/3068). ## 4.9.3 - TBD diff --git a/config.h.cmake.in b/config.h.cmake.in index 73c609b658..66e27db64f 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -157,6 +157,9 @@ are set when opening a binary file on Windows. */ /* if true, enable NCZARR */ #cmakedefine NETCDF_ENABLE_NCZARR 1 +/* if true, enable NCZARR Version 3*/ +#cmakedefine NETCDF_ENABLE_NCZARR_V3 1 + /* if true, enable nczarr filter support */ #cmakedefine NETCDF_ENABLE_NCZARR_FILTERS 1 @@ -193,6 +196,11 @@ are set when opening a binary file on Windows. */ /* S3 Working subtree path prefix*/ #define S3TESTSUBTREE "${S3TESTSUBTREE}" +/* S3 Test endpoint */ +#define S3ENDPOINT "${S3ENDPOINT}" + +/* S3 Test Bucket */ + /* if true, run extra tests which may not work yet */ #cmakedefine EXTRA_TESTS 1 diff --git a/configure.ac b/configure.ac index 6ff852c1e4..876775b567 100644 --- a/configure.ac +++ b/configure.ac @@ -237,6 +237,17 @@ AC_ARG_ENABLE([nczarr], test "x$enable_nczarr" = xno || enable_nczarr=yes AC_MSG_RESULT($enable_nczarr) +AC_MSG_CHECKING([whether netcdf zarr storage format version 3 should be enabled]) +AC_ARG_ENABLE([nczarr-v3], + [AS_HELP_STRING([--enable-nczarr-v3], + [enable netcdf zarr version 3 storage support])]) dnl +test "x$enable_nczarr_v3" = xyes || enable_nczarr_v3=no +AC_MSG_RESULT($enable_nczarr_v3) +if test "x$enable_nczarr_v3" = xyes ; then +AC_DEFINE([NETCDF_ENABLE_NCZARR_V3], [1], [if true, include NCZarr V3 support]) +fi +AM_CONDITIONAL([NETCDF_ENABLE_NCZARR_V3],[test "x$enable_nczarr_v3" = xyes]) + # HDF5 | HDF4 | NCZarr => netcdf-4 if test "x$enable_hdf5" = xyes || test "x$enable_hdf4" = xyes || test "x$enable_nczarr" = xyes ; then enable_netcdf4=yes @@ -248,11 +259,14 @@ AC_ARG_ENABLE([default-zarr-format_v3], [AS_HELP_STRING([--enable-default-zarr-format_v3], [Specify the default Zarr format.])]) test "x$enable_default_zarr_format_v3" = xyes | enable_default_zarr_format_v3=no # V2 is the default by default +if test "x$enable_nczarr_v3" = xno ; then + enable_default_zarr_format_v3=no +fi AC_MSG_RESULT([$enable_default_zarr_format_v3}]) if test "x$enable_default_zarr_format_v3" = xyes; then DFALTZARRFORMAT=3 else -DFALTZARRFORMAT=2 + DFALTZARRFORMAT=2 fi AC_DEFINE_UNQUOTED([DFALTZARRFORMAT], [$DFALTZARRFORMAT], [Default Zarr format]) AC_SUBST([DFALTZARRFORMAT],[$DFALTZARRFORMAT]) @@ -2135,6 +2149,7 @@ AC_SUBST(HAS_S3_AWS,[$enable_s3_aws]) AC_SUBST(HAS_S3_INTERNAL,[$enable_s3_internal]) AC_SUBST(HAS_HDF5_ROS3,[$has_hdf5_ros3]) AC_SUBST(HAS_NCZARR,[$enable_nczarr]) +AC_SUBST(HAS_NCZARR_V3,[$enable_nczarr_v3]) AC_SUBST(NETCDF_ENABLE_S3_TESTING,[$with_s3_testing]) AC_SUBST(HAS_NCZARR_ZIP,[$enable_nczarr_zip]) AC_SUBST(NCZARR_DEFAULT_FORMAT,[$DFALTZARRFORMAT]) @@ -2169,6 +2184,11 @@ AC_SUBST([S3TESTBUCKET],["unidata-zarr-test-data"]) AC_DEFINE([S3TESTSUBTREE], ["netcdf-c"], [S3 test path prefix]) AC_SUBST([S3TESTSUBTREE],[netcdf-c]) +# Additional S3 Test Endpoint +# WARNING: this must match the value in CMakeLists.txt +AC_DEFINE([S3ENDPOINT], ["s3.us-east-1.amazonaws.com"], [S3 test endpoint]) +AC_SUBST([S3ENDPOINT],["s3.us-east-1.amazonaws.com"]) + # Build a small unique id to avoid interference on same platform PLATFORMUID="$RANDOM" # Make sure uid > 0 @@ -2334,6 +2354,7 @@ AX_SET_META([NC_HAS_S3_AWS],[$enable_s3_aws],[yes]) AX_SET_META([NC_HAS_S3_INTERNAL],[$enable_s3_internal],[yes]) AX_SET_META([NC_HAS_HDF5_ROS3],[$has_hdf5_ros3],[yes]) AX_SET_META([NC_HAS_NCZARR],[$enable_nczarr],[yes]) +AX_SET_META([NC_HAS_NCZARR_V3],[$enable_nczarr_v3],[yes]) AX_SET_META([NC_HAS_LOGGING],[$enable_logging],[yes]) AX_SET_META([NC_HAS_QUANTIZE],[$enable_quantize],[yes]) AX_SET_META([NC_HAS_SZIP],[$enable_hdf5_szip],[yes]) @@ -2432,19 +2453,8 @@ AC_CONFIG_FILES([Makefile dap4_test/Makefile plugins/Makefile nczarr_test/Makefile - v3_nczarr_test/Makefile + v3_nczarr_test/Makefile ]) AC_OUTPUT() -# Initialize v3_nczarr_test -AC_MSG_CHECKING([@@@ Initializing v3_nczarr_test]) -set -x -mkdir -p v3_nczarr_test -cd v3_nczarr_test -find . -maxdepth 0 -type f -not \( -name CMakeLists.txt -or -name '*.am' -or -name Makefile.in \) -delete -cp v3only/* . -cd .. -set +x -AC_MSG_RESULT([done]) - cat libnetcdf.settings diff --git a/docs/nczarr.md b/docs/nczarr.md index 72c8b0a4df..8ec62ff833 100644 --- a/docs/nczarr.md +++ b/docs/nczarr.md @@ -35,8 +35,9 @@ NCZarr uses a data model that, by design, extends the Zarr Version 2 Specificati __Note Carefully__: a legal _NCZarr_ dataset is expected to also be a legal _Zarr_ dataset. The inverse is true also. A legal _Zarr_ dataset is expected to also be a legal _NCZarr_ dataset, where "legal" means it conforms to the Zarr version 2 or 3 specification. -In addition, certain non-Zarr features are allowed and used. -Specifically the XArray [7] ''\_ARRAY\_DIMENSIONS'' attribute is one such. +In addition, certain extra-Zarr features are allowed and used, namely: +1. the XArray [7] ''\_ARRAY\_DIMENSIONS'' attribute. +2. the .zmetadata conventions where all the JSON metadata is held in a single object. There are two other, secondary assumptions: @@ -151,11 +152,13 @@ the netcdf-c library was built. As an aside, it should be the case that zipping a _file_ format directory tree will produce a file readable by the -_zip_ storage format, and vice-versa. +_zip_ storage format, and vice-versa. This may change depending +on the outcome of current deliberations by the Zarr committee. By default, the XArray convention is supported for Zarr Version 2 and used for both NCZarr files and pure Zarr files. It is not -needed for Version 3 and is ignored. +needed for Version 3, which has an equivalent array metadata key +called "dimension_names" This means that every variable in the root group whose named dimensions are also in the root group will have an attribute called *\_ARRAY\_DIMENSIONS* that stores those dimension names. @@ -443,6 +446,39 @@ Any of the following conditions will cause ''_ARRAY_DIMENSIONS'' not to be writt Note that this attribute is not needed for Zarr Version 3, and is ignored. +## The ".zmetdata" Mode +The NCzarr implementation of Version 2 also support the ".zmetadata" convention. +This convention adds an extra, root-level object called ".zmetadata". +This object is a JSON dictionary with this form: +```` +{"metadata": + { + "": , + "": , + ... + "": + }, +"zarr_consolidated_format":1 +} +```` +Each <key> refers to a content-bearing object and the <contents> is the JSON content of that object. +An example might look as follows: +```` +{ + "metadata": + { + ".zgroup": {"zarr_format": 2}, + ".zattr": {"globalfloat": 1}, + "v/.zarray": {"zarr_format": 2, "shape": [1], "dtype": " #include "ncexternl.h" -#undef NCCATCH +#define NCCATCH #define NCENVLOGGING "NCLOGGING" #define NCENVTRACING "NCTRACING" diff --git a/include/netcdf.h b/include/netcdf.h index 199735dfef..0ab9fa3cb9 100644 --- a/include/netcdf.h +++ b/include/netcdf.h @@ -541,8 +541,9 @@ by the desired type. */ #define NC_ENOOBJECT (-141) /**< Some object not found */ #define NC_EPLUGIN (-142) /**< Unclassified failure in accessing a dynamically loaded plugin> */ #define NC_ENOTZARR (-143) /**< Malformed (NC)Zarr file */ - -#define NC4_LAST_ERROR (-143) /**< @internal All netCDF errors > this. */ +#define NC_EZARRMETA (-144) /**< Malformed (NC)Zarr file consolidated metadata */ + +#define NC4_LAST_ERROR (-144) /**< @internal All netCDF errors > this. */ /* Errors for all remote access methods(e.g. DAP and CDMREMOTE)*/ #define NC_EURL (NC_EDAPURL) /**< Malformed URL */ diff --git a/libdispatch/derror.c b/libdispatch/derror.c index 4b2ab80669..92a7f79203 100644 --- a/libdispatch/derror.c +++ b/libdispatch/derror.c @@ -287,7 +287,9 @@ const char *nc_strerror(int ncerr1) return "NetCDF: Unclassified failure in accessing a dynamically loaded plugin"; case NC_ENOTZARR: return "Malformed (NC)Zarr file"; - default: + case NC_EZARRMETA: + return "Malformed (NC)Zarr file consolidated metadata"; + default: #ifdef USE_PNETCDF /* The behavior of ncmpi_strerror here is to return NULL, not a string. This causes problems in (at least) diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index f1a06b6458..a0cbe241c9 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -170,6 +170,7 @@ static const struct MODEINFER modenegations[] = { {"bytes","nczarr"}, /* bytes negates (nc)zarr */ {"bytes","zarr"}, {"noxarray","xarray"}, +{"nozmetadata","zmetadata"}, {NULL,NULL} }; @@ -409,6 +410,7 @@ envvlist2string(NClist* envv, const char* delim) NCbytes* buf = NULL; char* result = NULL; + NC_UNUSED(delim); if(envv == NULL || nclistlength(envv) == 0) return NULL; buf = ncbytesnew(); for(i=0;isort != NCJ_DICT + NCjson* jprev = NULL; + int found; + + if(jdict == NULL + || NCJsort(jdict) != NCJ_DICT || key == NULL || jvalue == NULL) {stat = NCJTHROW(NCJ_ERR); goto done;} - if((stat = NCJnewstring(NCJ_STRING,key,&jkey))==NCJ_ERR) goto done; - if((stat=listsetalloc(&dict->list,dict->list.len+2))<0) goto done; - if((stat = NCJappend(dict,jkey))==NCJ_ERR) goto done; - if((stat = NCJappend(dict,jvalue))==NCJ_ERR) goto done; + for(found=(-1),i=0;i < NCJdictlength(jdict); i++) { + jkey = NCJdictkey(jdict,i); + if (jkey != NULL && strcmp(NCJstring(jkey), key) == 0) { + found = (int)i; + break; + } + } + if(found >= 0) { + jprev = NCJdictvalue(jdict,found); + // replace existing values for new key + NCJreclaim(jprev); // free old value + NCJdictvalue(jdict,found) = jvalue; jvalue = NULL; + jkey = NULL; /* avoid reclamation */ + } else { /* not found */ + if((stat=listsetalloc(&jdict->list,jdict->list.len+2))<0) goto done; + NCJcheck(NCJnewstring(NCJ_STRING, key, (NCjson**)&jkey)); + NCJcheck(NCJappend(jdict,jkey)); jkey = NULL; + NCJcheck(NCJappend(jdict,jvalue)); jvalue = NULL; + } done: + NCJreclaim(jkey); + NCJreclaim(jvalue); return NCJTHROW(stat); } diff --git a/libnczarr/CMakeLists.txt b/libnczarr/CMakeLists.txt index 38fbfc9c61..01c1d4410c 100644 --- a/libnczarr/CMakeLists.txt +++ b/libnczarr/CMakeLists.txt @@ -7,6 +7,10 @@ # The source files for the HDF5 dispatch layer. SET(libnczarr_SOURCES +zmetadata.c +zmetadata0.c +zmetadata2.c +zmetadata3.c zformat.c zformat3.c zformat2.c @@ -38,6 +42,7 @@ znc4.c zfilter.c zplugins.c zdebug.c +zmetadata.h zformat.h zarr.h zfill.h diff --git a/libnczarr/Makefile.am b/libnczarr/Makefile.am index df4b95f752..a1ad84a74c 100644 --- a/libnczarr/Makefile.am +++ b/libnczarr/Makefile.am @@ -29,8 +29,9 @@ noinst_LTLIBRARIES = libnczarr.la # The source files. libnczarr_la_SOURCES = \ -zformat.c \ -zformat3.c \ +zmetadata.c \ +zmetadata0.c \ +zmetadata2.c \ zformat2.c \ zsync.c \ zinfer.c \ @@ -57,9 +58,11 @@ zfilter.c \ zplugins.c \ zutil.c \ zvar.c \ +zformat.c \ zwalk.c \ znc4.c \ zdebug.c \ +zmetadata.h \ zformat.h \ zarr.h \ zfill.h \ @@ -76,6 +79,10 @@ zfilter.h \ znc4.h \ zdebug.h +#ifdef NETCDF_ENABLE_NCZARR_V3 +libnczarr_la_SOURCES += zmetadata3.c zformat3.c +#endif + if NETCDF_ENABLE_NCZARR_ZIP libnczarr_la_SOURCES += zmap_zip.c endif diff --git a/libnczarr/zarr.c b/libnczarr/zarr.c index bbfa6c2a31..9e5b064046 100644 --- a/libnczarr/zarr.c +++ b/libnczarr/zarr.c @@ -33,7 +33,7 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, NClist* urlcontrol NCjson* json = NULL; char* key = NULL; - ZTRACE(3,"file=%s root=%s urlcontrols=%s",file->hdr.name,root->hdr.name,(controls?nczprint_env(urlcontrols):"null")); + ZTRACE(3,"root=%s urlcontrols=%s",root->hdr.name,(urlcontrols?nczprint_envlist(urlcontrols):"null")); nc = (NC*)file->controller; @@ -74,7 +74,10 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, NClist* urlcontrol if((stat = NCZ_get_map(file,uri,(mode_t)nc->mode,zfile->flags,NULL,&zfile->map))) goto done; /* And get the format dispatcher */ - if((stat = NCZ_get_formatter(file, (const NCZ_Formatter**)&zfile->dispatcher))) goto done; + if((stat = NCZ_get_create_formatter(file, (const NCZ_Formatter**)&zfile->dispatcher))) goto done; + + /* And get the consolidated metadata handler */ + if((stat = NCZMD_set_metadata_handler(file))) goto done; done: ncurifree(uri); @@ -101,7 +104,7 @@ ncz_open_dataset(NC_FILE_INFO_T* file, NClist* urlcontrols) NCZ_FILE_INFO_T* zfile = NULL; NClist* modeargs = NULL; - ZTRACE(3,"file=%s controls=%s",file->hdr.name,(controls?nczprint_envv(controls):"null")); + ZTRACE(3,"file=%s urlcontrols=%s",file->hdr.name,(urlcontrols?nczprint_envlist(urlcontrols):"null")); /* Extract info reachable via file */ nc = (NC*)file->controller; @@ -139,8 +142,18 @@ ncz_open_dataset(NC_FILE_INFO_T* file, NClist* urlcontrols) /* initialize map handle*/ if((stat = NCZ_get_map(file,uri,(mode_t)nc->mode,zfile->flags,NULL,&zfile->map))) goto done; - /* And get the format dispatcher */ - if((stat = NCZ_get_formatter(file, (const NCZ_Formatter**)&zfile->dispatcher))) goto done; + /* Get the zarr_format */ + if((stat = NCZ_infer_open_zarr_format(file))) goto done; + + /* And add the consolidated metadata manager to file */ + /* Must follow NCZ_infer_ope_zarr_format because it uses the discovered zarr format */ + if((stat = NCZMD_set_metadata_handler(file))) goto done; + + /* Set the nczarr format; must follow set_metadata_handler because it needs to read metadata */ + if((stat = NCZ_infer_open_nczarr_format(file))) goto done; + + /* And get the format dispatcher: uses discovered zarr and nczarr formats and the metadata handler */ + if((stat = NCZ_get_open_formatter(file, (const NCZ_Formatter**)&zfile->dispatcher))) goto done; /* Load the meta-data */ if((stat = ncz_decode_file(file))) goto done; @@ -227,6 +240,19 @@ controllookup(NClist* controls, const char* key) return NULL; } +/** +Look to various sources to get control information +for a given dataset. Current sources: +1. From URL: + * "mode=..." + * "log" + * "show=..." +2. Environment variables: + +@param zinfo modified to add controls +@return NC_NOERR if success +@return NC_EXXX if failures +*/ static int applycontrols(NCZ_FILE_INFO_T* zinfo) { @@ -236,6 +262,7 @@ applycontrols(NCZ_FILE_INFO_T* zinfo) NClist* modelist = nclistnew(); size64_t noflags = 0; /* track non-default negative flags */ + /* Apply controls from URL mode=... */ if((value = controllookup(zinfo->urlcontrols,"mode")) != NULL) { if((stat = NCZ_comma_parse(value,modelist))) goto done; } @@ -255,6 +282,10 @@ applycontrols(NCZ_FILE_INFO_T* zinfo) zinfo->zarr.zarr_format = ZARRFORMAT2; else if(strcasecmp(p,ZARRFORMAT3_STRING)==0) zinfo->zarr.zarr_format = ZARRFORMAT3; + else if(strcasecmp(p,NOZMETADATACONTROL)==0) + zinfo->flags |= FLAG_NOCONSOLIDATED; + else if(strcasecmp(p,ZMETADATACONTROL)==0) + noflags |= FLAG_NOCONSOLIDATED; } /* Apply negative controls by turning off negative flags */ @@ -270,6 +301,9 @@ applycontrols(NCZ_FILE_INFO_T* zinfo) if(strcasecmp(value,"fetch")==0) zinfo->flags |= FLAG_SHOWFETCH; } + + /* Environment Variables */ + done: nclistfreeall(modelist); return stat; diff --git a/libnczarr/zattr.c b/libnczarr/zattr.c index 705f71baca..9affa0df80 100644 --- a/libnczarr/zattr.c +++ b/libnczarr/zattr.c @@ -1021,7 +1021,7 @@ NCZ_computeattrdata(NC_FILE_INFO_T* file, const NCjson* jdata, struct NCZ_AttrIn NC_UNUSED(file); - ZTRACE(3,"typehint=%d typeid=%d values=|%s|",ainfo->typehint,ainfo->nctype,NCJtotext(jdata)); + ZTRACE(3,"typeid=%d values=|%s|",ainfo->nctype,NCJtotext(jdata,0)); /* See if this is a simple vector (or scalar) of atomic types vs more complex json */ isjson = (ainfo->nctype == NC_JSON || NCZ_iscomplexjson(ainfo->name,jdata)); @@ -1066,7 +1066,7 @@ NCZ_attr_convert(const NCjson* src, nc_type typeid, size_t* countp, NCbytes* dst size_t i; size_t count = 0; - ZTRACE(3,"src=%s typeid=%d typelen=%u",NCJtotext(src),typeid,typelen); + ZTRACE(3,"src=%s typeid=%d",NCJtotext(src,0),typeid); /* 3 cases: (1) singleton atomic value diff --git a/libnczarr/zclose.c b/libnczarr/zclose.c index 5fad8d1c75..6c5388159e 100644 --- a/libnczarr/zclose.c +++ b/libnczarr/zclose.c @@ -38,7 +38,7 @@ ncz_close_file(NC_FILE_INFO_T* file, int abort) if(!abort) { /* Flush | create all chunks for all vars */ - if((stat=zwrite_vars(file->root_grp))) goto done; + if((stat=zwrite_vars(file->root_grp))) goto done; } /* Internal close to reclaim zarr annotations */ @@ -47,8 +47,13 @@ ncz_close_file(NC_FILE_INFO_T* file, int abort) zinfo = file->format_file_info; - if((stat = nczmap_close(zinfo->map,(abort && zinfo->creating)?1:0))) - goto done; + /* Reclaim the metadata handler contents */ + if(NCZMD_is_metadata_consolidated(file) == NC_NOERR) + NCZMD_close(file); + + /* Release the zmap handler */ + if((stat = nczmap_close(zinfo->map,(abort && zinfo->creating)?1:0))) goto done; + nclistfreeall(zinfo->urlcontrols); NC_authfree(zinfo->auth); diff --git a/libnczarr/zdebug.c b/libnczarr/zdebug.c index ad77e8116b..a4155e32a4 100644 --- a/libnczarr/zdebug.c +++ b/libnczarr/zdebug.c @@ -362,6 +362,29 @@ nczprint_envv(const char** envv) return capture(result); } +char* +nczprint_envlist(const NClist* l) +{ + char* result = NULL; + size_t i; + NCbytes* buf = ncbytesnew(); + + ncbytescat(buf,"("); + if(l) { + for(i=0;i 0) ncbytescat(buf,","); + ncbytescat(buf,"'"); + ncbytescat(buf,e); + ncbytescat(buf,"'"); + } + } + ncbytescat(buf,")"); + result = ncbytesextract(buf); + ncbytesfree(buf); + return capture(result); +} + void zdumpcommon(const struct Common* c) { diff --git a/libnczarr/zdebug.h b/libnczarr/zdebug.h index 8c19d9f47a..9d8c487482 100644 --- a/libnczarr/zdebug.h +++ b/libnczarr/zdebug.h @@ -6,7 +6,7 @@ #define ZDEBUG_H #define ZCATCH /* Warning: significant performance impact */ -#undef ZTRACING /* Warning: significant performance impact */ +#define ZTRACING /* Warning: significant performance impact */ #undef ZDEBUG /* general debug */ #undef ZDEBUG1 /* detailed debug */ @@ -62,6 +62,7 @@ EXTERNL char* nczprint_idvector(size_t,const int*); EXTERNL char* nczprint_paramvector(size_t,const unsigned*); EXTERNL char* nczprint_sizevector(size_t,const size_t*); EXTERNL char* nczprint_envv(const char** envv); +EXTERNL char* nczprint_envlist(const NClist* l); EXTERNL void zdumpcommon(const struct Common*); diff --git a/libnczarr/zfile.c b/libnczarr/zfile.c index 5be95813d9..07163b76c4 100644 --- a/libnczarr/zfile.c +++ b/libnczarr/zfile.c @@ -165,7 +165,7 @@ NCZ_sync(int ncid) /* do not do this if file is writeonce */ stat = ncz_sync_netcdf4_file(file,!ZCLOSE); - return stat; + return ZUNTRACE(stat); } /** @@ -217,7 +217,7 @@ NCZ_close(int ncid, void* params) if ((stat = nc4_find_grp_h5(ncid, NULL, &h5))) return stat; assert(h5); - return ncz_closeorabort(h5, params, 0); + return ZUNTRACE(ncz_closeorabort(h5, params, 0)); } /** diff --git a/libnczarr/zfilter.c b/libnczarr/zfilter.c index 52c4a0cb51..883a76ddbf 100644 --- a/libnczarr/zfilter.c +++ b/libnczarr/zfilter.c @@ -367,7 +367,7 @@ NCZ_addfilter(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, NCZ_Filter** fip) NCZ_Filter* oldfi = NULL; int exists = 0; - ZTRACE(6,"file=%s var=%s id=%u nparams=%u params=%p",file->hdr.name,var->hdr.name,hdf5->id,hdf5->visible.nparams,hdf5->visible.params); + ZTRACE(6,"file=%s var=%s",file->hdr.name,var->hdr.name); assert(fip != NULL && *fip != NULL); @@ -608,6 +608,7 @@ int NCZ_filter_finalize(void) { int stat = NC_NOERR; + ZTRACE(6,""); if(!NCZ_filter_initialized) goto done; NCZ_filter_initialized = 0; diff --git a/libnczarr/zformat.h b/libnczarr/zformat.h index bc11a6907c..a417892967 100644 --- a/libnczarr/zformat.h +++ b/libnczarr/zformat.h @@ -198,7 +198,12 @@ extern const NCZ_Formatter* NCZ_formatter3; /* NCZarr V3 dispatch table => Zarr /* Use inference to get map and the formatter */ extern int NCZ_get_map(NC_FILE_INFO_T* file, NCURI* url, mode_t mode, size64_t constraints, void* params, NCZMAP** mapp); -extern int NCZ_get_formatter(NC_FILE_INFO_T* file, const NCZ_Formatter** formatterp); + +extern int NCZ_get_create_formatter(NC_FILE_INFO_T* file, const NCZ_Formatter** formatterp); +extern int NCZ_get_open_formatter(NC_FILE_INFO_T* file, const NCZ_Formatter** formatterp); + +extern int NCZ_infer_open_zarr_format(NC_FILE_INFO_T* file); +extern int NCZ_infer_open_nczarr_format(NC_FILE_INFO_T* file); /**************************************************/ /* Misc. */ diff --git a/libnczarr/zformat2.c b/libnczarr/zformat2.c index 1f523067dc..7192c24e27 100644 --- a/libnczarr/zformat2.c +++ b/libnczarr/zformat2.c @@ -202,7 +202,6 @@ int ZF2_download_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, struct ZOBJ* zobj) { int stat = NC_NOERR; - NCZ_FILE_INFO_T* zinfo = (NCZ_FILE_INFO_T*)file->format_file_info; char* fullpath = NULL; char* key = NULL; @@ -211,10 +210,10 @@ ZF2_download_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, struct ZOBJ* zobj) /* Download .zgroup and .zattrs */ if((stat = NCZ_grpkey(grp,&fullpath))) goto done; if((stat = nczm_concat(fullpath,Z2GROUP,&key))) goto done; - if((stat = NCZ_downloadjson(zinfo->map,key,&zobj->jobj))) goto done; + if((stat = NCZMD_fetch_json_content(file,NCZMD_GROUP,key,&zobj->jobj))) goto done; nullfree(key); key = NULL; if((stat = nczm_concat(fullpath,Z2ATTRS,&key))) goto done; - if((stat = NCZ_downloadjson(zinfo->map,key,&zobj->jatts))) goto done; + if((stat = NCZMD_fetch_json_content(file,NCZMD_ATTRS,key,&zobj->jatts))) goto done; zobj->constjatts = 0; done: @@ -227,17 +226,16 @@ int ZF2_download_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, struct ZOBJ* zobj) { int stat = NC_NOERR; - NCZ_FILE_INFO_T* zinfo = (NCZ_FILE_INFO_T*)file->format_file_info; char* fullpath = NULL; char* key = NULL; /* Download .zgroup and .zattrs */ if((stat = NCZ_varkey(var,&fullpath))) goto done; if((stat = nczm_concat(fullpath,Z2ARRAY,&key))) goto done; - if((stat = NCZ_downloadjson(zinfo->map,key,&zobj->jobj))) goto done; + if((stat = NCZMD_fetch_json_content(file,NCZMD_GROUP,key,&zobj->jobj))) goto done; nullfree(key); if((stat = nczm_concat(fullpath,Z2ATTRS,&key))) goto done; - if((stat = NCZ_downloadjson(zinfo->map,key,&zobj->jatts))) goto done; + if((stat = NCZMD_fetch_json_content(file,NCZMD_ATTRS,key,&zobj->jatts))) goto done; nullfree(key); key = NULL; zobj->constjatts = 0; @@ -283,18 +281,19 @@ ZF2_decode_superblock(NC_FILE_INFO_T* file, const NCjson* jsuper, int* zformatp, if(zformatp) *zformatp = 0; if(nczformatp) *nczformatp = 0; - /* Extract the zarr format number and the nczarr format number */ + /* Extract the zarr format number */ NCJcheck(NCJdictget(jsuper,"zarr_format",(NCjson**)&format)); if(format != NULL) { if(NCJsort(format) != NCJ_INT) {stat = NC_ENOTZARR; goto done;} if(1!=sscanf(NCJstring(format),ZARR_FORMAT_VERSION_TEMPLATE,&zformat)) {stat = NC_ENOTZARR; goto done;} } + /* Extract the nczarr format number */ NCJcheck(NCJdictget(jsuper,"nczarr_format",(NCjson**)&format)); if(format != NULL) { if(NCJsort(format) != NCJ_INT) {stat = NC_ENOTZARR; goto done;} if(1!=sscanf(NCJstring(format),NCZARR_FORMAT_VERSION_TEMPLATE,&nczformat)) {stat = NC_ENOTZARR; goto done;} } - + if(zformatp) *zformatp = zformat; if(nczformatp) *nczformatp = nczformat; @@ -438,7 +437,7 @@ ZF2_decode_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, struct ZOBJ* zobj, NCli if(jncvar == NULL) {stat = NC_ENCZARR; goto done;} assert((NCJsort(jncvar) == NCJ_DICT)); /* Extract scalar flag */ - if((stat = NCJdictget(jncvar,"scalar",(NCjson**)&jvalue))<0) {stat = NC_EINVAL; goto done;} + NCJcheck(NCJdictget(jncvar,"scalar",(NCjson**)&jvalue)); if(jvalue != NULL) zvar->scalar = 1; /* Ignore storage flag and treat everything as chunked */ var->storage = NC_CHUNKED; @@ -483,7 +482,7 @@ ZF2_decode_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, struct ZOBJ* zobj, NCli NCglobalstate* ngs = NC_getglobalstate(); assert(ngs != NULL); zvar->dimension_separator = 0; - if((stat = NCJdictget(jvar,"dimension_separator",(NCjson**)&jvalue))<0) {stat = NC_EINVAL; goto done;} + NCJcheck(NCJdictget(jvar,"dimension_separator",(NCjson**)&jvalue)); if(jvalue != NULL) { /* Verify its value */ if(NCJsort(jvalue) == NCJ_STRING && NCJstring(jvalue) != NULL && strlen(NCJstring(jvalue)) == 1) @@ -519,7 +518,7 @@ ZF2_decode_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, struct ZOBJ* zobj, NCli /* chunks */ { - if((stat = NCJdictget(jvar,"chunks",(NCjson**)&jvalue))<0) {stat = NC_EINVAL; goto done;} + NCJcheck(NCJdictget(jvar,"chunks",(NCjson**)&jvalue)); if(jvalue != NULL && NCJsort(jvalue) != NCJ_ARRAY) {stat = (THROW(NC_ENCZARR)); goto done;} if(zarr_rank == 0) {stat = NC_ENCZARR; goto done;} @@ -532,7 +531,7 @@ ZF2_decode_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, struct ZOBJ* zobj, NCli /* Capture row vs column major; currently, column major not used*/ { - if((stat = NCJdictget(jvar,"order",(NCjson**)&jvalue))<0) {stat = NC_EINVAL; goto done;} + NCJcheck(NCJdictget(jvar,"order",(NCjson**)&jvalue)); if(strcmp(NCJstring(jvalue),"C") > 0) ((NCZ_VAR_INFO_T*)var->format_var_info)->order = 1; else ((NCZ_VAR_INFO_T*)var->format_var_info)->order = 0; @@ -547,7 +546,7 @@ ZF2_decode_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, struct ZOBJ* zobj, NCli if(var->filters == NULL) var->filters = (void*)nclistnew(); if((stat = NCZ_filter_initialize())) goto done; { - if((stat = NCJdictget(jvar,"filters",(NCjson**)&jvalue))<0) {stat = NC_EINVAL; goto done;} + NCJcheck(NCJdictget(jvar,"filters",(NCjson**)&jvalue)); if(jvalue != NULL && NCJsort(jvalue) != NCJ_NULL) { int k; if(NCJsort(jvalue) != NCJ_ARRAY) {stat = NC_EFILTER; goto done;} @@ -564,7 +563,7 @@ ZF2_decode_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, struct ZOBJ* zobj, NCli /* From V2 Spec: A JSON object identifying the primary compression codec and providing configuration parameters, or ``null`` if no compressor is to be used. */ { - if((stat = NCJdictget(jvar,"compressor",(NCjson**)&jfilter))<0) {stat = NC_EINVAL; goto done;} + NCJcheck(NCJdictget(jvar,"compressor",(NCjson**)&jfilter)); if(jfilter != NULL && NCJsort(jfilter) != NCJ_NULL) { if(NCJsort(jfilter) != NCJ_DICT) {stat = NC_EFILTER; goto done;} nclistpush(filtersj,jfilter); @@ -710,7 +709,6 @@ int ZF2_upload_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, struct ZOBJ* zobj) { int stat = NC_NOERR; - NCZ_FILE_INFO_T* zinfo = (NCZ_FILE_INFO_T*)file->format_file_info; char* fullpath = NULL; char* key = NULL; @@ -720,14 +718,14 @@ ZF2_upload_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, struct ZOBJ* zobj) /* build ZGROUP path */ if((stat = nczm_concat(fullpath,Z2GROUP,&key))) goto done; /* Write to map */ - if((stat=NCZ_uploadjson(zinfo->map,key,zobj->jobj))) goto done; + if((stat=NCZMD_update_json_content(file,NCZMD_GROUP,key,zobj->jobj))) goto done; nullfree(key); key = NULL; if(zobj->jatts != NULL) { /* build ZATTRS path */ if((stat = nczm_concat(fullpath,Z2ATTRS,&key))) goto done; /* Write to map */ - if((stat=NCZ_uploadjson(zinfo->map,key,zobj->jatts))) goto done; + if((stat=NCZMD_update_json_content(file,NCZMD_ATTRS,key,zobj->jatts))) goto done; nullfree(key); key = NULL; } @@ -741,7 +739,6 @@ int ZF2_upload_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, struct ZOBJ* zobj) { int stat = NC_NOERR; - NCZ_FILE_INFO_T* zinfo = (NCZ_FILE_INFO_T*)file->format_file_info; char* fullpath = NULL; char* key = NULL; @@ -751,14 +748,14 @@ ZF2_upload_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, struct ZOBJ* zobj) /* build ZARRAY path */ if((stat = nczm_concat(fullpath,Z2ARRAY,&key))) goto done; /* Write to map */ - if((stat=NCZ_uploadjson(zinfo->map,key,zobj->jobj))) goto done; + if((stat=NCZMD_update_json_content(file,NCZMD_ARRAY,key,zobj->jobj))) goto done; nullfree(key); key = NULL; if(zobj->jatts != NULL) { /* build ZATTRS path */ if((stat = nczm_concat(fullpath,Z2ATTRS,&key))) goto done; /* Write to map */ - if((stat=NCZ_uploadjson(zinfo->map,key,zobj->jatts))) goto done; + if((stat=NCZMD_update_json_content(file,NCZMD_GROUP,key,zobj->jatts))) goto done; nullfree(key); key = NULL; } @@ -1126,7 +1123,6 @@ ZF2_encode_attributes(NC_FILE_INFO_T* file, NC_OBJ* container, NCjson** jnczconp if((stat = ncz_insert_attr(jatts,jtypes,a->hdr.name,&jdata,d2name))) goto done; /* cleanup */ - NCJreclaim(jdata); jdata = NULL; nullfree(d2name); d2name = NULL; nullfree(d2attr); d2attr = NULL; } @@ -1279,7 +1275,6 @@ ZF2_searchobjects(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames, NC { int stat = NC_NOERR; size_t i; - NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; char* grpkey = NULL; char* subgrpkey = NULL; char* varkey = NULL; @@ -1289,7 +1284,7 @@ ZF2_searchobjects(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames, NC /* Compute the key for the grp */ if((stat = NCZ_grpkey(grp,&grpkey))) goto done; - if((stat = nczmap_list(zfile->map,grpkey,matches))) goto done; /* Shallow listing */ + if((stat = NCZMD_list(file,grpkey,matches))) goto done; /* Shallow listing */ /* Search grp for group-level .zxxx and for var-level .zxxx*/ for(i=0;imap,zarray)) == NC_NOERR) { + if((stat = NCZMD_exists(file,zarray)) == NC_NOERR) { nclistpush(varnames,strdup(name)); } else { /* See if name is a group by testing for name/.zgroup exists */ if((stat = nczm_concat(grpkey,name,&subgrpkey))) goto done; if((stat = nczm_concat(varkey,Z2GROUP,&zgroup))) goto done; - if((stat = nczmap_exists(zfile->map,zgroup)) == NC_NOERR) + if((stat = NCZMD_exists(file,zgroup)) == NC_NOERR) nclistpush(subgrpnames,strdup(name)); } stat = NC_NOERR; @@ -1705,7 +1700,7 @@ computeattrinfo(NC_FILE_INFO_T* file, nc_type typehint, const char* aname, const NCZ_FILE_INFO_T* zinfo = (NCZ_FILE_INFO_T*)file->format_file_info; const NCjson* jatype = NULL; - ZTRACE(3,"name=%s typehint=%d values=|%s|",att->name,att->typehint,NCJtotext(att->jdata)); + ZTRACE(3,"typehint=%d aname=%s typehint=%d",typehint,aname); assert(aname != NULL); @@ -1728,5 +1723,5 @@ computeattrinfo(NC_FILE_INFO_T* file, nc_type typehint, const char* aname, const if((stat = NCZ_computeattrdata(file,jdata,ainfo))) goto done; done: - return ZUNTRACEX(THROW(stat),"typeid=%d typelen=%d len=%u",ainfo->nctype,ainfo->typelen,ainfo->len); + return ZUNTRACEX(THROW(stat),"typeid=%d typelen=%d datalen=%u",ainfo->nctype,ainfo->typelen,ainfo->datalen); } diff --git a/libnczarr/zformat3.c b/libnczarr/zformat3.c index 1b6e1e74e9..ffccd82739 100644 --- a/libnczarr/zformat3.c +++ b/libnczarr/zformat3.c @@ -196,14 +196,13 @@ int ZF3_download_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, struct ZOBJ* zobj) { int stat = NC_NOERR; - NCZ_FILE_INFO_T* zinfo = (NCZ_FILE_INFO_T*)file->format_file_info; char* fullpath = NULL; char* key = NULL; /* Download zarr.json */ if((stat = NCZ_grpkey(grp,&fullpath))) goto done; if((stat = nczm_concat(fullpath,Z3GROUP,&key))) goto done; - if((stat = NCZ_downloadjson(zinfo->map,key,&zobj->jobj))) goto done; + if((stat = NCZMD_fetch_json_content(file,NCZMD_GROUP,key,&zobj->jobj))) goto done; nullfree(key); key = NULL; /* Verify that group zarr.json exists */ if(zobj->jobj == NULL) {stat = NC_ENOTZARR; goto done;} @@ -221,14 +220,13 @@ int ZF3_download_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, struct ZOBJ* zobj) { int stat = NC_NOERR; - NCZ_FILE_INFO_T* zinfo = (NCZ_FILE_INFO_T*)file->format_file_info; char* fullpath = NULL; char* key = NULL; /* Download zarr.json */ if((stat = NCZ_varkey(var,&fullpath))) goto done; if((stat = nczm_concat(fullpath,Z3ARRAY,&key))) goto done; - if((stat = NCZ_downloadjson(zinfo->map,key,&zobj->jobj))) goto done; + if((stat = NCZMD_fetch_json_content(file,NCZMD_ARRAY,key,&zobj->jobj))) goto done; nullfree(key); key = NULL; /* Verify that var zarr.json exists */ if(zobj->jobj == NULL) {stat = NC_ENOTZARR; goto done;} @@ -516,7 +514,7 @@ ZF3_decode_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, struct ZOBJ* zobj, NCli if((stat = NCZ_filter_initialize())) goto done; #endif { - if((stat = NCJdictget(jvar,"codecs",(NCjson**)&jcodecs))<0) {stat = NC_EINVAL; goto done;} + NCJcheck(NCJdictget(jvar,"codecs",(NCjson**)&jcodecs)); if(jcodecs == NULL || NCJsort(jcodecs) != NCJ_ARRAY || NCJarraylength(jcodecs) == 0) {stat = NC_ENOTZARR; goto done;} /* Get endianess from the first codec */ @@ -638,7 +636,6 @@ int ZF3_upload_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, struct ZOBJ* zobj) { int stat = NC_NOERR; - NCZ_FILE_INFO_T* zinfo = (NCZ_FILE_INFO_T*)file->format_file_info; char* fullpath = NULL; char* key = NULL; @@ -651,7 +648,7 @@ ZF3_upload_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, struct ZOBJ* zobj) /* build ZGROUP path */ if((stat = nczm_concat(fullpath,Z3GROUP,&key))) goto done; /* Write to map */ - if((stat=NCZ_uploadjson(zinfo->map,key,zobj->jobj))) goto done; + if((stat=NCZMD_update_json_content(file,NCZMD_GROUP,key,zobj->jobj))) goto done; nullfree(key); key = NULL; done: @@ -664,7 +661,6 @@ int ZF3_upload_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, struct ZOBJ* zobj) { int stat = NC_NOERR; - NCZ_FILE_INFO_T* zinfo = (NCZ_FILE_INFO_T*)file->format_file_info; char* fullpath = NULL; char* key = NULL; @@ -677,7 +673,7 @@ ZF3_upload_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, struct ZOBJ* zobj) /* build ZARRAY path */ if((stat = nczm_concat(fullpath,Z3ARRAY,&key))) goto done; /* Write to map */ - if((stat=NCZ_uploadjson(zinfo->map,key,zobj->jobj))) goto done; + if((stat=NCZMD_update_json_content(file,NCZMD_ARRAY,key,zobj->jobj))) goto done; nullfree(key); key = NULL; done: @@ -1204,47 +1200,60 @@ ZF3_searchobjects(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames, NC { int stat = NC_NOERR; size_t i; - NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; char* grpkey = NULL; - char* subgrpkey = NULL; - char* varkey = NULL; - char* zarray = NULL; - char* zgroup = NULL; NClist* matches = nclistnew(); + char* subkey = NULL; + char* objkey = NULL; + NCjson* jcontents = NULL; + const NCjson* jnodetype = NULL; /* Compute the key for the grp */ if((stat = NCZ_grpkey(grp,&grpkey))) goto done; - if((stat = nczmap_list(zfile->map,grpkey,matches))) goto done; /* Shallow listing */ - /* Search grp for group-level .zxxx and for var-level .zxxx*/ + if((stat = NCZMD_list(file,grpkey,matches))) goto done; /* Shallow listing */ + /* Search grp for zarr.json objects and for chunk objects */ + /* In order to tell if the name refers to an array, there are two ways to do it. + 1. we extend the objkey with "/c/ or "/c." to see if it exists as a prefix. + 2. we read the zarr.json and look at the node_type field. + In the absence of consolidated metadat, (1) is slightly faster, but requires + extending the zmap interface. + So, for now, we implement case (2). + */ for(i=0;imap,zarray)) == NC_NOERR) { - nclistpush(varnames,strdup(name)); - } else { - /* See if name is a group by testing for name/.zgroup exists */ - if((stat = nczm_concat(grpkey,name,&subgrpkey))) goto done; - if((stat = nczm_concat(varkey,Z3GROUP,&zgroup))) goto done; - if((stat = nczmap_exists(zfile->map,zgroup)) == NC_NOERR) + const char* ndtype = NULL; + if(strcmp(name,Z3OBJECT) == 0) continue; /* current group metadata */ + /* Look for a zarr.json */ + if((stat = nczm_concat(grpkey,name,&subkey))) goto done; + if((stat = nczm_concat(subkey,Z3OBJECT,&objkey))) goto done; + /* Read the zarr.json object */ + switch (stat = NCZMD_fetch_json_content(file,NCZMD_GROUP,objkey,&jcontents)) { + case NC_NOERR: { /* We found a zarr.json object */ + if(jcontents == NULL || NCJsort(jcontents) != NCJ_DICT) break; + NCJcheck(NCJdictget(jcontents,"node_type",(NCjson**)&jnodetype)); + if(jnodetype == NULL || NCJsort(jnodetype) != NCJ_STRING) {stat = NC_ENOTZARR; goto done;} + ndtype = NCJstring(jnodetype); + if(strcmp("array",ndtype)==0) + nclistpush(varnames,strdup(name)); + else if(strcmp("group",ndtype)==0) nclistpush(subgrpnames,strdup(name)); + else + {stat = NC_ENOTZARR; goto done;} + } break; + case NC_ENOOBJECT: break; /* keep searching */ + default: goto done; } stat = NC_NOERR; - nullfree(varkey); varkey = NULL; - nullfree(zarray); zarray = NULL; - nullfree(subgrpkey); subgrpkey = NULL; - nullfree(zgroup); zgroup = NULL; + nullfree(subkey); subkey = NULL; + nullfree(objkey); objkey = NULL; + NCJreclaim(jcontents); jcontents = NULL; } done: + nullfree(subkey); + nullfree(objkey); nullfree(grpkey); - nullfree(varkey); - nullfree(zarray); - nullfree(zgroup); - nullfree(subgrpkey); nclistfreeall(matches); + NCJreclaim(jcontents); return stat; } @@ -1672,7 +1681,7 @@ computeattrinfo(NC_FILE_INFO_T* file, NC_OBJ* container, const char* aname, cons int stat = NC_NOERR; const NCjson* jatype = NULL; - ZTRACE(3,"name=%s typehint=%d values=|%s|",att->name,att->typehint,NCJtotext(att->jdata)); + ZTRACE(3,"container=%s aname=%s",container->name,aname); assert(aname != NULL); @@ -1699,7 +1708,7 @@ computeattrinfo(NC_FILE_INFO_T* file, NC_OBJ* container, const char* aname, cons if((stat = NCZ_computeattrdata(file,jdata,ainfo))) goto done; done: - return ZUNTRACEX(THROW(stat),"typeid=%d typelen=%d len=%u",ainfo->nctype,ainfo->typelen,ainfo->len); + return ZUNTRACEX(THROW(stat),"typeid=%d typelen=%d datalen=%u",ainfo->nctype,ainfo->typelen,ainfo->datalen); } /* Build a {name,configuration} dict */ diff --git a/libnczarr/zincludes.h b/libnczarr/zincludes.h index dd95955b8a..b47a130b35 100644 --- a/libnczarr/zincludes.h +++ b/libnczarr/zincludes.h @@ -49,6 +49,7 @@ extern "C" { #include "ncproplist.h" #include "zmap.h" +#include "zmetadata.h" #include "zinternal.h" #include "zfilter.h" #include "zformat.h" diff --git a/libnczarr/zinfer.c b/libnczarr/zinfer.c index 06326e9960..6e70f288bc 100644 --- a/libnczarr/zinfer.c +++ b/libnczarr/zinfer.c @@ -43,53 +43,85 @@ The current rules are as follows. struct TagParam { int zarrformat; int nczarrformat; + int haszmetadata; +}; + +struct ZarrObjects { + const char* name; + int zarr_version; + int haszmetadata; +} zarrobjects[] = { +{"zarr.json", ZARRFORMAT3, 0}, +{".zgroup", ZARRFORMAT2, 0}, +{".zarray", ZARRFORMAT2, 0}, +{".zattrs", ZARRFORMAT2, 0}, +{".zmetadata", ZARRFORMAT2, 1}, +{NULL, 0, 0}, }; /**************************************************/ /*Forward*/ -static int NCZ_infer_format(NC_FILE_INFO_T* file, int* zarrformatp, int* nczarrformatp); -static int NCZ_infer_storage_type(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCURI* url, NCZM_IMPL* implp); - -static int infer_create_format(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, int* zarrformatp, int* nczarrformatp); -static int infer_open_format(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, int* zarrformatp, int* nczarrformatp); - +static int NCZ_infer_storage_type(NC_FILE_INFO_T* file, NCURI* url, NCZM_IMPL* implp); +static int infer_create_format(NC_FILE_INFO_T* file, int* zarrformatp, int* nczarrformatp); static int tagsearch(NCZMAP* map, const char* prefix, const char* segment, void* param); /**************************************************/ -static int -NCZ_infer_format(NC_FILE_INFO_T* file, int* zarrformatp, int* nczarrformatp) +/** +Figure out the formatter to use when creating a file +@param file +@param formatterp +@return NC_NOERR | NC_EXXX +*/ + +int +NCZ_get_create_formatter(NC_FILE_INFO_T* file, const NCZ_Formatter** formatterp) { int stat = NC_NOERR; - int create; + const NCZ_Formatter* formatter = NULL; NCZ_FILE_INFO_T* zfile = NULL; - NCZMAP* zmap = NULL; - int zarrformat = 0; - int nczarrformat = 0; + int zarr_format = 0; + int nczarr_format = 0; zfile = (NCZ_FILE_INFO_T*)file->format_file_info; assert(zfile != NULL); - create = zfile->creating; - zmap = zfile->map; - assert(zmap != NULL); - /* Assume controls exist and have been applied */ - - if(create) - stat = infer_create_format(file,zfile,zmap,&zarrformat,&nczarrformat); - else - stat = infer_open_format(file,zfile,zmap,&zarrformat,&nczarrformat); - if(zarrformatp) *zarrformatp = zarrformat; - if(nczarrformatp) *nczarrformatp = nczarrformat; + + /* Infer the zarr+nczarr formats */ + if((stat = infer_create_format(file,&zarr_format,&nczarr_format))) goto done; + zfile->zarr.zarr_format = zarr_format; + zfile->zarr.nczarr_format = nczarr_format; + + /* If the nczarr_format is NULL, then that implies pure zarr, + so use the zarr format instead. */ + if(nczarr_format != 0) { + switch(nczarr_format) { + case 2: formatter = NCZ_formatter2; break; + case 3: formatter = NCZ_formatter3; break; + default: stat = NC_ENCZARR; goto done; + } + } else { /* Decide based on zarr format plus the fact that it is pure zarr */ + switch(zarr_format) { + case 2: formatter = NCZ_formatter2; break; + case 3: formatter = NCZ_formatter3; break; + default: stat = NC_ENCZARR; goto done; + } + } + + if(formatterp) *formatterp = formatter; + +done: return THROW(stat); } static int -infer_create_format(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, int* zarrformatp, int* nczarrformatp) +infer_create_format(NC_FILE_INFO_T* file, int* zarrformatp, int* nczarrformatp) { int stat = NC_NOERR; int zarrformat = 0; int nczarrformat = NCZARRFORMAT0; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + NCZMAP* map = zfile->map; NC_UNUSED(file); NC_UNUSED(map); @@ -110,16 +142,58 @@ infer_create_format(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, i return THROW(stat); } -static int -infer_open_format(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, int* zarrformatp, int* nczarrformatp) +/** +Figure out the formatter to use when opening a file +@param file +@param formatterp +@return NC_NOERR | NC_EXXX +*/ + +int +NCZ_get_open_formatter(NC_FILE_INFO_T* file, const NCZ_Formatter** formatterp) +{ + int stat = NC_NOERR; + const NCZ_Formatter* formatter = NULL; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + int zarr_format = zfile->zarr.zarr_format; + int nczarr_format = zfile->zarr.nczarr_format; + + zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + assert(zfile != NULL); + + zfile->zarr.zarr_format = zarr_format; + zfile->zarr.nczarr_format = nczarr_format; + assert(zfile->zarr.zarr_format != 0 && zfile->zarr.nczarr_format != 0); + + /* If the nczarr_format is NULL, then that implies pure zarr, + so use the zarr format instead. */ + if(nczarr_format != 0) { + switch(nczarr_format) { + case 2: formatter = NCZ_formatter2; break; + case 3: formatter = NCZ_formatter3; break; + default: stat = NC_ENCZARR; goto done; + } + } else { /* Decide based on zarr format plus the fact that it is pure zarr */ + switch(zarr_format) { + case 2: formatter = NCZ_formatter2; break; + case 3: formatter = NCZ_formatter3; break; + default: stat = NC_ENCZARR; goto done; + } + } + + if(formatterp) *formatterp = formatter; + +done: + return THROW(stat); +} + +int +NCZ_infer_open_zarr_format(NC_FILE_INFO_T* file) { int stat = NC_NOERR; int zarrformat = 0; - int nczarrformat = 0; - NCjson* jrootgrp = NULL; - const NCjson* jsuperg = NULL; - const NCjson* jsupera = NULL; - struct TagParam param; + struct TagParam param = {0,0,0}; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; NC_UNUSED(file); @@ -127,28 +201,49 @@ infer_open_format(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, int if(zarrformat == 0) { /* We need to search subtree for a V2 or V3 tag */ - param.zarrformat = 0; param.nczarrformat = 0; - switch(stat = nczmap_walk(map,"/",tagsearch, ¶m)) { - case NC_NOERR: + switch(stat = nczmap_walk(zfile->map,"/",tagsearch, ¶m)) { + case NC_ENOOBJECT: /* No tag was found, so its not a zarr file */ stat = NC_ENOTZARR; goto done; - case NC_EOBJECT: /* Arbitrary error signaling found and format is in param */ - stat = NC_NOERR; + case NC_NOERR: /* found and format is in param */ switch(param.zarrformat) { - case ZARRFORMAT2: case ZARRFORMAT3: zarrformat = param.zarrformat; break; - default: stat = NC_ENOTZARR; goto done; + case ZARRFORMAT2: case ZARRFORMAT3: + zarrformat = param.zarrformat; + break; + default: + stat = NC_ENOTZARR; + goto done; } break; - default: stat = NC_ENOTZARR; goto done; + default: + stat = NC_ENOTZARR; + goto done; } } + if(zarrformat == 0) {stat = NC_ENOTZARR; goto done;} + zfile->zarr.zarr_format = zarrformat; - if(zarrformat == ZARRFORMAT2 && nczarrformat == 0) { - NCjson* jrootatts = NULL; - /* Download /.zattrs and /.zgroup */ - if((stat = NCZ_downloadjson(zfile->map, Z2ATTSROOT, &jrootgrp))) goto done; - if((stat = NCZ_downloadjson(zfile->map, Z2METAROOT, &jrootatts))) goto done; +done: + return THROW(stat); +} + +int +NCZ_infer_open_nczarr_format(NC_FILE_INFO_T* file) +{ + int stat = NC_NOERR; + NCjson* jrootgrp = NULL; + const NCjson* jsuperg = NULL; + const NCjson* jsupera = NULL; + NCjson* jrootatts = NULL; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + int zarrformat = zfile->zarr.zarr_format; + int nczarrformat = 0; + + if(zarrformat == ZARRFORMAT2) { + /* Download /.zgroup and /.zattrs */ + if((stat = NCZMD_fetch_json_content(file, NCZMD_GROUP, Z2METAROOT, &jrootgrp))) goto done; + if((stat = NCZMD_fetch_json_content(file, NCZMD_ATTRS, Z2ATTSROOT, &jrootatts))) goto done; /* Look for superblock */ if(jrootgrp != NULL) NCJdictget(jrootgrp,NC_NCZARR_SUPERBLOCK_ATTR,(NCjson**)&jsuperg); if(jrootatts != NULL) NCJdictget(jrootatts,NC_NCZARR_SUPERBLOCK_ATTR,(NCjson**)&jsupera); @@ -156,11 +251,11 @@ infer_open_format(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, int NCZ_reclaim_json(jrootgrp); jrootgrp = NULL; NCZ_reclaim_json(jrootatts); jrootatts = NULL; } - +#ifdef NETCDF_ENABLE_NCZARR_V3 if(zarrformat == ZARRFORMAT3 && nczarrformat == 0) { const NCjson* jrootatts = NULL; /* Look for "/zarr.json" */ - if((stat = NCZ_downloadjson(zfile->map, Z3METAROOT, &jrootgrp))) goto done; + if((stat = NCZMD_fetch_json_content(file, NCZMD_GROUP, Z3METAROOT, &jrootgrp))) goto done; if(jrootgrp == NULL || NCJsort(jrootgrp) != NCJ_DICT) { nczarrformat = NCZARRFORMAT0; } else { @@ -178,27 +273,30 @@ infer_open_format(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, int } NCZ_reclaim_json(jrootgrp); jrootgrp = NULL; } - - if(zarrformat == 0) {stat = NC_ENOTZARR; goto done;} - - if(zarrformatp) *zarrformatp = zarrformat; - if(nczarrformatp) *nczarrformatp = nczarrformat; +#else + {stat = NC_ENOTBUILT; goto done;} +#endif + + if(nczarrformat == 0) nczarrformat = zarrformat; + zfile->zarr.nczarr_format = nczarrformat; done: - if(stat == NC_ENOTZARR) { - if(zarrformatp) *zarrformatp = 0; - if(nczarrformatp) *nczarrformatp = NCZARRFORMAT0; - } NCZ_reclaim_json(jrootgrp); + NCZ_reclaim_json(jrootatts); return THROW(stat); } +/* +Figure out the zarr format based on the +top-level keys of the dataset. +*/ static int tagsearch(NCZMAP* map, const char* prefix, const char* key, void* param) { struct TagParam* formats = (struct TagParam*)param; const char* segment = NULL; size_t seglen = 0; + struct ZarrObjects* zo = NULL; NC_UNUSED(map); NC_UNUSED(prefix); @@ -209,19 +307,13 @@ tagsearch(NCZMAP* map, const char* prefix, const char* key, void* param) seglen = strlen(segment); if(seglen == 0) return NC_NOERR; - if(strcasecmp(segment,"zarr.json")==0) { - formats->zarrformat = ZARRFORMAT3; - return NC_EOBJECT; /* arbitrary error telling walker to stop */ - } - if(seglen < 2) return NC_NOERR; /* keep looking */ - if(segment[0] != '.' || segment[1] != 'z') return NC_NOERR; - if(strcasecmp(segment,Z2GROUP)==0 - || strcasecmp(segment,Z2ARRAY)==0 - || strcasecmp(segment,Z2ATTRS)==0) { - formats->zarrformat = ZARRFORMAT2; - return NC_EOBJECT; /* V2 object found */ + for(zo=zarrobjects;zo->name;zo++) { + if(strcasecmp(segment,zo->name)==0) { + formats->zarrformat = zo->zarr_version; + return NC_NOERR; /* tell walker to stop */ + } } - return NC_NOERR; /* Keep looking */ + return NC_ENOOBJECT; /* Keep looking */ } /**************************************************/ @@ -246,11 +338,12 @@ The current rules are as follows. */ static int -NCZ_infer_storage_type(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCURI* url, NCZM_IMPL* implp) +NCZ_infer_storage_type(NC_FILE_INFO_T* file, NCURI* url, NCZM_IMPL* implp) { int ret = NC_NOERR; int create; NCZM_IMPL impl = NCZM_UNDEF; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; NC_UNUSED(file); @@ -267,7 +360,7 @@ NCZ_infer_storage_type(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCURI* url, if(!create) { /* Reading a file of some kind */ if(strcasecmp(url->protocol,"file")==0) { struct stat buf; - /* Either file or zip */ + /* Storage: file,zip,... */ if(NCstat(url->path,&buf)<0) {ret = errno; goto done;} if(S_ISDIR(buf.st_mode)) impl = NCZM_FILE; /* only possibility */ @@ -289,7 +382,7 @@ NCZ_infer_storage_type(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCURI* url, } if(impl == NCZM_UNDEF) - {ret = NC_ENOTZARR; goto done;} + {ret = NC_EURL; goto done;} if(implp) *implp = impl; done: @@ -321,7 +414,7 @@ NCZ_get_map(NC_FILE_INFO_T* file, NCURI* url, mode_t mode, size64_t constraints, assert(zfile != NULL); create = zfile->creating; - if((stat = NCZ_infer_storage_type(file, zfile, url, &impl))) goto done; + if((stat = NCZ_infer_storage_type(file, url, &impl))) goto done; if((path = ncuribuild(url,NULL,NULL,NCURIALL))==NULL) {stat = NC_ENCZARR; goto done;} @@ -333,7 +426,11 @@ NCZ_get_map(NC_FILE_INFO_T* file, NCURI* url, mode_t mode, size64_t constraints, {if((stat = nczmap_open(impl,path,mode,constraints,params,&map))) goto done;} break; case NCZM_UNDEF: - default: stat = NC_ENOTZARR; goto done; + stat = NC_EURL; + goto done; + default: + stat = NC_ENOTZARR; + goto done; } if(mapp) {*mapp = map; map = NULL;} @@ -344,53 +441,3 @@ NCZ_get_map(NC_FILE_INFO_T* file, NCURI* url, mode_t mode, size64_t constraints, return THROW(stat); } - -/** -Figure out the formatter to use. - -@param file -@param formatterp - -@return NC_NOERR | NC_EXXX -*/ - -int -NCZ_get_formatter(NC_FILE_INFO_T* file, const NCZ_Formatter** formatterp) -{ - int stat = NC_NOERR; - const NCZ_Formatter* formatter = NULL; - NCZ_FILE_INFO_T* zfile = NULL; - int zarr_format = 0; - int nczarr_format = 0; - - zfile = (NCZ_FILE_INFO_T*)file->format_file_info; - assert(zfile != NULL); - - /* Infer the NCZarr+Zarr formats */ - if((stat = NCZ_infer_format(file, &zarr_format, &nczarr_format))) goto done; - - zfile->zarr.zarr_format = zarr_format; - zfile->zarr.nczarr_format = nczarr_format; - - /* If the nczarr_format is NULL, then that implies pure zarr, - so use the zarr format instead. */ - if(nczarr_format != 0) { - switch(nczarr_format) { - case 2: formatter = NCZ_formatter2; break; - case 3: formatter = NCZ_formatter3; break; - default: stat = NC_ENCZARR; goto done; - } - } else { /* Decide based on zarr format plus the fact that it is pure zarr */ - switch(zarr_format) { - case 2: formatter = NCZ_formatter2; break; - case 3: formatter = NCZ_formatter3; break; - default: stat = NC_ENCZARR; goto done; - } - } - - if(formatterp) *formatterp = formatter; - -done: - return THROW(stat); -} - diff --git a/libnczarr/zinternal.h b/libnczarr/zinternal.h index 316251ee97..d58f4280a6 100644 --- a/libnczarr/zinternal.h +++ b/libnczarr/zinternal.h @@ -36,10 +36,12 @@ /* Map the NCZarr Format version to a string */ #define NCZARR_FORMAT_VERSION_TEMPLATE "%d.0.0" - /* The name of the env var for changing default zarr format */ #define NCZARRDEFAULTFORMAT "NCZARRFORMAT" +/* The name of the env var for controlling .zmetadata use*/ +#define NCZARRDEFAULTNOMETA "NCNOZMETADATA" + /* These have to do with creating chunked datasets in ZARR. */ #define NCZ_CHUNKSIZE_FACTOR (10) #define NCZ_MIN_CHUNK_SIZE (2) @@ -60,7 +62,8 @@ # endif #endif -/* V2 Reserved Objects */ +/* V2 Reserved Objects */# +#define Z2METADATA "/.zmetadata" #define Z2METAROOT "/.zgroup" #define Z2ATTSROOT "/.zattrs" #define Z2GROUP ".zgroup" @@ -68,10 +71,12 @@ #define Z2ARRAY ".zarray" /* V3 Reserved Objects */ -#define Z3METAROOT "/zarr.json" +#define Z3METADATA "/zarr.json" +#define Z3METAROOT Z3METADATA #define Z3OBJECT "zarr.json" #define Z3GROUP Z3OBJECT #define Z3ARRAY Z3OBJECT +#define Z3CHUNK "c" /* Bytes codec name */ #define ZBYTES3 "bytes" @@ -201,8 +206,10 @@ Optionally Inserted into any group zarr.json or array zarr.json is the extra att #define DIMSCALAR "/_scalar_" #define FORMAT2CONTROL "v2" #define FORMAT3CONTROL "v3" +#define ZMETADATACONTROL "zmetadata" +#define NOZMETADATACONTROL "nozmetadata" -#define LEGAL_DIM_SEPARATORS "./" +#define LEGAL_DIM_SEPARATORS "/." #define DFALT_DIM_SEPARATOR_V2 '.' #define DFALT_DIM_SEPARATOR_V3 '/' @@ -260,6 +267,7 @@ typedef struct NCZ_FILE_INFO { NCZcommon common; struct NCZMAP* map; /* implementation */ struct NCauth* auth; + struct NCZ_Metadata metadata_handler; struct Zarrformat { int zarr_format; int nczarr_format; @@ -270,11 +278,12 @@ typedef struct NCZ_FILE_INFO { size_t default_maxstrlen; /* default max str size for variables of type string */ NClist* urlcontrols; /* controls specified by the file url fragment */ size64_t flags; -# define FLAG_PUREZARR 1 -# define FLAG_SHOWFETCH 2 -# define FLAG_LOGGING 4 -# define FLAG_XARRAYDIMS 8 -# define FLAG_NCZARR_KEY 16 /* _nczarr_xxx keys are stored in object and not in _nczarr_attrs */ +# define FLAG_PUREZARR 1 +# define FLAG_SHOWFETCH 2 +# define FLAG_LOGGING 4 +# define FLAG_XARRAYDIMS 8 +# define FLAG_NCZARR_KEY 16 /* _nczarr_xxx keys are stored in object and not in _nczarr_attrs */ +# define FLAG_NOCONSOLIDATED 32 /* Suppress consolidated metadata */ NCZM_IMPL mapimpl; struct NCZ_Formatter* dispatcher; struct NCZ_META_HDR* metastate; /* Hold per-format state */ diff --git a/libnczarr/zmap.c b/libnczarr/zmap.c index 7b6604d864..844c92cbb7 100644 --- a/libnczarr/zmap.c +++ b/libnczarr/zmap.c @@ -567,6 +567,14 @@ NCZ_mapkind(NCZM_IMPL impl) return "Unknown"; } +/** +Walk the content-bearing keys of a dataset and invoke +a specified function on each such key. +@param map +@param prefix only pass keys that have this prefix +@param fcn to process each key; return NC_NOERR to stop walk, NC_ENOOBJECT to keep going. +@param param arbitrary arg to give to fcn +*/ int nczmap_walk(NCZMAP* map, const char* prefix, NCZWALKFCN fcn, void* param) { @@ -574,6 +582,7 @@ nczmap_walk(NCZMAP* map, const char* prefix, NCZWALKFCN fcn, void* param) NCbytes* path = ncbytesnew(); NClist* subtree = nclistnew(); size_t i; + int stop; assert(prefix != NULL && strlen(prefix) > 0); if(prefix[0] != '/') ncbytescat(path,"/"); @@ -584,12 +593,15 @@ nczmap_walk(NCZMAP* map, const char* prefix, NCZWALKFCN fcn, void* param) if(nclistlength(subtree) == 0) goto done; /* empty subtree */ /* Apply fcn to all paths in subtree */ - for(i=0;ipath,0); /* leave root; ignore errors */ done: ncurifree(url); - return stat; + return ZUNTRACE(stat); } /**************************************************/ @@ -789,7 +789,7 @@ platformtestcontentbearing(const char* canonpath) int ret = 0; struct stat buf; - ZTRACE(6,"map=%s canonpath=%s",zfmap->map.url,canonpath); + ZTRACE(6,"canonpath=%s",canonpath); errno = 0; ret = NCstat(canonpath, &buf); @@ -813,7 +813,7 @@ platformcreatefile(mode_t mode, const char* canonpath, FD* fd) int createflags = 0; mode_t permissions = NC_DEFAULT_ROPEN_PERMS; - ZTRACE(6,"map=%s canonpath=%s",zfmap->map.url,canonpath); + ZTRACE(6,"mode=%d canonpath=%s",mode,canonpath); errno = 0; if(!fIsSet(mode, NC_WRITE)) @@ -854,7 +854,7 @@ platformopenfile(mode_t mode, const char* canonpath, FD* fd) int ioflags = 0; mode_t permissions = 0; - ZTRACE(6,"map=%s canonpath=%s",zfmap->map.url,canonpath); + ZTRACE(6,"mode=%d canonpath=%s",mode,canonpath); errno = 0; if(!fIsSet(mode, NC_WRITE)) { @@ -888,7 +888,7 @@ platformcreatedir(mode_t mode, const char* canonpath) { int ret = NC_NOERR; - ZTRACE(6,"map=%s canonpath=%s",zfmap->map.url,canonpath); + ZTRACE(6,"mode=%d canonpath=%s",mode,canonpath); errno = 0; /* Try to access file as if it exists */ @@ -920,7 +920,7 @@ platformopendir(mode_t mode, const char* canonpath) NC_UNUSED(mode); - ZTRACE(6,"map=%s canonpath=%s",zfmap->map.url,canonpath); + ZTRACE(6,"mode=%d canonpath=%s",mode,canonpath); errno = 0; /* Try to access file as if it exists */ @@ -958,7 +958,7 @@ platformdircontent(const char* canonpath, NClist* contents) size_t len; char* d = NULL; - ZTRACE(6,"map=%s canonpath=%s",zfmap->map.url,canonpath); + ZTRACE(6,"canonpath=%s",canonpath); switch (ret = platformtestcontentbearing(canonpath)) { case NC_EEMPTY: ret = NC_NOERR; break; /* directory */ @@ -1020,7 +1020,7 @@ platformdircontent(const char* canonpath, NClist* contents) errno = 0; DIR* dir = NULL; - ZTRACE(6,"map=%s canonpath=%s",zfmap->map.url,canonpath); + ZTRACE(6,"canonpath=%s",canonpath); switch (ret = platformtestcontentbearing(canonpath)) { case NC_EEMPTY: ret = NC_NOERR; break; /* directory */ @@ -1061,7 +1061,7 @@ platformdeleter(NCbytes* canonpath, int depth) char* local = NULL; local = ncbytescontents(canonpath); - ZTRACE(6,"map=%s canonpath=%s delroot=%d depth=%d",zfmap->map.url,local,delroot,depth); + ZTRACE(6,"canonpath=%s depth=%d",canonpath,depth); ret = platformdircontent(local, subfiles); #ifdef DEBUG @@ -1131,7 +1131,7 @@ platformdelete(const char* rootpath, int delroot) int stat = NC_NOERR; NCbytes* canonpath = ncbytesnew(); - ZTRACE(6,"map=%s rootpath=%s delroot=%d",zfmap->map.url,rootpath,delroot); + ZTRACE(6,"rootpath=%s delroot=%d",rootpath,delroot); if(rootpath == NULL || strlen(rootpath) == 0) goto done; ncbytescat(canonpath,rootpath); @@ -1165,7 +1165,7 @@ platformseek(FD* fd, int pos, size64_t* sizep) assert(fd && fd->fd >= 0); - ZTRACE(6,"map=%s fd=%d pos=%d",zfmap->map.url,(fd?fd->fd:-1),pos); + ZTRACE(6,"fd=%d pos=%d",(fd?fd->fd:-1),pos); errno = 0; ret = NCfstat(fd->fd, &statbuf); @@ -1188,7 +1188,7 @@ platformread(FD* fd, size64_t count, void* content) assert(fd && fd->fd >= 0); - ZTRACE(6,"map=%s fd=%d count=%llu",zfmap->map.url,(fd?fd->fd:-1),count); + ZTRACE(6,"fd=%d count=%llu",(fd?fd->fd:-1),count); while(need > 0) { ssize_t red; @@ -1211,7 +1211,7 @@ platformwrite(FD* fd, size64_t count, const void* content) assert(fd && fd->fd >= 0); - ZTRACE(6,"map=%s fd=%d count=%llu",zfmap->map.url,(fd?fd->fd:-1),count); + ZTRACE(6,"fd=%d count=%llu",(fd?fd->fd:-1),count); while(need > 0) { ssize_t red = 0; @@ -1230,7 +1230,7 @@ platformwrite(FD* fd, size64_t count, const void* content) static void platformrelease(FD* fd) { - ZTRACE(6,"map=%s fd=%d",zfmap->map.url,(fd?fd->fd:-1)); + ZTRACE(6,"fd=%d",(fd?fd->fd:-1)); if(fd->fd >=0) NCclose(fd->fd); fd->fd = -1; (void)ZUNTRACE(NC_NOERR); diff --git a/libnczarr/zmap_s3sdk.c b/libnczarr/zmap_s3sdk.c index d70e7b5575..9eb8d39097 100644 --- a/libnczarr/zmap_s3sdk.c +++ b/libnczarr/zmap_s3sdk.c @@ -263,7 +263,7 @@ zs3truncate(const char *s3url) ncurifree(url); ncurifree(purl); (void)NC_s3clear(&info); - return stat; + return ZUNTRACE(stat); } /**************************************************/ @@ -314,6 +314,37 @@ zs3len(NCZMAP* map, const char* key, size64_t* lenp) return ZUNTRACE(stat); } +/* +@return NC_NOERR if key is a prefix for some existing object. +@return NC_EEMPTY if key is not such a prefix. +@return NC_EXXX return true error +*/ +static int +zs3keyexists(NCZMAP* map, const char* key) +{ + int stat = NC_NOERR; + ZS3MAP* z3map = (ZS3MAP*)map; + char* truekey = NULL; + + ZTRACE(6,"map=%s key=%s",map->url,key); + + if((stat = maketruekey(z3map->s3.rootkey,key,&truekey))) goto done; + + switch (stat = NC_s3sdkinfo(z3map->s3client,z3map->s3.bucket,truekey,lenp,&z3map->errmsg)) { + case NC_NOERR: break; + case NC_EEMPTY: stat = NC_ENOOBJECT; /* fall thru */ + case NC_ENOOBJECT: + if(lenp) *lenp = 0; + goto done; + default: + goto done; + } +done: + nullfree(truekey); + reporterr(z3map); + return ZUNTRACE(stat); +} + /* @return NC_NOERR if object at key was read @return NC_ENOOBJECT if object at key does not exist diff --git a/libnczarr/zmap_zip.c b/libnczarr/zmap_zip.c index d3eefa6c9f..13a3cb2999 100644 --- a/libnczarr/zmap_zip.c +++ b/libnczarr/zmap_zip.c @@ -284,7 +284,7 @@ ziptruncate(const char* surl) zip_close(zip); done: ncurifree(url); - return stat; + return ZUNTRACE(stat); } /**************************************************/ diff --git a/libnczarr/zmetadata.c b/libnczarr/zmetadata.c new file mode 100644 index 0000000000..b9e3698a06 --- /dev/null +++ b/libnczarr/zmetadata.c @@ -0,0 +1,411 @@ +/********************************************************************* + * Copyright 2018, UCAR/Unidata + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + *********************************************************************/ + +#include "zincludes.h" + +/**************************************************/ + +extern int NCZMD2_initialize(void); +extern int NCZMD2_finalize(void); + +/**************************************************/ +//////////////////////////////////////////////////// + +int NCZMD_initialize(void) +{ + int stat = NC_NOERR; + if((stat=NCZMD2_initialize())) goto done; +done: + return THROW(stat); +} + +int NCZMD_finalize(void) +{ + + int stat = NC_NOERR; + if((stat=NCZMD2_finalize())) goto done; +done: + return THROW(stat); +} + +///////////////////////////////////////////////////////////////////// +// Fetch list of subnodes of .zmetadata or storage +///////////////////////////////////////////////////////////////////// + +/** +Return a vector of names (not keys) representing the +next segment of legal objects that are immediately contained by the prefix key. +@param file -- the containing file +@param prefix -- the key into the tree where the search is to occur +@param matches -- return the set of names in this list; might be empty +@return NC_NOERR if the operation succeeded +@return NC_EXXX if the operation failed for one of several possible reasons +*/ +int +NCZMD_list(NC_FILE_INFO_T* file, const char* prefix, struct NClist* matches) +{ + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + return zfile->metadata_handler.dispatcher->list(file, prefix, matches); +} + +/** +Return a vector of keys representing the +list of all objects whose key is prefixed by the specified prefix arg. +In effect it returns the complete subtree below a specified prefix. +@param file -- the containing file +@param prefix -- the key into the tree whose subtree of keys is to be returned. +@param matches -- return the set of keys in this list; might be empty +@return NC_NOERR if the operation succeeded +@return NC_EXXX if the operation failed for one of several possible reasons +*/ +int +NCZMD_listall(NC_FILE_INFO_T* file, const char* prefix, struct NClist* matches) +{ + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + return zfile->metadata_handler.dispatcher->listall(file, prefix, matches); +} + +/** +Test if key refers to content-bearing object. +@param file -- the containing file +@param key -- the key into the tree to test +@return NC_NOERR if the key exists +@return NC_ENOOBJECT if the key does not exists +@return NC_EXXX if the operation failed for one of several possible reasons +*/ +int +NCZMD_exists(NC_FILE_INFO_T* file, const char* key) +{ + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + return zfile->metadata_handler.dispatcher->exists(file, key); +} + +/** +Open a consolidated metadata object +@param file -- the containing file +@return NC_NOERR if the operation succeeded +@return NC_EXXX if the operation failed for one of several possible reasons +*/ +int +NCZMD_open(NC_FILE_INFO_T* file) +{ + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + return zfile->metadata_handler.dispatcher->open(file); +} + +/** +Create a consolidated metadata object +@param file -- the containing file +@return NC_NOERR if the operation succeeded +@return NC_EXXX if the operation failed for one of several possible reasons +*/ +int +NCZMD_create(NC_FILE_INFO_T* file) +{ + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + return zfile->metadata_handler.dispatcher->create(file); +} + +/** +Close a consolidated metadata object +@param file -- the containing file +@return NC_NOERR if the operation succeeded +@return NC_EXXX if the operation failed for one of several possible reasons +*/ +int +NCZMD_close(NC_FILE_INFO_T* file) +{ + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + return zfile->metadata_handler.dispatcher->close(file); +} + +/** +Write a consolidated metadata object +@param file -- the containing file +@return NC_NOERR if the operation succeeded +@return NC_EXXX if the operation failed for one of several possible reasons +*/ +int +NCZMD_consolidate(NC_FILE_INFO_T* file) +{ + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + return zfile->metadata_handler.dispatcher->consolidate(file); +} + +///////////////////////////////////////////////////////////////////// +// Fetch JSON content from .zmetadata or storage +///////////////////////////////////////////////////////////////////// + +int +NCZMD_fetch_json_content(NC_FILE_INFO_T *file, NCZMD_MetadataType mdtype, const char *key, NCjson **jobjp) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + if((stat = zfile->metadata_handler.dispatcher->fetch_json_content(file, mdtype, key, jobjp))) goto done; + +done: + return THROW(stat); +} + +#if 0 +int +NCZMD_fetch_json_group(NC_FILE_INFO_T *file, NC_GRP_INFO_T *grp, const char *name, NCjson **jgroup) +{ + int stat = NC_NOERR; + char *group= NULL; + char *key = NULL; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + + if (grp && ((stat = NCZ_grpkey(grp, &group)) != NC_NOERR)) + goto done; + if ((stat = nczm_concat(group, name, &key))) + goto done; + if((stat = zfile->metadata_handler.dispatcher->fetch_json_content(file, NCZMD_GROUP, key, jgroup))) + goto done; + +done: + nullfree(group); + nullfree(key); + return stat; +} + +int +NCZMD_fetch_json_attrs(NC_FILE_INFO_T *file, NC_GRP_INFO_T *grp, const char *name, NCjson **jattrs) +{ + int stat = NC_NOERR; + char *group= NULL; + char *key = NULL; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + + if (grp && ((stat = NCZ_grpkey(grp, &group)) != NC_NOERR)) + goto done; + if ((stat = nczm_concat(group, name, &key))) + goto done; + if((stat = zfile->metadata_handler.dispatcher->fetch_json_content(file, NCZMD_ATTRS, key , jattrs))) + goto done; + +done: + nullfree(group); + nullfree(key); + return stat; +} + +int +NCZMD_fetch_json_array(NC_FILE_INFO_T *file, NC_GRP_INFO_T *grp, const char *name, NCjson **jarray) +{ + int stat = NC_NOERR; + char *group= NULL; + char *key = NULL; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + + if (grp && ((stat = NCZ_grpkey(grp, &group)) != NC_NOERR)) + goto done; + + if ((stat = nczm_concat(group, name, &key))) + goto done; + if((stat = zfile->metadata_handler.dispatcher->fetch_json_content(file, NCZMD_ARRAY, key, jarray))) + goto done; + +done: + nullfree(group); + nullfree(key); + return stat; +} +#endif /*0*/ + +//////////////////////////////////////////////////////////////////////////////// +// Update in-memory + storage JSON content +//////////////////////////////////////////////////////////////////////////////// + +int +NCZMD_update_json_content(NC_FILE_INFO_T *file, NCZMD_MetadataType mdtype, const char *key, const NCjson* jobj) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + if((stat = zfile->metadata_handler.dispatcher->update_json_content(file, mdtype, key, jobj))) goto done; + +done: + return THROW(stat); +} + +#if 0 +int +NCZMD_update_json_group(NC_FILE_INFO_T *file, NC_GRP_INFO_T *grp, const char *name, const NCjson *jgroup) +{ + int stat = NC_NOERR; + char *group= NULL; + char *key = NULL; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + + if (grp && ((stat = NCZ_grpkey(grp, &group)) != NC_NOERR)) + goto done; + if ((stat = nczm_concat(group, name, &key))) + goto done; + if((stat = zfile->metadata_handler.dispatcher->update_json_content(file, NCZMD_GROUP, key, jgroup))) + goto done; + +done: + nullfree(group); + nullfree(key); + return stat; +} + +int +NCZMD_update_json_attrs(NC_FILE_INFO_T *file, NC_GRP_INFO_T *grp, const char *name, const NCjson *jattrs) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + char *group= NULL; + char *key = NULL; + + if (grp && ((stat = NCZ_grpkey(grp, &group)) != NC_NOERR)) + goto done; + if ((stat = nczm_concat(group, name, &key))) + goto done; + if((stat = zfile->metadata_handler.dispatcher->update_json_content(file, NCZMD_ATTRS, key , jattrs))) + goto done; + +done: + nullfree(group); + nullfree(key); + return stat; +} + +int +NCZMD_update_json_array(NC_FILE_INFO_T *file, NC_GRP_INFO_T *grp, const char *name, const NCjson *jarray) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + char *group= NULL; + char *key = NULL; + + if (grp && ((stat = NCZ_grpkey(grp, &group)) != NC_NOERR)) + goto done; + if ((stat = nczm_concat(group, name, &key))) + goto done; + if((stat = zfile->metadata_handler.dispatcher->update_json_content(file, NCZMD_ARRAY, key, jarray))) + goto done; + +done: + nullfree(group); + nullfree(key); + return stat; +} +#endif /*0*/ + +//////////////////////////////////////////////////////////////////////////// + +int +NCZMD_is_metadata_consolidated(NC_FILE_INFO_T* file) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + NCZ_Metadata *zmd = &zfile->metadata_handler; + if(zmd->jcsl == NULL || + NCJsort(zmd->jcsl) != NCJ_DICT || + (zfile->flags & FLAG_NOCONSOLIDATED)) + {stat = NC_ENOOBJECT; goto done;} +done: + return stat; +} + +int +NCZMD_get_metadata_format(NC_FILE_INFO_T* file, int* zarrformat) +{ + // Only pure Zarr is determined + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + NCZ_Metadata *zmd = &zfile->metadata_handler; + + if(!zmd->dispatcher ) {stat = NC_EFILEMETA; goto done;} + if(zmd->dispatcher->zarr_format >= ZARRFORMAT2) { + *zarrformat = zmd->dispatcher->zarr_format; + goto done; + } + /* Last thing to do is to look for: + .zattrs, .zgroup or .zarray + */ + if(!nczmap_exists(zfile->map, "/" Z2ATTRS) + && !nczmap_exists(zfile->map, "/" Z2GROUP) + && !nczmap_exists(zfile->map, "/" Z2ARRAY)) + goto done; + *zarrformat = ZARRFORMAT2; +done: + return THROW(stat); +} + +/* Inference of the metadata handler */ +int +NCZMD_set_metadata_handler(NC_FILE_INFO_T *file) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + NCZ_Metadata* zmd = &zfile->metadata_handler; + const NCZ_Metadata_Dispatcher *zmd_dispatcher = NULL; + int disallowzmetadata = 0; + NCjson* jcsl = NULL; + const NCjson* jmeta = NULL; + + /* Override from env var or mode flag */ + if((zfile->flags & FLAG_NOCONSOLIDATED) || getenv(NCZARRDEFAULTNOMETA) != NULL) disallowzmetadata = 1; + + /* First, figure out and set the dispatcher */ + if(disallowzmetadata) { + zmd_dispatcher = NCZ_metadata_handler; + } else if(zfile->creating) { + switch (zfile->zarr.zarr_format) { + case 2: + zmd_dispatcher = NCZ_csl_metadata_handler2; + break; + case 3: + zmd_dispatcher = NCZ_csl_metadata_handler3; + break; + default: + zmd_dispatcher = NCZ_metadata_handler; + break; + } + } else if(zmd_dispatcher == NULL) { /* opening a file */ + /* See if /.zmetadata exists */ + switch (zfile->zarr.zarr_format) { + case 2: /* Try to download .zmetadata */ + if((stat = NCZ_downloadjson(zfile->map,Z2METADATA,&jcsl))) goto done; + if(jcsl != NULL) + NCJcheck(NCJdictget(jcsl,"metadata",(NCjson**)&jmeta)); + break; +#ifdef NETCDF_ENABLE_NCZARR_V3 + case 3: /* For V3, we need to look inside the root group's zarr.json */ + if((stat = NCZ_downloadjson(zfile->map,Z3METADATA,&jcsl))) goto done; + if(jcsl != NULL) + NCJcheck(NCJdictget(jcsl,"metadata",(NCjson**)&jmeta)); + break; +#endif + default: + break; + } + if(jmeta != NULL && zfile->zarr.zarr_format == 2) + zmd_dispatcher = NCZ_csl_metadata_handler2; +#ifdef NETCDF_ENABLE_NCZARR_V3 + else if(jmeta && zfile->zarr.zarr_format == 3) + zmd_dispatcher = NCZ_csl_metadata_handler3; +#endif + else + zmd_dispatcher = NCZ_metadata_handler; + } + zmd->dispatcher = zmd_dispatcher; + assert(zmd->dispatcher != NULL); + + zmd->jcsl = jcsl; jcsl = NULL; + zmd->jmeta = jmeta; jmeta = NULL; + + /* Now open/create the consolidated metadata object */ + if(zfile->creating) { + if((stat=NCZMD_create(file))) goto done; + } else { + if((stat=NCZMD_open(file))) goto done; + } + +done: + return THROW(stat); +} diff --git a/libnczarr/zmetadata.h b/libnczarr/zmetadata.h new file mode 100644 index 0000000000..3dee5bf26c --- /dev/null +++ b/libnczarr/zmetadata.h @@ -0,0 +1,111 @@ +/* Copyright 2018-2018 University Corporation for Atmospheric + Research/Unidata. */ + +/* +Zarr Metadata Handling + +Encapsulates Zarr metadata operations across versions, supporting both +consolidated access and per-file access. Provides a common interface +for metadata operations. + +The dispatcher is defined by the type NCZ_Metadata_Dispatcher. +It offers several types of operations that allow decoupling/abstract +filesystem access, content reading of the JSON metadata files +1. Retrieve JSON representation of (sub)groups, arrays and attributes. + Directly read from filesystem/objectstore or retrieve the JSON + object from the consolidated view respective to the group or variable +2. Wrappers for selected zmap operations that are key based. + +Note: This will also be the case of zarr v3 +(the elements will be extracted from zarr.json instead) +*/ + +#ifndef ZMETADATA_H +#define ZMETADATA_H + +#if defined(__cplusplus) +extern "C" +{ +#endif +/* This is the version of the metadata table. It should be changed + * when new functions are added to the metadata table. */ +#ifndef NCZ_METADATA_VERSION +#define NCZ_METADATA_VERSION 1 +#endif /*NCZ_METADATA_VERSION*/ + +/* The keys in this dict are all stored in root group's container for consolidated metadata */ +#define MINIMIM_CSL_REP2_RAW "{\"zarr_consolidated_format\":1, \"metadata\":{}}" +#define MINIMIM_CSL_REP3_RAW "{\"zarr_format\": 3, \"node_type\": \"group\", \"attributes\": {}, \"kind\": \"inline\", \"must_understand\": False, \"metadata\": {}}" + +typedef enum NCZMD_MetadataType { + NCZMD_NULL, + NCZMD_GROUP, + NCZMD_ATTRS, + NCZMD_ARRAY +} NCZMD_MetadataType; + +typedef struct NCZ_Metadata_Dispatcher +{ + int zarr_format; /* Zarr format version */ + int dispatch_version; /* Dispatch table version*/ + size64_t flags; /* Metadata handling flags */ +#define ZARR_NOT_CONSOLIDATED 0 +#define ZARR_CONSOLIDATED 1 + int (*open)(NC_FILE_INFO_T* file); + int (*create)(NC_FILE_INFO_T* file); + int (*close)(NC_FILE_INFO_T* file); + int (*consolidate)(NC_FILE_INFO_T* file); + int (*fetch_json_content)(NC_FILE_INFO_T *, NCZMD_MetadataType, const char* key, NCjson** jobj); + int (*update_json_content)(NC_FILE_INFO_T *, NCZMD_MetadataType, const char *key, const NCjson *jobj); + + /* zmap wrappers */ + int (*list)(NC_FILE_INFO_T*, const char* prefix, NClist* matches); + int (*listall)(NC_FILE_INFO_T*, const char* prefix, NClist* matches); + int (*exists)(NC_FILE_INFO_T* file, const char* prefix); +} NCZ_Metadata_Dispatcher; + +typedef struct NCZ_Metadata +{ + NCjson *jcsl; /* Consolidated JSON container: .zmetadata for V2, + or root group zarr.json (minus "metadata" dict) for V3 */ + const NCjson *jmeta; /* "metadata" dict from jcsl (or NULL) */ + int dirty; /* The consolidated metadata was modified */ + const NCZ_Metadata_Dispatcher *dispatcher; +} NCZ_Metadata; + +/* Handler when not using consolidated metadata */ +extern const NCZ_Metadata_Dispatcher *NCZ_metadata_handler; +/* Consolidated metadata handler for Zarr version 2*/ +extern const NCZ_Metadata_Dispatcher *NCZ_csl_metadata_handler2; +/* Consolidated metadata handler for Zarr version 3*/ +extern const NCZ_Metadata_Dispatcher *NCZ_csl_metadata_handler3; + +/* Called by nc_initialize and nc_finalize respectively */ +extern int NCZMD_initialize(void); +extern int NCZMD_finalize(void); + +extern int NCZMD_open(NC_FILE_INFO_T *file); +extern int NCZMD_create(NC_FILE_INFO_T *file); +extern int NCZMD_close(NC_FILE_INFO_T *file); +extern int NCZMD_consolidate(NC_FILE_INFO_T *file); + +extern int NCZMD_fetch_json_content(NC_FILE_INFO_T *, NCZMD_MetadataType, const char* key, NCjson** jobjp); +extern int NCZMD_update_json_content(NC_FILE_INFO_T *, NCZMD_MetadataType, const char *key, const NCjson *jobj); + +extern int NCZMD_list(NC_FILE_INFO_T*, const char* prefix, struct NClist* matches); +extern int NCZMD_listall(NC_FILE_INFO_T*, const char* prefix, struct NClist* matches); +extern int NCZMD_exists(NC_FILE_INFO_T* file, const char* prefix); + +/**************************************************/ + +/* Inference for the Metadata handler */ +extern int NCZMD_is_metadata_consolidated(NC_FILE_INFO_T* file); +extern int NCZMD_get_metadata_format(NC_FILE_INFO_T *zfile, int *zarrformat); /* Only pure Zarr is determined */ +extern int NCZMD_set_metadata_handler(NC_FILE_INFO_T *zfile); +extern void NCZMD_clear_metadata_handler(NCZ_Metadata * zmd); + +#if defined(__cplusplus) +} +#endif + +#endif /* ZMETADATA_H */ diff --git a/libnczarr/zmetadata0.c b/libnczarr/zmetadata0.c new file mode 100644 index 0000000000..6637077afa --- /dev/null +++ b/libnczarr/zmetadata0.c @@ -0,0 +1,147 @@ +/********************************************************************* + * Copyright 2018, UCAR/Unidata + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + *********************************************************************/ + +#include "zincludes.h" + +/**************************************************/ + +extern int NCZF0_initialize(void); +extern int NCZF0_finalize(void); + +static int list_nodes(NC_FILE_INFO_T*, const char* prefix, struct NClist* matches); +static int listall_nodes(NC_FILE_INFO_T*, const char* prefix, struct NClist* matches); +static int exists_nodes(NC_FILE_INFO_T* file, const char* prefix); + +static int fetch_json_content(NC_FILE_INFO_T* file, NCZMD_MetadataType zarr_obj_type, const char *key, NCjson **jobj); +static int update_json_content(NC_FILE_INFO_T* file, NCZMD_MetadataType zobj_t, const char *prefix, const NCjson *jobj); + +static int open_noop(NC_FILE_INFO_T* file); +static int create_noop(NC_FILE_INFO_T* file); +static int close_noop(NC_FILE_INFO_T* file); +static int consolidate_noop(NC_FILE_INFO_T*); + +/**************************************************/ + +static const NCZ_Metadata_Dispatcher NCZ_md_table = { /* Common to V2 and V3 */ + ZARRFORMAT2, + NCZ_METADATA_VERSION, /* Version of the dispatch table */ + ZARR_NOT_CONSOLIDATED, /* Flags*/ + .list = list_nodes, + .listall = listall_nodes, + .exists = exists_nodes, + .open = open_noop, + .create = create_noop, + .close = close_noop, + .consolidate = consolidate_noop, + .fetch_json_content = fetch_json_content, + .update_json_content = update_json_content, +}; +const NCZ_Metadata_Dispatcher *NCZ_metadata_handler= &NCZ_md_table; + +/******************************************************/ + +int +NCZMD0_initialize(void) +{ + return NC_NOERR; +} + +int +NCZMD0_finalize(void) +{ + return NC_NOERR; +} + +/*/////////////////////////////////////////////////// +// .zmetadata dispatch functions +///////////////////////////////////////////////////*/ + +static int +list_nodes(NC_FILE_INFO_T* file, const char* prefix, NClist* matches) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + if((stat = nczmap_list(zfile->map,prefix,matches))) goto done; +done: + return stat; +} + +static int +listall_nodes(NC_FILE_INFO_T* file, const char* prefix, NClist* matches) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + if((stat = nczmap_listall(zfile->map,prefix,matches))) goto done; +done: + return stat; +} + +static int +exists_nodes(NC_FILE_INFO_T* file, const char* prefix) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + if((stat = nczmap_exists(zfile->map,prefix))) goto done; +done: + return stat; +} + +static int +fetch_json_content(NC_FILE_INFO_T* file, NCZMD_MetadataType zobj_t, const char *key, NCjson **jobjp) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + NCjson* jobj = NULL; + + NC_UNUSED(zobj_t); + if((stat = NCZ_downloadjson(zfile->map, key, &jobj))) goto done; + if(jobjp) {*jobjp = jobj; jobj = NULL;} +done: + NCJreclaim(jobj); + return stat; +} + +static int +update_json_content(NC_FILE_INFO_T* file, NCZMD_MetadataType zobj_t, const char* key, const NCjson *jobj) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + + NC_UNUSED(zobj_t); + if((stat = NCZ_uploadjson(zfile->map, key, jobj))) goto done; + zfile->metadata_handler.dirty = 0; /* after writing */ + +done: + return stat; +} + + +static int +open_noop(NC_FILE_INFO_T* file) +{ + NC_UNUSED(file); + return NC_NOERR; +} + +static int +create_noop(NC_FILE_INFO_T* file) +{ + NC_UNUSED(file); + return NC_NOERR; +} + +static int +close_noop(NC_FILE_INFO_T* file) +{ + NC_UNUSED(file); + return NC_NOERR; +} + +static int +consolidate_noop(NC_FILE_INFO_T* file) +{ + NC_UNUSED(file); + return NC_NOERR; +} diff --git a/libnczarr/zmetadata2.c b/libnczarr/zmetadata2.c new file mode 100644 index 0000000000..cdae7d9c84 --- /dev/null +++ b/libnczarr/zmetadata2.c @@ -0,0 +1,273 @@ +/********************************************************************* + * Copyright 2018, UCAR/Unidata + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + *********************************************************************/ + +#include "zincludes.h" + +/**************************************************/ + +extern int NCZF2_initialize(void); +extern int NCZF2_finalize(void); + +static int list_nodes_csl_v2(NC_FILE_INFO_T*, const char* prefix, struct NClist* matches); +static int listall_nodes_csl_v2(NC_FILE_INFO_T*, const char* prefix, struct NClist* matches); +static int exists_key_csl_v2(NC_FILE_INFO_T* file, const char* prefix); + +static int fetch_csl_json_content_v2(NC_FILE_INFO_T* file, NCZMD_MetadataType zarr_obj_type, const char *key, NCjson **jobj); +static int update_csl_json_content_v2(NC_FILE_INFO_T* file, NCZMD_MetadataType zobj_t, const char *prefix, const NCjson *jobj); + +static int open_csl_v2(NC_FILE_INFO_T* file); +static int create_csl_v2(NC_FILE_INFO_T* file); +static int close_csl_v2(NC_FILE_INFO_T* file); +static int consolidate_csl_v2(NC_FILE_INFO_T*); + +/**************************************************/ +static const NCZ_Metadata_Dispatcher NCZ_csl_md2_table = { + ZARRFORMAT2, + NCZ_METADATA_VERSION, /* Version of the dispatch table */ + ZARR_CONSOLIDATED, /* Flags*/ + .list = list_nodes_csl_v2, + .listall = listall_nodes_csl_v2, + .exists = exists_key_csl_v2, + .consolidate = consolidate_csl_v2, + .close = close_csl_v2, + .open = open_csl_v2, + .create = create_csl_v2, + .fetch_json_content = fetch_csl_json_content_v2, + .update_json_content = update_csl_json_content_v2, +}; +const NCZ_Metadata_Dispatcher *NCZ_csl_metadata_handler2 = &NCZ_csl_md2_table; + +/******************************************************/ + +int +NCZMD2_initialize(void) +{ + return NC_NOERR; +} + +int +NCZMD2_finalize(void) +{ + return NC_NOERR; +} + +/*/////////////////////////////////////////////////// +// .zmetadata dispatch functions +///////////////////////////////////////////////////*/ + +static int +list_nodes_csl_v2(NC_FILE_INFO_T* file, const char* prefix, NClist* matches) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + size_t i,plen; + NCZ_Metadata* zmd = &zfile->metadata_handler; + char* seg1 = NULL; + NClist* segments = nclistnew(); + + assert(zmd->jmeta != NULL && NCJsort(zmd->jmeta)==NCJ_DICT); + if(prefix[0] == '/') prefix++; /* drop leading '/' for search purposes */ + plen = strlen(prefix); + /* Walk the metadata nodes and collect the matches */ + for(i=0;ijmeta);i++) { + const NCjson* jkey = NCJdictkey(zmd->jmeta,i); + const char* skey = NCJstring(jkey); + size_t slen = strlen(skey); + size_t j, found; + /* Check for prefix taking root key into acct. */ + if((plen == 0 && slen > 0) || strncmp(skey,prefix,plen) == 0) { + const char* suffix = NULL; + /* This is a match */ + suffix = &skey[plen]; /* point past the prefix */ + assert(strlen(suffix) > 0); + nclistclearall(segments); + ncz_splitkey(suffix,segments); + if(nclistlength(segments) > 0) { /* test if just the prefix */ + seg1 = (char*)nclistremove(segments,0); + /* suppress duplicates */ + for(found=0,j=0;jformat_file_info; + size_t i,plen; + NCZ_Metadata* zmd = &zfile->metadata_handler; + NCbytes* key = ncbytesnew(); + + assert(zmd->jmeta != NULL && NCJsort(zmd->jmeta)==NCJ_DICT); + if(prefix[0] == '/') prefix++; /* drop leading '/' for search purposes */ + plen = strlen(prefix); + ncbytescat(key,"/"); + /* Walk the metadata nodes and collect the matches (with leading '/') */ + for(i=0;ijmeta);i++) { + NCjson* jkey = NCJdictkey(zmd->jmeta,i); + const char* skey = NCJstring(jkey); + if(strncmp(skey,prefix,plen) > 0) { + /* This is a match and is not just the prefix*/ + ncbytessetlength(key,1); + ncbytescat(key,prefix); /* add leading '/' */ + nclistpush(matches,strdup(ncbytescontents(key))); + } + } + return stat; +} + +static int +exists_key_csl_v2(NC_FILE_INFO_T* file, const char* prefix) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + size_t i,plen; + NCZ_Metadata* zmd = &zfile->metadata_handler; + + assert(zmd->jmeta != NULL && NCJsort(zmd->jmeta)==NCJ_DICT); + if(prefix[0] == '/') prefix++; /* drop leading '/' for search purposes */ + plen = strlen(prefix); + /* Walk the metadata nodes and see if there is a prefix match */ + for(i=0;ijmeta);i++) { + NCjson* jkey = NCJdictkey(zmd->jmeta,i); + const char* skey = NCJstring(jkey); + if(strncmp(skey,prefix,plen) == 0) {stat = NC_NOERR; goto done;} + } + stat = NC_ENOOBJECT; +done: + return stat; +} + +static int +fetch_csl_json_content_v2(NC_FILE_INFO_T* file, NCZMD_MetadataType zobj_t, const char* key, NCjson **jobjp) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + NCjson* jobj = NULL; + NCjson *jkey = NULL; + NCZ_Metadata* zmd = &zfile->metadata_handler; + + NC_UNUSED(zobj_t); + assert(zmd->jmeta != NULL); + if(key[0] == '/') key++; /* remove any leading key */ + /* Meta-data is stored a mostly flat format using the whole key (with leading / removed) */ + if ((stat = NCJdictget(zmd->jmeta, key, (NCjson**)&jkey))) goto done; + NCJcheck(NCJclone(jkey, &jobj)); + if(jobj != NULL) + {if(jobjp) {*jobjp = jobj; jobj = NULL;}} +done: + return stat; +} + +static int +update_csl_json_content_v2(NC_FILE_INFO_T* file, NCZMD_MetadataType zobj_t, const char *key, const NCjson *jobj) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + NCjson* jclone = NULL; + NCZ_Metadata* zmd = &zfile->metadata_handler; + + NC_UNUSED(zobj_t); + if(key[0] == '/') key++; + + /* Create the metadata skeleton if it does not exist */ + if(zmd->jcsl == NULL) { + if((stat = NCJparse(MINIMIM_CSL_REP2_RAW,0,&zmd->jcsl))) goto done; + NCJcheck(NCJdictget(zmd->jcsl,"metadata",(NCjson**)&zmd->jmeta)); + } + /* Insert/overwrite the key+value */ + NCJcheck(NCJclone(jobj,&jclone)); + NCJcheck(NCJinsert((NCjson*)zmd->jmeta,key,jclone)); + zmd->dirty = 1; + +#if 0 /* Do we need this? */ + // Allocating representation if doesn't exist + // Updating the internal JSON representation to be synced later + NCjson * jrep = NULL; + if ((stat = NCJdictget(zfile->metadata_handler.jcsl,"metadata", (NCjson**)&jrep)) || jrep == NULL) { + goto done; + } +#endif /*0*/ + +done: + return stat; +} + +//////////////////////////////////////////////////////////////////////////// +/* Writes .zmetadata file into storage */ + +static int +consolidate_csl_v2(NC_FILE_INFO_T* file) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + NCZ_Metadata* zmd = &zfile->metadata_handler; + + if (zmd->dirty) { + if(zmd->jcsl == NULL) {stat = NC_EZARRMETA; goto done;} + stat = NCZ_uploadjson(zfile->map, Z2METADATA ,zmd->jcsl); + zmd->dirty = 0; + } +done: + return stat; +} + +static int +close_csl_v2(NC_FILE_INFO_T* file) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + NCZ_Metadata* zmd = &zfile->metadata_handler; + NCJreclaim(zmd->jcsl); + zmd->jcsl = NULL; + zmd->jmeta = NULL; + return stat; +} + +static int +open_csl_v2(NC_FILE_INFO_T* file) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + NCZ_Metadata* zmd = &zfile->metadata_handler; + + /* Read /.zmetadata */ + if(zmd->jcsl == NULL) { + if((stat = NCZ_downloadjson(zfile->map,Z2METADATA,&zmd->jcsl))) goto done; + } + if(zmd->jcsl == NULL || NCJsort(zmd->jcsl) != NCJ_DICT) {stat = NC_EZARRMETA; goto done;} + /* Pull out the "metadata" key and save it */ + NCJcheck(NCJdictget(zmd->jcsl,"metadata",(NCjson**)&zmd->jmeta)); + if(zmd->jmeta == NULL || NCJsort(zmd->jmeta) != NCJ_DICT) {stat = NC_EZARRMETA; goto done;} +done: + return stat; +} + +static int +create_csl_v2(NC_FILE_INFO_T* file) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + NCZ_Metadata* zmd = &zfile->metadata_handler; + + /* Create the JSON skeleton */ + if((stat = NCJparse(MINIMIM_CSL_REP2_RAW,0,&zmd->jcsl))) goto done; /* Create the metadata skeleton */ + /* Pull out the "metadata" key and save it */ + NCJcheck(NCJdictget(zmd->jcsl,"metadata",(NCjson**)&zmd->jmeta)); + +done: + return stat; +} + diff --git a/libnczarr/zmetadata3.c b/libnczarr/zmetadata3.c new file mode 100644 index 0000000000..0d3a5298e5 --- /dev/null +++ b/libnczarr/zmetadata3.c @@ -0,0 +1,282 @@ +/********************************************************************* + * Copyright 2018, UCAR/Unidata + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + *********************************************************************/ + +#include "zincludes.h" + +/**************************************************/ + +extern int NCZF3_initialize(void); +extern int NCZF3_finalize(void); + +static int list_nodes_csl_v3(NC_FILE_INFO_T*, const char* prefix, struct NClist* matches); +static int listall_nodes_csl_v3(NC_FILE_INFO_T*, const char* prefix, struct NClist* matches); +static int exists_key_csl_v3(NC_FILE_INFO_T* file, const char* prefix); + +static int fetch_csl_json_content_v3(NC_FILE_INFO_T* file, NCZMD_MetadataType zarr_obj_type, const char *key, NCjson **jobj); +static int update_csl_json_content_v3(NC_FILE_INFO_T* file, NCZMD_MetadataType zobj_t, const char *prefix, const NCjson *jobj); + +static int open_csl_v3(NC_FILE_INFO_T* file); +static int create_csl_v3(NC_FILE_INFO_T* file); +static int close_csl_v3(NC_FILE_INFO_T* file); +static int consolidate_csl_v3(NC_FILE_INFO_T*); + +/**************************************************/ + +static const NCZ_Metadata_Dispatcher NCZ_csl_md3_table = { + ZARRFORMAT3, + NCZ_METADATA_VERSION, /* Version of the dispatch table */ + ZARR_CONSOLIDATED, /* Flags*/ + .list = list_nodes_csl_v3, + .listall = listall_nodes_csl_v3, + .exists = exists_key_csl_v3, + .fetch_json_content = fetch_csl_json_content_v3, + .update_json_content = update_csl_json_content_v3, + .open = open_csl_v3, + .create = create_csl_v3, + .close = close_csl_v3, + .consolidate = consolidate_csl_v3, +}; + +const NCZ_Metadata_Dispatcher *NCZ_csl_metadata_handler3 = &NCZ_csl_md3_table; + +/******************************************************/ + +int +NCZMD3_initialize(void) +{ + return NC_NOERR; +} + +int +NCZMD3_finalize(void) +{ + return NC_NOERR; +} + +/*/////////////////////////////////////////////////// +// .zmetadata dispatch functions +///////////////////////////////////////////////////*/ + +static int +list_nodes_csl_v3(NC_FILE_INFO_T* file, const char* prefix, NClist* matches) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + size_t i,plen; + NCZ_Metadata* zmd = &zfile->metadata_handler; + char* seg1 = NULL; + NClist* segments = nclistnew(); + + assert(zmd->jmeta != NULL && NCJsort(zmd->jmeta)==NCJ_DICT); + if(prefix[0] == '/') prefix++; /* drop leading '/' for search purposes */ + plen = strlen(prefix); + /* Walk the metadata nodes and collect the matches */ + for(i=0;ijmeta);i++) { + const NCjson* jkey = NCJdictkey(zmd->jmeta,i); + const char* skey = NCJstring(jkey); + size_t slen = strlen(skey); + size_t j, found; + /* Check for prefix taking root key into acct. */ + if((plen == 0 && slen > 0) || strncmp(skey,prefix,plen) > 0) { + const char* suffix = NULL; + /* This is a match and is not just the prefix*/ + /* truncate any segments beyond the first */ + suffix = &skey[plen]; + assert(strlen(suffix) > 0); + nclistclearall(segments); + ncz_splitkey(suffix,segments); + if(nclistlength(segments) > 0) { + seg1 = (char*)nclistremove(segments,0); + /* suppress duplicates */ + for(found=0,j=0;jformat_file_info; + size_t i,plen; + NCZ_Metadata* zmd = &zfile->metadata_handler; + NCbytes* key = ncbytesnew(); + + assert(zmd->jmeta != NULL && NCJsort(zmd->jmeta)==NCJ_DICT); + if(prefix[0] == '/') prefix++; /* drop leading '/' for search purposes */ + plen = strlen(prefix); + ncbytescat(key,"/"); + /* Walk the metadata nodes and collect the matches (with leading '/') */ + for(i=0;ijmeta);i++) { + NCjson* jkey = NCJdictkey(zmd->jmeta,i); + const char* skey = NCJstring(jkey); + if(strncmp(skey,prefix,plen) > 0) { + /* This is a match and is not just the prefix*/ + ncbytessetlength(key,1); + ncbytescat(key,prefix); /* add leading '/' */ + nclistpush(matches,strdup(ncbytescontents(key))); + } + } + return stat; +} + +static int +fetch_csl_json_content_v3(NC_FILE_INFO_T* file, NCZMD_MetadataType zobj_t, const char* key, NCjson **jobjp) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + NCjson* jobj = NULL; + NCjson *jkey = NULL; + NCZ_Metadata* zmd = &zfile->metadata_handler; + + NC_UNUSED(zobj_t); + assert(zmd->jmeta != NULL); + /* If asking for /zarr.json, then short circuit to read from metadata handler */ + if(strcmp(key,Z2METAROOT)==0) { + jobj = zmd->jcsl; + goto retval; + } + if(key[0] == '/') key++; /* remove any leading key */ + /* Meta-data is stored a mostly flat format using the whole key (with leading / removed) */ + if ((stat = NCJdictget(zmd->jmeta, key, (NCjson**)&jkey))) goto done; + NCJcheck(NCJclone(jkey, &jobj)); +retval: + if(jobj != NULL) + {if(jobjp) {*jobjp = jobj; jobj = NULL;}} +done: + return stat; +} + +/*/////////////////////////////////////////////////////////////////////////// +// Write to internal JSON pointer and/or directly to storage +///////////////////////////////////////////////////////////////////////////*/ + +static int +update_csl_json_content_v3(NC_FILE_INFO_T* file, NCZMD_MetadataType zobj_t, const char *key, const NCjson *jobj) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + NCjson* jclone = NULL; + NCZ_Metadata* zmd = &zfile->metadata_handler; + + NC_UNUSED(zobj_t); + + assert(zmd->jcsl != NULL && zmd->jmeta != NULL); + /* clone the key value */ + NCJcheck(NCJclone(jobj,&jclone)); + + /* If writing to /zarr.json, then short circuit */ + if(strcmp(key,Z2METAROOT)==0) { + NCJcheck(NCJinsert((NCjson*)zmd->jcsl,key,jclone)); + } else { + if(key[0] == '/') key++; + NCJcheck(NCJinsert((NCjson*)zmd->jmeta,key,jclone)); + } + zmd->dirty = 1; + +#if 0 /* Do we need this? */ + // Allocating representation if doesn't exist + // Updating the internal JSON representation to be synced later + NCjson * jrep = NULL; + if ((stat = NCJdictget(zfile->metadata_handler.jcsl,"metadata", (NCjson**)&jrep)) || jrep == NULL) { + goto done; + } +#endif /*0*/ + +done: + return stat; +} + +static int +consolidate_csl_v3(NC_FILE_INFO_T* file) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + NCZ_Metadata* zmd = &zfile->metadata_handler; + if (zmd->dirty) { + if(zmd->jcsl == NULL) {stat = NC_EZARRMETA; goto done;} + if(zmd->jmeta == NULL) {stat = NC_EZARRMETA; goto done;} + stat = NCZ_uploadjson(zfile->map, Z3METADATA ,zmd->jcsl); + zmd->dirty = 0; + } +done: + return stat; +} + +static int +close_csl_v3(NC_FILE_INFO_T* file) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + NCZ_Metadata* zmd = &zfile->metadata_handler; + NCJreclaim(zmd->jcsl); zmd->jcsl = NULL; + zmd->jmeta = NULL; + return stat; +} + +static int +open_csl_v3(NC_FILE_INFO_T* file) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + NCZ_Metadata* zmd = &zfile->metadata_handler; + + /* Read /zarr.json */ + if(zmd->jcsl == NULL) { + if((stat = NCZ_downloadjson(zfile->map,Z3METADATA,&zmd->jcsl))) goto done; + } + if(zmd->jcsl == NULL || NCJsort(zmd->jcsl) != NCJ_DICT) {stat = NC_EZARRMETA; goto done;} + /* Pull out the "metadata" key and save it */ + NCJcheck(NCJdictget(zmd->jcsl,"metadata",(NCjson**)&zmd->jmeta)); + if(zmd->jmeta == NULL || NCJsort(zmd->jmeta) != NCJ_DICT) {stat = NC_EZARRMETA; goto done;} +done: + return stat; +} + +static int +create_csl_v3(NC_FILE_INFO_T* file) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + NCZ_Metadata* zmd = &zfile->metadata_handler; + + /* Create the JSON skeleton */ + NCJcheck(NCJparse(MINIMIM_CSL_REP3_RAW,0,&zmd->jcsl)); /* Create the metadata skeleton */ + /* Pull out the "metadata" key and save it */ + NCJcheck(NCJdictget(zmd->jcsl,"metadata",(NCjson**)&zmd->jmeta)); +done: + return stat; +} + +static int +exists_key_csl_v3(NC_FILE_INFO_T* file, const char* prefix) +{ + int stat = NC_NOERR; + NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info; + size_t i,plen; + NCZ_Metadata* zmd = &zfile->metadata_handler; + + assert(zmd->jmeta != NULL && NCJsort(zmd->jmeta)==NCJ_DICT); + if(prefix[0] == '/') prefix++; /* drop leading '/' for search purposes */ + plen = strlen(prefix); + /* Walk the metadata nodes and see if there is a prefix match */ + for(i=0;ijmeta);i++) { + NCjson* jkey = NCJdictkey(zmd->jmeta,i); + const char* skey = NCJstring(jkey); + if(strncmp(skey,prefix,plen) == 0) {stat = NC_NOERR; goto done;} + } + stat = NC_ENOOBJECT; +done: + return stat; +} diff --git a/libnczarr/zopen.c b/libnczarr/zopen.c index 2c05a747cc..e1599c9c44 100644 --- a/libnczarr/zopen.c +++ b/libnczarr/zopen.c @@ -74,7 +74,7 @@ ncz_open_file(const char *path, int mode, NClist* controls, int ncid) LOG((3, "%s: path %s mode %d", __func__, path, mode)); assert(path); - ZTRACE(2,"path=%s,mode=%d,ncid=%d,controls=%s)",path,mode,ncid,(controls?nczprint_envv(controls):"null")); + ZTRACE(2,"path=%s,mode=%d,ncid=%d,controls=%s)",path,mode,ncid,(controls?nczprint_envlist(controls):"null")); /* Convert ncid to an NC* structure pointer */ if((stat = NC_check_id(ncid,&nc))) goto exit; diff --git a/libnczarr/zplugins.c b/libnczarr/zplugins.c index 477326df2a..23bdc06294 100644 --- a/libnczarr/zplugins.c +++ b/libnczarr/zplugins.c @@ -82,8 +82,10 @@ int NCZ_plugin_path_finalize(void) { int stat = NC_NOERR; - size_t i; struct NCglobalstate* gs = NC_getglobalstate(); +#ifdef NETCDF_ENABLE_NCZARR_FILTERS + size_t i; +#endif #ifdef NETCDF_ENABLE_NCZARR_FILTERS /* Reclaim all loaded filters */ diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c index dc9b04ad8f..4a46bc1db6 100644 --- a/libnczarr/zsync.c +++ b/libnczarr/zsync.c @@ -70,6 +70,8 @@ ncz_encode_file(NC_FILE_INFO_T* file, int isclose) if((stat = ncz_encode_grp(file, file->root_grp))) goto done; + if((stat = NCZMD_consolidate(file))) goto done; + done: return ZUNTRACE(stat); } @@ -310,9 +312,9 @@ ncz_flush_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var) size64_t* indices = nczodom_indices(chunkodom); /* Convert to key */ if((stat = NCZ_buildchunkpath(zvar->cache,indices,&key))) goto done; - switch (stat = nczmap_exists(map,key)) { + switch (stat = NCZMD_exists(file,key)) { case NC_NOERR: goto next; /* already exists */ - case NC_EEMPTY: break; /* does not exist, create it with fill */ + case NC_ENOOBJECT: break; /* does not exist, create it with fill */ default: goto done; /* some other error */ } /* If we reach here, then chunk does not exist, create it with fill */ @@ -361,6 +363,15 @@ ncz_decode_file(NC_FILE_INFO_T* file) /* Download the root group object and associated attributes */ root = file->root_grp; if((stat = NCZF_download_grp(file, root, &zobj))) goto done; + +#if 0 +Is this code needed? + switch(stat = NCZMD_is_metadata_consolidated(file)) { + case NC_NOERR: break; + case NC_ENOOBJECT: stat = NC_NOERR; break; + default: goto done; + } +#endif /* Decode the group metadata to get only the superblock */ if((stat = NCZF_decode_group(file,root,&zobj,NULL,(NCjson**)&jsuper))) goto done; @@ -434,7 +445,7 @@ ncz_decode_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, struct ZOBJ* zobj) int purezarr = 0; const NCjson* jnczgrp = NULL; - ZTRACE(3,"file=%s parent=%s",file->controller->path,(parent?parent->hdr.name:"NULL")); + ZTRACE(3,"grp=%s",grp->hdr.name); TESTPUREZARR; @@ -474,35 +485,6 @@ ncz_decode_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, struct ZOBJ* zobj) return ZUNTRACE(THROW(stat)); } -#if 0 -/** - * @internal Materialize dimensions into memory - * - * @param file Pointer to file info struct. - * @param parent Pointer to parent grp info struct. - * @param diminfo vector of struct NCZ_DimInfo* - * - * @return ::NC_NOERR No error. - * @author Dennis Heimbigner - */ -static int -ncz_decode_dims(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, NClist* diminfo) -{ - int stat = NC_NOERR; - size_t i; - - ZTRACE(3,"file=%s parent=%s |diminfo|=%u",file->controller->path,parent->hdr.name,nclistlength(diminfo)); - - for(i=0;iname,dim->shape,dim->unlimited,NULL))) goto done; - } - -done: - return ZUNTRACE(THROW(stat)); -} -#endif /*0*/ - /** * @internal Materialize single var into memory; * Take xarray and purezarr into account. @@ -587,7 +569,7 @@ ncz_decode_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, const char* varname nclistfree(filters); ncbytesfree(fqn); NCZ_clear_zobj(&zobj); - return THROW(stat); + return ZUNTRACE(THROW(stat)); } /** @@ -607,7 +589,7 @@ ncz_decode_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, NClist* varnames) int stat = NC_NOERR; size_t i; - ZTRACE(3,"file=%s grp=%s |varnames|=%u",file->controller->path,grp->hdr.name,nclistlength(varnames)); + ZTRACE(3,"parent=%s |varnames|=%u",parent->hdr.name,nclistlength(varnames)); /* Load each var in turn */ for(i = 0; i < nclistlength(varnames); i++) { @@ -655,12 +637,7 @@ ncz_decode_atts(NC_FILE_INFO_T* file, NC_OBJ* container, const NCjson* jatts) /* If we have not read a _FillValue attribute, then go ahead and create it */ if(stat == NC_ENOTATT) { stat = NC_NOERR; /*reset*/ - ainfo.name = NC_FillValue; - ainfo.nctype = var->type_info->hdr.id; - if((stat = NC4_inq_atomic_type(ainfo.nctype, NULL, &ainfo.typelen))) goto done; - ainfo.datalen = 1; - if((stat = NC_copy_data_all(file->controller,ainfo.nctype,var->fill_value,ainfo.datalen,&ainfo.data))) goto done; - if((stat = ncz_makeattr(file,(NC_OBJ*)var,&ainfo,&special))) goto done; + if((stat = NCZ_sync_dual_att(file,(NC_OBJ*)var,NC_FillValue, DA_FILLVALUE, FIXATT))) goto done; } else if(stat != NC_NOERR) goto done; } @@ -781,7 +758,7 @@ get_group_content_pure(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varname { int stat = NC_NOERR; - ZTRACE(3,"zinfo=%s grp=%s |varnames|=%u |subgrps|=%u",zinfo->common.file->controller->path,grp->hdr.name,(unsigned)nclistlength(varnames),(unsigned)nclistlength(subgrps)); + ZTRACE(3,"grp=%s |varnames|=%u |subgrps|=%u",grp->hdr.name,(unsigned)nclistlength(varnames),(unsigned)nclistlength(subgrps)); nclistclear(varnames); if((stat = NCZF_searchobjects(file,grp,varnames,subgrps))) goto done; @@ -790,82 +767,6 @@ get_group_content_pure(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varname return ZUNTRACE(THROW(stat)); } -#if 0 -/** - * @internal Get the metadata for a variable. - * - * @param var Pointer to var info struct. - * - * @return ::NC_NOERR No error. - * @return ::NC_EBADID Bad ncid. - * @return ::NC_ENOMEM Out of memory. - * @return ::NC_EHDFERR HDF5 returned error. - * @return ::NC_EVARMETA Error with var metadata. - * @author Ed Hartnett - */ -static int -ncz_get_var_meta(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var) -{ - int retval = NC_NOERR; - - assert(file && var && var->format_var_info); - LOG((3, "%s: var %s", __func__, var->hdr.name)); - ZTRACE(3,"file=%s var=%s",file->controller->path,var->hdr.name); - - /* Have we already read the var metadata? */ - if (var->meta_read) - goto done; - -#ifdef LOOK - /* Get the current chunk cache settings. */ - if ((access_pid = H5Dget_access_plist(hdf5_var->hdf_datasetid)) < 0) - BAIL(NC_EVARMETA); - - /* Learn about current chunk cache settings. */ - if ((H5Pget_chunk_cache(access_pid, &(var->chunk_cache_nelems), - &(var->chunk_cache_size), &rdcc_w0)) < 0) - BAIL(NC_EHDFERR); - var->chunk_cache_preemption = rdcc_w0; - - /* Get the dataset creation properties. */ - if ((propid = H5Dget_create_plist(hdf5_var->hdf_datasetid)) < 0) - BAIL(NC_EHDFERR); - - /* Get var chunking info. */ - if ((retval = get_chunking_info(propid, var))) - BAIL(retval); - - /* Get filter info for a var. */ - if ((retval = get_filter_info(propid, var))) - BAIL(retval); - - /* Get fill value, if defined. */ - if ((retval = get_fill_info(propid, var))) - BAIL(retval); - - /* Is this a deflated variable with a chunksize greater than the - * current cache size? */ - if ((retval = nc4_adjust_var_cache(var))) - BAIL(retval); - - /* Is there an attribute which means quantization was used? */ - if ((retval = get_quantize_info(var))) - BAIL(retval); - - if (var->coords_read && !var->dimscale) - if ((retval = get_attached_info(var, hdf5_var, var->ndims, hdf5_var->hdf_datasetid))) - goto done;; -#endif - - /* Remember that we have read the metadata for this var. */ - var->meta_read = NC_TRUE; -done: -#if 0 -#endif - return ZUNTRACE(retval); -} -#endif /*0*/ - /** Insert an attribute into a list of attribute, including typing Takes control of javalue but not atype @@ -890,210 +791,6 @@ ncz_insert_attr(NCjson* jatts, NCjson* jtypes, const char* aname, NCjson** javal } /**************************************************/ -#if 0 -/* Convert an attribute "types list to an envv style list */ -static int -jtypesatypes(NCjson* jtypes, NClist* atypes) -{ - int stat = NC_NOERR; - size_t i; - for(i=0;iformat_file_info; - int validate = 0; - NCbytes* prefix = ncbytesnew(); - NClist* queue = nclistnew(); - NClist* nextlevel = nclistnew(); - NCZMAP* map = zinfo->map; - char* path = NULL; - char* segment = NULL; - size_t seglen; - - ZTRACE(3,"file=%s",file->controller->path); - - path = strdup("/"); - nclistpush(queue,path); - path = NULL; - do { - nullfree(path); path = NULL; - /* This should be full path key */ - path = nclistremove(queue,0); /* remove from front of queue */ - /* get list of next level segments (partial keys) */ - assert(nclistlength(nextlevel)==0); - if((stat=nczmap_search(map,path,nextlevel))) {validate = 0; goto done;} - /* For each s in next level, test, convert to full path, and push onto queue */ - while(nclistlength(nextlevel) > 0) { - segment = nclistremove(nextlevel,0); - seglen = nulllen(segment); - if((seglen >= 2 && memcmp(segment,".z",2)==0) || (seglen >= 4 && memcmp(segment,".ncz",4)==0)) { - validate = 1; - goto done; - } - /* Convert to full path */ - ncbytesclear(prefix); - ncbytescat(prefix,path); - if(strlen(path) > 1) ncbytescat(prefix,"/"); - ncbytescat(prefix,segment); - /* push onto queue */ - nclistpush(queue,ncbytesextract(prefix)); - nullfree(segment); segment = NULL; - } - } while(nclistlength(queue) > 0); -done: - if(!validate) stat = NC_ENOTNC; - nullfree(path); - nullfree(segment); - nclistfreeall(queue); - nclistfreeall(nextlevel); - ncbytesfree(prefix); - return ZUNTRACE(THROW(stat)); -} -#endif - -#if 0 -/** -Insert _nczarr_attrs into .zattrs -Take control of jtypes -@param jatts -@param jtypes -*/ -static int -insert_nczarr_attrs(NCjson* jatts, NCjson* jtypes) -{ - NCjson* jdict = NULL; - if(jatts != NULL && jtypes != NULL) { - NCJinsertstring(jtypes,NCZ_ATTRS,NC_JSON_DTYPE); /* type for _nczarr_attrs */ - NCJnew(NCJ_DICT,&jdict); - NCJinsert(jdict,"types",jtypes); - NCJinsert(jatts,NCZ_ATTR,jdict); - jdict = NULL; - } - return NC_NOERR; -} -#endif /*0*/ - -#if 0 -/** -Upload a .zattrs object -Optionally take control of jatts and jtypes -@param file -@param container -@param jattsp -@param jtypesp -*/ -static int -upload_attrs(NC_FILE_INFO_T* file, NC_OBJ* container, NCjson* jatts) -{ - int stat = NC_NOERR; - NCZ_FILE_INFO_T* zinfo = NULL; - NC_VAR_INFO_T* var = NULL; - NC_GRP_INFO_T* grp = NULL; - NCZMAP* map = NULL; - char* fullpath = NULL; - char* key = NULL; - - ZTRACE(3,"file=%s grp=%s",file->controller->path,grp->hdr.name); - - if(jatts == NULL) goto done; - - zinfo = file->format_file_info; - map = zinfo->map; - - if(container->sort == NCVAR) { - var = (NC_VAR_INFO_T*)container; - } else if(container->sort == NCGRP) { - grp = (NC_GRP_INFO_T*)container; - } - - /* Construct container path */ - if(container->sort == NCGRP) - stat = NCZ_grpkey(grp,&fullpath); - else - stat = NCZ_varkey(var,&fullpath); - if(stat) goto done; - - /* write .zattrs*/ - if((stat = nczm_concat(fullpath,ZATTRS,&key))) goto done; - if((stat=NCZ_uploadjson(map,key,jatts))) goto done; - nullfree(key); key = NULL; - -done: - nullfree(key); - nullfree(fullpath); - return ZUNTRACE(THROW(stat)); -} -#endif /*0*/ - -#if 0 -/** -@internal Get contents of a meta object; fail it it does not exist -@param zmap - [in] map -@param key - [in] key of the object -@param jsonp - [out] return parsed json || NULL if not exists -@return NC_NOERR -@return NC_EXXX -@author Dennis Heimbigner -*/ -static int -readarray(NCZMAP* zmap, const char* key, NCjson** jsonp) -{ - int stat = NC_NOERR; - NCjson* json = NULL; - - if((stat = NCZ_downloadjson(zmap,key,&json))) goto done; - if(json != NULL && NCJsort(json) != NCJ_ARRAY) {stat = NC_ENCZARR; goto done;} - if(jsonp) {*jsonp = json; json = NULL;} -done: - NCZ_reclaim_json(json); - return stat; -} -#endif - -#if 0 -static int -downloadzarrobj(NC_FILE_INFO_T* file, struct ZARROBJ* zobj, const char* fullpath, const char* objname) -{ - int stat = NC_NOERR; - char* key = NULL; - NCZMAP* map = ((NCZ_FILE_INFO_T*)file->format_file_info)->map; - - /* Download .zXXX and .zattrs */ - nullfree(zobj->prefix); - zobj->prefix = strdup(fullpath); - NCZ_reclaim_json(zobj->obj); zobj->obj = NULL; - NCZ_reclaim_json(zobj->atts); zobj->obj = NULL; - if((stat = nczm_concat(fullpath,objname,&key))) goto done; - if((stat=NCZ_downloadjson(map,key,&zobj->obj))) goto done; - nullfree(key); key = NULL; - if((stat = nczm_concat(fullpath,ZATTRS,&key))) goto done; - if((stat=NCZ_downloadjson(map,key,&zobj->atts))) goto done; -done: - nullfree(key); - return THROW(stat); -} -#endif /*0*/ /* Convert dimrefs to dimension declarations (possibly creating them) */ static int diff --git a/libnczarr/zutil.c b/libnczarr/zutil.c index 3fdd31019f..9881ca3d94 100644 --- a/libnczarr/zutil.c +++ b/libnczarr/zutil.c @@ -292,54 +292,6 @@ fprintf(stderr,">>>> uploadjson: %s: %s\n",key,content); return ZUNTRACE(stat); } -/** -@internal Get contents of a meta object; fail it it does not exist -@param zmap - [in] map -@param key - [in] key of the object -@param jsonp - [out] return parsed json -@return NC_NOERR -@return NC_ENOOBJECT [object did not exist] -@author Dennis Heimbigner -*/ -int -NCZ_readdict(NCZMAP* zmap, const char* key, NCjson** jsonp) -{ - int stat = NC_NOERR; - NCjson* json = NULL; - - if((stat = NCZ_downloadjson(zmap,key,&json))) goto done; - if(json != NULL) { - if(NCJsort(json) != NCJ_DICT) {stat = NC_ENCZARR; goto done;} - } - if(jsonp) {*jsonp = json; json = NULL;} -done: - NCZ_reclaim_json(json); - return stat; -} - -/** -@internal Get contents of a meta object; fail it it does not exist -@param zmap - [in] map -@param key - [in] key of the object -@param jsonp - [out] return parsed json -@return NC_NOERR -@return NC_ENOOBJECT [object did not exist] -@author Dennis Heimbigner -*/ -int -NCZ_readarray(NCZMAP* zmap, const char* key, NCjson** jsonp) -{ - int stat = NC_NOERR; - NCjson* json = NULL; - - if((stat = NCZ_downloadjson(zmap,key,&json))) goto done; - if(NCJsort(json) != NCJ_ARRAY) {stat = NC_ENCZARR; goto done;} - if(jsonp) {*jsonp = json; json = NULL;} -done: - NCZ_reclaim_json(json); - return stat; -} - /** @internal Given an nc_type, produce the corresponding fill value JSON type @@ -369,7 +321,7 @@ NCZ_isLittleEndian(void) return (u.bytes[0] == 1 ? 1 : 0); } - +#if 0 /* Given a path to a group, return the list of objects that contain another object with the name of the tag. @@ -418,6 +370,7 @@ NCZ_subobjects(NCZMAP* map, const char* prefix, const char* tag, char dimsep, NC ncbytesfree(path); return stat; } +#endif /*0*/ /* Infer the attribute's type based on its value(s).*/ int diff --git a/libnczarr/zvar.c b/libnczarr/zvar.c index 4fc840351d..2e5fbcc9fd 100644 --- a/libnczarr/zvar.c +++ b/libnczarr/zvar.c @@ -317,7 +317,6 @@ var->type_info->rc++; var->ndims, var->hdr.name)); if(var->ndims > 0) { assert(var->chunksizes == NULL); -//if(var->chunksizes != NULL) {free(var->chunksizes); var->chunksizes = NULL;} if (!(var->chunksizes = calloc(var->ndims, sizeof(size_t)))) {stat = NC_ENOMEM; goto done;} if ((stat = ncz_find_default_chunksizes2(grp, var))) goto done; } else { @@ -2229,7 +2228,7 @@ NCZ_write_var_data(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var) size64_t* indices = nczodom_indices(chunkodom); /* Convert to key */ if((stat = NCZ_buildchunkpath(zvar->cache,indices,&key))) goto done; - switch (stat = nczmap_exists(map,key)) { + switch (stat = NCZMD_exists(file,key)) { case NC_NOERR: goto next; /* already exists */ case NC_ENOOBJECT: break; /* does not exist, create it with fill */ default: goto done; /* some other error */ diff --git a/libnczarr/zwalk.c b/libnczarr/zwalk.c index 7ba8473942..1d90a3c191 100644 --- a/libnczarr/zwalk.c +++ b/libnczarr/zwalk.c @@ -459,13 +459,13 @@ transfern(const struct Common* common, unsigned char* slpptr, unsigned char* mem if(common->reading) { if(slpstride == 1) { if((stat=NCZ_copy_data(common->file,common->var,slpptr,avail,common->reading,memptr))) goto done; -/// memcpy(memptr,slpptr,len); /* straight copy */ +//// memcpy(memptr,slpptr,len); /* straight copy */ } else { for(m=0,s=0;sfile,common->var,slpptr+soffset,1,common->reading,memptr+moffset))) goto done; -/// memcpy(memptr+moffset,slpptr+soffset,typesize); +//// memcpy(memptr+moffset,slpptr+soffset,typesize); } } if(common->swap && xtype < NC_STRING) diff --git a/libnetcdf.settings.in b/libnetcdf.settings.in index c925a5ccdb..a7962b9bcf 100644 --- a/libnetcdf.settings.in +++ b/libnetcdf.settings.in @@ -44,6 +44,7 @@ S3 Support: @HAS_S3@ S3 SDK: @WHICH_S3_SDK@ NCZarr Support: @HAS_NCZARR@ +NCZarr Support V3: @HAS_NCZARR_V3@ NCZarr File Support: yes NCZarr Zip Support: @HAS_NCZARR_ZIP@ NCZarr S3 Support: @HAS_S3@ diff --git a/libsrc4/nc4internal.c b/libsrc4/nc4internal.c index 4832077591..3969810499 100644 --- a/libsrc4/nc4internal.c +++ b/libsrc4/nc4internal.c @@ -27,6 +27,7 @@ #include #include #include "ncrc.h" +#include "nclog.h" /** @internal Number of reserved attributes. These attributes are * hidden from the netcdf user, but exist in the implementation @@ -174,31 +175,31 @@ nc4_check_name(const char *name, char *norm_name) /* Check for NULL. */ if (!name) - return NC_EINVAL; + return NCTHROW(NC_EINVAL); /* Make sure this is a valid netcdf name. This should be done * before the name is normalized, because it gives better error * codes for bad utf8 strings. */ if ((retval = NC_check_name(name))) - return retval; + return NCTHROW(retval); /* Normalize the name. */ if ((retval = nc_utf8_normalize((const unsigned char *)name, (unsigned char **)&temp))) - return retval; + return NCTHROW(retval); /* Check length of normalized name. */ if (strlen(temp) > NC_MAX_NAME) { free(temp); - return NC_EMAXNAME; + return NCTHROW(NC_EMAXNAME); } /* Copy the normalized name. */ strcpy(norm_name, temp); free(temp); - return NC_NOERR; + return NCTHROW(NC_NOERR); } /** diff --git a/ncdump/tst_nccopy4.sh b/ncdump/tst_nccopy4.sh index 07521da73e..a59426f251 100755 --- a/ncdump/tst_nccopy4.sh +++ b/ncdump/tst_nccopy4.sh @@ -9,6 +9,7 @@ isolate "testdir_nccopy4" THISDIR=`pwd` cd $ISOPATH +set -x set -e # For a netCDF-4 build, test nccopy on netCDF files in this directory diff --git a/nczarr_test/CMakeLists.txt b/nczarr_test/CMakeLists.txt index 624e7b0385..501a4ac348 100644 --- a/nczarr_test/CMakeLists.txt +++ b/nczarr_test/CMakeLists.txt @@ -23,7 +23,11 @@ remove_definitions(-DDLL_EXPORT) file(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.sh ${CMAKE_CURRENT_SOURCE_DIR}/ref*.cdl ${CMAKE_CURRENT_SOURCE_DIR}/ref*.txt - ${CMAKE_CURRENT_SOURCE_DIR}/ref*.zmap) + ${CMAKE_CURRENT_SOURCE_DIR}/ref*.zmap + ${CMAKE_CURRENT_SOURCE_DIR}/ref*.baseline + ${CMAKE_CURRENT_SOURCE_DIR}/ref*.baseline + ${CMAKE_CURRENT_SOURCE_DIR}/ref*.tgz + ) file(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE) @@ -181,6 +185,7 @@ IF(NETCDF_ENABLE_TESTS) add_sh_test(nczarr_test run_external) add_sh_test(nczarr_test run_quantize) add_sh_test(nczarr_test run_notzarr) + add_sh_test(nczarr_test run_consolidated_zarr) if(USE_HDF5) add_sh_test(nczarr_test run_fillonlyz) diff --git a/nczarr_test/Makefile.am b/nczarr_test/Makefile.am index b26db09ab3..e8f0f34fff 100644 --- a/nczarr_test/Makefile.am +++ b/nczarr_test/Makefile.am @@ -4,13 +4,13 @@ # Put together AM_CPPFLAGS and AM_LDFLAGS. include $(top_srcdir)/lib_flags.am -AM_TESTS_ENVIRONMENT += export NCZARRFORMAT=2; +AM_TESTS_ENVIRONMENT += export NCZARRFORMAT=2;export NCNOZMETADATA=1; TESTS_ENVIRONMENT = TEST_EXTENSIONS = .sh -#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose -#TESTS_ENVIRONMENT += export SETX=1; +SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +TESTS_ENVIRONMENT += export SETX=1; AM_CFLAGS += -I${top_srcdir} AM_CPPFLAGS += -I${top_srcdir} @@ -20,17 +20,12 @@ LDADD = ${top_builddir}/liblib/libnetcdf.la AM_LDFLAGS += ${top_builddir}/liblib/libnetcdf.la AM_CXXFLAGS = -BUILT_SOURCES = EXTRA_DIST = DISTCLEANFILES = TESTS = check_PROGRAMS = -TESTINPUTS = - -#include $(top_srcdir)/nczarr_test/Makefile.files if NETCDF_BUILD_UTILITIES -# Test v2 specific tests # Test fill issues TESTS += run_nczarr_fill.sh @@ -85,6 +80,7 @@ TESTS += run_strings.sh TESTS += run_scalar.sh TESTS += run_nulls.sh TESTS += run_notzarr.sh +TESTS += run_consolidated_zarr.sh if NETCDF_ENABLE_EXTERNAL_SERVER_TESTS TESTS += run_external.sh @@ -207,19 +203,20 @@ endif CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc tmp*.dump tmp*.tmp tmp*.zmap tmp_ngc.c ref_zarr_test_data.cdl tst_*.nc.zip ref_quotes.zip ref_power_901_constants.zip \ tmp_*.nc tmp_*.zarr tst_quantize*.zarr rcmiscdir ref_power_901_constants.file -#results.file results.s3 results.zip +# *** WARNING *** Most of the files in nczarr_test have been duplicated into v3_nczarr_test. +# If any file here is changed, then it may need to be propagated to the other copy. + +EXTRA_DIST += CMakeLists.txt test_nczarr.sh + +EXTRA_DIST += run_chunkcases.sh run_corrupt.sh run_external.sh run_fillonlyz.sh run_filterinstall.sh run_filter_misc.sh run_filter.sh run_filter_vlen.sh run_interop.sh run_jsonconvention.sh run_misc.sh run_mud.sh run_nan.sh run_nccopy5.sh run_nccopyz.sh run_ncgen4.sh run_nczarr_fill.sh run_nczarr_fill.sh run_nczfilter.sh run_notzarr.sh run_nulls.sh run_perf_chunks1.sh run_purezarr.sh run_quantize.sh run_scalar.sh run_specific_filters.sh run_strings.sh run_unknown.sh run_unlim_io.sh run_ut_mapapi.sh run_ut_map.sh run_ut_misc.sh run_consolidated_zarr.sh -EXTRA_DIST += CMakeLists.txt test_nczarr.sh run_corrupt.sh run_external.sh run_interop.sh run_nczarr_fill.sh run_chunkcases.sh run_fillonlyz.sh run_filter.sh run_nccopyz.sh run_nczarr_fill.sh run_nczfilter.sh run_newformat.sh run_perf_chunks1.sh run_unlim_io.sh run_ut_map.sh run_ut_mapapi.sh run_ncgen4.sh run_nan.sh run_quantize.sh run_purezarr.sh run_misc.sh run_jsonconvention.sh run_strings.sh run_scalar.sh run_nulls.sh run_notzarr.sh run_ut_misc.sh run_unknown.sh run_specific_filters.sh run_filter_vlen.sh run_filterinstall.sh test_unlim_vars.c test_h5_endians.c test_put_vars_two_unlim_dim.c test_put_vars_two_unlim_dim.c run_mud.sh run_nccopy5.sh run_filter_misc.sh \ -timer_utils.c timer_utils.h bm_utils.c test_utils.c bm_utils.h bm_timer.h +EXTRA_DIST += bm_utils.c test_h5_endians.c test_put_vars_two_unlim_dim.c test_put_vars_two_unlim_dim.c test_unlim_vars.c test_utils.c timer_utils.c bm_timer.h bm_utils.h timer_utils.h -EXTRA_DIST += ref_noshape.file.zip ref_power_901_constants.cdl ref_power_901_constants_orig.zip ref_avail1.cdl ref_avail1.dmp ref_avail1.txt ref_byte.cdl ref_byte_fill_value_null.cdl ref_byte_fill_value_null.zarr.zip ref_byte.zarr.zip ref_fillonly.cdl ref_groups.h5 ref_misc1.cdl ref_misc1.dmp ref_ndims.cdl ref_ndims.dmp ref_newformatpure.cdl ref_notzarr.tar.gz ref_nulls.cdl ref_oldformat.cdl ref_oldformat.zip ref_perdimspecs.cdl ref_purezarr_base.cdl ref_quotes.cdl ref_quotes_orig.zip ref_rem.cdl ref_rem.dmp ref_scalar.cdl ref_scalar_nczarr.cdl ref_skip.cdl ref_skip.txt ref_skipw.cdl ref_string.cdl ref_t_meta_dim1.cdl ref_t_meta_var1.cdl ref_ut_json_build.txt ref_ut_json_parse.txt ref_ut_mapapi_create.cdl ref_ut_mapapi_data.cdl ref_ut_mapapi_meta.cdl ref_ut_mapapi_search.txt ref_ut_map_create.cdl ref_ut_map_readmeta2.txt ref_ut_map_readmeta.txt ref_ut_map_search.txt ref_ut_map_writedata.cdl ref_ut_map_writemeta2.cdl ref_ut_map_writemeta.cdl ref_ut_proj.txt ref_ut_testmap_create.cdl ref_whole.cdl ref_whole.txt ref_zarr_test_data.cdl.gz ref_bzip2.cdl ref_filtered.cdl ref_purezarr.cdl ref_jsonconvention.cdl ref_groups_regular.cdl ref_zarr_test_data_2d.cdl.gz ref_misc2.cdl ref_string_zarr.baseline ref_string_nczarr.baseline ref_nulls_zarr.baseline ref_nulls_nczarr.baseline ref_any.cdl ref_xarray.cdl ref_jsonconvention.zmap ref_nczarr2zarr.cdl ref_multi.cdl ref_tst_mud4.cdl ref_tst_mud4-bc.cdl ref_tst_mud4_chars.cdl ref_tst_nans.dmp ref_cmip6.zmap +EXTRA_DIST += ref_noshape.file.zip ref_power_901_constants.cdl ref_power_901_constants_orig.zip ref_avail1.cdl ref_avail1.dmp ref_avail1.txt ref_byte.cdl ref_byte_fill_value_null.cdl ref_byte_fill_value_null.zarr.zip ref_byte.zarr.zip ref_fillonly.cdl ref_groups.h5 ref_misc1.cdl ref_misc1.dmp ref_ndims.cdl ref_ndims.dmp ref_notzarr.tar.gz ref_nulls.cdl ref_perdimspecs.cdl ref_purezarr_base.cdl ref_quotes.cdl ref_quotes_orig.zip ref_rem.cdl ref_rem.dmp ref_scalar.cdl ref_scalar_nczarr.cdl ref_skip.cdl ref_skip.txt ref_skipw.cdl ref_string.cdl ref_t_meta_dim1.cdl ref_t_meta_var1.cdl ref_ut_json_build.txt ref_ut_json_parse.txt ref_ut_mapapi_create.cdl ref_ut_mapapi_data.cdl ref_ut_mapapi_meta.cdl ref_ut_mapapi_search.txt ref_ut_map_create.cdl ref_ut_map_readmeta2.txt ref_ut_map_readmeta.txt ref_ut_map_search.txt ref_ut_map_writedata.cdl ref_ut_map_writemeta2.cdl ref_ut_map_writemeta.cdl ref_ut_proj.txt ref_ut_testmap_create.cdl ref_whole.cdl ref_whole.txt ref_zarr_test_data.cdl.gz ref_bzip2.cdl ref_filtered.cdl ref_purezarr.cdl ref_jsonconvention.cdl ref_groups_regular.cdl ref_zarr_test_data_2d.cdl.gz ref_misc2.cdl ref_string_zarr.baseline ref_string_nczarr.baseline ref_nulls_zarr.baseline ref_nulls_nczarr.baseline ref_any.cdl ref_xarray.cdl ref_jsonconvention.zmap ref_nczarr2zarr.cdl ref_multi.cdl ref_tst_mud4.cdl ref_tst_mud4-bc.cdl ref_tst_mud4_chars.cdl ref_tst_nans.dmp ref_cmip6.zmap ref_consolidated_zarr.cdl ref_consolidated_zarr_base.cdl ref_consolidated_zarr_2.18.2_python.zarr.tgz -DISTCLEANFILES += findplugin.sh ${BUILT_SOURCES} +OLDFORMAT = ref_oldformat.cdl ref_oldformat.zip ref_oldformat_only_consolidated.zip run_newformat.sh ref_newformatpure.cdl -all-local: - if ! test -f "${builddir}/test_nczarr.sh" ; then \ - cp ${top_srcdir}/nczarr_test/test_nczarr.sh ${builddir}/test_nczarr.sh ; \ - fi +DISTCLEANFILES += findplugin.sh clean-local: rm -fr tmp*.file diff --git a/nczarr_test/ref_cmip6.zmap b/nczarr_test/ref_cmip6.zmap index 98d96fcf0c..c31ce78b10 100644 --- a/nczarr_test/ref_cmip6.zmap +++ b/nczarr_test/ref_cmip6.zmap @@ -1,4 +1,69 @@ -[0] /.zattrs : (3182) |{"Conventions": "CF-1.7 CMIP-6.2", "activity_id": "CMIP", "branch_method": "standard", "branch_time_in_child": 0.0, "branch_time_in_parent": 0.0, "cmor_version": "3.3.2", "contact": "Kenneth Lo (cdkkl@giss.nasa.gov)", "coordinates": "lat_bnds lon_bnds time_bnds", "creation_date": "2018-11-20T18:45:56Z", "data_specs_version": "01.00.23", "experiment": "all-forcing simulation of the recent past", "experiment_id": "historical", "external_variables": "areacella", "forcing_index": 1, "frequency": "day", "further_info_url": "https://furtherinfo.es-doc.org/CMIP6.NASA-GISS.GISS-E2-1-G.historical.none.r1i1p1f1", "grid": "atmospheric grid: 144x90, ocean grid: 288x180", "grid_label": "gn", "history": "2018-11-20T18:45:56Z ; CMOR rewrote data to be consistent with CMIP6, CF-1.7 CMIP-6.2 and CF standards.", "initialization_index": 1, "institution": "Goddard Institute for Space Studies, New York, NY 10025, USA", "institution_id": "NASA-GISS", "license": "CMIP6 model data produced by NASA Goddard Institute for Space Studies is licensed under a Creative Commons Attribution ShareAlike 4.0 International License (https://creativecommons.org/licenses). Consult https://pcmdi.llnl.gov/CMIP6/TermsOfUse for terms of use governing CMIP6 output, including citation requirements and proper acknowledgment. Further information about this data, including some limitations, can be found via the further_info_url (recorded as a global attribute in this file) and at https:///pcmdi.llnl.gov/. The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law.", "mip_era": "CMIP6", "model_id": "E200f10aF40oQ40", "nominal_resolution": "250 km", "parent_activity_id": "CMIP", "parent_experiment_id": "piControl", "parent_experiment_rip": "r1i1p1", "parent_mip_era": "CMIP6", "parent_source_id": "GISS-E2-1-G", "parent_time_units": "days since 4150-1-1", "parent_variant_label": "r1i1p1f1", "physics_index": 1, "product": "model-output", "realization_index": 1, "realm": "atmos", "references": "https://data.giss.nasa.gov/modelE/cmip6", "source": "GISS-E2.1G (2016): \naerosol: Varies with physics-version (p==1 none, p==3 OMA, p==4 TOMAS, p==5 MATRIX)\natmos: GISS-E2.1 (2.5x2 degree; 144 x 90 longitude/latitude; 40 levels; top level 0.1 hPa)\natmosChem: Varies with physics-version (p==1 Non-interactive, p>1 GPUCCINI)\nland: GISS LSM\nlandIce: none\nocean: GISS Ocean (1.25x1 degree; 288 x 180 longitude/latitude; 32 levels; top grid cell 0-10 m)\nocnBgchem: none\nseaIce: GISS SI", "source_id": "GISS-E2-1-G", "source_type": "AOGCM", "sub_experiment": "none", "sub_experiment_id": "none", "table_id": "day", "table_info": "Creation Date:(21 March 2018) MD5:f76dbc1e8bf6b7e4aee30573a09e5454", "title": "GISS-E2-1-G output prepared for CMIP6", "tracking_id": "hdl:21.14100/364b8aea-de19-4e29-ac36-0b26406e9a29", "variable_id": "tasmin", "variant_label": "r1i1p1f1", "status": "2019-10-25;created;by nhn2@columbia.edu"}| +[0] /.zattrs : (3182) |{"Conventions": "CF-1.7 CMIP-6.2", +"activity_id": "CMIP", +"branch_method": "standard", +"branch_time_in_child": 0.0, +"branch_time_in_parent": 0.0, +"cmor_version": "3.3.2", +"contact": "Kenneth Lo (cdkkl@giss.nasa.gov)", +"coordinates": "lat_bnds lon_bnds time_bnds", +"creation_date": "2018-11-20T18:45:56Z", +"data_specs_version": "01.00.23", +"experiment": "all-forcing simulation of the recent past", +"experiment_id": "historical", +"external_variables": "areacella", +"forcing_index": 1, +"frequency": "day", +"further_info_url": "https://furtherinfo.es-doc.org/CMIP6.NASA-GISS.GISS-E2-1-G.historical.none.r1i1p1f1", +"grid": "atmospheric grid: 144x90, +ocean grid: 288x180", +"grid_label": "gn", +"history": "2018-11-20T18:45:56Z ; CMOR rewrote data to be consistent with CMIP6, +CF-1.7 CMIP-6.2 and CF standards.", +"initialization_index": 1, +"institution": "Goddard Institute for Space Studies, +New York, +NY 10025, +USA", +"institution_id": "NASA-GISS", +"license": "CMIP6 model data produced by NASA Goddard Institute for Space Studies is licensed under a Creative Commons Attribution ShareAlike 4.0 International License (https://creativecommons.org/licenses). Consult https://pcmdi.llnl.gov/CMIP6/TermsOfUse for terms of use governing CMIP6 output, +including citation requirements and proper acknowledgment. Further information about this data, +including some limitations, +can be found via the further_info_url (recorded as a global attribute in this file) and at https:///pcmdi.llnl.gov/. The data producers and data providers make no warranty, +either express or implied, +including, +but not limited to, +warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law.", +"mip_era": "CMIP6", +"model_id": "E200f10aF40oQ40", +"nominal_resolution": "250 km", +"parent_activity_id": "CMIP", +"parent_experiment_id": "piControl", +"parent_experiment_rip": "r1i1p1", +"parent_mip_era": "CMIP6", +"parent_source_id": "GISS-E2-1-G", +"parent_time_units": "days since 4150-1-1", +"parent_variant_label": "r1i1p1f1", +"physics_index": 1, +"product": "model-output", +"realization_index": 1, +"realm": "atmos", +"references": "https://data.giss.nasa.gov/modelE/cmip6", +"source": "GISS-E2.1G (2016): \naerosol: Varies with physics-version (p==1 none, +p==3 OMA, +p==4 TOMAS, +p==5 MATRIX)\natmos: GISS-E2.1 (2.5x2 degree; 144 x 90 longitude/latitude; 40 levels; top level 0.1 hPa)\natmosChem: Varies with physics-version (p==1 Non-interactive, +p>1 GPUCCINI)\nland: GISS LSM\nlandIce: none\nocean: GISS Ocean (1.25x1 degree; 288 x 180 longitude/latitude; 32 levels; top grid cell 0-10 m)\nocnBgchem: none\nseaIce: GISS SI", +"source_id": "GISS-E2-1-G", +"source_type": "AOGCM", +"sub_experiment": "none", +"sub_experiment_id": "none", +"table_id": "day", +"table_info": "Creation Date:(21 March 2018) MD5:f76dbc1e8bf6b7e4aee30573a09e5454", +"title": "GISS-E2-1-G output prepared for CMIP6", +"tracking_id": "hdl:21.14100/364b8aea-de19-4e29-ac36-0b26406e9a29", +"variable_id": "tasmin", +"variant_label": "r1i1p1f1", +"status": "2019-10-25;created;by nhn2@columbia.edu"}| [1] /.zgroup : (24) |{ "zarr_format": 2 }| @@ -21,13 +86,25 @@ "forcing_index": 1, "frequency": "day", "further_info_url": "https://furtherinfo.es-doc.org/CMIP6.NASA-GISS.GISS-E2-1-G.historical.none.r1i1p1f1", - "grid": "atmospheric grid: 144x90, ocean grid: 288x180", + "grid": "atmospheric grid: 144x90, +ocean grid: 288x180", "grid_label": "gn", - "history": "2018-11-20T18:45:56Z ; CMOR rewrote data to be consistent with CMIP6, CF-1.7 CMIP-6.2 and CF standards.", + "history": "2018-11-20T18:45:56Z ; CMOR rewrote data to be consistent with CMIP6, +CF-1.7 CMIP-6.2 and CF standards.", "initialization_index": 1, - "institution": "Goddard Institute for Space Studies, New York, NY 10025, USA", + "institution": "Goddard Institute for Space Studies, +New York, +NY 10025, +USA", "institution_id": "NASA-GISS", - "license": "CMIP6 model data produced by NASA Goddard Institute for Space Studies is licensed under a Creative Commons Attribution ShareAlike 4.0 International License (https://creativecommons.org/licenses). Consult https://pcmdi.llnl.gov/CMIP6/TermsOfUse for terms of use governing CMIP6 output, including citation requirements and proper acknowledgment. Further information about this data, including some limitations, can be found via the further_info_url (recorded as a global attribute in this file) and at https:///pcmdi.llnl.gov/. The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law.", + "license": "CMIP6 model data produced by NASA Goddard Institute for Space Studies is licensed under a Creative Commons Attribution ShareAlike 4.0 International License (https://creativecommons.org/licenses). Consult https://pcmdi.llnl.gov/CMIP6/TermsOfUse for terms of use governing CMIP6 output, +including citation requirements and proper acknowledgment. Further information about this data, +including some limitations, +can be found via the further_info_url (recorded as a global attribute in this file) and at https:///pcmdi.llnl.gov/. The data producers and data providers make no warranty, +either express or implied, +including, +but not limited to, +warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law.", "mip_era": "CMIP6", "model_id": "E200f10aF40oQ40", "nominal_resolution": "250 km", @@ -43,7 +120,11 @@ "realization_index": 1, "realm": "atmos", "references": "https://data.giss.nasa.gov/modelE/cmip6", - "source": "GISS-E2.1G (2016): \naerosol: Varies with physics-version (p==1 none, p==3 OMA, p==4 TOMAS, p==5 MATRIX)\natmos: GISS-E2.1 (2.5x2 degree; 144 x 90 longitude/latitude; 40 levels; top level 0.1 hPa)\natmosChem: Varies with physics-version (p==1 Non-interactive, p>1 GPUCCINI)\nland: GISS LSM\nlandIce: none\nocean: GISS Ocean (1.25x1 degree; 288 x 180 longitude/latitude; 32 levels; top grid cell 0-10 m)\nocnBgchem: none\nseaIce: GISS SI", + "source": "GISS-E2.1G (2016): \naerosol: Varies with physics-version (p==1 none, +p==3 OMA, +p==4 TOMAS, +p==5 MATRIX)\natmos: GISS-E2.1 (2.5x2 degree; 144 x 90 longitude/latitude; 40 levels; top level 0.1 hPa)\natmosChem: Varies with physics-version (p==1 Non-interactive, +p>1 GPUCCINI)\nland: GISS LSM\nlandIce: none\nocean: GISS Ocean (1.25x1 degree; 288 x 180 longitude/latitude; 32 levels; top grid cell 0-10 m)\nocnBgchem: none\nseaIce: GISS SI", "source_id": "GISS-E2-1-G", "source_type": "AOGCM", "sub_experiment": "none", @@ -227,7 +308,8 @@ ], "cell_measures": "area: areacella", "cell_methods": "area: mean time: minimum", - "comment": "minimum near-surface (usually, 2 meter) air temperature (add cell_method attribute 'time: min')", + "comment": "minimum near-surface (usually, +2 meter) air temperature (add cell_method attribute 'time: min')", "coordinates": "height", "history": "2018-11-20T18:45:56Z altered by CMOR: Treated scalar dimension: 'height'. 2018-11-20T18:45:56Z altered by CMOR: replaced missing value flag (-1e+30) with standard missing value (1e+20).", "long_name": "Daily Minimum Near-Surface Air Temperature", @@ -469,7 +551,8 @@ ], "cell_measures": "area: areacella", "cell_methods": "area: mean time: minimum", - "comment": "minimum near-surface (usually, 2 meter) air temperature (add cell_method attribute 'time: min')", + "comment": "minimum near-surface (usually, +2 meter) air temperature (add cell_method attribute 'time: min')", "coordinates": "height", "history": "2018-11-20T18:45:56Z altered by CMOR: Treated scalar dimension: 'height'. 2018-11-20T18:45:56Z altered by CMOR: replaced missing value flag (-1e+30) with standard missing value (1e+20).", "long_name": "Daily Minimum Near-Surface Air Temperature", diff --git a/nczarr_test/ref_consolidated_zarr.cdl b/nczarr_test/ref_consolidated_zarr.cdl new file mode 100644 index 0000000000..f37bd3946f --- /dev/null +++ b/nczarr_test/ref_consolidated_zarr.cdl @@ -0,0 +1,12 @@ +netcdf tmp_consolidated_zarr { +dimensions: + _Anonymous_Dim_2 = 2 ; + _Anonymous_Dim_5 = 5 ; +variables: + int i(_Anonymous_Dim_2, _Anonymous_Dim_5) ; +data: + + i = + _, _, _, _, _, + _, _, _, _, _ ; +} diff --git a/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr.tgz b/nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr.tgz new file mode 100644 index 0000000000000000000000000000000000000000..1afaa7e44e296a85fd5b6d748a49e6aeb594288e GIT binary patch literal 1144 zcmV-;1c&<{iwFqW$aQA`19D|%Ut@1>b8l>EWMOn=WM6t=a&liXE-^SRGGB0cbZBpG zE_z{daxQdXascg`X>*%K7{`%gCysNr&C#Yu2yJsJG7AFSByG~>?3Frc&dFpnSV9pA z5*k2GC6C|aRlaUMLA%feBR~?1o%PsyejetrcG+Fv+5fWpu#VBJ)$NXJTV_M|j7Dvs zJ5Eiiprwjb>-N2t-KqFVi(GL;jHW5RMVcbUZ3u^|nk=g#s;Y<+k*Xn$3xjdpA7d6V*YC{AAg-$jMH0KNY;q^jV57SQ#5ebjRFFH(YkUzW+qcDw5su4_BEz*=;cuwvQuH?BD_aI&~jTBuvbs$tZl+bLe#Bu)d4*x!ekJtG4cbO~A?3tb0yKjE~!hwT_4j(!C$)}$k zJAUHi=U<%q@~f}EInAFDzWwg|v**rVxOl02xw0rCNvH3YIf4O<< z_MN-;e*NwDKmNS`*Ybmhe?NNsR&-A{UZtV&jOm1W0qqx=!^+>P=Xaovr~ z?D%e!pB?9ovXk-NkoCkpWW;}?ysS8IJTny!jxtkm;n;O01=@g~pE9{me@IB_Y%90*m+|26T3j{0b&mldx+S>#V#AzA}GH96kGgph1P$P z4C_B8!1`~xq0oP^%^zHz{U2IN{|e0inGo&&$n`-Nh6Mls00000000000002kN&E-B KOa;dPPyhh2p<<2z literal 0 HcmV?d00001 diff --git a/nczarr_test/ref_consolidated_zarr_base.cdl b/nczarr_test/ref_consolidated_zarr_base.cdl new file mode 100644 index 0000000000..34c0a525d8 --- /dev/null +++ b/nczarr_test/ref_consolidated_zarr_base.cdl @@ -0,0 +1,7 @@ +netcdf ref_purezarr { +dimensions: + x = 2; + y = 5; +variables: + int i(x,y) ; +} diff --git a/nczarr_test/ref_filtered.cdl b/nczarr_test/ref_filtered.cdl index 6fbb062637..43609129c0 100644 --- a/nczarr_test/ref_filtered.cdl +++ b/nczarr_test/ref_filtered.cdl @@ -11,7 +11,6 @@ dimensions: group: g { variables: float var(dim0, dim1, dim2, dim3) ; - var:_FillValue = 9.96921e+36f ; var:_Storage = "chunked" ; var:_ChunkSizes = 4, 4, 4, 4 ; var:_Filter = "307,9" ; diff --git a/nczarr_test/ref_nulls_nczarr.baseline b/nczarr_test/ref_nulls_nczarr.baseline index 0030498209..263561c0e0 100644 --- a/nczarr_test/ref_nulls_nczarr.baseline +++ b/nczarr_test/ref_nulls_nczarr.baseline @@ -6,7 +6,6 @@ variables: test:space_sng = " " ; test:zero_sng = "0" ; test:nul0 = 0b ; - test:_FillValue = -2147483647 ; data: test = 1 ; diff --git a/nczarr_test/ref_nulls_zarr.baseline b/nczarr_test/ref_nulls_zarr.baseline index cc4db43153..e3e830d211 100644 --- a/nczarr_test/ref_nulls_zarr.baseline +++ b/nczarr_test/ref_nulls_zarr.baseline @@ -8,7 +8,6 @@ variables: test:space_sng = " " ; test:zero_sng = "0" ; test:nul0 = 0 ; - test:_FillValue = -2147483647 ; data: test = 1 ; diff --git a/nczarr_test/ref_oldformat_only_consolidated.zip b/nczarr_test/ref_oldformat_only_consolidated.zip new file mode 100644 index 0000000000000000000000000000000000000000..29550a5481b6d116448ad075445736eee9492d52 GIT binary patch literal 1103 zcmWIWW@Zs#U}E54P-^Un^w}7@T%3V{K>>(Wfw(9&Ej~XdB`v=wH?br>KQE^;J~=ur$&X{Ee061FU=ZhH zV9+AMWWB1~)RIJ?Es1ZZ#b@6(5IAuEvu4x$*yE?HZcR-SxW#J~VJ0OZohos5-;QOQ zQdKM`l>B@*b*fsa&dK1(eTO3EHNEru{=NKK?Rx$CI6fxzh=liR>eu^S$=q9h>jL-CJ7#%5A%i2_x1 zulH>=Samn^-jue8DF=F$iXGlBU-9(FvopW{YW|s=bt)s&>eVN&wvXKxe&i)Dc)7mJ zFKk*;&K#*#ibpE~pI0XLwI(dA{=1U>o$AatzIS)7yVb`XrBf93@rBXp#lb5pUsJFkDe~dJ(j=m(~8ztK|Lb8Z?c~~*!rP7!2ja6W1rvr5dER2dB6RA z-n(6yRi`B9-jJ?j5WN()?5R`Bw3bBKt=rDs6S(^KZPE*ly#~(u6YhLomFxI^@tIGv zbx-Ps9}9cy7k4Cqji*b(imhrvZAy)~tH`d#1FWeQp}L{XES=9;UjAFXJksH?L}d2M zO-p3H&3yANY;7HnZLz26*{9i$tS8EQJXY|0Sa;^1agE%WN6DPJ3VrL>AGcq0di8mE zwacsZC$BW*_&f2MeW`UNL~0QW%hb!&(qxvlYI8MC{{`Z zU5WlGhn5(>N&Br}2PQ%XM&c4u&oh9 z5tWP3jYQ8d5Ca()wlzi(VI(5^pqq>CcZ9jmf!ULgQxPE%;LXYg(#s5jz;HBU2JryP C^t*ci literal 0 HcmV?d00001 diff --git a/nczarr_test/ref_purezarr.cdl b/nczarr_test/ref_purezarr.cdl index 056328eebb..d0cfb4f0cf 100644 --- a/nczarr_test/ref_purezarr.cdl +++ b/nczarr_test/ref_purezarr.cdl @@ -4,7 +4,6 @@ dimensions: _Anonymous_Dim_5 = 5 ; variables: int i(_Anonymous_Dim_2, _Anonymous_Dim_5) ; - i:_FillValue = -2147483647 ; data: i = diff --git a/nczarr_test/ref_string_nczarr.baseline b/nczarr_test/ref_string_nczarr.baseline index 9091ea045a..4af4a481a9 100644 --- a/nczarr_test/ref_string_nczarr.baseline +++ b/nczarr_test/ref_string_nczarr.baseline @@ -3,7 +3,6 @@ dimensions: d2 = 2 ; variables: char c(d2) ; - c:_FillValue = "" ; string v(d2) ; string v:_FillValue = "" ; string truncated ; diff --git a/nczarr_test/ref_string_zarr.baseline b/nczarr_test/ref_string_zarr.baseline index 646664e0ba..737ba8cb80 100644 --- a/nczarr_test/ref_string_zarr.baseline +++ b/nczarr_test/ref_string_zarr.baseline @@ -4,7 +4,6 @@ dimensions: _scalar_ = 1 ; variables: char c(d2) ; - c:_FillValue = "" ; string v(d2) ; string v:_FillValue = "" ; string truncated(_scalar_) ; diff --git a/nczarr_test/ref_xarray.cdl b/nczarr_test/ref_xarray.cdl index 9689897b4d..69ef9f8e8b 100644 --- a/nczarr_test/ref_xarray.cdl +++ b/nczarr_test/ref_xarray.cdl @@ -4,7 +4,6 @@ dimensions: y = 5 ; variables: int i(x, y) ; - i:_FillValue = -2147483647 ; data: i = diff --git a/nczarr_test/run_cachetest.sh b/nczarr_test/run_cachetest.sh index ce181bee2a..6a554e19dd 100755 --- a/nczarr_test/run_cachetest.sh +++ b/nczarr_test/run_cachetest.sh @@ -3,7 +3,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. "${builddir}/test_nczarr.sh" +. "${srcdir}/test_nczarr.sh" set -e diff --git a/nczarr_test/run_chunkcases.sh b/nczarr_test/run_chunkcases.sh index 7a8228e681..d8f60cd62f 100755 --- a/nczarr_test/run_chunkcases.sh +++ b/nczarr_test/run_chunkcases.sh @@ -13,7 +13,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. ${builddir}/test_nczarr.sh +. ${srcdir}/test_nczarr.sh set -e diff --git a/nczarr_test/run_consolidated_zarr.sh b/nczarr_test/run_consolidated_zarr.sh new file mode 100755 index 0000000000..d11c71c157 --- /dev/null +++ b/nczarr_test/run_consolidated_zarr.sh @@ -0,0 +1,76 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "$srcdir/test_nczarr.sh" + +set -e + +s3isolate "testdir_consolidated_zarr" +THISDIR=`pwd` +cd $ISOPATH + +# Force use of consolidated metadata +unset NCNOZMETADATA + +# Locate the plugin path +if ! findplugin h5blosc ; then exit 0; fi +if ! avail blosc; then exit 0; fi + +# This shell script tests support for: +# 1. consolidated zarr (noxarray) read-only +# 2. xarray read + +testcase() { + zext=$1 + echo "*** Test: consolidated zarr write then read; format=$zext" + fileargs tmp_consolidated_zarr "mode=zarr,noxarray,zmetadata,$zext" + deletemap $zext $file + ${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_consolidated_zarr_base.cdl + ${NCDUMP} $fileurl > tmp_consolidated_zarr_${zext}.cdl + diff -b ${srcdir}/ref_consolidated_zarr.cdl tmp_consolidated_zarr_${zext}.cdl + + echo "*** Test: xarray zarr write then read; format=$zext" + fileargs tmp_xarray "mode=zarr,$zext" + #deletemap $zext $file + ${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_consolidated_zarr_base.cdl + ${NCDUMP} $fileurl > tmp_xarray_${zext}.cdl + diff -b ${srcdir}/ref_xarray.cdl tmp_xarray_${zext}.cdl + + echo "*** Test: consolidated zarr reading nczarr; format=$zext" + fileargs tmp_nczarr "mode=nczarr,noxarray,zmetadata,$zext" + deletemap $zext $file + ${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_whole.cdl + fileargs tmp_nczarr "mode=zarr,$zext" + ${NCDUMP} -n nczarr2zarr $fileurl > tmp_nczarr_${zext}.cdl + diff -b ${srcdir}/ref_nczarr2zarr.cdl tmp_nczarr_${zext}.cdl +} + +testcase_csl_vs_no(){ + zext=$1 + echo "*** Test: consolidated pure python zarr read; format=$zext" + deletemap $zext $file + tar -zxf $srcdir/ref_consolidated_zarr_2.18.2_python.zarr.tgz + cp -r ref_consolidated_zarr_2.18.2_python.zarr ref_consolidated_zarr_2.18.2_python.zarr.$zext + cp -r ref_consolidated_zarr_2.18.2_python.zarr ref_zarr_2.18.2_python.zarr.$zext + rm -f ref_zarr_2.18.2_python.zarr.$zext/.zmetadata + fileargs ref_consolidated_zarr_2.18.2_python.zarr "mode=zarr,${zext}" +echo ">>> (1)" +P=`pwd` +ls -lrtd *.file +${ZMD} -h $fileurl +export NCTRACING=10 + ${NCDUMP} -n same_name $fileurl |tee tmp_consolidated_python_zarr_${zext}.cdl + fileargs ref_zarr_2.18.2_python.zarr "mode=zarr,$zext" +echo ">>> (2)" +ls -lrtd *.zarr + ${NCDUMP} -n same_name $fileurl > tmp_python_zarr_${zext}.cdl + rm -f diff.txt + diff -b tmp_consolidated_python_zarr_${zext}.cdl tmp_python_zarr_${zext}.cdl > diff.txt +} + +testcase file +testcase_csl_vs_no file +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi +if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi diff --git a/nczarr_test/run_corrupt.sh b/nczarr_test/run_corrupt.sh index 95df15cb64..1943f600d6 100755 --- a/nczarr_test/run_corrupt.sh +++ b/nczarr_test/run_corrupt.sh @@ -6,7 +6,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. "${builddir}/test_nczarr.sh" +. "${srcdir}/test_nczarr.sh" set -e diff --git a/nczarr_test/run_external.sh b/nczarr_test/run_external.sh index d74c90dace..62a9920ef5 100755 --- a/nczarr_test/run_external.sh +++ b/nczarr_test/run_external.sh @@ -5,7 +5,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi set -e -. "${builddir}/test_nczarr.sh" +. "${srcdir}/test_nczarr.sh" s3isolate "testdir_external" THISDIR=`pwd` diff --git a/nczarr_test/run_fillonlyz.sh b/nczarr_test/run_fillonlyz.sh index 60df1bab1d..e9ab3e8b5d 100755 --- a/nczarr_test/run_fillonlyz.sh +++ b/nczarr_test/run_fillonlyz.sh @@ -3,7 +3,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. "${builddir}/test_nczarr.sh" +. "${srcdir}/test_nczarr.sh" # This shell script tests bug reported in github issue # https://github.com/Unidata/netcdf-c/issues/1826 diff --git a/nczarr_test/run_filter.sh b/nczarr_test/run_filter.sh index 5a4ca37d98..20eaba526d 100755 --- a/nczarr_test/run_filter.sh +++ b/nczarr_test/run_filter.sh @@ -3,7 +3,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. "${builddir}/test_nczarr.sh" +. "${srcdir}/test_nczarr.sh" set -e diff --git a/nczarr_test/run_filter_misc.sh b/nczarr_test/run_filter_misc.sh index 1ef1f14a70..4f094c4a39 100755 --- a/nczarr_test/run_filter_misc.sh +++ b/nczarr_test/run_filter_misc.sh @@ -7,7 +7,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh if test "x$TESTNCZARR" = x1; then -. ${builddir}/test_nczarr.sh +. ${srcdir}/test_nczarr.sh fi set -e diff --git a/nczarr_test/run_filter_vlen.sh b/nczarr_test/run_filter_vlen.sh index 0f8b276a5e..6c38ee1832 100755 --- a/nczarr_test/run_filter_vlen.sh +++ b/nczarr_test/run_filter_vlen.sh @@ -9,7 +9,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh if test "x$TESTNCZARR" = x1; then -. ${builddir}/test_nczarr.sh +. ${srcdir}/test_nczarr.sh fi set -e diff --git a/nczarr_test/run_filterinstall.sh b/nczarr_test/run_filterinstall.sh index 187ad69da7..f5cc612784 100755 --- a/nczarr_test/run_filterinstall.sh +++ b/nczarr_test/run_filterinstall.sh @@ -11,7 +11,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh if test "x$TESTNCZARR" = x1; then -. ${builddir}/test_nczarr.sh +. ${srcdir}/test_nczarr.sh fi set -e diff --git a/nczarr_test/run_interop.sh b/nczarr_test/run_interop.sh index 4bee6ae5c8..6cdbf8c5d9 100755 --- a/nczarr_test/run_interop.sh +++ b/nczarr_test/run_interop.sh @@ -3,7 +3,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. "${builddir}/test_nczarr.sh" +. "${srcdir}/test_nczarr.sh" set -e diff --git a/nczarr_test/run_it_chunks1.sh b/nczarr_test/run_it_chunks1.sh index 535dd0ce25..e02d8b132b 100755 --- a/nczarr_test/run_it_chunks1.sh +++ b/nczarr_test/run_it_chunks1.sh @@ -6,7 +6,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. "${builddir}/test_nczarr.sh" +. "${srcdir}/test_nczarr.sh" set -e diff --git a/nczarr_test/run_jsonconvention.sh b/nczarr_test/run_jsonconvention.sh index aea0c47d74..5a906c07c6 100755 --- a/nczarr_test/run_jsonconvention.sh +++ b/nczarr_test/run_jsonconvention.sh @@ -3,7 +3,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. "${builddir}/test_nczarr.sh" +. "${srcdir}/test_nczarr.sh" set -e diff --git a/nczarr_test/run_misc.sh b/nczarr_test/run_misc.sh index 52d5584fb7..feb4c26f5d 100755 --- a/nczarr_test/run_misc.sh +++ b/nczarr_test/run_misc.sh @@ -3,7 +3,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. "${builddir}/test_nczarr.sh" +. "${srcdir}/test_nczarr.sh" set -e diff --git a/nczarr_test/run_mud.sh b/nczarr_test/run_mud.sh index 0c9152b561..320492b767 100755 --- a/nczarr_test/run_mud.sh +++ b/nczarr_test/run_mud.sh @@ -12,7 +12,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi set -e if test "x$TESTNCZARR" = x1 ; then -. ${builddir}/test_nczarr.sh +. ${srcdir}/test_nczarr.sh s3isolate "testdir_mud4" else isolate "testdir_mud4" diff --git a/nczarr_test/run_nan.sh b/nczarr_test/run_nan.sh index c0fa825457..c3bc5248e4 100755 --- a/nczarr_test/run_nan.sh +++ b/nczarr_test/run_nan.sh @@ -8,7 +8,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. "${builddir}/test_nczarr.sh" +. "${srcdir}/test_nczarr.sh" # Isolate both test and S3 s3isolate "testdir_nan" @@ -16,6 +16,7 @@ THISDIR=`pwd` cd $ISOPATH set -x +export SETX=1 set -e # Location constants @@ -43,9 +44,11 @@ for t in tst_nans ; do rm -fr ${t}.$zext rm -f tmp_${t}.dmp fileargs $t +export NCTRACING=10 ${NCGEN} -4 -lb -o ${fileurl} ${cdl}/${ref}.cdl -${ZMD} -t float ${fileurl} ${NCDUMP} ${headflag} ${specflag} -n ${ref} ${fileurl} > tmp_${t}.dmp +unset NCTRACING +${ZMD} -t float ${fileurl} # compare against expected diff -b -w ${expected}/${ref}.dmp ./tmp_${t}.dmp echo "*** SUCCEED: ${t}" diff --git a/nczarr_test/run_nccopy5.sh b/nczarr_test/run_nccopy5.sh index be1bdcd4e0..2d89abb3d0 100755 --- a/nczarr_test/run_nccopy5.sh +++ b/nczarr_test/run_nccopy5.sh @@ -9,7 +9,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi set -e if test "x$TESTNCZARR" = x1 ; then -. ${builddir}/test_nczarr.sh +. ${srcdir}/test_nczarr.sh s3isolate "testdir_nccopy5" else isolate testdir_ncccopy5 diff --git a/nczarr_test/run_ncgen4.sh b/nczarr_test/run_ncgen4.sh index 83fc0f68a0..a19d256204 100755 --- a/nczarr_test/run_ncgen4.sh +++ b/nczarr_test/run_ncgen4.sh @@ -8,7 +8,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. "${builddir}/test_nczarr.sh" +. "${srcdir}/test_nczarr.sh" # Isolate both test and S3 s3isolate "testdir_ncgen4" diff --git a/nczarr_test/run_nczarr_fill.sh b/nczarr_test/run_nczarr_fill.sh index 85b434a68b..9856c611c0 100755 --- a/nczarr_test/run_nczarr_fill.sh +++ b/nczarr_test/run_nczarr_fill.sh @@ -3,7 +3,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. "${builddir}/test_nczarr.sh" +. "${srcdir}/test_nczarr.sh" s3isolate "testdir_nczarr_fill" THISDIR=`pwd` diff --git a/nczarr_test/run_nczfilter.sh b/nczarr_test/run_nczfilter.sh index e7f3985818..2ce44ccd11 100755 --- a/nczarr_test/run_nczfilter.sh +++ b/nczarr_test/run_nczfilter.sh @@ -3,7 +3,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. "${builddir}/test_nczarr.sh" +. "${srcdir}/test_nczarr.sh" # This shell script runs test_nczfilter.c diff --git a/nczarr_test/run_newformat.sh b/nczarr_test/run_newformat.sh index fc894b848f..bc5691476d 100755 --- a/nczarr_test/run_newformat.sh +++ b/nczarr_test/run_newformat.sh @@ -3,7 +3,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. "${builddir}/test_nczarr.sh" +. "${srcdir}/test_nczarr.sh" set -e diff --git a/nczarr_test/run_notzarr.sh b/nczarr_test/run_notzarr.sh index eafed26d02..7bc0415073 100755 --- a/nczarr_test/run_notzarr.sh +++ b/nczarr_test/run_notzarr.sh @@ -3,7 +3,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. "${builddir}/test_nczarr.sh" +. "${srcdir}/test_nczarr.sh" set -e @@ -27,7 +27,7 @@ testfailed() { } # Make test sets -cp ${srcdir}/ref_notzarr.tar.gz . +cp ${abs_top_srcdir}/nczarr_test//ref_notzarr.tar.gz ${ISOPATH} gunzip ref_notzarr.tar.gz tar -xf ref_notzarr.tar if test "x$FEATURE_S3TESTS" = xyes ; then @@ -59,6 +59,6 @@ if test 1 = 0 ; then testfailed "$RET" fi echo "Test non-zarr S3 file" -RET=`${execdir}/test_notzarr "https://$URL/${S3ISOPATH}/notzarr.s3#mode=zarr,s3"` +RET=`${execdir}/test_notzarr "https://$URL/${ISOPATH}/notzarr.s3#mode=zarr,s3"` testfailed "$RET" fi diff --git a/nczarr_test/run_nulls.sh b/nczarr_test/run_nulls.sh index 040c668c18..0df59554dc 100755 --- a/nczarr_test/run_nulls.sh +++ b/nczarr_test/run_nulls.sh @@ -3,7 +3,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. "${builddir}/test_nczarr.sh" +. "${srcdir}/test_nczarr.sh" set -e @@ -32,9 +32,9 @@ deletemap $zext $nczarrfile # Create alternate ref files echo "*** create pure zarr file" -${NCGEN} -4 -b -o "$zarrurl" $srcdir/ref_nulls.cdl +${NCGEN} -4 -b -o "$zarrurl" $srcdir/../nczarr_test/ref_nulls.cdl echo "*** create nczarr file" -${NCGEN} -4 -b -o "$nczarrurl" $srcdir/ref_nulls.cdl +${NCGEN} -4 -b -o "$nczarrurl" $srcdir/../nczarr_test/ref_nulls.cdl echo "*** read purezarr" ${NCDUMP} -n ref_nulls $zarrurl > tmp_nulls_zarr_${zext}.cdl diff --git a/nczarr_test/run_perf_chunks1.sh b/nczarr_test/run_perf_chunks1.sh index 9d30f21507..e08e1417f1 100755 --- a/nczarr_test/run_perf_chunks1.sh +++ b/nczarr_test/run_perf_chunks1.sh @@ -9,7 +9,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. "${builddir}/test_nczarr.sh" +. "${srcdir}/test_nczarr.sh" s3isolate "testdir_perf_chunks1" THISDIR=`pwd` diff --git a/nczarr_test/run_purezarr.sh b/nczarr_test/run_purezarr.sh index 0b646c8b6a..efe177468c 100755 --- a/nczarr_test/run_purezarr.sh +++ b/nczarr_test/run_purezarr.sh @@ -3,7 +3,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. "${builddir}/test_nczarr.sh" +. "${srcdir}/test_nczarr.sh" set -e diff --git a/nczarr_test/run_quantize.sh b/nczarr_test/run_quantize.sh index 9c1c2a7ab0..1907c62da7 100755 --- a/nczarr_test/run_quantize.sh +++ b/nczarr_test/run_quantize.sh @@ -3,7 +3,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. "${builddir}/test_nczarr.sh" +. "${srcdir}/test_nczarr.sh" # Construct both ISOPATH and S3ISOPATH s3isolate "testdir_quantize" diff --git a/nczarr_test/run_scalar.sh b/nczarr_test/run_scalar.sh index 2fc7cd4972..cb601e0558 100755 --- a/nczarr_test/run_scalar.sh +++ b/nczarr_test/run_scalar.sh @@ -3,7 +3,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. "${builddir}/test_nczarr.sh" +. "${srcdir}/test_nczarr.sh" set -e diff --git a/nczarr_test/run_specific_filters.sh b/nczarr_test/run_specific_filters.sh index 89c0366d19..bad0623030 100755 --- a/nczarr_test/run_specific_filters.sh +++ b/nczarr_test/run_specific_filters.sh @@ -14,7 +14,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi set -e if test "x$TESTNCZARR" = x1 ; then -. ${builddir}/test_nczarr.sh +. ${srcdir}/test_nczarr.sh s3isolate "testdir_specific_filters" THISDIR=`pwd` cd $ISOPATH diff --git a/nczarr_test/run_strings.sh b/nczarr_test/run_strings.sh index c2653e1842..e27248afba 100755 --- a/nczarr_test/run_strings.sh +++ b/nczarr_test/run_strings.sh @@ -3,7 +3,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. "${builddir}/test_nczarr.sh" +. "${srcdir}/test_nczarr.sh" # This shell script tests support for the NC_STRING type diff --git a/nczarr_test/run_unknown.sh b/nczarr_test/run_unknown.sh index 63f63dc97e..1112bf60d1 100755 --- a/nczarr_test/run_unknown.sh +++ b/nczarr_test/run_unknown.sh @@ -11,7 +11,7 @@ THISDIR=`pwd` cd $ISOPATH if test "x$TESTNCZARR" = x1 ; then -. ${builddir}/test_nczarr.sh +. ${srcdir}/test_nczarr.sh s3isolate fi diff --git a/nczarr_test/run_unlim_io.sh b/nczarr_test/run_unlim_io.sh index 7f00edbe05..aec2320003 100755 --- a/nczarr_test/run_unlim_io.sh +++ b/nczarr_test/run_unlim_io.sh @@ -3,7 +3,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. ${builddir}/test_nczarr.sh +. ${srcdir}/test_nczarr.sh set -e diff --git a/nczarr_test/run_ut_map.sh b/nczarr_test/run_ut_map.sh index c8cff9a5a0..4f2743577a 100755 --- a/nczarr_test/run_ut_map.sh +++ b/nczarr_test/run_ut_map.sh @@ -3,8 +3,9 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. ${builddir}/test_nczarr.sh +. ${srcdir}/test_nczarr.sh +set -x set -e s3isolate "testdir_utmap" @@ -32,6 +33,7 @@ testmapcreate() { $CMD -k$1 -x create -o $output cdl="ut_${tag}_create_${zext}.cdl" ref="ref_ut_${tag}_create.cdl" +echo "@@@@" dumpmap $zext $output ./$cdl diff -wb ${srcdir}/$ref ./$cdl # delete the test file diff --git a/nczarr_test/run_ut_mapapi.sh b/nczarr_test/run_ut_mapapi.sh index c05c164288..2e39132349 100755 --- a/nczarr_test/run_ut_mapapi.sh +++ b/nczarr_test/run_ut_mapapi.sh @@ -3,7 +3,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. "${builddir}/test_nczarr.sh" +. "${srcdir}/test_nczarr.sh" #TR="-T10" diff --git a/nczarr_test/run_ut_misc.sh b/nczarr_test/run_ut_misc.sh index 3b72f4ba1b..4abbf1c0ff 100755 --- a/nczarr_test/run_ut_misc.sh +++ b/nczarr_test/run_ut_misc.sh @@ -3,9 +3,9 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. "${builddir}/test_nczarr.sh" +. "${srcdir}/test_nczarr.sh" -set -e +#set -e # Test misc functionality @@ -17,14 +17,16 @@ TESTS=json testjson() { file="ut_json_build.txt" - rm -f $file + reffile="ref_${file}" + rm -f ${builddir}/$file CMD="${execdir}/ut_json${ext}" - $CMD -x build > $file - diff -wb ${srcdir}/ref_$file ./$file + $CMD -x build > ${builddir}/$file + diff -wb ${srcdir}/$reffile ${builddir}/$file file="ut_json_parse.txt" - rm -f $file - $CMD -x parse > $file - diff -wb ${srcdir}/ref_$file ./$file + rm -f ${builddir}/$file + reffile="ref_${file}" + $CMD -x parse > ${builddir}/$file + diff -wb ${srcdir}/$reffile ${builddir}/$file } echo "" diff --git a/nczarr_test/test_nczarr.sh b/nczarr_test/test_nczarr.sh new file mode 100644 index 0000000000..b2934229e8 --- /dev/null +++ b/nczarr_test/test_nczarr.sh @@ -0,0 +1,233 @@ +#!/bin/sh + +# This file must kept in sync with +# nczarr_test/test_nczarr.sh +# and +# v3_nczarr_test/test_nczarr.sh + +# Load only once +if test "x$TEST_NCZARR_SH" = x ; then +export TEST_NCZARR_SH=1 + +if test "x$SETX" != x; then set -x; fi + +# Get the directory in which we are running +TMP=`pwd` +NCZARRDIR=`basename $TMP` +if test "x$NCZARRDIR" = xnczarr_test ; then + export NCZARRFORMAT=2; export NCNOZMETADATA=1 +elif test "x$NCZARRDIR" = x3_nczarr_test ; then + export NCZARRFORMAT=3; export NCNOZMETADATA=1 +else + unset NCZARRFORMAT; unset NCNOZMETADATA +fi +echo "@@@ NCZARRFORMAT=$NCZARRFORMAT NCNOZMETADATA=$NCNOZMETADATA" + +# Figure out which cloud repo to use +if test "x$NCZARR_S3_TEST_HOST" = x ; then +# export NCZARR_S3_TEST_HOST=stratus.ucar.edu + export NCZARR_S3_TEST_HOST="${S3ENDPOINT:-s3.us-east-1.amazonaws.com}" +fi +if test "x$NCZARR_S3_TEST_BUCKET" = x ; then + export NCZARR_S3_TEST_BUCKET="${S3TESTBUCKET}" +fi +export NCZARR_S3_TEST_URL="https://${NCZARR_S3_TEST_HOST}/${NCZARR_S3_TEST_BUCKET}" + +# TAG for zarr format to use; uses the environment variable NCZARRFORMAT +if test "x${NCZARRFORMAT}" = x3 ; then + export ZDF="_v3" +else + export ZDF="" +fi + +# Fix execdir +NCZARRDIR="${execdir}/../nczarr_test" + +ZMD="${NCZARRDIR}/${DL}zmapio" +S3UTIL="${NCZARRDIR}/${DL}s3util" +ZS3PARSE="${NCZARRDIR}/${DL}zs3parse" +NCDUMPCHUNKS="${NCZARRDIR}/${DL}ncdumpchunks" +ZHEX="${NCZARRDIR}/${DL}zhex" +ZISJSON="${NCZARRDIR}/${DL}zisjson" + +# Check settings +checksetting() { +if test -f ${TOPBUILDDIR}/libnetcdf.settings ; then + local PATTERN + PATTERN="${1}:[ ]*yes" + if grep "$PATTERN" <${TOPBUILDDIR}/libnetcdf.settings ; then + HAVE_SETTING=1 + else + unset HAVE_SETTING + fi +fi +} + +checkprops() { + specflag= + headflag= + isxfail= + # determine if this is an xfailtest + for t in ${XFAILTESTS} ; do + if test "x${t}" = "x${x}" ; then isxfail=1; fi + done + for t in ${SPECTESTS} ; do + if test "x${t}" = "x${f}" ; then specflag="-s"; fi + done + for t in ${HEADTESTS} ; do + if test "x${t}" = "x${f}" ; then headflag="-h"; fi + done +} + +extfor() { + case "$1" in + file) zext="file" ;; + zip) zext="zip" ;; + s3) zext="s3" ;; + *) echo "unknown kind: $1" ; exit 1;; + esac +} + +deletemap() { + case "$1" in + file) rm -fr $2;; + zip) rm -f $2;; + s3) S3KEY=`${ZS3PARSE} -k $2`; s3sdkdelete $S3KEY ;; + *) echo "unknown kind: $1" ; exit 1;; + esac +} + +mapstillexists() { + mapstillexists=0 + if "${ZMD} $fileurl" &> /dev/null ; then + echo "delete failed: $1" + mapstillexists=1 + fi +} + +fileargs() { + f="$1" + frag="$2" + if test "x$frag" = x ; then frag="mode=nczarr,$zext" ; fi + case "$zext" in + s3) + S3PATH="${NCZARR_S3_TEST_URL}/${S3ISOPATH}" + fileurl="${S3PATH}/${f}#${frag}" + file=$fileurl + S3HOST=`${ZS3PARSE} -h $S3PATH` + S3BUCKET=`${ZS3PARSE} -b $S3PATH` + S3PREFIX=`${ZS3PARSE} -k $S3PATH` + ;; + *) + file="${f}.$zext" + fileurl="file://${f}.$zext#${frag}" + ;; + esac +} + +dumpmap() { + zext=$1 + zbase=`basename $2 ".$zext"` + fileargs $zbase +df -h +export NCTRACING=10 + ${ZMD} -t int -x objdump $fileurl > $3 +unset NCTRACING +} + +# Function to remove selected -s attributes from file; +# These attributes might be platform dependent +sclean() { +sed -i.bak -e '/:_IsNetcdf4/d' $1 +sed -i.bak -e '/:_Endianness/d' $1 +sed -i.bak -e '/_NCProperties/d' $1 +sed -i.bak -e '/_SuperblockVersion/d' $1 +} + +# s3clean plus remove additional lines +scleanplus() { +sclean $1 +sed -i.bak -e '/_Format/d' $1 +sed -i.bak -e '/_global attributes:/d' $1 +} + +# Function to rewrite selected key values in a zmapio output. +# because these values might be platform dependent +zmapclean() { +sed -i.bak -e 's|^\([^(]*\)([0-9][0-9]*)|\1()|' $1 +sed -i.bak -e 's/"_NCProperties":[ ]*"version=\([0-9]\),[^"]*"/"_NCProperties": "version=\1,netcdf=0.0.0,nczarr=0.0.0"/g' $1 +sed -i.bak -e 's/"_nczarr_superblock":[ ]*{[^}]*}/"_nczarr_superblock": {"version": "0.0.0", "format": 2}/g' $1 +sed -i.bak -e 's/"_nczarr_superblock":[ ]*{[^}]*}/"_nczarr_superblock": {"version": "0.0.0", "format": 2}/g' $1 +} + +# Make sure execdir and srcdir absolute paths are available +WD=`pwd` +cd $srcdir ; abs_srcdir=`pwd` ; cd $WD +cd $execdir ; abs_execdir=`pwd` ; cd $WD + +# Clear out any existing .rc files +WD=`pwd` +if test "x$NCAUTH_HOMETEST" != x ; then RCHOME=1; fi + +# Set plugin path + +if test "x$FP_USEPLUGINS" = xyes; then +# Load the findplugins function +. ${builddir}/findplugin.sh +echo "findplugin.sh loaded" + +# Locate the plugin path and the library names; argument order is critical +# Find misc in order to determine HDF5_PLUGIN+PATH. +# Assume all test filters are in same plugin dir +if ! findplugin h5misc ; then exit 0; fi + +echo "final HDF5_PLUGIN_DIR=${HDF5_PLUGIN_DIR}" +export HDF5_PLUGIN_PATH="${HDF5_PLUGIN_DIR}" +fi # USEPLUGINS + +resetrc() { + if test "x$RCHOME" = x1 ; then + rm -f ${HOME}/.dodsrc ${HOME}/.daprc ${HOME}/.ncrc + fi + rm -f ${WD}/.dodsrc ${WD}/.daprc ${WD}/.ncrc + unset NCRCENV_IGNORE + unset NCRCENV_RC + unset DAPRCFILE +} + +s3sdkdelete() { +if test -f ${S3UTIL} ; then + ${S3UTIL} ${PROFILE} -u "${NCZARR_S3_TEST_URL}" -k "$1" clear +elif which aws ; then + aws s3api delete-object --endpoint-url=https://${NCZARR_S3_TEST_HOST} --bucket=${NCZARR_S3_TEST_BUCKET} --key="/${S3ISOPATH}/$1" +else + echo "**** Could not delete ${NCZAR_S3_TEST_URL}" +fi +} + +s3sdkcleanup() { +if test -f ${S3UTIL} ; then + ${S3UTIL} ${PROFILE} -u "${NCZARR_S3_TEST_URL}" -k "$1" clear +elif which aws ; then + aws s3api delete-object --endpoint-url=https://${NCZARR_S3_TEST_HOST} --bucket=${NCZARR_S3_TEST_BUCKET} --key="/${S3ISOPATH}/$1" +else + echo "**** Could not delete ${NCZAR_S3_TEST_URL}" +fi +} + +# Create an isolation path for S3; build on the isolation directory +s3isolate() { + if test "x${S3ISOPATH}" = x ; then + if test "x${ISOPATH}" = x ; then isolate "$1"; fi + # Need isolation path to include the test directory + BNAME=`basename $srcdir` + S3ISODIR="${BNAME}_${TESTUID}/${ISODIR}" + S3ISOPATH="${S3TESTSUBTREE}/${S3ISODIR}" + fi +} + +GDBB="gdb -batch -ex r -ex bt -ex q --args" + +resetrc + +fi #TEST_NCZARR_SH diff --git a/nczarr_test/test_unlim_vars.c b/nczarr_test/test_unlim_vars.c index 4e1e7aafa6..3e3dc1d6d7 100644 --- a/nczarr_test/test_unlim_vars.c +++ b/nczarr_test/test_unlim_vars.c @@ -108,7 +108,7 @@ main(int argc, char **argv) if (nc_inq_var(ncid, 0, name_in, &xtype_in, &ndims_in, dimids_in, &natts_in)) ERR; if (strcmp(name_in, SFC_TEMP_NAME) || xtype_in != NC_FLOAT || - ndims_in != 3 || natts_in != 1) ERR; /* _FillValue is defined att */ + ndims_in != 3 || natts_in != 0) ERR; /* _FillValue is defined att */ for (d = 0; d < NDIM3; d++) if (dimids_in[d] != dimids[d]) ERR; if (nc_inq_dim(ncid, 0, name_in, &len_in)) ERR; @@ -128,7 +128,7 @@ main(int argc, char **argv) if (nc_inq_var(ncid, 0, name_in, &xtype_in, &ndims_in, dimids_in, &natts_in)) ERR; if (strcmp(name_in, SFC_TEMP_NAME) || xtype_in != NC_FLOAT || - ndims_in != 3 || natts_in != 1) ERR; + ndims_in != 3 || natts_in != 0) ERR; for (d = 0; d < NDIM3; d++) if (dimids_in[d] != dimids[d]) ERR; if (nc_inq_dim(ncid, 0, name_in, &len_in)) ERR; @@ -154,7 +154,7 @@ main(int argc, char **argv) if (nc_inq_var(ncid, 0, name_in, &xtype_in, &ndims_in, dimids_in, &natts_in)) ERR; if (strcmp(name_in, SFC_TEMP_NAME) || xtype_in != NC_FLOAT || - ndims_in != 3 || natts_in != 1) ERR; + ndims_in != 3 || natts_in != 0) ERR; for (d = 0; d < NDIM3; d++) if (dimids_in[d] != dimids[d]) ERR; if (nc_inq_dim(ncid, 0, name_in, &len_in)) ERR; diff --git a/test_common.in b/test_common.in index bef085bff7..fc9aeb9e8c 100644 --- a/test_common.in +++ b/test_common.in @@ -61,6 +61,9 @@ S3TESTBUCKET=@S3TESTBUCKET@ # All S3 tests should use this to store intermediate results. S3TESTSUBTREE=@S3TESTSUBTREE@ +# Additional S3 test endpoint +S3ENDPOINT=@S3ENDPOINT@ + TESTUID=@TESTUID@ set -e diff --git a/unit_test/run_s3sdk.sh b/unit_test/run_s3sdk.sh index 5b0bb65059..ce7fb9d092 100755 --- a/unit_test/run_s3sdk.sh +++ b/unit_test/run_s3sdk.sh @@ -7,7 +7,7 @@ set -e #CMD="valgrind --leak-check=full" -URL="https://s3.us-east-1.amazonaws.com/${S3TESTBUCKET}" +URL="https://${S3ENDPOINT}/${S3TESTBUCKET}" isolate "testdir_uts3sdk" diff --git a/v3_nczarr_test/CMakeLists.txt b/v3_nczarr_test/CMakeLists.txt index b975154565..9696149a8a 100644 --- a/v3_nczarr_test/CMakeLists.txt +++ b/v3_nczarr_test/CMakeLists.txt @@ -7,15 +7,26 @@ set(abs_top_srcdir ${CMAKE_CURRENT_SOURCE_DIR}) -include(v3_setup) -v3_setup() +NCZARR_C_TEST(tst_quantize test_quantize nc_test4) +NCZARR_C_TEST(tst_filter_vlen test_filter_vlen nc_test4) +NCZARR_C_TEST(tst_unlim_vars test_unlim_vars nc_test4) +NCZARR_C_TEST(tst_h5_endians test_endians nc_test4) +NCZARR_C_TEST(tst_put_vars_two_unlim_dim test_put_vars_two_unlim_dim nc_test4) +NCZARR_C_TEST(tst_chunking test_chunking ncdump) + +NCZARR_SH_TEST(specific_filters nc_test4) +NCZARR_SH_TEST(unknown nc_test4) +NCZARR_SH_TEST(filter_vlen nc_test4) remove_definitions(-DDLL_EXPORT) file(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.sh -${CMAKE_CURRENT_SOURCE_DIR}/ref*.cdl -${CMAKE_CURRENT_SOURCE_DIR}/ref*.txt -${CMAKE_CURRENT_SOURCE_DIR}/ref*.zmap) + ${CMAKE_CURRENT_SOURCE_DIR}/ref*.cdl + ${CMAKE_CURRENT_SOURCE_DIR}/ref*.txt + ${CMAKE_CURRENT_SOURCE_DIR}/ref*.zmap + ${CMAKE_CURRENT_SOURCE_DIR}/ref*.baseline + ${CMAKE_CURRENT_SOURCE_DIR}/ref*.tgz + ) file(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE) @@ -23,19 +34,34 @@ if(USE_X_GETOPT) set(XGETOPTSRC "${CMAKE_CURRENT_SOURCE_DIR}/../libdispatch/XGetopt.c") endif() -if(NETCDF_ENABLE_TESTS) +# Unlimited tests +NCZARR_SH_TEST(mud ncdump) +NCZARR_SH_TEST(nccopy5 ncdump) + +FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.sh +${CMAKE_CURRENT_SOURCE_DIR}/ref*.cdl +${CMAKE_CURRENT_SOURCE_DIR}/ref*.txt +${CMAKE_CURRENT_SOURCE_DIR}/ref*.zmap) + +FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE) + +IF(USE_X_GETOPT) +SET(XGETOPTSRC "${CMAKE_CURRENT_SOURCE_DIR}/../libdispatch/XGetopt.c") +ENDIF() - add_library(v3_ut_util STATIC ut_util.c ut_test.c ut_includes.h ut_test.h ${XGETOPTSRC}) - if(NETCDF_ENABLE_DLL) - target_compile_definitions(v3_ut_util PUBLIC -DDLL_EXPORT -DDLL_NETCDF) - endif(NETCDF_ENABLE_DLL) - target_include_directories(v3_ut_util PUBLIC ../libnczarr ../plugins ${CMAKE_CURRENT_LIST_DIR}) - target_link_libraries(v3_ut_util PUBLIC netcdf ${ALL_TLL_LIBS}) +IF(NETCDF_ENABLE_TESTS) + + add_library(ut_util STATIC ut_util.c ut_test.c ut_includes.h ut_test.h ${XGETOPTSRC}) + IF(NETCDF_ENABLE_DLL) + target_compile_definitions(ut_util PUBLIC -DDLL_EXPORT -DDLL_NETCDF) + ENDIF(NETCDF_ENABLE_DLL) + target_include_directories(ut_util PUBLIC ../libnczarr ../plugins ${CMAKE_CURRENT_LIST_DIR}) + target_link_libraries(ut_util PUBLIC netcdf ${ALL_TLL_LIBS}) add_library(test_utils STATIC test_utils.c test_utils.h ${XGETOPTSRC}) - if(NETCDF_ENABLE_DLL) + IF(NETCDF_ENABLE_DLL) target_compile_definitions(test_utils PUBLIC -DDLL_EXPORT -DDLL_NETCDF) - endif(NETCDF_ENABLE_DLL) + ENDIF(NETCDF_ENABLE_DLL) target_include_directories(test_utils PUBLIC ../libnczarr ../plugins ${CMAKE_CURRENT_LIST_DIR}) target_link_libraries(test_utils PUBLIC netcdf ${ALL_TLL_LIBS}) @@ -45,6 +71,23 @@ if(NETCDF_ENABLE_TESTS) RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR} ) + macro(build_bin_test_with_util_lib F UTIL_LIB) + build_bin_test(${F}) + IF(NETCDF_ENABLE_DLL) + target_compile_definitions(${F} PUBLIC -DDLL_NETCDF) + ENDIF(NETCDF_ENABLE_DLL) + target_link_libraries(${F} ${UTIL_LIB} ${ALL_TLL_LIBS}) + endmacro() + + macro(add_bin_test_with_util_lib PREFIX F UTIL_LIB) + add_bin_test(${PREFIX} ${F}) + IF(NETCDF_ENABLE_DLL) + target_compile_definitions(${PREFIX}_${F} PUBLIC -DDLL_NETCDF) + ENDIF(NETCDF_ENABLE_DLL) + target_link_libraries(${PREFIX}_${F} ${UTIL_LIB} ${ALL_TLL_LIBS}) + + endmacro() + # Base tests # The tests are set up as a combination of shell scripts and executables that # must be run in a particular order. It is painful but will use macros to help @@ -58,7 +101,7 @@ if(NETCDF_ENABLE_TESTS) build_bin_test_with_util_lib(test_quantize test_utils) build_bin_test_with_util_lib(test_notzarr test_utils) -# add_bin_test(nczarr_test test_endians ${TSTCOMMONSRC}) +# ADD_BIN_TEST(nczarr_test test_endians ${TSTCOMMONSRC}) # Unlimited Tests if(USE_HDF5) @@ -75,25 +118,37 @@ if(NETCDF_ENABLE_TESTS) add_sh_test(nczarr_test run_mud) endif() + if(FALSE) # Obsolete tests + build_bin_test(ut_projections ${COMMONSRC}) + build_bin_test(ut_chunking ${COMMONSRC}) + build_bin_test(ut_walk ${COMMONSRC}) + endif() + # Helper programs for testing + build_bin_test(zhex) build_bin_test_with_util_lib(zisjson ut_util) target_include_directories(zisjson PUBLIC ../libnczarr) build_bin_test_with_util_lib(zs3parse ut_util) target_include_directories(zs3parse PUBLIC ../libnczarr) build_bin_test_with_util_lib(zmapio ut_util) + if(USE_HDF5) + build_bin_test_with_util_lib(ncdumpchunks ut_util) + endif() - if(NETCDF_ENABLE_S3 AND NOT WITH_S3_TESTING STREQUAL "NO") + IF(NETCDF_ENABLE_S3 AND NOT WITH_S3_TESTING STREQUAL "NO") + # Helper programs for testing + build_bin_test(s3util ${COMMONSRC}) # Pure AWS Test - if(NETCDF_ENABLE_S3_AWS) - set(TMP_CMAKE_CXX_STANDARD ${MAKE_CXX_STANDARD}) - set(CMAKE_CXX_STANDARD 11) - add_executable(tst_pure_awssdk tst_pure_awssdk.cpp) - target_link_libraries(tst_pure_awssdk ${AWSSDK_CORE_LIB_FILE}) - add_test(tst_pure_awssdk ${EXECUTABLE_PUTPUT_PATH}/tst_pure_awssdk) - set(F tst_pure_awssdk) - if(MSVC) - set_property(TEST ${F} PROPERTY FOLDER "tests/") - set_target_properties(${F} PROPERTIES RUNTIME_OUTPUT_DIRECTORY + IF(NETCDF_ENABLE_S3_AWS) + SET(TMP_CMAKE_CXX_STANDARD ${MAKE_CXX_STANDARD}) + SET(CMAKE_CXX_STANDARD 11) + ADD_EXECUTABLE(tst_pure_awssdk tst_pure_awssdk.cpp) + TARGET_LINK_LIBRARIES(tst_pure_awssdk ${AWSSDK_CORE_LIB_FILE}) + ADD_TEST(tst_pure_awssdk ${EXECUTABLE_PUTPUT_PATH}/tst_pure_awssdk) + SET(F tst_pure_awssdk) + IF(MSVC) + SET_PROPERTY(TEST ${F} PROPERTY FOLDER "tests/") + SET_TARGET_PROPERTIES(${F} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) set_target_properties(${F} PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_BINARY_DIR}) @@ -101,11 +156,12 @@ if(NETCDF_ENABLE_TESTS) ${CMAKE_CURRENT_BINARY_DIR}) endif(MSVC) - SET(CMAKE_CXX_STANDARD ${TMP_CMAKE_CXX_STANDARD}) + set(CMAKE_CXX_STANDARD ${TMP_CMAKE_CXX_STANDARD}) endif(NETCDF_ENABLE_S3_AWS) + target_include_directories(s3util PUBLIC ../libnczarr) endif() - if(BUILD_UTILITIES) + IF(NETCDF_BUILD_UTILITIES) add_sh_test(nczarr_test run_ut_map) add_sh_test(nczarr_test run_ut_mapapi) add_sh_test(nczarr_test run_ut_misc) @@ -128,12 +184,8 @@ if(NETCDF_ENABLE_TESTS) add_sh_test(nczarr_test run_external) add_sh_test(nczarr_test run_quantize) add_sh_test(nczarr_test run_notzarr) + add_sh_test(nczarr_test run_consolidated_zarr) - # This has timeout under CMake -# if(NOT ISCMAKE) - add_sh_test(nczarr_test run_interop) -# endif() - if(USE_HDF5) add_sh_test(nczarr_test run_fillonlyz) endif() @@ -147,6 +199,7 @@ if(NETCDF_ENABLE_TESTS) build_bin_test(testfilter_multi) build_bin_test(testfilter_order) build_bin_test(testfilter_repeat) + add_sh_test(nczarr_test run_interop) add_sh_test(nczarr_test run_nczfilter) add_sh_test(nczarr_test run_filter) add_sh_test(nczarr_test run_specific_filters) @@ -160,20 +213,19 @@ if(NETCDF_ENABLE_TESTS) endif(NETCDF_ENABLE_NCZARR_FILTERS) if(NETCDF_ENABLE_NCZARR_ZIP) - add_sh_test(nczarr_test run_newformat) # Test various corrupted files - add_sh_test(nczarr_test run_corrupt.sh) + add_sh_test(nczarr_test run_corrupt) endif() if(FALSE) # Obsolete tests add_sh_test(nczarr_test run_ut_chunk) endif() - endif(BUILD_UTILITIES) + endif(NETCDF_BUILD_UTILITIES) endif(NETCDF_ENABLE_TESTS) ## Specify files to be distributed by 'make dist' FILE(GLOB CUR_EXTRA_DIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*.c ${CMAKE_CURRENT_SOURCE_DIR}/*.h ${CMAKE_CURRENT_SOURCE_DIR}/*.sh ${CMAKE_CURRENT_SOURCE_DIR}/*.cdl) -SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} CMakeLists.txt Makefile.am) +SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} CMakeLists.txt Makefile.am zmapio.c zhex.c ncdumpchunks.c) ADD_EXTRA_DIST("${CUR_EXTRA_DIST}") diff --git a/v3_nczarr_test/Makefile.am b/v3_nczarr_test/Makefile.am index ddc7d4bc2d..250b059615 100644 --- a/v3_nczarr_test/Makefile.am +++ b/v3_nczarr_test/Makefile.am @@ -4,7 +4,7 @@ # Put together AM_CPPFLAGS and AM_LDFLAGS. include $(top_srcdir)/lib_flags.am -AM_TESTS_ENVIRONMENT += export NCZARRFORMAT=3; +AM_TESTS_ENVIRONMENT += export NCZARRFORMAT=3;export NCNOZMETADATA=1; TESTS_ENVIRONMENT = TEST_EXTENSIONS = .sh #SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose @@ -20,12 +20,24 @@ LDADD = ${top_builddir}/liblib/libnetcdf.la AM_LDFLAGS += ${top_builddir}/liblib/libnetcdf.la AM_CXXFLAGS = -BUILT_SOURCES = EXTRA_DIST = DISTCLEANFILES = TESTS = check_PROGRAMS = -NCZARRFILES = + +if NETCDF_BUILD_UTILITIES + +# Test fill issues +TESTS += run_nczarr_fill.sh + +# Test various corrupted files +TESTS += run_corrupt.sh + +if NETCDF_ENABLE_EXTERNAL_SERVER_TESTS +TESTS += run_external.sh +endif + +endif #BUILD_UTILITIES testcommonsrc = test_utils.c test_utils.h commonsrc = ut_util.c ut_test.c ut_includes.h ut_test.h ut_util.h test_nczarr_utils.h @@ -68,6 +80,11 @@ TESTS += run_strings.sh TESTS += run_scalar.sh TESTS += run_nulls.sh TESTS += run_notzarr.sh +TESTS += run_consolidated_zarr.sh + +if NETCDF_ENABLE_EXTERNAL_SERVER_TESTS +TESTS += run_external.sh +endif # Unlimited Dimension tests (at least in part) TESTS += run_mud.sh @@ -84,18 +101,14 @@ TESTS += run_unlim_io.sh run_nccopyz.sh endif if LARGE_FILE_TESTS - check_PROGRAMS += test_writecaching test_readcaching - TESTS += run_cachetest.sh +check_PROGRAMS += test_writecaching test_readcaching +TESTS += run_cachetest.sh endif endif #NETCDF_BUILD_UTILITIES if NETCDF_BUILD_UTILITIES -if NETCDF_ENABLE_NCZARR_ZIP - TESTS += run_newformat.sh -endif - if LARGE_FILE_TESTS check_PROGRAMS += test_chunkcases test_chunkcases_SOURCES = test_chunkcases.c ${testcommonsrc} @@ -111,7 +124,7 @@ endif if BUILD_BENCHMARKS -UTILSRC = bm_utils.c timer_utils.c test_utils.c +UTILSRC = bm_utils.c timer_utils.c test_utils.c bm_utils.h bm_timer.h bm_chunks3_SOURCES = bm_chunks3.c ${UTILSRC} @@ -122,11 +135,14 @@ if AX_DISABLE TESTS += run_perf_chunks1.sh endif -endif # NETCDF_BUILD_BENCHMARKS +endif # BUILD_BENCHMARKS if NETCDF_ENABLE_NCZARR_FILTERS if NETCDF_ENABLE_FILTER_TESTING +# Test interoperability of V2 files +TESTS += run_interop.sh + check_PROGRAMS += test_filter_avail test_filter_vlen TESTS += run_filter_vlen.sh run_filter_misc.sh @@ -155,23 +171,23 @@ endif #NETCDF_ENABLE_NCZARR_FILTERS endif #NETCDF_BUILD_UTILITIES # These programs are used by the test cases -#noinst_PROGRAMS = zhex -#zhex_SOURCES = zhex.c -#noinst_PROGRAMS += zisjson -#zisjson_SOURCES = zisjson.c -#noinst_PROGRAMS += zmapio -#zmapio_SOURCES = zmapio.c -#noinst_PROGRAMS += zs3parse -#zs3parse_SOURCES = zs3parse.c +noinst_PROGRAMS = zhex +zhex_SOURCES = zhex.c +noinst_PROGRAMS += zisjson +zisjson_SOURCES = zisjson.c +noinst_PROGRAMS += zmapio +zmapio_SOURCES = zmapio.c +noinst_PROGRAMS += zs3parse +zs3parse_SOURCES = zs3parse.c if NETCDF_ENABLE_S3 -#noinst_PROGRAMS += s3util -#s3util_SOURCES = s3util.c +noinst_PROGRAMS += s3util +s3util_SOURCES = s3util.c if NETCDF_ENABLE_S3_TESTALL if NETCDF_ENABLE_S3_AWS -#check_PROGRAMS += tst_pure_awssdk -#tst_pure_awssdk_SOURCES = tst_pure_awssdk.cpp -#AM_CXXFLAGS += -std=c++11 +check_PROGRAMS += tst_pure_awssdk +tst_pure_awssdk_SOURCES = tst_pure_awssdk.cpp +AM_CXXFLAGS += -std=c++11 endif endif endif @@ -179,59 +195,40 @@ endif if USE_HDF5 # Given a netcdf4|NCZarr file, dump the actual chunk contents. # Used to validate nczarr chunking code. -#AM_CPPFLAGS += -I$(top_srcdir)/libnczarr -#noinst_PROGRAMS += ncdumpchunks -#ncdumpchunks_SOURCES = ncdumpchunks.c +AM_CPPFLAGS += -I$(top_srcdir)/libnczarr +noinst_PROGRAMS += ncdumpchunks +ncdumpchunks_SOURCES = ncdumpchunks.c endif -CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc tmp*.dump tmp*.tmp tmp*.zmap tmp_ngc.c ref_zarr_test_data.cdl tst_*.nc.zip ref_quotes.zip ref_power_901_constants.zip +CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc tmp*.dump tmp*.tmp tmp*.zmap tmp_ngc.c ref_zarr_test_data.cdl tst_*.nc.zip ref_quotes.zip ref_power_901_constants.zip \ +tmp_*.nc tmp_*.zarr tst_quantize*.zarr rcmiscdir ref_power_901_constants.file + +# *** WARNING *** Most of the files in nczarr_test have been duplicated into v3_nczarr_test. +# If any file here is changed, then it may need to be propagated to the other copy. + +EXTRA_DIST += CMakeLists.txt test_nczarr.sh -EXTRA_DIST += CMakeLists.txt v3manifest.am +EXTRA_DIST += run_chunkcases.sh run_corrupt.sh run_external.sh run_fillonlyz.sh run_filterinstall.sh run_filter_misc.sh run_filter.sh run_filter_vlen.sh run_interop.sh run_jsonconvention.sh run_misc.sh run_mud.sh run_nan.sh run_nccopy5.sh run_nccopyz.sh run_ncgen4.sh run_nczarr_fill.sh run_nczarr_fill.sh run_nczfilter.sh run_newformat.sh run_notzarr.sh run_nulls.sh run_perf_chunks1.sh run_purezarr.sh run_quantize.sh run_scalar.sh run_specific_filters.sh run_strings.sh run_unknown.sh run_unlim_io.sh run_ut_mapapi.sh run_ut_map.sh run_ut_misc.sh run_consolidated_zarr.sh -include v3manifest.am +EXTRA_DIST += bm_utils.c test_h5_endians.c test_put_vars_two_unlim_dim.c test_put_vars_two_unlim_dim.c test_unlim_vars.c test_utils.c timer_utils.c bm_timer.h bm_utils.h timer_utils.h -EXTRA_DIST += ${V3FILES} ${V3DATA} CMakeLists.txt +EXTRA_DIST += ref_noshape.file.zip ref_power_901_constants.cdl ref_power_901_constants_orig.zip ref_avail1.cdl ref_avail1.dmp ref_avail1.txt ref_byte.cdl ref_byte_fill_value_null.cdl ref_byte_fill_value_null.zarr.zip ref_byte.zarr.zip ref_fillonly.cdl ref_groups.h5 ref_misc1.cdl ref_misc1.dmp ref_ndims.cdl ref_ndims.dmp ref_newformatpure.cdl ref_notzarr.tar.gz ref_nulls.cdl ref_oldformat.cdl ref_oldformat.zip ref_perdimspecs.cdl ref_purezarr_base.cdl ref_quotes.cdl ref_quotes_orig.zip ref_rem.cdl ref_rem.dmp ref_scalar.cdl ref_scalar_nczarr.cdl ref_skip.cdl ref_skip.txt ref_skipw.cdl ref_string.cdl ref_t_meta_dim1.cdl ref_t_meta_var1.cdl ref_ut_json_build.txt ref_ut_json_parse.txt ref_ut_mapapi_create.cdl ref_ut_mapapi_data.cdl ref_ut_mapapi_meta.cdl ref_ut_mapapi_search.txt ref_ut_map_create.cdl ref_ut_map_readmeta2.txt ref_ut_map_readmeta.txt ref_ut_map_search.txt ref_ut_map_writedata.cdl ref_ut_map_writemeta2.cdl ref_ut_map_writemeta.cdl ref_ut_proj.txt ref_ut_testmap_create.cdl ref_whole.cdl ref_whole.txt ref_zarr_test_data.cdl.gz ref_bzip2.cdl ref_filtered.cdl ref_purezarr.cdl ref_jsonconvention.cdl ref_groups_regular.cdl ref_zarr_test_data_2d.cdl.gz ref_misc2.cdl ref_string_zarr.baseline ref_string_nczarr.baseline ref_nulls_zarr.baseline ref_nulls_nczarr.baseline ref_any.cdl ref_xarray.cdl ref_jsonconvention.zmap ref_nczarr2zarr.cdl ref_multi.cdl ref_tst_mud4.cdl ref_tst_mud4-bc.cdl ref_tst_mud4_chars.cdl ref_tst_nans.dmp ref_cmip6.zmap ref_consolidated_zarr.cdl ref_consolidated_zarr_base.cdl ref_consolidated_zarr_2.18.2_python.zarr.tgz DISTCLEANFILES += findplugin.sh -DISTCLEANFILES += ${TESTFILES_NCZARR_C} ${TESTDATA_NCZARR} - -# Note: v3_nczarr_test is initialized with the files specified by the -# list ${V3FILES} and ${V3DATA} in v3manifest.am. The actual -# files are kept in the file ${top_srcdir}/v3init.tgz and they -# are copied into v3_nczarr_test in configure.ac. - -BUILT_SOURCES = ${TESTFILES_NCZARR_SH} ${TESTFILES_NCZARR_C} ${TESTDATA_NCZARR} - -dist-hook: setup -all-local: setup - -SRC = ${abs_top_srcdir}/nczarr_test -DST = ${abs_top_srcdir}/v3_nczarr_test -#DST = . - -.PHONY: setup -setup ${TESTFILES_NCZARR_SH} ${TESTFILES_NCZARR_C} ${TESTDATA_NCZARR}: - set -x ; for u in ${TESTFILES_NCZARR_SH}; do \ - cp -f ${SRC}/$$u ${DST}/$$u ; \ - chmod a+x ${DST}/$$u ; \ - done - set -x ; for u in ${TESTFILES_NCZARR_C} ; do \ - cp -f ${SRC}/$$u ${DST}/$$u ; \ - done - set -x; for u in ${TESTDATA_NCZARR}; do \ - cp -f ${SRC}/$$u ${DST}/$$u ; \ - done # Remove directories clean-local: + rm -fr tmp*.file +if NETCDF_ENABLE_S3_TESTALL + rm -fr testdir_* testset_* rm -fr tmp_*.nc tmp_*.zarr tst_quantize*.zarr tmp*.file results.file results.s3 results.zip rm -fr rcmiscdir ref_power_901_constants.file - rm -fr v3_*.log v3_*.trs +endif + if NETCDF_ENABLE_S3_TESTALL +check-local: bash ${abs_top_builddir}/s3cleanup.sh endif # If valgrind is present, add valgrind targets. @VALGRIND_CHECK_RULES@ - -################################################## diff --git a/v3_nczarr_test/bm_chunks3.c b/v3_nczarr_test/bm_chunks3.c new file mode 100644 index 0000000000..66d0b64a0d --- /dev/null +++ b/v3_nczarr_test/bm_chunks3.c @@ -0,0 +1,422 @@ +/* This is part of the netCDF package. Copyright 2005-2018 University + Corporation for Atmospheric Research/Unidata See COPYRIGHT file for + conditions of use. + + Runs benchmarks on different chunking sizes. + + Russ Rew, Ed Hartnett, Dennis Heimbigner +*/ + +#include +#include +#include + +#ifdef HAVE_UNISTD_H +#include /* for sysconf */ +#endif +#include + +#include "nc_tests.h" /* The ERR macro is here... */ +#include "netcdf.h" + +#include "bm_utils.h" +#include "timer_utils.h" + +#undef NOREPEAT +#undef NOCONTIG + +/* + * The following timing macros can be used by including the necessary + * declarations with + * + * TIMING_DECLS(seconds) + * + * and surrounding sections of code to be timed with the "statements" + * + * TIMING_START + * [code to be timed goes here] + * TIMING_END(seconds) + * + * The macros assume the user has stored a description of what is + * being timed in a 100-char string time_mess, and has included + * and . The timing message printed by + * TIMING_END is not terminated by a new-line, to permit appending + * additional text to that line, so user must at least printf("\n") + * after that. + */ + +#define TIMING_DECLS(seconds) \ + long TMreps; /* counts repetitions of timed code */ \ + long TMrepeats; /* repetitions needed to exceed 0.1 second */ \ + Nanotime bnano,enano,delta; /* start/stop times in nanoseconds */ \ + char time_mess[100]; \ + double seconds; \ + NCT_inittimer(); + +#ifndef NOREPEAT +#define TIMING_START \ + TMrepeats = 1; \ + do { \ + NCT_marktime(&bnano); \ + for(TMreps=0; TMreps < TMrepeats; TMreps++) { + +#define TIMING_END(time_mess,seconds) \ + } \ + NCT_marktime(&enano); \ + NCT_elapsedtime(&bnano,&enano,&delta); \ + TMrepeats *= 2; \ + } while (NCT_nanoseconds(delta) < 100000000 ); \ + seconds = ((double)NCT_nanoseconds(delta)) / (1000000000.0 * TMreps); \ + printf("%-45.45s %#08.6F sec", time_mess, seconds); + +#else /*NOREPEAT*/ + +#define TIMING_START \ + do { \ +fprintf(stderr,"TIME_START\n"); \ + NCT_marktime(&bnano); \ + { + +#define TIMING_END(time_mess,seconds) \ + } \ + NCT_marktime(&enano); \ + NCT_elapsedtime(&bnano,&enano,&delta); \ + } while (0); \ +fprintf(stderr,"TIME_END\n"); \ + seconds = ((double)NCT_nanoseconds(delta)) / (1000000000.0 * TMreps); \ + printf("%-45.45s %#08.6F sec", time_mess, seconds); + +#endif /*NOREPEAT*/ + + +#define NC_COMPRESSED 1 + +/* This macro prints an error message with line number and name of + * test program. */ +#define ERR1(n) do { \ +fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ +fprintf(stderr, "Sorry! Unexpected result, %s, line: %d - %s\n", \ + __FILE__, __LINE__, nc_strerror(n)); \ +return n; \ +} while (0) + +void * +emalloc(size_t bytes) { + size_t *memory; + memory = malloc(bytes); + if(memory == 0) { + printf("malloc failed\n"); + exit(2); + } + return memory; +} + +typedef enum Tag {tag_contiguous=0, tag_chunked=1, tag_compressed=2} Tag; + +static const char* tagnames[] = {"contiguous", "chunked ", "compressed"}; +static const char* rwnames[] = {"write", "read "}; + +static float test(Tag tag, int read, int ncid, int varid, int rank, int index, size_t* dims, size_t* chunks, unsigned* data); +static void testtrio(int read, int ncid, int index, unsigned* data, const char* slab); + +/* Globals */ +static int ncid; /* netCDF id */ +static int dim1id, dim2id, dim3id; +static int varid_g; /* varid for contiguous */ +static int varid_k; /* varid for chunked */ +static int varid_x; /* varid for compressed */ +static unsigned *varxy, *varxz, *varyz; /* 2D memory slabs used for I/O */ +static size_t szyz=0, szxz=0, szxy=0; + +static const char* +dataname(void* data) +{ + if(data == varyz) return "yz"; + if(data == varxz) return "xz"; + if(data == varxy) return "xy"; + return "?"; +} + +/* compare contiguous, chunked, and compressed performance */ +int +main(int argc, char *argv[]) +{ + int stat; /* return status */ + int i, j, k; + int mm; + int deflate_level = 0; /* default compression level, 9 is + * better and slower. If negative, + * turn on shuffle filter also. */ + int shuffle = NC_NOSHUFFLE; + size_t cache_size_def; + size_t cache_hash_def; + float cache_pre_def; + size_t cache_size = 0; /* use library default */ + size_t cache_hash = 0; /* use library default */ + float cache_pre = -1.0f; /* use library default */ + char* path = NULL; + int fill = -1; + + size_t* dims = NULL; /* alias */ + size_t* chunks = NULL; /* alias */ + + + /* rank (number of dimensions) for each variable */ +# define RANK 3 + + /* variable shapes */ + int var_dims[RANK]; + + NCCHECK(bm_getoptions(&argc,&argv,&bmoptions)); + NCCHECK(nc4_buildpath(&bmoptions,&path)); + + if(bmoptions.debug) { + reportoptions(&bmoptions); + reportmetaoptions(&bmoptions.meta); + } + + if(bmoptions.meta.dims.count != RANK) + ERR1(NC_EINVAL); + if(bmoptions.meta.dims.count != bmoptions.meta.chunks.count) + ERR1(NC_EINVAL); + dims = bmoptions.meta.dims.list; + chunks = bmoptions.meta.chunks.list; + + if(bmoptions.meta.cachesize > 0) { + /* get cache defaults, then set cache parameters that are not default */ + if((stat = nc_get_chunk_cache(&cache_size_def, &cache_hash_def, + &cache_pre_def))) + ERR1(stat); + if(cache_size == 0) + cache_size = cache_size_def; + if(cache_hash == 0) + cache_hash = cache_hash_def; + if(cache_pre == -1.0f) + cache_pre = cache_pre_def; + if((stat = nc_set_chunk_cache(cache_size, cache_hash, cache_pre))) + ERR1(stat); + printf("cache: %3.2f MBytes %ld objs %3.2f preempt, ", + cache_size/1.e6, cache_hash, cache_pre); + } + + if(bmoptions.debug) { + if(deflate_level == 0) { + printf("compression level: uncompressed"); + } else { + printf("compression level: %d ", deflate_level); + } + } + if(shuffle == 1) { + printf(", shuffled"); + } + printf("\n\n"); + + /* initialize 2D slabs for writing along each axis with phony data */ + szyz = 1 * dims[1] * dims[2]; + szxz = dims[0] * 1 * dims[2]; + szxy = dims[0] * dims[1] * 1; + if(bmoptions.debug) + fprintf(stderr,"|yz|=%d |xz|=%d |xy|=%d\n",(int)szyz,(int)szxz,(int)szxy); + varyz = (unsigned *) emalloc(sizeof(unsigned) * szyz); + varxz = (unsigned *) emalloc(sizeof(unsigned) * szxz); + varxy = (unsigned *) emalloc(sizeof(unsigned) * szxy); + + mm = 0; + for(j = 0; j < dims[1]; j++) { + for(k = 0; k < dims[2]; k++) { + varyz[mm++] = k + dims[2]*j; + } + } + mm = 0; + for(i = 0; i < dims[0]; i++) { + for(k = 0; k < dims[2]; k++) { + varxz[mm++] = k + dims[2]*i; + } + } + mm = 0; + for(i = 0; i < dims[0]; i++) { + for(j = 0; j < dims[1]; j++) { + varxy[mm++] = j + dims[1]*i; + } + } + + if((stat = nc_create(path, NC_NETCDF4, &ncid))) + ERR1(stat); + /* define dimensions */ + if((stat = nc_def_dim(ncid, "dim1", dims[0], &dim1id))) + ERR1(stat); + if((stat = nc_def_dim(ncid, "dim2", dims[1], &dim2id))) + ERR1(stat); + if((stat = nc_def_dim(ncid, "dim3", dims[2], &dim3id))) + ERR1(stat); + + /* define variables */ + var_dims[0] = dim1id; + var_dims[1] = dim2id; + var_dims[2] = dim3id; + if((stat = nc_def_var(ncid, "var_contiguous", NC_INT, RANK, + var_dims, &varid_g))) + ERR1(stat); + if((stat = nc_def_var(ncid, "var_chunked", NC_INT, RANK, + var_dims, &varid_k))) + ERR1(stat); + if((stat = nc_def_var(ncid, "var_compressed", NC_INT, RANK, + var_dims, &varid_x))) + ERR1(stat); + + if((stat = nc_def_var_fill(ncid, varid_g, NC_FILL, &fill))) + ERR1(stat); + + if((stat = nc_def_var_fill(ncid, varid_k, NC_FILL, &fill))) + ERR1(stat); + + if((stat = nc_def_var_fill(ncid, varid_x, NC_FILL, &fill))) + ERR1(stat); + +#ifndef NOCONTIG + if((stat = nc_def_var_chunking(ncid, varid_g, NC_CONTIGUOUS, 0))) + ERR1(stat); +#else + if((stat = nc_def_var_chunking(ncid, varid_g, NC_CHUNKED, dims))) + ERR1(stat); +#endif + + if((stat = nc_def_var_chunking(ncid, varid_k, NC_CHUNKED, chunks))) + ERR1(stat); + + if((stat = nc_def_var_chunking(ncid, varid_x, NC_CHUNKED, chunks))) + ERR1(stat); + + if (bmoptions.meta.deflatelevel != 0) { + if((stat = nc_def_var_deflate(ncid, varid_x, shuffle, NC_COMPRESSED, bmoptions.meta.deflatelevel))) + ERR1(stat); + } + + /* leave define mode */ + if((stat = nc_enddef (ncid))) + ERR1(stat); + + printf("Parameters: path=%s format=%s\n",path,formatname(&bmoptions)); + printf(" mode | R/W | dims | chunked | time | speedup |\n"); + printf("--------------------------------------------------------------------------\n"); + + /* write each variable one yz slab at a time */ + testtrio(0,ncid,0,varyz,"yz"); + + /* write each variable one xz slab at a time */ + testtrio(0,ncid,1,varxz,"xz"); + + /* write each variable one xy slab at a time */ + testtrio(0,ncid,2,varxy,"xy"); + + /* read each variable one yz slab at a time */ + testtrio(1,ncid,0,varyz,"yz"); + + /* read each variable one xz slab at a time */ + testtrio(1,ncid,1,varxz,"xz"); + + /* read each variable one xy slab at a time */ + testtrio(1,ncid,2,varxy,"xy"); + + if((stat = nc_close(ncid))) + ERR1(stat); + + nullfree(path); + clearoptions(&bmoptions); + + return 0; +} + +static float +test(Tag tag, int reading, int ncid, int varid, int rank, int index, size_t* dims, size_t* chunks, unsigned* data) +{ + int stat = NC_NOERR; + int i; + size_t start[NC_MAX_VAR_DIMS]; + size_t count[NC_MAX_VAR_DIMS]; + float time; + TIMING_DECLS(TMsec) ; + + /* do each variable one slab at a time */ + start[0] = 0; + start[1] = 0; + start[2] = 0; + count[0] = dims[0]; + count[1] = dims[1]; + count[2] = dims[2]; + + count[index] = 1; + + if(chunks != NULL) + snprintf(time_mess, sizeof(time_mess),"%s %s %3ld %3ld %3ld %3ld %3ld %3ld", tagnames[tag], rwnames[reading], + count[0], count[1], count[2], chunks[0], chunks[1], chunks[2]); + else + snprintf(time_mess, sizeof(time_mess),"%s %s %3ld %3ld %3ld", tagnames[tag], rwnames[reading], + count[0], count[1], count[2]); + + TIMING_START ; + for(i = 0; i < dims[index]; i++) { + start[index] = i; +#if 0 +if(reading){ + int k,x; + size_t prod = 1; + size_t st = 0; + for(k=0;k= 1.0) printf(" %5.2g x faster\n", ratio); else printf(" %5.2g x slower\n", 1.0/ratio); + if(bmoptions.debug) + fprintf(stderr,"testtrio: case=%s done\n","chunked"); + + if (bmoptions.meta.deflatelevel != 0) { + compressed_time = test(tag_compressed, reading, ncid, varid_x, RANK, index, dims, chunks, data); + ratio = contig_time/compressed_time; + if(ratio >= 1.0) printf(" %5.2g x faster\n", ratio); else printf(" %5.2g x slower\n", 1.0/ratio); + } +} + diff --git a/v3_nczarr_test/ncdumpchunks.c b/v3_nczarr_test/ncdumpchunks.c new file mode 100644 index 0000000000..5b4cd80e42 --- /dev/null +++ b/v3_nczarr_test/ncdumpchunks.c @@ -0,0 +1,607 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#ifdef HAVE_UNISTD_H +#include +#endif + +#ifdef HAVE_GETOPT_H +#include +#endif + +#if defined(_WIN32) && !defined(__MINGW32__) +#include "XGetopt.h" +#endif + +#include "netcdf.h" +#include "ncpathmgr.h" +#include "nclog.h" + +#ifdef USE_HDF5 +#include +#include +#endif + +#ifdef NETCDF_ENABLE_NCZARR +#include "zincludes.h" +#endif + +#undef DEBUG + +/* Short Aliases */ +#ifdef USE_HDF5 +#define H5 +#endif +#ifdef NETCDF_ENABLE_NCZARR +#define NZ +#endif + +typedef struct Format { + int format; + char file_name[NC_MAX_NAME]; + char var_name[NC_MAX_NAME]; + int fillvalue; + int debug; + int linear; + int holevalue; + size_t rank; + size_t dimlens[NC_MAX_VAR_DIMS]; + size_t chunklens[NC_MAX_VAR_DIMS]; + size_t chunkcounts[NC_MAX_VAR_DIMS]; + size_t chunkprod; + size_t dimprod; + nc_type xtype; +} Format; + +typedef struct Odometer { + size_t rank; /*rank */ + size_t start[NC_MAX_VAR_DIMS]; + size_t stop[NC_MAX_VAR_DIMS]; + size_t max[NC_MAX_VAR_DIMS]; /* max size of ith index */ + size_t index[NC_MAX_VAR_DIMS]; /* current value of the odometer*/ +} Odometer; + +#define floordiv(x,y) ((x) / (y)) +#define ceildiv(x,y) (((x) % (y)) == 0 ? ((x) / (y)) : (((x) / (y)) + 1)) + +static char* captured[4096]; +static size_t ncap = 0; + +extern int nc__testurl(const char*,char**); + +Odometer* odom_new(size_t rank, const size_t* stop, const size_t* max); +void odom_free(Odometer* odom); +int odom_more(Odometer* odom); +int odom_next(Odometer* odom); +size_t* odom_indices(Odometer* odom); +size_t odom_offset(Odometer* odom); +const char* odom_print(Odometer* odom); + +static void +usage(int err) +{ + if(err != 0) { + fprintf(stderr,"Error: (%d) %s\n",err,nc_strerror(err)); + } + fprintf(stderr,"usage: ncdumpchunks [-b] -v \n"); + fflush(stderr); + exit(1); +} + + +const char* +printvector(size_t rank, size_t* vec) +{ + char svec[NC_MAX_VAR_DIMS*3+1]; + size_t i; + svec[0] = '\0'; + for(i=0;i 0) strlcat(svec,",",sizeof(svec)); + snprintf(s,sizeof(s),"%u",(unsigned)vec[i]); + strlcat(svec,s,sizeof(svec)); + } + captured[ncap++] = strdup(svec); + return captured[ncap-1]; +} + +void +cleanup(void) +{ + size_t i; + for(i=0;irank = rank; + for(i=0;istart[i] = 0; + odom->stop[i] = stop[i]; + odom->max[i] = max[i]; + odom->index[i] = 0; + } + return odom; +} + +void +odom_free(Odometer* odom) +{ + if(odom) free(odom); +} + +int +odom_more(Odometer* odom) +{ + return (odom->index[0] < odom->stop[0]); +} + +int +odom_next(Odometer* odom) +{ + size_t i; + for(i=odom->rank-1;i>=0;i--) { + odom->index[i]++; + if(odom->index[i] < odom->stop[i]) break; + if(i == 0) return 0; /* leave the 0th entry if it overflows */ + odom->index[i] = 0; /* reset this position */ + } + return 1; +} + +/* Get the value of the odometer */ +size_t* +odom_indices(Odometer* odom) +{ + return odom->index; +} + +size_t +odom_offset(Odometer* odom) +{ + size_t i,offset; + + offset = 0; + for(i=0;irank;i++) { + offset *= odom->max[i]; + offset += odom->index[i]; + } + return offset; +} + +const char* +odom_print(Odometer* odom) +{ + static char s[4096]; + static char tmp[4096]; + const char* sv; + + s[0] = '\0'; + snprintf(tmp,sizeof(tmp),"{rank=%u",(unsigned)odom->rank); + strcat(s,tmp); + strcat(s," start=("); sv = printvector(odom->rank,odom->start); strcat(s,sv); strcat(s,")"); + strcat(s," stop=("); sv = printvector(odom->rank,odom->stop); strcat(s,sv); strcat(s,")"); + strcat(s," max=("); sv = printvector(odom->rank,odom->max); strcat(s,sv); strcat(s,")"); + snprintf(tmp,sizeof(tmp)," offset=%u",(unsigned)odom_offset(odom)); strcat(s,tmp); + strcat(s," indices=("); sv = printvector(odom->rank,odom->index); strcat(s,sv); strcat(s,")"); + strcat(s,"}"); + return s; +} + +#ifdef DEBUG +char* +chunk_key(size_t formatrank, size_t* indices) +{ + char key[NC_MAX_VAR_DIMS*3+1]; + size_t i; + key[0] = '\0'; + for(i=0;i 0) strlcat(key,".",sizeof(key)); + snprintf(s,sizeof(s),"%u",(unsigned)indices[i]); + strlcat(key,s,sizeof(key)); + } + return strdup(key); +} +#endif + +void +setoffset(Odometer* odom, size_t* chunksizes, size_t* offset) +{ + size_t i; + for(i=0;irank;i++) + offset[i] = odom->index[i] * chunksizes[i]; +} + +static void +printindent(size_t indent) +{ + while(indent-- > 0) printf(" "); +} + +int +printchunklinear(Format* format, int* chunkdata, size_t indent) +{ + size_t i; + NC_UNUSED(indent); + for(i=0;ichunkprod;i++) { + if(chunkdata[i] == format->fillvalue) + printf(" _"); + else + printf(" %02d", chunkdata[i]); + } + printf("\n"); + return NC_NOERR; +} + +static void +printchunk2d(Format* format, int* chunkdata, size_t indent) +{ + size_t pos; + size_t row; + pos = 0; + for(row=0;rowchunklens[0];row++) { + size_t col; + if(row > 0) printindent(indent); + for(col=0;colchunklens[1];col++,pos++) { + if(chunkdata[pos] == format->fillvalue) printf(" _"); else printf(" %02d", chunkdata[pos]); + } + printf("\n"); + } +} + +static void +printchunk(Format* format, int* chunkdata, size_t indent) +{ + size_t k[3]; + size_t rank = format->rank; + size_t cols[3], pos; + size_t* chl = format->chunklens; + + memset(cols,0,sizeof(cols)); + + if(format->xtype == NC_UBYTE) rank = 0; + switch (rank) { + case 1: + cols[0] = 1; + cols[1] = 1; + cols[2] = chl[0]; + break; + case 2: + cols[0] = 1; + cols[1] = chl[0]; + cols[2] = chl[1]; + break; + case 3: + cols[0] = chl[0]; + cols[1] = chl[1]; + cols[2] = chl[2]; + break; + default: + cols[0] = 1; + cols[1] = 1; + cols[2] = format->chunkprod; + break; + } +// offset = (((k0*chl[0])+k1)*chl[1])+k2; + pos = 0; + for(k[0]=0;k[0] 0) printindent(indent); + k[1] = 0; k[2] = 0; /* reset */ + for(k[1]=0;k[1] 0) printf(" |"); + for(k[2]=0;k[2]xtype == NC_UBYTE) { + size_t l; + unsigned char* bchunkdata = (unsigned char*)(&chunkdata[pos]); + for(l=0;lfillvalue) + printf(" _"); + else + printf(" %02u", bchunkdata[l]); + } + } else { + if(chunkdata[pos] == format->fillvalue) + printf(" _"); + else + printf(" %02d", chunkdata[pos]); + } + pos++; + } + } + printf("\n"); + } + +#if 0 + for(k=0;kchunkprod;k++) { + if(k > 0 && k % cols == 0) printf(" |"); + printf(" %02d", chunkdata[k]); + } + printf("\n"); +#endif +} + + +int +dump(Format* format) +{ + void* chunkdata = NULL; /*[CHUNKPROD];*/ + Odometer* odom = NULL; + size_t r; + size_t offset[NC_MAX_VAR_DIMS]; + int holechunk = 0; + char sindices[64]; +#ifdef H5 + size_t i; + hid_t fileid, grpid, datasetid; + hid_t dxpl_id = H5P_DEFAULT; /*data transfer property list */ + unsigned int filter_mask = 0; + hsize_t hoffset[NC_MAX_VAR_DIMS]; +#endif +#ifdef NZ + int stat = NC_NOERR; + size64_t zindices[NC_MAX_VAR_DIMS]; + int ncid, varid; +#endif + +#ifdef H5 + if(format->debug) { + H5Eset_auto2(H5E_DEFAULT,(H5E_auto2_t)H5Eprint1,stderr); + } +#endif + + memset(offset,0,sizeof(offset)); +#ifdef H5 + memset(hoffset,0,sizeof(hoffset)); +#endif + + switch (format->format) { +#ifdef H5 + case NC_FORMATX_NC_HDF5: + if ((fileid = H5Fopen(format->file_name, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) usage(NC_EHDFERR); + if ((grpid = H5Gopen1(fileid, "/")) < 0) usage(NC_EHDFERR); + if ((datasetid = H5Dopen1(grpid, format->var_name)) < 0) usage(NC_EHDFERR); + break; +#endif +#ifdef NZ + case NC_FORMATX_NCZARR: + if((stat=nc_open(format->file_name,0,&ncid))) usage(stat); + if((stat=nc_inq_varid(ncid,format->var_name,&varid))) usage(stat); + break; +#endif + default: usage(NC_EINVAL); + } + + if((odom = odom_new(format->rank,format->chunkcounts,format->dimlens))==NULL) usage(NC_ENOMEM); + + if((chunkdata = calloc(sizeof(int),format->chunkprod))==NULL) usage(NC_ENOMEM); + + printf("rank=%zu dims=(%s) chunks=(%s)\n",format->rank,printvector(format->rank,format->dimlens), + printvector(format->rank,format->chunklens)); + + while(odom_more(odom)) { + setoffset(odom,format->chunklens,offset); + +#ifdef DEBUG + fprintf(stderr,"odom=%s\n",odom_print(odom)); + fprintf(stderr,"offset=("); + for(i=0;irank;i++) + fprintf(stderr,"%s%lu",(i > 0 ? "," : ""),(unsigned long)offset[i]); + fprintf(stderr,")\n"); + fflush(stderr); +#endif + + if(format->debug) { + fprintf(stderr,"chunk: %s\n",printvector(format->rank,offset)); + } + + holechunk = 0; + switch (format->format) { +#ifdef H5 + case NC_FORMATX_NC_HDF5: { + for(i=0;irank;i++) hoffset[i] = (hsize_t)offset[i]; + if(H5Dread_chunk(datasetid, dxpl_id, hoffset, &filter_mask, chunkdata) < 0) + holechunk = 1; + } break; +#endif +#ifdef NZ + case NC_FORMATX_NCZARR: + for(r=0;rrank;r++) zindices[r] = (size64_t)odom->index[r]; + switch (stat=NCZ_read_chunk(ncid, varid, zindices, chunkdata)) { + case NC_NOERR: break; + case NC_ENOOBJECT: holechunk = 1; break; + default: usage(stat); + } + break; +#endif + default: usage(NC_EINVAL); + } + if(holechunk) { + /* Hole chunk: use holevalue */ + size_t i = 0; + int* idata = (int*)chunkdata; + for(i=0;ichunkprod;i++) + idata[i] = format->holevalue; + } + sindices[0] = '\0'; + for(r=0;rrank;r++) { + char sstep[64]; + snprintf(sstep,sizeof(sstep),"[%lu/%lu]",(unsigned long)odom->index[r],(unsigned long)offset[r]); + strcat(sindices,sstep); + } + strcat(sindices," ="); + printf("%s",sindices); + if(format->linear) + printchunklinear(format,chunkdata,strlen(sindices)); + else if(format->rank == 2) + printchunk2d(format,chunkdata,strlen(sindices)); + else + printchunk(format,chunkdata,strlen(sindices)); + fflush(stdout); + odom_next(odom); + } + + /* Close up. */ + switch (format->format) { +#ifdef H5 + case NC_FORMATX_NC_HDF5: + if (H5Dclose(datasetid) < 0) abort(); + if (H5Gclose(grpid) < 0) abort(); + if (H5Fclose(fileid) < 0) abort(); + break; +#endif +#ifdef NZ + case NC_FORMATX_NCZARR: + if((stat=nc_close(ncid))) usage(stat); + break; +#endif + default: usage(NC_EINVAL); + } + /* Cleanup */ + free(chunkdata); + odom_free(odom); + return 0; +} + +static const char* urlexts[] = {"file", "zip", NULL}; + +static const char* +filenamefor(const char* f0) +{ + static char result[4096]; + const char** extp; + char* p; + + strcpy(result,f0); /* default */ + if(nc__testurl(f0,NULL)) goto done; + /* Not a URL */ + p = strrchr(f0,'.'); /* look at the extension, if any */ + if(p == NULL) goto done; /* No extension */ + p++; + for(extp=urlexts;*extp;extp++) { + if(strcmp(p,*extp)==0) break; + } + if(*extp == NULL) goto done; /* not found */ + /* Assemble the url */ + strcpy(result,"file://"); + strcat(result,f0); /* core path */ + strcat(result,"#mode=nczarr,"); + strcat(result,*extp); +done: + return result; +} + +int +main(int argc, char** argv) +{ + int stat = NC_NOERR; + size_t i; + Format format; + int ncid, varid, dimids[NC_MAX_VAR_DIMS]; + int vtype, storage; + int mode; + int c; + + memset(&format,0,sizeof(format)); + + /* Init some format fields */ + format.xtype = NC_INT; + format.holevalue = -2; + + while ((c = getopt(argc, argv, "bhv:DT:L")) != EOF) { + switch(c) { + case 'b': + format.xtype = NC_UBYTE; + break; + case 'h': + usage(0); + break; + case 'v': + strcpy(format.var_name,optarg); + break; + case 'D': + format.debug = 1; + break; + case 'L': + format.linear = 1; + break; + case 'T': + nctracelevel(atoi(optarg)); + break; + case '?': + fprintf(stderr,"unknown option: '%c'\n",c); + exit(1); + } + } + + /* get file argument */ + argc -= optind; + argv += optind; + + if (argc == 0) { + fprintf(stderr, "no input file specified\n"); + exit(1); + } + + { + char* s = NC_shellUnescape(argv[0]); + strcpy(format.file_name,filenamefor(s)); + nullfree(s); + } + + if(strlen(format.file_name) == 0) { + fprintf(stderr, "no input file specified\n"); + exit(1); + } + + if(strlen(format.var_name) == 0) { + fprintf(stderr, "no input var specified\n"); + exit(1); + } + + /* Get info about the file type */ + if((stat=nc_open(format.file_name,0,&ncid))) usage(stat); + + if((stat=nc_inq_format_extended(ncid,&format.format,&mode))) usage(stat); + + /* Get the info about the var */ + if((stat=nc_inq_varid(ncid,format.var_name,&varid))) usage(stat); + { int frank = (int)format.rank; + if((stat=nc_inq_var(ncid,varid,NULL,&vtype,&frank,dimids,NULL))) usage(stat); + format.rank = (size_t)frank; + } + if(format.rank == 0) usage(NC_EDIMSIZE); + if((stat=nc_inq_var_chunking(ncid,varid,&storage,format.chunklens))) usage(stat); + if(storage != NC_CHUNKED) usage(NC_EBADCHUNK); + if((stat=nc_get_att(ncid,varid,NC_FillValue,&format.fillvalue))) usage(stat); + + for(i=0;iIv_Y209^i_c!GQw((W1{9~?S&SOeqCi>HkkhM5FUl`10K0mX zmyTAK|7rbGzB+-Mf(#zPf`zx^?K4TBPI0i=j7(e%xZ@G11PP$p1&LLJe)NC^#VZ0B z0GX)zp|Q-!B*K7*XXH2o#S8)@0-0D0fW~@&H^Ll{Ly!Xy)tvP>%>l(XvN@pOMh+}c ea3eq#Bf5K_fgj+_3QBbhtUwq7RCEl885jWL(!y5& literal 0 HcmV?d00001 diff --git a/v3_nczarr_test/ref_byte_fill_value_null.cdl b/v3_nczarr_test/ref_byte_fill_value_null.cdl new file mode 100644 index 0000000000..4841e6c127 --- /dev/null +++ b/v3_nczarr_test/ref_byte_fill_value_null.cdl @@ -0,0 +1,35 @@ +netcdf ref_byte_fill_value_null { +dimensions: + _Anonymous_Dim_20 = 20 ; +variables: + ubyte byt(_Anonymous_Dim_20, _Anonymous_Dim_20) ; + byt:_Storage = "chunked" ; + byt:_ChunkSizes = 20, 20 ; + byt:_NoFill = "true" ; + +// global attributes: + :_Format = "netCDF-4" ; +data: + + byt = + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ; +} diff --git a/v3_nczarr_test/ref_byte_fill_value_null.zarr.zip b/v3_nczarr_test/ref_byte_fill_value_null.zarr.zip new file mode 100644 index 0000000000000000000000000000000000000000..a532016ba4b423489447cdbb415c812ab6dabe41 GIT binary patch literal 1676 zcmWIWW@h1H00HNE$zU)8O2{$DFchVx#V1vkq{gRZ=H$eeCFYc-#^;shjg?VQL6WOJ1WnhUfMVs_lp+o{b!jKypb zpphJ4Hy_~omzoOXwF9vxQD%c(pI8ZZ{v%B1`*!>CIT#4ItykJSk2SVyZgPi*vvksm zGh2j?++Epmqt^bt?1t=T$E@aUX}1aFxb*tyosB`7tG2kk`16e?poK@^mfe}fe;BO= zOjj=xSZOJsXz`B6eR)sYY%w#XzWx8h_9RyRu4Dv-5yBt*y`7yLKtD`iWMEJw${z-L z2KfB&`+vO%j~W95!Wv~@RG9UMr5FOOaU>|Xnn3G60BvfJfh-YEoh z4+sA9ZJ%ZBt$>J9aU4WK>FfkS;!UF)5 z+_9vWYNdET=YXR8g4Cjt%+z9_l}cr)Ma7x)Ic7wt0Nn-$atsnswgAHqP}C93nyiqliIH^>c3@^q zWII547Jv4KWK*C$plpiO9z^0s&7=@}fa#oJL*o{r?15)leBMG>hnkm=t*ZgrO4wWA zoJH6w%>0CG)nlTp0{RS+xPaDyau?QsL?jB-tb%M^ITInD!7~m%pCPP5%_PWHy&%FW cgs(tZ2E$jZY`}PDUpF literal 0 HcmV?d00001 diff --git a/v3_nczarr_test/ref_cmip6.zmap b/v3_nczarr_test/ref_cmip6.zmap new file mode 100644 index 0000000000..98d96fcf0c --- /dev/null +++ b/v3_nczarr_test/ref_cmip6.zmap @@ -0,0 +1,555 @@ +[0] /.zattrs : (3182) |{"Conventions": "CF-1.7 CMIP-6.2", "activity_id": "CMIP", "branch_method": "standard", "branch_time_in_child": 0.0, "branch_time_in_parent": 0.0, "cmor_version": "3.3.2", "contact": "Kenneth Lo (cdkkl@giss.nasa.gov)", "coordinates": "lat_bnds lon_bnds time_bnds", "creation_date": "2018-11-20T18:45:56Z", "data_specs_version": "01.00.23", "experiment": "all-forcing simulation of the recent past", "experiment_id": "historical", "external_variables": "areacella", "forcing_index": 1, "frequency": "day", "further_info_url": "https://furtherinfo.es-doc.org/CMIP6.NASA-GISS.GISS-E2-1-G.historical.none.r1i1p1f1", "grid": "atmospheric grid: 144x90, ocean grid: 288x180", "grid_label": "gn", "history": "2018-11-20T18:45:56Z ; CMOR rewrote data to be consistent with CMIP6, CF-1.7 CMIP-6.2 and CF standards.", "initialization_index": 1, "institution": "Goddard Institute for Space Studies, New York, NY 10025, USA", "institution_id": "NASA-GISS", "license": "CMIP6 model data produced by NASA Goddard Institute for Space Studies is licensed under a Creative Commons Attribution ShareAlike 4.0 International License (https://creativecommons.org/licenses). Consult https://pcmdi.llnl.gov/CMIP6/TermsOfUse for terms of use governing CMIP6 output, including citation requirements and proper acknowledgment. Further information about this data, including some limitations, can be found via the further_info_url (recorded as a global attribute in this file) and at https:///pcmdi.llnl.gov/. The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law.", "mip_era": "CMIP6", "model_id": "E200f10aF40oQ40", "nominal_resolution": "250 km", "parent_activity_id": "CMIP", "parent_experiment_id": "piControl", "parent_experiment_rip": "r1i1p1", "parent_mip_era": "CMIP6", "parent_source_id": "GISS-E2-1-G", "parent_time_units": "days since 4150-1-1", "parent_variant_label": "r1i1p1f1", "physics_index": 1, "product": "model-output", "realization_index": 1, "realm": "atmos", "references": "https://data.giss.nasa.gov/modelE/cmip6", "source": "GISS-E2.1G (2016): \naerosol: Varies with physics-version (p==1 none, p==3 OMA, p==4 TOMAS, p==5 MATRIX)\natmos: GISS-E2.1 (2.5x2 degree; 144 x 90 longitude/latitude; 40 levels; top level 0.1 hPa)\natmosChem: Varies with physics-version (p==1 Non-interactive, p>1 GPUCCINI)\nland: GISS LSM\nlandIce: none\nocean: GISS Ocean (1.25x1 degree; 288 x 180 longitude/latitude; 32 levels; top grid cell 0-10 m)\nocnBgchem: none\nseaIce: GISS SI", "source_id": "GISS-E2-1-G", "source_type": "AOGCM", "sub_experiment": "none", "sub_experiment_id": "none", "table_id": "day", "table_info": "Creation Date:(21 March 2018) MD5:f76dbc1e8bf6b7e4aee30573a09e5454", "title": "GISS-E2-1-G output prepared for CMIP6", "tracking_id": "hdl:21.14100/364b8aea-de19-4e29-ac36-0b26406e9a29", "variable_id": "tasmin", "variant_label": "r1i1p1f1", "status": "2019-10-25;created;by nhn2@columbia.edu"}| +[1] /.zgroup : (24) |{ + "zarr_format": 2 +}| +[2] /.zmetadata : (10410) |{ + "metadata": { + ".zattrs": { + "Conventions": "CF-1.7 CMIP-6.2", + "activity_id": "CMIP", + "branch_method": "standard", + "branch_time_in_child": 0.0, + "branch_time_in_parent": 0.0, + "cmor_version": "3.3.2", + "contact": "Kenneth Lo (cdkkl@giss.nasa.gov)", + "coordinates": "lat_bnds lon_bnds time_bnds", + "creation_date": "2018-11-20T18:45:56Z", + "data_specs_version": "01.00.23", + "experiment": "all-forcing simulation of the recent past", + "experiment_id": "historical", + "external_variables": "areacella", + "forcing_index": 1, + "frequency": "day", + "further_info_url": "https://furtherinfo.es-doc.org/CMIP6.NASA-GISS.GISS-E2-1-G.historical.none.r1i1p1f1", + "grid": "atmospheric grid: 144x90, ocean grid: 288x180", + "grid_label": "gn", + "history": "2018-11-20T18:45:56Z ; CMOR rewrote data to be consistent with CMIP6, CF-1.7 CMIP-6.2 and CF standards.", + "initialization_index": 1, + "institution": "Goddard Institute for Space Studies, New York, NY 10025, USA", + "institution_id": "NASA-GISS", + "license": "CMIP6 model data produced by NASA Goddard Institute for Space Studies is licensed under a Creative Commons Attribution ShareAlike 4.0 International License (https://creativecommons.org/licenses). Consult https://pcmdi.llnl.gov/CMIP6/TermsOfUse for terms of use governing CMIP6 output, including citation requirements and proper acknowledgment. Further information about this data, including some limitations, can be found via the further_info_url (recorded as a global attribute in this file) and at https:///pcmdi.llnl.gov/. The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law.", + "mip_era": "CMIP6", + "model_id": "E200f10aF40oQ40", + "nominal_resolution": "250 km", + "parent_activity_id": "CMIP", + "parent_experiment_id": "piControl", + "parent_experiment_rip": "r1i1p1", + "parent_mip_era": "CMIP6", + "parent_source_id": "GISS-E2-1-G", + "parent_time_units": "days since 4150-1-1", + "parent_variant_label": "r1i1p1f1", + "physics_index": 1, + "product": "model-output", + "realization_index": 1, + "realm": "atmos", + "references": "https://data.giss.nasa.gov/modelE/cmip6", + "source": "GISS-E2.1G (2016): \naerosol: Varies with physics-version (p==1 none, p==3 OMA, p==4 TOMAS, p==5 MATRIX)\natmos: GISS-E2.1 (2.5x2 degree; 144 x 90 longitude/latitude; 40 levels; top level 0.1 hPa)\natmosChem: Varies with physics-version (p==1 Non-interactive, p>1 GPUCCINI)\nland: GISS LSM\nlandIce: none\nocean: GISS Ocean (1.25x1 degree; 288 x 180 longitude/latitude; 32 levels; top grid cell 0-10 m)\nocnBgchem: none\nseaIce: GISS SI", + "source_id": "GISS-E2-1-G", + "source_type": "AOGCM", + "sub_experiment": "none", + "sub_experiment_id": "none", + "table_id": "day", + "table_info": "Creation Date:(21 March 2018) MD5:f76dbc1e8bf6b7e4aee30573a09e5454", + "title": "GISS-E2-1-G output prepared for CMIP6", + "tracking_id": "hdl:21.14100/364b8aea-de19-4e29-ac36-0b26406e9a29", + "variable_id": "tasmin", + "variant_label": "r1i1p1f1", + "status": "2019-10-25;created;by nhn2@columbia.edu", + "netcdf_tracking_ids": "hdl:21.14100/364b8aea-de19-4e29-ac36-0b26406e9a29", + "version_id": "v20181015" + }, + ".zgroup": { + "zarr_format": 2 + }, + "height/.zarray": { + "chunks": [], + "compressor": null, + "dtype": "2upe6j#!uP1A#`+?NnH6lYgwDCJBb+=Q#ZFpNjBJdzx+nSS9D|1N(4 z?8+-emSssRCu-b0&B$x*(?0F~-g|rec0H@pY`J~kb?mkoSncN6^t`6rz}pSEIT!_9 zx8I;yTSD~pIxK4zJ;{zdt; z%n?U#z;Ayt04NaubvQtE`-js4b5x>|)c(4zCdd6~9zaD%!`k%tFDo+A1QC_cz3?k% zeEk1wQzVzv>JIyNeCdYxYxqp_^y~3otc8Dno8(9>w>R)C-*-JyV4@&Ya^SeFJH9=( zNH#huEwvo$&~iwomdxy%J^G^Lj8!R-Z?|bCQRTOi*?xD}={U3i4(oE4)ukPb2BDgp zcBDq?*pAaYG@W6hA~Z9wJle$m&~aF4*K1p}5z-F(mgINMP+u7w!l`n{^?GJNbjX`` z7k0+_6;zLZ)F>ODG{*DdpA7e@_?I!ne?7jix`NlbD}q4pzsV=Bk^6UY|FE%~meeLx(-n{-+V+&zf zQ8nGze*4BdH{X5l{SQ9;XlM81Pxe0j?DH?a{Oap(_P_n^`yYP1_0!M4ge_Ya{@oA% zz|XxzfIt2l+DE^U>TK~&flga$QZ2!VhmDyKe{HNpp{VYzvo;p~;O=LHsL7 z)zk5>lkGwn|GAOoFTmnIvMuQVCOILYM<`y};ZJ3Tf?dmRdG;W%-9GIyb`l%NBArTz zf#nBM%pWqcj_C}SkFz?{QxN(#y^@8=CS*>X3c9d~x(tPjRcNv+#^%PAn>EoA?X56w zeO6lrPa8EW!PiDvjJJ&*8Gjo+&f{@o`5ZnsmS^$0N5X7=HE#`DsQ<#*Yd2=jIp8gUz&j02T3Au;CisGM~_DaRSs_L-+bF6VbPuc2< zvhgPo()*vpOXN6Mc&)PZ981?&y3W$`EWN_WT!;63oxIzy_%FGk uyPR{}lSdK000;mAfB+x>2mk_r03ZMe00MvjAOHve0)PM@00^8;1pWt-DU6^1 literal 0 HcmV?d00001 diff --git a/v3_nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr.tgz b/v3_nczarr_test/ref_consolidated_zarr_2.18.2_python.zarr.tgz new file mode 100644 index 0000000000000000000000000000000000000000..1afaa7e44e296a85fd5b6d748a49e6aeb594288e GIT binary patch literal 1144 zcmV-;1c&<{iwFqW$aQA`19D|%Ut@1>b8l>EWMOn=WM6t=a&liXE-^SRGGB0cbZBpG zE_z{daxQdXascg`X>*%K7{`%gCysNr&C#Yu2yJsJG7AFSByG~>?3Frc&dFpnSV9pA z5*k2GC6C|aRlaUMLA%feBR~?1o%PsyejetrcG+Fv+5fWpu#VBJ)$NXJTV_M|j7Dvs zJ5Eiiprwjb>-N2t-KqFVi(GL;jHW5RMVcbUZ3u^|nk=g#s;Y<+k*Xn$3xjdpA7d6V*YC{AAg-$jMH0KNY;q^jV57SQ#5ebjRFFH(YkUzW+qcDw5su4_BEz*=;cuwvQuH?BD_aI&~jTBuvbs$tZl+bLe#Bu)d4*x!ekJtG4cbO~A?3tb0yKjE~!hwT_4j(!C$)}$k zJAUHi=U<%q@~f}EInAFDzWwg|v**rVxOl02xw0rCNvH3YIf4O<< z_MN-;e*NwDKmNS`*Ybmhe?NNsR&-A{UZtV&jOm1W0qqx=!^+>P=Xaovr~ z?D%e!pB?9ovXk-NkoCkpWW;}?ysS8IJTny!jxtkm;n;O01=@g~pE9{me@IB_Y%90*m+|26T3j{0b&mldx+S>#V#AzA}GH96kGgph1P$P z4C_B8!1`~xq0oP^%^zHz{U2IN{|e0inGo&&$n`-Nh6Mls00000000000002kN&E-B KOa;dPPyhh2p<<2z literal 0 HcmV?d00001 diff --git a/v3_nczarr_test/ref_consolidated_zarr_base.cdl b/v3_nczarr_test/ref_consolidated_zarr_base.cdl new file mode 100644 index 0000000000..34c0a525d8 --- /dev/null +++ b/v3_nczarr_test/ref_consolidated_zarr_base.cdl @@ -0,0 +1,7 @@ +netcdf ref_purezarr { +dimensions: + x = 2; + y = 5; +variables: + int i(x,y) ; +} diff --git a/v3_nczarr_test/ref_fillonly.cdl b/v3_nczarr_test/ref_fillonly.cdl new file mode 100644 index 0000000000..4ef1325fa6 --- /dev/null +++ b/v3_nczarr_test/ref_fillonly.cdl @@ -0,0 +1,20 @@ +netcdf ref_fillonly { +dimensions: +// When UNLIMITED is implemented, then use this value for x +// x = UNLIMITED ; // (6 currently) + x = 6; +variables: + int i(x) ; + i:_Storage = "chunked" ; + i:_ChunkSizes = 6 ; + float f(x) ; + f:_FillValue = -9999.f ; + f:_Storage = "chunked" ; + f:_ChunkSizes = 6 ; + +// global attributes: + :_Format = "netCDF-4" ; +data: + + i = 1, 2, 3, 4, 5, 6 ; +} diff --git a/v3_nczarr_test/ref_filtered.cdl b/v3_nczarr_test/ref_filtered.cdl index a9e1a8e642..b7e5187bba 100644 --- a/v3_nczarr_test/ref_filtered.cdl +++ b/v3_nczarr_test/ref_filtered.cdl @@ -11,11 +11,10 @@ dimensions: group: g { variables: float var(dim0, dim1, dim2, dim3) ; - var:_FillValue = 9.96921e+36f ; - var:_Storage = "chunked" ; + var:_Storage = "chunked" ; var:_ChunkSizes = 4, 4, 4, 4 ; var:_Filter = "307,9" ; - var:_Codecs = "[{\"name\": \"bz2\", \"configuration\": {\"level\": 9}}]" ; + var:_Codecs = "[{\"name\": \"bz2\", \"configuration\": {\"level\": 9}}]" ; // group attributes: data: diff --git a/v3_nczarr_test/ref_groups.h5 b/v3_nczarr_test/ref_groups.h5 new file mode 100644 index 0000000000000000000000000000000000000000..e1ce0ece7f5b288f0225fd2e95c99a70fb861270 GIT binary patch literal 9836 zcmeI0y-veG49A_MEupAVK|)jrP$tw7c!Qz_y6_=jV1og+N>qGEbnD8-BXs0Z7#VpC z#%yrT`43IhP>?DWQb%fhcd;+>ujg~l53S}-rMz4wzI>kwR3$I&*xS2Y?e%0IDt%FQ zWS}RFbLqFj#$nu)@hbF0LhN;SudnVpgn8cZAm{SrC=AJX4bG5o`?O?RJ_b}y6#<9b zI3@z|dfvn*$4D;~XMlou+mL}kWK@C1a5-un*6j{Kwc+MEnsWkK>(DP)$6p4Mh8a5E zKWI8;rDETz%RYwvJc34gy*6oQ}6Njri1c)*vgl>Eh<{cGD=O*I(LB`$|9wC;=s)1eC!4Bj8ZU z=6Ff;Me}9PO~-ZP5l`w&Rr8&1kN83Cq$3p|$F)A=*bnJb5YOX!cH(IDOOJJ=TtEwQ zl3=aXu-hQIw~7`g~X#bs_QKh)Xuq8&^5BR2+5>Nt4KnW-TC6G%3 EAGt7vi~s-t literal 0 HcmV?d00001 diff --git a/v3_nczarr_test/ref_misc1.cdl b/v3_nczarr_test/ref_misc1.cdl new file mode 100644 index 0000000000..375f472f8e --- /dev/null +++ b/v3_nczarr_test/ref_misc1.cdl @@ -0,0 +1,84 @@ +netcdf tmp_misc1 { +dimensions: + d0 = 6 ; + d1 = 12 ; + d2 = 4 ; +variables: + int v(d0, d1, d2) ; + v:_FillValue = -1 ; +data: + + v = + 0, 1, 2, 3, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + 4, 5, 6, 7, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + 8, 9, 10, 11, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + 12, 13, 14, 15, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + 16, 17, 18, 19, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + 20, 21, 22, 23, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _, + _, _, _, _ ; +} diff --git a/v3_nczarr_test/ref_misc1.dmp b/v3_nczarr_test/ref_misc1.dmp new file mode 100644 index 0000000000..7d80574031 --- /dev/null +++ b/v3_nczarr_test/ref_misc1.dmp @@ -0,0 +1,97 @@ +rank=3 dims=(6,12,4) chunks=(2,3,1) +[0/0][0/0][0/0] = 00 | _ | _ + 04 | _ | _ +[0/0][0/0][1/1] = 01 | _ | _ + 05 | _ | _ +[0/0][0/0][2/2] = 02 | _ | _ + 06 | _ | _ +[0/0][0/0][3/3] = 03 | _ | _ + 07 | _ | _ +[0/0][1/3][0/0] = _ | _ | _ + _ | _ | _ +[0/0][1/3][1/1] = _ | _ | _ + _ | _ | _ +[0/0][1/3][2/2] = _ | _ | _ + _ | _ | _ +[0/0][1/3][3/3] = _ | _ | _ + _ | _ | _ +[0/0][2/6][0/0] = _ | _ | _ + _ | _ | _ +[0/0][2/6][1/1] = _ | _ | _ + _ | _ | _ +[0/0][2/6][2/2] = _ | _ | _ + _ | _ | _ +[0/0][2/6][3/3] = _ | _ | _ + _ | _ | _ +[0/0][3/9][0/0] = _ | _ | _ + _ | _ | _ +[0/0][3/9][1/1] = _ | _ | _ + _ | _ | _ +[0/0][3/9][2/2] = _ | _ | _ + _ | _ | _ +[0/0][3/9][3/3] = _ | _ | _ + _ | _ | _ +[1/2][0/0][0/0] = 08 | _ | _ + 12 | _ | _ +[1/2][0/0][1/1] = 09 | _ | _ + 13 | _ | _ +[1/2][0/0][2/2] = 10 | _ | _ + 14 | _ | _ +[1/2][0/0][3/3] = 11 | _ | _ + 15 | _ | _ +[1/2][1/3][0/0] = _ | _ | _ + _ | _ | _ +[1/2][1/3][1/1] = _ | _ | _ + _ | _ | _ +[1/2][1/3][2/2] = _ | _ | _ + _ | _ | _ +[1/2][1/3][3/3] = _ | _ | _ + _ | _ | _ +[1/2][2/6][0/0] = _ | _ | _ + _ | _ | _ +[1/2][2/6][1/1] = _ | _ | _ + _ | _ | _ +[1/2][2/6][2/2] = _ | _ | _ + _ | _ | _ +[1/2][2/6][3/3] = _ | _ | _ + _ | _ | _ +[1/2][3/9][0/0] = _ | _ | _ + _ | _ | _ +[1/2][3/9][1/1] = _ | _ | _ + _ | _ | _ +[1/2][3/9][2/2] = _ | _ | _ + _ | _ | _ +[1/2][3/9][3/3] = _ | _ | _ + _ | _ | _ +[2/4][0/0][0/0] = 16 | _ | _ + 20 | _ | _ +[2/4][0/0][1/1] = 17 | _ | _ + 21 | _ | _ +[2/4][0/0][2/2] = 18 | _ | _ + 22 | _ | _ +[2/4][0/0][3/3] = 19 | _ | _ + 23 | _ | _ +[2/4][1/3][0/0] = _ | _ | _ + _ | _ | _ +[2/4][1/3][1/1] = _ | _ | _ + _ | _ | _ +[2/4][1/3][2/2] = _ | _ | _ + _ | _ | _ +[2/4][1/3][3/3] = _ | _ | _ + _ | _ | _ +[2/4][2/6][0/0] = _ | _ | _ + _ | _ | _ +[2/4][2/6][1/1] = _ | _ | _ + _ | _ | _ +[2/4][2/6][2/2] = _ | _ | _ + _ | _ | _ +[2/4][2/6][3/3] = _ | _ | _ + _ | _ | _ +[2/4][3/9][0/0] = _ | _ | _ + _ | _ | _ +[2/4][3/9][1/1] = _ | _ | _ + _ | _ | _ +[2/4][3/9][2/2] = _ | _ | _ + _ | _ | _ +[2/4][3/9][3/3] = _ | _ | _ + _ | _ | _ diff --git a/v3_nczarr_test/ref_ndims.cdl b/v3_nczarr_test/ref_ndims.cdl new file mode 100644 index 0000000000..643051a1f5 --- /dev/null +++ b/v3_nczarr_test/ref_ndims.cdl @@ -0,0 +1,525 @@ +netcdf tmp_ndims { +dimensions: + d0 = 8 ; + d1 = 8 ; + d2 = 8 ; + d3 = 8 ; +variables: + int v(d0, d1, d2, d3) ; + v:_FillValue = -1 ; +data: + + v = + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, 69, 70, 71, + 72, 73, 74, 75, 76, 77, 78, 79, + 80, 81, 82, 83, 84, 85, 86, 87, + 88, 89, 90, 91, 92, 93, 94, 95, + 96, 97, 98, 99, 100, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, + 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, + 128, 129, 130, 131, 132, 133, 134, 135, + 136, 137, 138, 139, 140, 141, 142, 143, + 144, 145, 146, 147, 148, 149, 150, 151, + 152, 153, 154, 155, 156, 157, 158, 159, + 160, 161, 162, 163, 164, 165, 166, 167, + 168, 169, 170, 171, 172, 173, 174, 175, + 176, 177, 178, 179, 180, 181, 182, 183, + 184, 185, 186, 187, 188, 189, 190, 191, + 192, 193, 194, 195, 196, 197, 198, 199, + 200, 201, 202, 203, 204, 205, 206, 207, + 208, 209, 210, 211, 212, 213, 214, 215, + 216, 217, 218, 219, 220, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, 231, + 232, 233, 234, 235, 236, 237, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 250, 251, 252, 253, 254, 255, + 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, 268, 269, 270, 271, + 272, 273, 274, 275, 276, 277, 278, 279, + 280, 281, 282, 283, 284, 285, 286, 287, + 288, 289, 290, 291, 292, 293, 294, 295, + 296, 297, 298, 299, 300, 301, 302, 303, + 304, 305, 306, 307, 308, 309, 310, 311, + 312, 313, 314, 315, 316, 317, 318, 319, + 320, 321, 322, 323, 324, 325, 326, 327, + 328, 329, 330, 331, 332, 333, 334, 335, + 336, 337, 338, 339, 340, 341, 342, 343, + 344, 345, 346, 347, 348, 349, 350, 351, + 352, 353, 354, 355, 356, 357, 358, 359, + 360, 361, 362, 363, 364, 365, 366, 367, + 368, 369, 370, 371, 372, 373, 374, 375, + 376, 377, 378, 379, 380, 381, 382, 383, + 384, 385, 386, 387, 388, 389, 390, 391, + 392, 393, 394, 395, 396, 397, 398, 399, + 400, 401, 402, 403, 404, 405, 406, 407, + 408, 409, 410, 411, 412, 413, 414, 415, + 416, 417, 418, 419, 420, 421, 422, 423, + 424, 425, 426, 427, 428, 429, 430, 431, + 432, 433, 434, 435, 436, 437, 438, 439, + 440, 441, 442, 443, 444, 445, 446, 447, + 448, 449, 450, 451, 452, 453, 454, 455, + 456, 457, 458, 459, 460, 461, 462, 463, + 464, 465, 466, 467, 468, 469, 470, 471, + 472, 473, 474, 475, 476, 477, 478, 479, + 480, 481, 482, 483, 484, 485, 486, 487, + 488, 489, 490, 491, 492, 493, 494, 495, + 496, 497, 498, 499, 500, 501, 502, 503, + 504, 505, 506, 507, 508, 509, 510, 511, + 512, 513, 514, 515, 516, 517, 518, 519, + 520, 521, 522, 523, 524, 525, 526, 527, + 528, 529, 530, 531, 532, 533, 534, 535, + 536, 537, 538, 539, 540, 541, 542, 543, + 544, 545, 546, 547, 548, 549, 550, 551, + 552, 553, 554, 555, 556, 557, 558, 559, + 560, 561, 562, 563, 564, 565, 566, 567, + 568, 569, 570, 571, 572, 573, 574, 575, + 576, 577, 578, 579, 580, 581, 582, 583, + 584, 585, 586, 587, 588, 589, 590, 591, + 592, 593, 594, 595, 596, 597, 598, 599, + 600, 601, 602, 603, 604, 605, 606, 607, + 608, 609, 610, 611, 612, 613, 614, 615, + 616, 617, 618, 619, 620, 621, 622, 623, + 624, 625, 626, 627, 628, 629, 630, 631, + 632, 633, 634, 635, 636, 637, 638, 639, + 640, 641, 642, 643, 644, 645, 646, 647, + 648, 649, 650, 651, 652, 653, 654, 655, + 656, 657, 658, 659, 660, 661, 662, 663, + 664, 665, 666, 667, 668, 669, 670, 671, + 672, 673, 674, 675, 676, 677, 678, 679, + 680, 681, 682, 683, 684, 685, 686, 687, + 688, 689, 690, 691, 692, 693, 694, 695, + 696, 697, 698, 699, 700, 701, 702, 703, + 704, 705, 706, 707, 708, 709, 710, 711, + 712, 713, 714, 715, 716, 717, 718, 719, + 720, 721, 722, 723, 724, 725, 726, 727, + 728, 729, 730, 731, 732, 733, 734, 735, + 736, 737, 738, 739, 740, 741, 742, 743, + 744, 745, 746, 747, 748, 749, 750, 751, + 752, 753, 754, 755, 756, 757, 758, 759, + 760, 761, 762, 763, 764, 765, 766, 767, + 768, 769, 770, 771, 772, 773, 774, 775, + 776, 777, 778, 779, 780, 781, 782, 783, + 784, 785, 786, 787, 788, 789, 790, 791, + 792, 793, 794, 795, 796, 797, 798, 799, + 800, 801, 802, 803, 804, 805, 806, 807, + 808, 809, 810, 811, 812, 813, 814, 815, + 816, 817, 818, 819, 820, 821, 822, 823, + 824, 825, 826, 827, 828, 829, 830, 831, + 832, 833, 834, 835, 836, 837, 838, 839, + 840, 841, 842, 843, 844, 845, 846, 847, + 848, 849, 850, 851, 852, 853, 854, 855, + 856, 857, 858, 859, 860, 861, 862, 863, + 864, 865, 866, 867, 868, 869, 870, 871, + 872, 873, 874, 875, 876, 877, 878, 879, + 880, 881, 882, 883, 884, 885, 886, 887, + 888, 889, 890, 891, 892, 893, 894, 895, + 896, 897, 898, 899, 900, 901, 902, 903, + 904, 905, 906, 907, 908, 909, 910, 911, + 912, 913, 914, 915, 916, 917, 918, 919, + 920, 921, 922, 923, 924, 925, 926, 927, + 928, 929, 930, 931, 932, 933, 934, 935, + 936, 937, 938, 939, 940, 941, 942, 943, + 944, 945, 946, 947, 948, 949, 950, 951, + 952, 953, 954, 955, 956, 957, 958, 959, + 960, 961, 962, 963, 964, 965, 966, 967, + 968, 969, 970, 971, 972, 973, 974, 975, + 976, 977, 978, 979, 980, 981, 982, 983, + 984, 985, 986, 987, 988, 989, 990, 991, + 992, 993, 994, 995, 996, 997, 998, 999, + 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, + 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, + 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, + 1024, 1025, 1026, 1027, 1028, 1029, 1030, 1031, + 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039, + 1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047, + 1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055, + 1056, 1057, 1058, 1059, 1060, 1061, 1062, 1063, + 1064, 1065, 1066, 1067, 1068, 1069, 1070, 1071, + 1072, 1073, 1074, 1075, 1076, 1077, 1078, 1079, + 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087, + 1088, 1089, 1090, 1091, 1092, 1093, 1094, 1095, + 1096, 1097, 1098, 1099, 1100, 1101, 1102, 1103, + 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1111, + 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119, + 1120, 1121, 1122, 1123, 1124, 1125, 1126, 1127, + 1128, 1129, 1130, 1131, 1132, 1133, 1134, 1135, + 1136, 1137, 1138, 1139, 1140, 1141, 1142, 1143, + 1144, 1145, 1146, 1147, 1148, 1149, 1150, 1151, + 1152, 1153, 1154, 1155, 1156, 1157, 1158, 1159, + 1160, 1161, 1162, 1163, 1164, 1165, 1166, 1167, + 1168, 1169, 1170, 1171, 1172, 1173, 1174, 1175, + 1176, 1177, 1178, 1179, 1180, 1181, 1182, 1183, + 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, + 1192, 1193, 1194, 1195, 1196, 1197, 1198, 1199, + 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, + 1208, 1209, 1210, 1211, 1212, 1213, 1214, 1215, + 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, + 1224, 1225, 1226, 1227, 1228, 1229, 1230, 1231, + 1232, 1233, 1234, 1235, 1236, 1237, 1238, 1239, + 1240, 1241, 1242, 1243, 1244, 1245, 1246, 1247, + 1248, 1249, 1250, 1251, 1252, 1253, 1254, 1255, + 1256, 1257, 1258, 1259, 1260, 1261, 1262, 1263, + 1264, 1265, 1266, 1267, 1268, 1269, 1270, 1271, + 1272, 1273, 1274, 1275, 1276, 1277, 1278, 1279, + 1280, 1281, 1282, 1283, 1284, 1285, 1286, 1287, + 1288, 1289, 1290, 1291, 1292, 1293, 1294, 1295, + 1296, 1297, 1298, 1299, 1300, 1301, 1302, 1303, + 1304, 1305, 1306, 1307, 1308, 1309, 1310, 1311, + 1312, 1313, 1314, 1315, 1316, 1317, 1318, 1319, + 1320, 1321, 1322, 1323, 1324, 1325, 1326, 1327, + 1328, 1329, 1330, 1331, 1332, 1333, 1334, 1335, + 1336, 1337, 1338, 1339, 1340, 1341, 1342, 1343, + 1344, 1345, 1346, 1347, 1348, 1349, 1350, 1351, + 1352, 1353, 1354, 1355, 1356, 1357, 1358, 1359, + 1360, 1361, 1362, 1363, 1364, 1365, 1366, 1367, + 1368, 1369, 1370, 1371, 1372, 1373, 1374, 1375, + 1376, 1377, 1378, 1379, 1380, 1381, 1382, 1383, + 1384, 1385, 1386, 1387, 1388, 1389, 1390, 1391, + 1392, 1393, 1394, 1395, 1396, 1397, 1398, 1399, + 1400, 1401, 1402, 1403, 1404, 1405, 1406, 1407, + 1408, 1409, 1410, 1411, 1412, 1413, 1414, 1415, + 1416, 1417, 1418, 1419, 1420, 1421, 1422, 1423, + 1424, 1425, 1426, 1427, 1428, 1429, 1430, 1431, + 1432, 1433, 1434, 1435, 1436, 1437, 1438, 1439, + 1440, 1441, 1442, 1443, 1444, 1445, 1446, 1447, + 1448, 1449, 1450, 1451, 1452, 1453, 1454, 1455, + 1456, 1457, 1458, 1459, 1460, 1461, 1462, 1463, + 1464, 1465, 1466, 1467, 1468, 1469, 1470, 1471, + 1472, 1473, 1474, 1475, 1476, 1477, 1478, 1479, + 1480, 1481, 1482, 1483, 1484, 1485, 1486, 1487, + 1488, 1489, 1490, 1491, 1492, 1493, 1494, 1495, + 1496, 1497, 1498, 1499, 1500, 1501, 1502, 1503, + 1504, 1505, 1506, 1507, 1508, 1509, 1510, 1511, + 1512, 1513, 1514, 1515, 1516, 1517, 1518, 1519, + 1520, 1521, 1522, 1523, 1524, 1525, 1526, 1527, + 1528, 1529, 1530, 1531, 1532, 1533, 1534, 1535, + 1536, 1537, 1538, 1539, 1540, 1541, 1542, 1543, + 1544, 1545, 1546, 1547, 1548, 1549, 1550, 1551, + 1552, 1553, 1554, 1555, 1556, 1557, 1558, 1559, + 1560, 1561, 1562, 1563, 1564, 1565, 1566, 1567, + 1568, 1569, 1570, 1571, 1572, 1573, 1574, 1575, + 1576, 1577, 1578, 1579, 1580, 1581, 1582, 1583, + 1584, 1585, 1586, 1587, 1588, 1589, 1590, 1591, + 1592, 1593, 1594, 1595, 1596, 1597, 1598, 1599, + 1600, 1601, 1602, 1603, 1604, 1605, 1606, 1607, + 1608, 1609, 1610, 1611, 1612, 1613, 1614, 1615, + 1616, 1617, 1618, 1619, 1620, 1621, 1622, 1623, + 1624, 1625, 1626, 1627, 1628, 1629, 1630, 1631, + 1632, 1633, 1634, 1635, 1636, 1637, 1638, 1639, + 1640, 1641, 1642, 1643, 1644, 1645, 1646, 1647, + 1648, 1649, 1650, 1651, 1652, 1653, 1654, 1655, + 1656, 1657, 1658, 1659, 1660, 1661, 1662, 1663, + 1664, 1665, 1666, 1667, 1668, 1669, 1670, 1671, + 1672, 1673, 1674, 1675, 1676, 1677, 1678, 1679, + 1680, 1681, 1682, 1683, 1684, 1685, 1686, 1687, + 1688, 1689, 1690, 1691, 1692, 1693, 1694, 1695, + 1696, 1697, 1698, 1699, 1700, 1701, 1702, 1703, + 1704, 1705, 1706, 1707, 1708, 1709, 1710, 1711, + 1712, 1713, 1714, 1715, 1716, 1717, 1718, 1719, + 1720, 1721, 1722, 1723, 1724, 1725, 1726, 1727, + 1728, 1729, 1730, 1731, 1732, 1733, 1734, 1735, + 1736, 1737, 1738, 1739, 1740, 1741, 1742, 1743, + 1744, 1745, 1746, 1747, 1748, 1749, 1750, 1751, + 1752, 1753, 1754, 1755, 1756, 1757, 1758, 1759, + 1760, 1761, 1762, 1763, 1764, 1765, 1766, 1767, + 1768, 1769, 1770, 1771, 1772, 1773, 1774, 1775, + 1776, 1777, 1778, 1779, 1780, 1781, 1782, 1783, + 1784, 1785, 1786, 1787, 1788, 1789, 1790, 1791, + 1792, 1793, 1794, 1795, 1796, 1797, 1798, 1799, + 1800, 1801, 1802, 1803, 1804, 1805, 1806, 1807, + 1808, 1809, 1810, 1811, 1812, 1813, 1814, 1815, + 1816, 1817, 1818, 1819, 1820, 1821, 1822, 1823, + 1824, 1825, 1826, 1827, 1828, 1829, 1830, 1831, + 1832, 1833, 1834, 1835, 1836, 1837, 1838, 1839, + 1840, 1841, 1842, 1843, 1844, 1845, 1846, 1847, + 1848, 1849, 1850, 1851, 1852, 1853, 1854, 1855, + 1856, 1857, 1858, 1859, 1860, 1861, 1862, 1863, + 1864, 1865, 1866, 1867, 1868, 1869, 1870, 1871, + 1872, 1873, 1874, 1875, 1876, 1877, 1878, 1879, + 1880, 1881, 1882, 1883, 1884, 1885, 1886, 1887, + 1888, 1889, 1890, 1891, 1892, 1893, 1894, 1895, + 1896, 1897, 1898, 1899, 1900, 1901, 1902, 1903, + 1904, 1905, 1906, 1907, 1908, 1909, 1910, 1911, + 1912, 1913, 1914, 1915, 1916, 1917, 1918, 1919, + 1920, 1921, 1922, 1923, 1924, 1925, 1926, 1927, + 1928, 1929, 1930, 1931, 1932, 1933, 1934, 1935, + 1936, 1937, 1938, 1939, 1940, 1941, 1942, 1943, + 1944, 1945, 1946, 1947, 1948, 1949, 1950, 1951, + 1952, 1953, 1954, 1955, 1956, 1957, 1958, 1959, + 1960, 1961, 1962, 1963, 1964, 1965, 1966, 1967, + 1968, 1969, 1970, 1971, 1972, 1973, 1974, 1975, + 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, + 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1991, + 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, + 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, + 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, + 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, + 2024, 2025, 2026, 2027, 2028, 2029, 2030, 2031, + 2032, 2033, 2034, 2035, 2036, 2037, 2038, 2039, + 2040, 2041, 2042, 2043, 2044, 2045, 2046, 2047, + 2048, 2049, 2050, 2051, 2052, 2053, 2054, 2055, + 2056, 2057, 2058, 2059, 2060, 2061, 2062, 2063, + 2064, 2065, 2066, 2067, 2068, 2069, 2070, 2071, + 2072, 2073, 2074, 2075, 2076, 2077, 2078, 2079, + 2080, 2081, 2082, 2083, 2084, 2085, 2086, 2087, + 2088, 2089, 2090, 2091, 2092, 2093, 2094, 2095, + 2096, 2097, 2098, 2099, 2100, 2101, 2102, 2103, + 2104, 2105, 2106, 2107, 2108, 2109, 2110, 2111, + 2112, 2113, 2114, 2115, 2116, 2117, 2118, 2119, + 2120, 2121, 2122, 2123, 2124, 2125, 2126, 2127, + 2128, 2129, 2130, 2131, 2132, 2133, 2134, 2135, + 2136, 2137, 2138, 2139, 2140, 2141, 2142, 2143, + 2144, 2145, 2146, 2147, 2148, 2149, 2150, 2151, + 2152, 2153, 2154, 2155, 2156, 2157, 2158, 2159, + 2160, 2161, 2162, 2163, 2164, 2165, 2166, 2167, + 2168, 2169, 2170, 2171, 2172, 2173, 2174, 2175, + 2176, 2177, 2178, 2179, 2180, 2181, 2182, 2183, + 2184, 2185, 2186, 2187, 2188, 2189, 2190, 2191, + 2192, 2193, 2194, 2195, 2196, 2197, 2198, 2199, + 2200, 2201, 2202, 2203, 2204, 2205, 2206, 2207, + 2208, 2209, 2210, 2211, 2212, 2213, 2214, 2215, + 2216, 2217, 2218, 2219, 2220, 2221, 2222, 2223, + 2224, 2225, 2226, 2227, 2228, 2229, 2230, 2231, + 2232, 2233, 2234, 2235, 2236, 2237, 2238, 2239, + 2240, 2241, 2242, 2243, 2244, 2245, 2246, 2247, + 2248, 2249, 2250, 2251, 2252, 2253, 2254, 2255, + 2256, 2257, 2258, 2259, 2260, 2261, 2262, 2263, + 2264, 2265, 2266, 2267, 2268, 2269, 2270, 2271, + 2272, 2273, 2274, 2275, 2276, 2277, 2278, 2279, + 2280, 2281, 2282, 2283, 2284, 2285, 2286, 2287, + 2288, 2289, 2290, 2291, 2292, 2293, 2294, 2295, + 2296, 2297, 2298, 2299, 2300, 2301, 2302, 2303, + 2304, 2305, 2306, 2307, 2308, 2309, 2310, 2311, + 2312, 2313, 2314, 2315, 2316, 2317, 2318, 2319, + 2320, 2321, 2322, 2323, 2324, 2325, 2326, 2327, + 2328, 2329, 2330, 2331, 2332, 2333, 2334, 2335, + 2336, 2337, 2338, 2339, 2340, 2341, 2342, 2343, + 2344, 2345, 2346, 2347, 2348, 2349, 2350, 2351, + 2352, 2353, 2354, 2355, 2356, 2357, 2358, 2359, + 2360, 2361, 2362, 2363, 2364, 2365, 2366, 2367, + 2368, 2369, 2370, 2371, 2372, 2373, 2374, 2375, + 2376, 2377, 2378, 2379, 2380, 2381, 2382, 2383, + 2384, 2385, 2386, 2387, 2388, 2389, 2390, 2391, + 2392, 2393, 2394, 2395, 2396, 2397, 2398, 2399, + 2400, 2401, 2402, 2403, 2404, 2405, 2406, 2407, + 2408, 2409, 2410, 2411, 2412, 2413, 2414, 2415, + 2416, 2417, 2418, 2419, 2420, 2421, 2422, 2423, + 2424, 2425, 2426, 2427, 2428, 2429, 2430, 2431, + 2432, 2433, 2434, 2435, 2436, 2437, 2438, 2439, + 2440, 2441, 2442, 2443, 2444, 2445, 2446, 2447, + 2448, 2449, 2450, 2451, 2452, 2453, 2454, 2455, + 2456, 2457, 2458, 2459, 2460, 2461, 2462, 2463, + 2464, 2465, 2466, 2467, 2468, 2469, 2470, 2471, + 2472, 2473, 2474, 2475, 2476, 2477, 2478, 2479, + 2480, 2481, 2482, 2483, 2484, 2485, 2486, 2487, + 2488, 2489, 2490, 2491, 2492, 2493, 2494, 2495, + 2496, 2497, 2498, 2499, 2500, 2501, 2502, 2503, + 2504, 2505, 2506, 2507, 2508, 2509, 2510, 2511, + 2512, 2513, 2514, 2515, 2516, 2517, 2518, 2519, + 2520, 2521, 2522, 2523, 2524, 2525, 2526, 2527, + 2528, 2529, 2530, 2531, 2532, 2533, 2534, 2535, + 2536, 2537, 2538, 2539, 2540, 2541, 2542, 2543, + 2544, 2545, 2546, 2547, 2548, 2549, 2550, 2551, + 2552, 2553, 2554, 2555, 2556, 2557, 2558, 2559, + 2560, 2561, 2562, 2563, 2564, 2565, 2566, 2567, + 2568, 2569, 2570, 2571, 2572, 2573, 2574, 2575, + 2576, 2577, 2578, 2579, 2580, 2581, 2582, 2583, + 2584, 2585, 2586, 2587, 2588, 2589, 2590, 2591, + 2592, 2593, 2594, 2595, 2596, 2597, 2598, 2599, + 2600, 2601, 2602, 2603, 2604, 2605, 2606, 2607, + 2608, 2609, 2610, 2611, 2612, 2613, 2614, 2615, + 2616, 2617, 2618, 2619, 2620, 2621, 2622, 2623, + 2624, 2625, 2626, 2627, 2628, 2629, 2630, 2631, + 2632, 2633, 2634, 2635, 2636, 2637, 2638, 2639, + 2640, 2641, 2642, 2643, 2644, 2645, 2646, 2647, + 2648, 2649, 2650, 2651, 2652, 2653, 2654, 2655, + 2656, 2657, 2658, 2659, 2660, 2661, 2662, 2663, + 2664, 2665, 2666, 2667, 2668, 2669, 2670, 2671, + 2672, 2673, 2674, 2675, 2676, 2677, 2678, 2679, + 2680, 2681, 2682, 2683, 2684, 2685, 2686, 2687, + 2688, 2689, 2690, 2691, 2692, 2693, 2694, 2695, + 2696, 2697, 2698, 2699, 2700, 2701, 2702, 2703, + 2704, 2705, 2706, 2707, 2708, 2709, 2710, 2711, + 2712, 2713, 2714, 2715, 2716, 2717, 2718, 2719, + 2720, 2721, 2722, 2723, 2724, 2725, 2726, 2727, + 2728, 2729, 2730, 2731, 2732, 2733, 2734, 2735, + 2736, 2737, 2738, 2739, 2740, 2741, 2742, 2743, + 2744, 2745, 2746, 2747, 2748, 2749, 2750, 2751, + 2752, 2753, 2754, 2755, 2756, 2757, 2758, 2759, + 2760, 2761, 2762, 2763, 2764, 2765, 2766, 2767, + 2768, 2769, 2770, 2771, 2772, 2773, 2774, 2775, + 2776, 2777, 2778, 2779, 2780, 2781, 2782, 2783, + 2784, 2785, 2786, 2787, 2788, 2789, 2790, 2791, + 2792, 2793, 2794, 2795, 2796, 2797, 2798, 2799, + 2800, 2801, 2802, 2803, 2804, 2805, 2806, 2807, + 2808, 2809, 2810, 2811, 2812, 2813, 2814, 2815, + 2816, 2817, 2818, 2819, 2820, 2821, 2822, 2823, + 2824, 2825, 2826, 2827, 2828, 2829, 2830, 2831, + 2832, 2833, 2834, 2835, 2836, 2837, 2838, 2839, + 2840, 2841, 2842, 2843, 2844, 2845, 2846, 2847, + 2848, 2849, 2850, 2851, 2852, 2853, 2854, 2855, + 2856, 2857, 2858, 2859, 2860, 2861, 2862, 2863, + 2864, 2865, 2866, 2867, 2868, 2869, 2870, 2871, + 2872, 2873, 2874, 2875, 2876, 2877, 2878, 2879, + 2880, 2881, 2882, 2883, 2884, 2885, 2886, 2887, + 2888, 2889, 2890, 2891, 2892, 2893, 2894, 2895, + 2896, 2897, 2898, 2899, 2900, 2901, 2902, 2903, + 2904, 2905, 2906, 2907, 2908, 2909, 2910, 2911, + 2912, 2913, 2914, 2915, 2916, 2917, 2918, 2919, + 2920, 2921, 2922, 2923, 2924, 2925, 2926, 2927, + 2928, 2929, 2930, 2931, 2932, 2933, 2934, 2935, + 2936, 2937, 2938, 2939, 2940, 2941, 2942, 2943, + 2944, 2945, 2946, 2947, 2948, 2949, 2950, 2951, + 2952, 2953, 2954, 2955, 2956, 2957, 2958, 2959, + 2960, 2961, 2962, 2963, 2964, 2965, 2966, 2967, + 2968, 2969, 2970, 2971, 2972, 2973, 2974, 2975, + 2976, 2977, 2978, 2979, 2980, 2981, 2982, 2983, + 2984, 2985, 2986, 2987, 2988, 2989, 2990, 2991, + 2992, 2993, 2994, 2995, 2996, 2997, 2998, 2999, + 3000, 3001, 3002, 3003, 3004, 3005, 3006, 3007, + 3008, 3009, 3010, 3011, 3012, 3013, 3014, 3015, + 3016, 3017, 3018, 3019, 3020, 3021, 3022, 3023, + 3024, 3025, 3026, 3027, 3028, 3029, 3030, 3031, + 3032, 3033, 3034, 3035, 3036, 3037, 3038, 3039, + 3040, 3041, 3042, 3043, 3044, 3045, 3046, 3047, + 3048, 3049, 3050, 3051, 3052, 3053, 3054, 3055, + 3056, 3057, 3058, 3059, 3060, 3061, 3062, 3063, + 3064, 3065, 3066, 3067, 3068, 3069, 3070, 3071, + 3072, 3073, 3074, 3075, 3076, 3077, 3078, 3079, + 3080, 3081, 3082, 3083, 3084, 3085, 3086, 3087, + 3088, 3089, 3090, 3091, 3092, 3093, 3094, 3095, + 3096, 3097, 3098, 3099, 3100, 3101, 3102, 3103, + 3104, 3105, 3106, 3107, 3108, 3109, 3110, 3111, + 3112, 3113, 3114, 3115, 3116, 3117, 3118, 3119, + 3120, 3121, 3122, 3123, 3124, 3125, 3126, 3127, + 3128, 3129, 3130, 3131, 3132, 3133, 3134, 3135, + 3136, 3137, 3138, 3139, 3140, 3141, 3142, 3143, + 3144, 3145, 3146, 3147, 3148, 3149, 3150, 3151, + 3152, 3153, 3154, 3155, 3156, 3157, 3158, 3159, + 3160, 3161, 3162, 3163, 3164, 3165, 3166, 3167, + 3168, 3169, 3170, 3171, 3172, 3173, 3174, 3175, + 3176, 3177, 3178, 3179, 3180, 3181, 3182, 3183, + 3184, 3185, 3186, 3187, 3188, 3189, 3190, 3191, + 3192, 3193, 3194, 3195, 3196, 3197, 3198, 3199, + 3200, 3201, 3202, 3203, 3204, 3205, 3206, 3207, + 3208, 3209, 3210, 3211, 3212, 3213, 3214, 3215, + 3216, 3217, 3218, 3219, 3220, 3221, 3222, 3223, + 3224, 3225, 3226, 3227, 3228, 3229, 3230, 3231, + 3232, 3233, 3234, 3235, 3236, 3237, 3238, 3239, + 3240, 3241, 3242, 3243, 3244, 3245, 3246, 3247, + 3248, 3249, 3250, 3251, 3252, 3253, 3254, 3255, + 3256, 3257, 3258, 3259, 3260, 3261, 3262, 3263, + 3264, 3265, 3266, 3267, 3268, 3269, 3270, 3271, + 3272, 3273, 3274, 3275, 3276, 3277, 3278, 3279, + 3280, 3281, 3282, 3283, 3284, 3285, 3286, 3287, + 3288, 3289, 3290, 3291, 3292, 3293, 3294, 3295, + 3296, 3297, 3298, 3299, 3300, 3301, 3302, 3303, + 3304, 3305, 3306, 3307, 3308, 3309, 3310, 3311, + 3312, 3313, 3314, 3315, 3316, 3317, 3318, 3319, + 3320, 3321, 3322, 3323, 3324, 3325, 3326, 3327, + 3328, 3329, 3330, 3331, 3332, 3333, 3334, 3335, + 3336, 3337, 3338, 3339, 3340, 3341, 3342, 3343, + 3344, 3345, 3346, 3347, 3348, 3349, 3350, 3351, + 3352, 3353, 3354, 3355, 3356, 3357, 3358, 3359, + 3360, 3361, 3362, 3363, 3364, 3365, 3366, 3367, + 3368, 3369, 3370, 3371, 3372, 3373, 3374, 3375, + 3376, 3377, 3378, 3379, 3380, 3381, 3382, 3383, + 3384, 3385, 3386, 3387, 3388, 3389, 3390, 3391, + 3392, 3393, 3394, 3395, 3396, 3397, 3398, 3399, + 3400, 3401, 3402, 3403, 3404, 3405, 3406, 3407, + 3408, 3409, 3410, 3411, 3412, 3413, 3414, 3415, + 3416, 3417, 3418, 3419, 3420, 3421, 3422, 3423, + 3424, 3425, 3426, 3427, 3428, 3429, 3430, 3431, + 3432, 3433, 3434, 3435, 3436, 3437, 3438, 3439, + 3440, 3441, 3442, 3443, 3444, 3445, 3446, 3447, + 3448, 3449, 3450, 3451, 3452, 3453, 3454, 3455, + 3456, 3457, 3458, 3459, 3460, 3461, 3462, 3463, + 3464, 3465, 3466, 3467, 3468, 3469, 3470, 3471, + 3472, 3473, 3474, 3475, 3476, 3477, 3478, 3479, + 3480, 3481, 3482, 3483, 3484, 3485, 3486, 3487, + 3488, 3489, 3490, 3491, 3492, 3493, 3494, 3495, + 3496, 3497, 3498, 3499, 3500, 3501, 3502, 3503, + 3504, 3505, 3506, 3507, 3508, 3509, 3510, 3511, + 3512, 3513, 3514, 3515, 3516, 3517, 3518, 3519, + 3520, 3521, 3522, 3523, 3524, 3525, 3526, 3527, + 3528, 3529, 3530, 3531, 3532, 3533, 3534, 3535, + 3536, 3537, 3538, 3539, 3540, 3541, 3542, 3543, + 3544, 3545, 3546, 3547, 3548, 3549, 3550, 3551, + 3552, 3553, 3554, 3555, 3556, 3557, 3558, 3559, + 3560, 3561, 3562, 3563, 3564, 3565, 3566, 3567, + 3568, 3569, 3570, 3571, 3572, 3573, 3574, 3575, + 3576, 3577, 3578, 3579, 3580, 3581, 3582, 3583, + 3584, 3585, 3586, 3587, 3588, 3589, 3590, 3591, + 3592, 3593, 3594, 3595, 3596, 3597, 3598, 3599, + 3600, 3601, 3602, 3603, 3604, 3605, 3606, 3607, + 3608, 3609, 3610, 3611, 3612, 3613, 3614, 3615, + 3616, 3617, 3618, 3619, 3620, 3621, 3622, 3623, + 3624, 3625, 3626, 3627, 3628, 3629, 3630, 3631, + 3632, 3633, 3634, 3635, 3636, 3637, 3638, 3639, + 3640, 3641, 3642, 3643, 3644, 3645, 3646, 3647, + 3648, 3649, 3650, 3651, 3652, 3653, 3654, 3655, + 3656, 3657, 3658, 3659, 3660, 3661, 3662, 3663, + 3664, 3665, 3666, 3667, 3668, 3669, 3670, 3671, + 3672, 3673, 3674, 3675, 3676, 3677, 3678, 3679, + 3680, 3681, 3682, 3683, 3684, 3685, 3686, 3687, + 3688, 3689, 3690, 3691, 3692, 3693, 3694, 3695, + 3696, 3697, 3698, 3699, 3700, 3701, 3702, 3703, + 3704, 3705, 3706, 3707, 3708, 3709, 3710, 3711, + 3712, 3713, 3714, 3715, 3716, 3717, 3718, 3719, + 3720, 3721, 3722, 3723, 3724, 3725, 3726, 3727, + 3728, 3729, 3730, 3731, 3732, 3733, 3734, 3735, + 3736, 3737, 3738, 3739, 3740, 3741, 3742, 3743, + 3744, 3745, 3746, 3747, 3748, 3749, 3750, 3751, + 3752, 3753, 3754, 3755, 3756, 3757, 3758, 3759, + 3760, 3761, 3762, 3763, 3764, 3765, 3766, 3767, + 3768, 3769, 3770, 3771, 3772, 3773, 3774, 3775, + 3776, 3777, 3778, 3779, 3780, 3781, 3782, 3783, + 3784, 3785, 3786, 3787, 3788, 3789, 3790, 3791, + 3792, 3793, 3794, 3795, 3796, 3797, 3798, 3799, + 3800, 3801, 3802, 3803, 3804, 3805, 3806, 3807, + 3808, 3809, 3810, 3811, 3812, 3813, 3814, 3815, + 3816, 3817, 3818, 3819, 3820, 3821, 3822, 3823, + 3824, 3825, 3826, 3827, 3828, 3829, 3830, 3831, + 3832, 3833, 3834, 3835, 3836, 3837, 3838, 3839, + 3840, 3841, 3842, 3843, 3844, 3845, 3846, 3847, + 3848, 3849, 3850, 3851, 3852, 3853, 3854, 3855, + 3856, 3857, 3858, 3859, 3860, 3861, 3862, 3863, + 3864, 3865, 3866, 3867, 3868, 3869, 3870, 3871, + 3872, 3873, 3874, 3875, 3876, 3877, 3878, 3879, + 3880, 3881, 3882, 3883, 3884, 3885, 3886, 3887, + 3888, 3889, 3890, 3891, 3892, 3893, 3894, 3895, + 3896, 3897, 3898, 3899, 3900, 3901, 3902, 3903, + 3904, 3905, 3906, 3907, 3908, 3909, 3910, 3911, + 3912, 3913, 3914, 3915, 3916, 3917, 3918, 3919, + 3920, 3921, 3922, 3923, 3924, 3925, 3926, 3927, + 3928, 3929, 3930, 3931, 3932, 3933, 3934, 3935, + 3936, 3937, 3938, 3939, 3940, 3941, 3942, 3943, + 3944, 3945, 3946, 3947, 3948, 3949, 3950, 3951, + 3952, 3953, 3954, 3955, 3956, 3957, 3958, 3959, + 3960, 3961, 3962, 3963, 3964, 3965, 3966, 3967, + 3968, 3969, 3970, 3971, 3972, 3973, 3974, 3975, + 3976, 3977, 3978, 3979, 3980, 3981, 3982, 3983, + 3984, 3985, 3986, 3987, 3988, 3989, 3990, 3991, + 3992, 3993, 3994, 3995, 3996, 3997, 3998, 3999, + 4000, 4001, 4002, 4003, 4004, 4005, 4006, 4007, + 4008, 4009, 4010, 4011, 4012, 4013, 4014, 4015, + 4016, 4017, 4018, 4019, 4020, 4021, 4022, 4023, + 4024, 4025, 4026, 4027, 4028, 4029, 4030, 4031, + 4032, 4033, 4034, 4035, 4036, 4037, 4038, 4039, + 4040, 4041, 4042, 4043, 4044, 4045, 4046, 4047, + 4048, 4049, 4050, 4051, 4052, 4053, 4054, 4055, + 4056, 4057, 4058, 4059, 4060, 4061, 4062, 4063, + 4064, 4065, 4066, 4067, 4068, 4069, 4070, 4071, + 4072, 4073, 4074, 4075, 4076, 4077, 4078, 4079, + 4080, 4081, 4082, 4083, 4084, 4085, 4086, 4087, + 4088, 4089, 4090, 4091, 4092, 4093, 4094, 4095 ; +} diff --git a/v3_nczarr_test/ref_ndims.dmp b/v3_nczarr_test/ref_ndims.dmp new file mode 100644 index 0000000000..321e13254d --- /dev/null +++ b/v3_nczarr_test/ref_ndims.dmp @@ -0,0 +1,37 @@ +rank=4 dims=(8,8,8,8) chunks=(3,3,4,4) +[0/0][0/0][0/0][0/0] = 00 01 02 03 08 09 10 11 16 17 18 19 24 25 26 27 64 65 66 67 72 73 74 75 80 81 82 83 88 89 90 91 128 129 130 131 136 137 138 139 144 145 146 147 152 153 154 155 512 513 514 515 520 521 522 523 528 529 530 531 536 537 538 539 576 577 578 579 584 585 586 587 592 593 594 595 600 601 602 603 640 641 642 643 648 649 650 651 656 657 658 659 664 665 666 667 1024 1025 1026 1027 1032 1033 1034 1035 1040 1041 1042 1043 1048 1049 1050 1051 1088 1089 1090 1091 1096 1097 1098 1099 1104 1105 1106 1107 1112 1113 1114 1115 1152 1153 1154 1155 1160 1161 1162 1163 1168 1169 1170 1171 1176 1177 1178 1179 +[0/0][0/0][0/0][1/4] = 04 05 06 07 12 13 14 15 20 21 22 23 28 29 30 31 68 69 70 71 76 77 78 79 84 85 86 87 92 93 94 95 132 133 134 135 140 141 142 143 148 149 150 151 156 157 158 159 516 517 518 519 524 525 526 527 532 533 534 535 540 541 542 543 580 581 582 583 588 589 590 591 596 597 598 599 604 605 606 607 644 645 646 647 652 653 654 655 660 661 662 663 668 669 670 671 1028 1029 1030 1031 1036 1037 1038 1039 1044 1045 1046 1047 1052 1053 1054 1055 1092 1093 1094 1095 1100 1101 1102 1103 1108 1109 1110 1111 1116 1117 1118 1119 1156 1157 1158 1159 1164 1165 1166 1167 1172 1173 1174 1175 1180 1181 1182 1183 +[0/0][0/0][1/4][0/0] = 32 33 34 35 40 41 42 43 48 49 50 51 56 57 58 59 96 97 98 99 104 105 106 107 112 113 114 115 120 121 122 123 160 161 162 163 168 169 170 171 176 177 178 179 184 185 186 187 544 545 546 547 552 553 554 555 560 561 562 563 568 569 570 571 608 609 610 611 616 617 618 619 624 625 626 627 632 633 634 635 672 673 674 675 680 681 682 683 688 689 690 691 696 697 698 699 1056 1057 1058 1059 1064 1065 1066 1067 1072 1073 1074 1075 1080 1081 1082 1083 1120 1121 1122 1123 1128 1129 1130 1131 1136 1137 1138 1139 1144 1145 1146 1147 1184 1185 1186 1187 1192 1193 1194 1195 1200 1201 1202 1203 1208 1209 1210 1211 +[0/0][0/0][1/4][1/4] = 36 37 38 39 44 45 46 47 52 53 54 55 60 61 62 63 100 101 102 103 108 109 110 111 116 117 118 119 124 125 126 127 164 165 166 167 172 173 174 175 180 181 182 183 188 189 190 191 548 549 550 551 556 557 558 559 564 565 566 567 572 573 574 575 612 613 614 615 620 621 622 623 628 629 630 631 636 637 638 639 676 677 678 679 684 685 686 687 692 693 694 695 700 701 702 703 1060 1061 1062 1063 1068 1069 1070 1071 1076 1077 1078 1079 1084 1085 1086 1087 1124 1125 1126 1127 1132 1133 1134 1135 1140 1141 1142 1143 1148 1149 1150 1151 1188 1189 1190 1191 1196 1197 1198 1199 1204 1205 1206 1207 1212 1213 1214 1215 +[0/0][1/3][0/0][0/0] = 192 193 194 195 200 201 202 203 208 209 210 211 216 217 218 219 256 257 258 259 264 265 266 267 272 273 274 275 280 281 282 283 320 321 322 323 328 329 330 331 336 337 338 339 344 345 346 347 704 705 706 707 712 713 714 715 720 721 722 723 728 729 730 731 768 769 770 771 776 777 778 779 784 785 786 787 792 793 794 795 832 833 834 835 840 841 842 843 848 849 850 851 856 857 858 859 1216 1217 1218 1219 1224 1225 1226 1227 1232 1233 1234 1235 1240 1241 1242 1243 1280 1281 1282 1283 1288 1289 1290 1291 1296 1297 1298 1299 1304 1305 1306 1307 1344 1345 1346 1347 1352 1353 1354 1355 1360 1361 1362 1363 1368 1369 1370 1371 +[0/0][1/3][0/0][1/4] = 196 197 198 199 204 205 206 207 212 213 214 215 220 221 222 223 260 261 262 263 268 269 270 271 276 277 278 279 284 285 286 287 324 325 326 327 332 333 334 335 340 341 342 343 348 349 350 351 708 709 710 711 716 717 718 719 724 725 726 727 732 733 734 735 772 773 774 775 780 781 782 783 788 789 790 791 796 797 798 799 836 837 838 839 844 845 846 847 852 853 854 855 860 861 862 863 1220 1221 1222 1223 1228 1229 1230 1231 1236 1237 1238 1239 1244 1245 1246 1247 1284 1285 1286 1287 1292 1293 1294 1295 1300 1301 1302 1303 1308 1309 1310 1311 1348 1349 1350 1351 1356 1357 1358 1359 1364 1365 1366 1367 1372 1373 1374 1375 +[0/0][1/3][1/4][0/0] = 224 225 226 227 232 233 234 235 240 241 242 243 248 249 250 251 288 289 290 291 296 297 298 299 304 305 306 307 312 313 314 315 352 353 354 355 360 361 362 363 368 369 370 371 376 377 378 379 736 737 738 739 744 745 746 747 752 753 754 755 760 761 762 763 800 801 802 803 808 809 810 811 816 817 818 819 824 825 826 827 864 865 866 867 872 873 874 875 880 881 882 883 888 889 890 891 1248 1249 1250 1251 1256 1257 1258 1259 1264 1265 1266 1267 1272 1273 1274 1275 1312 1313 1314 1315 1320 1321 1322 1323 1328 1329 1330 1331 1336 1337 1338 1339 1376 1377 1378 1379 1384 1385 1386 1387 1392 1393 1394 1395 1400 1401 1402 1403 +[0/0][1/3][1/4][1/4] = 228 229 230 231 236 237 238 239 244 245 246 247 252 253 254 255 292 293 294 295 300 301 302 303 308 309 310 311 316 317 318 319 356 357 358 359 364 365 366 367 372 373 374 375 380 381 382 383 740 741 742 743 748 749 750 751 756 757 758 759 764 765 766 767 804 805 806 807 812 813 814 815 820 821 822 823 828 829 830 831 868 869 870 871 876 877 878 879 884 885 886 887 892 893 894 895 1252 1253 1254 1255 1260 1261 1262 1263 1268 1269 1270 1271 1276 1277 1278 1279 1316 1317 1318 1319 1324 1325 1326 1327 1332 1333 1334 1335 1340 1341 1342 1343 1380 1381 1382 1383 1388 1389 1390 1391 1396 1397 1398 1399 1404 1405 1406 1407 +[0/0][2/6][0/0][0/0] = 384 385 386 387 392 393 394 395 400 401 402 403 408 409 410 411 448 449 450 451 456 457 458 459 464 465 466 467 472 473 474 475 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 896 897 898 899 904 905 906 907 912 913 914 915 920 921 922 923 960 961 962 963 968 969 970 971 976 977 978 979 984 985 986 987 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 1408 1409 1410 1411 1416 1417 1418 1419 1424 1425 1426 1427 1432 1433 1434 1435 1472 1473 1474 1475 1480 1481 1482 1483 1488 1489 1490 1491 1496 1497 1498 1499 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +[0/0][2/6][0/0][1/4] = 388 389 390 391 396 397 398 399 404 405 406 407 412 413 414 415 452 453 454 455 460 461 462 463 468 469 470 471 476 477 478 479 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 900 901 902 903 908 909 910 911 916 917 918 919 924 925 926 927 964 965 966 967 972 973 974 975 980 981 982 983 988 989 990 991 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 1412 1413 1414 1415 1420 1421 1422 1423 1428 1429 1430 1431 1436 1437 1438 1439 1476 1477 1478 1479 1484 1485 1486 1487 1492 1493 1494 1495 1500 1501 1502 1503 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +[0/0][2/6][1/4][0/0] = 416 417 418 419 424 425 426 427 432 433 434 435 440 441 442 443 480 481 482 483 488 489 490 491 496 497 498 499 504 505 506 507 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 928 929 930 931 936 937 938 939 944 945 946 947 952 953 954 955 992 993 994 995 1000 1001 1002 1003 1008 1009 1010 1011 1016 1017 1018 1019 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 1440 1441 1442 1443 1448 1449 1450 1451 1456 1457 1458 1459 1464 1465 1466 1467 1504 1505 1506 1507 1512 1513 1514 1515 1520 1521 1522 1523 1528 1529 1530 1531 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +[0/0][2/6][1/4][1/4] = 420 421 422 423 428 429 430 431 436 437 438 439 444 445 446 447 484 485 486 487 492 493 494 495 500 501 502 503 508 509 510 511 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 932 933 934 935 940 941 942 943 948 949 950 951 956 957 958 959 996 997 998 999 1004 1005 1006 1007 1012 1013 1014 1015 1020 1021 1022 1023 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 1444 1445 1446 1447 1452 1453 1454 1455 1460 1461 1462 1463 1468 1469 1470 1471 1508 1509 1510 1511 1516 1517 1518 1519 1524 1525 1526 1527 1532 1533 1534 1535 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +[1/3][0/0][0/0][0/0] = 1536 1537 1538 1539 1544 1545 1546 1547 1552 1553 1554 1555 1560 1561 1562 1563 1600 1601 1602 1603 1608 1609 1610 1611 1616 1617 1618 1619 1624 1625 1626 1627 1664 1665 1666 1667 1672 1673 1674 1675 1680 1681 1682 1683 1688 1689 1690 1691 2048 2049 2050 2051 2056 2057 2058 2059 2064 2065 2066 2067 2072 2073 2074 2075 2112 2113 2114 2115 2120 2121 2122 2123 2128 2129 2130 2131 2136 2137 2138 2139 2176 2177 2178 2179 2184 2185 2186 2187 2192 2193 2194 2195 2200 2201 2202 2203 2560 2561 2562 2563 2568 2569 2570 2571 2576 2577 2578 2579 2584 2585 2586 2587 2624 2625 2626 2627 2632 2633 2634 2635 2640 2641 2642 2643 2648 2649 2650 2651 2688 2689 2690 2691 2696 2697 2698 2699 2704 2705 2706 2707 2712 2713 2714 2715 +[1/3][0/0][0/0][1/4] = 1540 1541 1542 1543 1548 1549 1550 1551 1556 1557 1558 1559 1564 1565 1566 1567 1604 1605 1606 1607 1612 1613 1614 1615 1620 1621 1622 1623 1628 1629 1630 1631 1668 1669 1670 1671 1676 1677 1678 1679 1684 1685 1686 1687 1692 1693 1694 1695 2052 2053 2054 2055 2060 2061 2062 2063 2068 2069 2070 2071 2076 2077 2078 2079 2116 2117 2118 2119 2124 2125 2126 2127 2132 2133 2134 2135 2140 2141 2142 2143 2180 2181 2182 2183 2188 2189 2190 2191 2196 2197 2198 2199 2204 2205 2206 2207 2564 2565 2566 2567 2572 2573 2574 2575 2580 2581 2582 2583 2588 2589 2590 2591 2628 2629 2630 2631 2636 2637 2638 2639 2644 2645 2646 2647 2652 2653 2654 2655 2692 2693 2694 2695 2700 2701 2702 2703 2708 2709 2710 2711 2716 2717 2718 2719 +[1/3][0/0][1/4][0/0] = 1568 1569 1570 1571 1576 1577 1578 1579 1584 1585 1586 1587 1592 1593 1594 1595 1632 1633 1634 1635 1640 1641 1642 1643 1648 1649 1650 1651 1656 1657 1658 1659 1696 1697 1698 1699 1704 1705 1706 1707 1712 1713 1714 1715 1720 1721 1722 1723 2080 2081 2082 2083 2088 2089 2090 2091 2096 2097 2098 2099 2104 2105 2106 2107 2144 2145 2146 2147 2152 2153 2154 2155 2160 2161 2162 2163 2168 2169 2170 2171 2208 2209 2210 2211 2216 2217 2218 2219 2224 2225 2226 2227 2232 2233 2234 2235 2592 2593 2594 2595 2600 2601 2602 2603 2608 2609 2610 2611 2616 2617 2618 2619 2656 2657 2658 2659 2664 2665 2666 2667 2672 2673 2674 2675 2680 2681 2682 2683 2720 2721 2722 2723 2728 2729 2730 2731 2736 2737 2738 2739 2744 2745 2746 2747 +[1/3][0/0][1/4][1/4] = 1572 1573 1574 1575 1580 1581 1582 1583 1588 1589 1590 1591 1596 1597 1598 1599 1636 1637 1638 1639 1644 1645 1646 1647 1652 1653 1654 1655 1660 1661 1662 1663 1700 1701 1702 1703 1708 1709 1710 1711 1716 1717 1718 1719 1724 1725 1726 1727 2084 2085 2086 2087 2092 2093 2094 2095 2100 2101 2102 2103 2108 2109 2110 2111 2148 2149 2150 2151 2156 2157 2158 2159 2164 2165 2166 2167 2172 2173 2174 2175 2212 2213 2214 2215 2220 2221 2222 2223 2228 2229 2230 2231 2236 2237 2238 2239 2596 2597 2598 2599 2604 2605 2606 2607 2612 2613 2614 2615 2620 2621 2622 2623 2660 2661 2662 2663 2668 2669 2670 2671 2676 2677 2678 2679 2684 2685 2686 2687 2724 2725 2726 2727 2732 2733 2734 2735 2740 2741 2742 2743 2748 2749 2750 2751 +[1/3][1/3][0/0][0/0] = 1728 1729 1730 1731 1736 1737 1738 1739 1744 1745 1746 1747 1752 1753 1754 1755 1792 1793 1794 1795 1800 1801 1802 1803 1808 1809 1810 1811 1816 1817 1818 1819 1856 1857 1858 1859 1864 1865 1866 1867 1872 1873 1874 1875 1880 1881 1882 1883 2240 2241 2242 2243 2248 2249 2250 2251 2256 2257 2258 2259 2264 2265 2266 2267 2304 2305 2306 2307 2312 2313 2314 2315 2320 2321 2322 2323 2328 2329 2330 2331 2368 2369 2370 2371 2376 2377 2378 2379 2384 2385 2386 2387 2392 2393 2394 2395 2752 2753 2754 2755 2760 2761 2762 2763 2768 2769 2770 2771 2776 2777 2778 2779 2816 2817 2818 2819 2824 2825 2826 2827 2832 2833 2834 2835 2840 2841 2842 2843 2880 2881 2882 2883 2888 2889 2890 2891 2896 2897 2898 2899 2904 2905 2906 2907 +[1/3][1/3][0/0][1/4] = 1732 1733 1734 1735 1740 1741 1742 1743 1748 1749 1750 1751 1756 1757 1758 1759 1796 1797 1798 1799 1804 1805 1806 1807 1812 1813 1814 1815 1820 1821 1822 1823 1860 1861 1862 1863 1868 1869 1870 1871 1876 1877 1878 1879 1884 1885 1886 1887 2244 2245 2246 2247 2252 2253 2254 2255 2260 2261 2262 2263 2268 2269 2270 2271 2308 2309 2310 2311 2316 2317 2318 2319 2324 2325 2326 2327 2332 2333 2334 2335 2372 2373 2374 2375 2380 2381 2382 2383 2388 2389 2390 2391 2396 2397 2398 2399 2756 2757 2758 2759 2764 2765 2766 2767 2772 2773 2774 2775 2780 2781 2782 2783 2820 2821 2822 2823 2828 2829 2830 2831 2836 2837 2838 2839 2844 2845 2846 2847 2884 2885 2886 2887 2892 2893 2894 2895 2900 2901 2902 2903 2908 2909 2910 2911 +[1/3][1/3][1/4][0/0] = 1760 1761 1762 1763 1768 1769 1770 1771 1776 1777 1778 1779 1784 1785 1786 1787 1824 1825 1826 1827 1832 1833 1834 1835 1840 1841 1842 1843 1848 1849 1850 1851 1888 1889 1890 1891 1896 1897 1898 1899 1904 1905 1906 1907 1912 1913 1914 1915 2272 2273 2274 2275 2280 2281 2282 2283 2288 2289 2290 2291 2296 2297 2298 2299 2336 2337 2338 2339 2344 2345 2346 2347 2352 2353 2354 2355 2360 2361 2362 2363 2400 2401 2402 2403 2408 2409 2410 2411 2416 2417 2418 2419 2424 2425 2426 2427 2784 2785 2786 2787 2792 2793 2794 2795 2800 2801 2802 2803 2808 2809 2810 2811 2848 2849 2850 2851 2856 2857 2858 2859 2864 2865 2866 2867 2872 2873 2874 2875 2912 2913 2914 2915 2920 2921 2922 2923 2928 2929 2930 2931 2936 2937 2938 2939 +[1/3][1/3][1/4][1/4] = 1764 1765 1766 1767 1772 1773 1774 1775 1780 1781 1782 1783 1788 1789 1790 1791 1828 1829 1830 1831 1836 1837 1838 1839 1844 1845 1846 1847 1852 1853 1854 1855 1892 1893 1894 1895 1900 1901 1902 1903 1908 1909 1910 1911 1916 1917 1918 1919 2276 2277 2278 2279 2284 2285 2286 2287 2292 2293 2294 2295 2300 2301 2302 2303 2340 2341 2342 2343 2348 2349 2350 2351 2356 2357 2358 2359 2364 2365 2366 2367 2404 2405 2406 2407 2412 2413 2414 2415 2420 2421 2422 2423 2428 2429 2430 2431 2788 2789 2790 2791 2796 2797 2798 2799 2804 2805 2806 2807 2812 2813 2814 2815 2852 2853 2854 2855 2860 2861 2862 2863 2868 2869 2870 2871 2876 2877 2878 2879 2916 2917 2918 2919 2924 2925 2926 2927 2932 2933 2934 2935 2940 2941 2942 2943 +[1/3][2/6][0/0][0/0] = 1920 1921 1922 1923 1928 1929 1930 1931 1936 1937 1938 1939 1944 1945 1946 1947 1984 1985 1986 1987 1992 1993 1994 1995 2000 2001 2002 2003 2008 2009 2010 2011 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2432 2433 2434 2435 2440 2441 2442 2443 2448 2449 2450 2451 2456 2457 2458 2459 2496 2497 2498 2499 2504 2505 2506 2507 2512 2513 2514 2515 2520 2521 2522 2523 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2944 2945 2946 2947 2952 2953 2954 2955 2960 2961 2962 2963 2968 2969 2970 2971 3008 3009 3010 3011 3016 3017 3018 3019 3024 3025 3026 3027 3032 3033 3034 3035 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +[1/3][2/6][0/0][1/4] = 1924 1925 1926 1927 1932 1933 1934 1935 1940 1941 1942 1943 1948 1949 1950 1951 1988 1989 1990 1991 1996 1997 1998 1999 2004 2005 2006 2007 2012 2013 2014 2015 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2436 2437 2438 2439 2444 2445 2446 2447 2452 2453 2454 2455 2460 2461 2462 2463 2500 2501 2502 2503 2508 2509 2510 2511 2516 2517 2518 2519 2524 2525 2526 2527 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2948 2949 2950 2951 2956 2957 2958 2959 2964 2965 2966 2967 2972 2973 2974 2975 3012 3013 3014 3015 3020 3021 3022 3023 3028 3029 3030 3031 3036 3037 3038 3039 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +[1/3][2/6][1/4][0/0] = 1952 1953 1954 1955 1960 1961 1962 1963 1968 1969 1970 1971 1976 1977 1978 1979 2016 2017 2018 2019 2024 2025 2026 2027 2032 2033 2034 2035 2040 2041 2042 2043 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2464 2465 2466 2467 2472 2473 2474 2475 2480 2481 2482 2483 2488 2489 2490 2491 2528 2529 2530 2531 2536 2537 2538 2539 2544 2545 2546 2547 2552 2553 2554 2555 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2976 2977 2978 2979 2984 2985 2986 2987 2992 2993 2994 2995 3000 3001 3002 3003 3040 3041 3042 3043 3048 3049 3050 3051 3056 3057 3058 3059 3064 3065 3066 3067 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +[1/3][2/6][1/4][1/4] = 1956 1957 1958 1959 1964 1965 1966 1967 1972 1973 1974 1975 1980 1981 1982 1983 2020 2021 2022 2023 2028 2029 2030 2031 2036 2037 2038 2039 2044 2045 2046 2047 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2468 2469 2470 2471 2476 2477 2478 2479 2484 2485 2486 2487 2492 2493 2494 2495 2532 2533 2534 2535 2540 2541 2542 2543 2548 2549 2550 2551 2556 2557 2558 2559 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 2980 2981 2982 2983 2988 2989 2990 2991 2996 2997 2998 2999 3004 3005 3006 3007 3044 3045 3046 3047 3052 3053 3054 3055 3060 3061 3062 3063 3068 3069 3070 3071 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +[2/6][0/0][0/0][0/0] = 3072 3073 3074 3075 3080 3081 3082 3083 3088 3089 3090 3091 3096 3097 3098 3099 3136 3137 3138 3139 3144 3145 3146 3147 3152 3153 3154 3155 3160 3161 3162 3163 3200 3201 3202 3203 3208 3209 3210 3211 3216 3217 3218 3219 3224 3225 3226 3227 3584 3585 3586 3587 3592 3593 3594 3595 3600 3601 3602 3603 3608 3609 3610 3611 3648 3649 3650 3651 3656 3657 3658 3659 3664 3665 3666 3667 3672 3673 3674 3675 3712 3713 3714 3715 3720 3721 3722 3723 3728 3729 3730 3731 3736 3737 3738 3739 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +[2/6][0/0][0/0][1/4] = 3076 3077 3078 3079 3084 3085 3086 3087 3092 3093 3094 3095 3100 3101 3102 3103 3140 3141 3142 3143 3148 3149 3150 3151 3156 3157 3158 3159 3164 3165 3166 3167 3204 3205 3206 3207 3212 3213 3214 3215 3220 3221 3222 3223 3228 3229 3230 3231 3588 3589 3590 3591 3596 3597 3598 3599 3604 3605 3606 3607 3612 3613 3614 3615 3652 3653 3654 3655 3660 3661 3662 3663 3668 3669 3670 3671 3676 3677 3678 3679 3716 3717 3718 3719 3724 3725 3726 3727 3732 3733 3734 3735 3740 3741 3742 3743 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +[2/6][0/0][1/4][0/0] = 3104 3105 3106 3107 3112 3113 3114 3115 3120 3121 3122 3123 3128 3129 3130 3131 3168 3169 3170 3171 3176 3177 3178 3179 3184 3185 3186 3187 3192 3193 3194 3195 3232 3233 3234 3235 3240 3241 3242 3243 3248 3249 3250 3251 3256 3257 3258 3259 3616 3617 3618 3619 3624 3625 3626 3627 3632 3633 3634 3635 3640 3641 3642 3643 3680 3681 3682 3683 3688 3689 3690 3691 3696 3697 3698 3699 3704 3705 3706 3707 3744 3745 3746 3747 3752 3753 3754 3755 3760 3761 3762 3763 3768 3769 3770 3771 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +[2/6][0/0][1/4][1/4] = 3108 3109 3110 3111 3116 3117 3118 3119 3124 3125 3126 3127 3132 3133 3134 3135 3172 3173 3174 3175 3180 3181 3182 3183 3188 3189 3190 3191 3196 3197 3198 3199 3236 3237 3238 3239 3244 3245 3246 3247 3252 3253 3254 3255 3260 3261 3262 3263 3620 3621 3622 3623 3628 3629 3630 3631 3636 3637 3638 3639 3644 3645 3646 3647 3684 3685 3686 3687 3692 3693 3694 3695 3700 3701 3702 3703 3708 3709 3710 3711 3748 3749 3750 3751 3756 3757 3758 3759 3764 3765 3766 3767 3772 3773 3774 3775 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +[2/6][1/3][0/0][0/0] = 3264 3265 3266 3267 3272 3273 3274 3275 3280 3281 3282 3283 3288 3289 3290 3291 3328 3329 3330 3331 3336 3337 3338 3339 3344 3345 3346 3347 3352 3353 3354 3355 3392 3393 3394 3395 3400 3401 3402 3403 3408 3409 3410 3411 3416 3417 3418 3419 3776 3777 3778 3779 3784 3785 3786 3787 3792 3793 3794 3795 3800 3801 3802 3803 3840 3841 3842 3843 3848 3849 3850 3851 3856 3857 3858 3859 3864 3865 3866 3867 3904 3905 3906 3907 3912 3913 3914 3915 3920 3921 3922 3923 3928 3929 3930 3931 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +[2/6][1/3][0/0][1/4] = 3268 3269 3270 3271 3276 3277 3278 3279 3284 3285 3286 3287 3292 3293 3294 3295 3332 3333 3334 3335 3340 3341 3342 3343 3348 3349 3350 3351 3356 3357 3358 3359 3396 3397 3398 3399 3404 3405 3406 3407 3412 3413 3414 3415 3420 3421 3422 3423 3780 3781 3782 3783 3788 3789 3790 3791 3796 3797 3798 3799 3804 3805 3806 3807 3844 3845 3846 3847 3852 3853 3854 3855 3860 3861 3862 3863 3868 3869 3870 3871 3908 3909 3910 3911 3916 3917 3918 3919 3924 3925 3926 3927 3932 3933 3934 3935 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +[2/6][1/3][1/4][0/0] = 3296 3297 3298 3299 3304 3305 3306 3307 3312 3313 3314 3315 3320 3321 3322 3323 3360 3361 3362 3363 3368 3369 3370 3371 3376 3377 3378 3379 3384 3385 3386 3387 3424 3425 3426 3427 3432 3433 3434 3435 3440 3441 3442 3443 3448 3449 3450 3451 3808 3809 3810 3811 3816 3817 3818 3819 3824 3825 3826 3827 3832 3833 3834 3835 3872 3873 3874 3875 3880 3881 3882 3883 3888 3889 3890 3891 3896 3897 3898 3899 3936 3937 3938 3939 3944 3945 3946 3947 3952 3953 3954 3955 3960 3961 3962 3963 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +[2/6][1/3][1/4][1/4] = 3300 3301 3302 3303 3308 3309 3310 3311 3316 3317 3318 3319 3324 3325 3326 3327 3364 3365 3366 3367 3372 3373 3374 3375 3380 3381 3382 3383 3388 3389 3390 3391 3428 3429 3430 3431 3436 3437 3438 3439 3444 3445 3446 3447 3452 3453 3454 3455 3812 3813 3814 3815 3820 3821 3822 3823 3828 3829 3830 3831 3836 3837 3838 3839 3876 3877 3878 3879 3884 3885 3886 3887 3892 3893 3894 3895 3900 3901 3902 3903 3940 3941 3942 3943 3948 3949 3950 3951 3956 3957 3958 3959 3964 3965 3966 3967 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +[2/6][2/6][0/0][0/0] = 3456 3457 3458 3459 3464 3465 3466 3467 3472 3473 3474 3475 3480 3481 3482 3483 3520 3521 3522 3523 3528 3529 3530 3531 3536 3537 3538 3539 3544 3545 3546 3547 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 3968 3969 3970 3971 3976 3977 3978 3979 3984 3985 3986 3987 3992 3993 3994 3995 4032 4033 4034 4035 4040 4041 4042 4043 4048 4049 4050 4051 4056 4057 4058 4059 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +[2/6][2/6][0/0][1/4] = 3460 3461 3462 3463 3468 3469 3470 3471 3476 3477 3478 3479 3484 3485 3486 3487 3524 3525 3526 3527 3532 3533 3534 3535 3540 3541 3542 3543 3548 3549 3550 3551 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 3972 3973 3974 3975 3980 3981 3982 3983 3988 3989 3990 3991 3996 3997 3998 3999 4036 4037 4038 4039 4044 4045 4046 4047 4052 4053 4054 4055 4060 4061 4062 4063 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +[2/6][2/6][1/4][0/0] = 3488 3489 3490 3491 3496 3497 3498 3499 3504 3505 3506 3507 3512 3513 3514 3515 3552 3553 3554 3555 3560 3561 3562 3563 3568 3569 3570 3571 3576 3577 3578 3579 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 4000 4001 4002 4003 4008 4009 4010 4011 4016 4017 4018 4019 4024 4025 4026 4027 4064 4065 4066 4067 4072 4073 4074 4075 4080 4081 4082 4083 4088 4089 4090 4091 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +[2/6][2/6][1/4][1/4] = 3492 3493 3494 3495 3500 3501 3502 3503 3508 3509 3510 3511 3516 3517 3518 3519 3556 3557 3558 3559 3564 3565 3566 3567 3572 3573 3574 3575 3580 3581 3582 3583 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 4004 4005 4006 4007 4012 4013 4014 4015 4020 4021 4022 4023 4028 4029 4030 4031 4068 4069 4070 4071 4076 4077 4078 4079 4084 4085 4086 4087 4092 4093 4094 4095 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ diff --git a/v3_nczarr_test/ref_newformatpure.cdl b/v3_nczarr_test/ref_newformatpure.cdl new file mode 100644 index 0000000000..dac8be5002 --- /dev/null +++ b/v3_nczarr_test/ref_newformatpure.cdl @@ -0,0 +1,33 @@ +netcdf ref_oldformat { +dimensions: + _Anonymous_Dim_8 = 8 ; + _Anonymous_Dim_10 = 10 ; +variables: + int lat(_Anonymous_Dim_8) ; + lat:_FillValue = -1 ; + string lat:lat_attr = "latitude" ; +data: + + lat = 1, 2, 3, 4, 5, 6, 7, 8 ; + +group: g1 { + variables: + int pos(_Anonymous_Dim_8, _Anonymous_Dim_10) ; + pos:_FillValue = -1 ; + string pos:pos_attr = "latXlon" ; + + // group attributes: + :g1_attr = 17 ; + data: + + pos = + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _ ; + } // group g1 +} diff --git a/v3_nczarr_test/ref_noshape.file.zip b/v3_nczarr_test/ref_noshape.file.zip new file mode 100644 index 0000000000000000000000000000000000000000..91d1bd6afbea46fac94671926ea0401ee048a71c GIT binary patch literal 1482 zcmWIWW@h1H0D;PM$8azMN(eH@FchVx#pmT0XCxM+>ZN7or0R!;a56A2Jt&##xAJjn zX$3a}Bg+eB1_m$@05__wHYozxC}C_yl|c;Kv-)xB6(Gi9mPuTJ2mAcISR0b1ugbUau$Ek2vnxM#{FF9ZLR zYL{I&PxznHID1;J&3k$3vnNNUt3I33bmfH4Yn%U)H-U*5uNc><)GkR6& zMfs%#a0f)M!U(x)B~a+br{x#rCYAs#GD5TCPOj}77oZ&=EP*Y4fU$$(Nvw9n`#A>` zElbVGFUU>JE78eI2D!}E zNY6meKndt7MkYCCMDPQh0|$cO)CuAWF#K=?(O7a1D6(SrcUSZ^SW zVJs*o0SyM_B&-JG%11DR85lM+&cJRkFjwJoGOip1F_{r)=7z=}*iDA#C!qOQ@)B~8 zfKnN1LPs`#BhVTwK>|+^_}q_(Nz_z~Y`zH-7W3h0nmE%@(=W2=Gq9TuP0hGWXJrEh RDg!$ZwgL6)0ZU*81_1a`d87aU literal 0 HcmV?d00001 diff --git a/v3_nczarr_test/ref_notzarr.tar.gz b/v3_nczarr_test/ref_notzarr.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..efc55444e5ed0a8aba861c3492d0e4b8ccbad842 GIT binary patch literal 433 zcmV;i0Z#rOiwFp!_8nvZ19D|%Uv6)7dSP;ME_7jX0PU7NPQx%9#hs?DO6h>a!T=~E z6SY&jS%DRV7=S%ef~Z746jD*TaSm1_?!Z~NLI(zRoTh<ufRN@FhXqU32rU132>B{SjamtA$lvR{sd>^`jWf8A` zTX^63aNz5xM>GE1|8Np2d`4t&m-AKpx0xUoRanzE@L&2r?Tr*Zqs)Js+j;*f{MW(7 zsab8#M-RQpb^4b&*^R%B_SNMEF+V={yr_*Qo5ZNjOp{cNiYfnZRQAnf3OkF@7MUGK zeWF*3^>SLZdUTf|k2BdHDSdMgF&KtZHwcs_fIsa24j0(}b&&BtIazZYDBb@V|K&fo zDb{~21n$5a_ASp1-R6JS0`!wid6fyUZ2pT*Zv7X6VgBo2dH!q2{I~czN)ljNq4oZD z(30b*ZKo7cTT78xdO0eJmW^Dr{3&%=xXh|$3yswb*0Ub900 b000000000000000!0LPfhZ2_h04M+e8Di27 literal 0 HcmV?d00001 diff --git a/v3_nczarr_test/ref_nulls.cdl b/v3_nczarr_test/ref_nulls.cdl new file mode 100644 index 0000000000..a16ca34a24 --- /dev/null +++ b/v3_nczarr_test/ref_nulls.cdl @@ -0,0 +1,12 @@ +netcdf ref_nulls { +variables: + int test; + test:nul_sng = '\0'; + test:empty_sng = ""; + test:space_sng = " "; + test:zero_sng = "0"; + test:nul0 = '\0' ; +data: + + test = 1; +} diff --git a/v3_nczarr_test/ref_nulls_nczarr.baseline b/v3_nczarr_test/ref_nulls_nczarr.baseline new file mode 100644 index 0000000000..263561c0e0 --- /dev/null +++ b/v3_nczarr_test/ref_nulls_nczarr.baseline @@ -0,0 +1,12 @@ +netcdf ref_nulls { +variables: + int test ; + test:nul_sng = 0b ; + test:empty_sng = "" ; + test:space_sng = " " ; + test:zero_sng = "0" ; + test:nul0 = 0b ; +data: + + test = 1 ; +} diff --git a/v3_nczarr_test/ref_nulls_zarr.baseline b/v3_nczarr_test/ref_nulls_zarr.baseline index 8baefe949a..2d5e25b141 100644 --- a/v3_nczarr_test/ref_nulls_zarr.baseline +++ b/v3_nczarr_test/ref_nulls_zarr.baseline @@ -6,7 +6,6 @@ variables: test:space_sng = " " ; test:zero_sng = "0" ; test:nul0 = 0 ; - test:_FillValue = -2147483647 ; data: test = 1 ; diff --git a/v3_nczarr_test/ref_oldformat.cdl b/v3_nczarr_test/ref_oldformat.cdl new file mode 100644 index 0000000000..5efd2abb03 --- /dev/null +++ b/v3_nczarr_test/ref_oldformat.cdl @@ -0,0 +1,33 @@ +netcdf ref_oldformat { +dimensions: + _Anonymous_Dim_8 = 8 ; + _Anonymous_Dim_10 = 10 ; +variables: + int lat(_Anonymous_Dim_8) ; + lat:_FillValue = -1 ; + lat:lat_attr = "latitude" ; +data: + + lat = 1, 2, 3, 4, 5, 6, 7, 8 ; + +group: g1 { + variables: + int pos(_Anonymous_Dim_8, _Anonymous_Dim_10) ; + pos:_FillValue = -1 ; + pos:pos_attr = "latXlon" ; + + // group attributes: + :g1_attr = 17 ; + data: + + pos = + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _ ; + } // group g1 +} diff --git a/v3_nczarr_test/ref_oldformat.zip b/v3_nczarr_test/ref_oldformat.zip new file mode 100644 index 0000000000000000000000000000000000000000..41a49399577335e3e0738e9b2c8e807631d810cf GIT binary patch literal 3342 zcmWIWW@h1H00G5GypdoAl;C5KVJJ#Xi_gzVNy{(FO)Sw54dG;9uKT__&GyIcw9*Q0 z21b?_%nS@*A^>iVXCJCLLKx=cz|6AzxH~Nhh_RR@0(1oj1IR3&jj_wcfjk8u7RG9p z0oY+U3`fNws7is~vh( ziA6<;m0+9VzwS;;2Vyj5uMS(~rK8p5e_H>PuTJ2mAcISRTIW1;JXnnu>n4>>3G&i9 zD5E;tM_5&M>64yId)_3?NQp}G^Q(zW^3(KaUAM3=s4mbeOVDGhu}5xQo?~EEVWFLe z%6b!Z_B9Jtwyuykwf-p+n!7(eoUFed=w>NKJnk+jDJsV2?p%LIu0sX_dwz4>EOzCV zb8*oKS=b~RCE%jcQg$_Amd7NytTXq0Ump$P=ia^9_;3&Frqns|n(GCpJ@%AfSl$+p zJjMR@+?l&vI2v3c#pZgQZ*Yhfo^LBVi{~M0~tqQ!Y$$9g{Ival(ehzbIX}IbAD_(GAC$)QmnJ=(m#u) zTv3|9mOkOrpF2&9t~`2FGr=P+(~mF0*)uE9a7z5?qpV1Y7nHHTN&Br}2RhvWn0O^{ zIp07Jk-JTRMGso|{{CMt5`dg59D%|7Tw;6M0-(VxK!fpx3p``o0w#<5m`NkI&yf3& zfk;bvee0c=i%rcP98DHGxRg3?3E&Z{5nt#Em2YWU?`XuF6nEOM``$+#vbC-tv*XyjlByJ1{{!%%8_aA<3x!as| zRo14*EGn1~i7N;#CtwXu{#vB)1NTaafWF4_2Jzq?dV* zzmBG65U`9ixgOYc-g_Om;M6~*uglOCs=YMj*|KM%>Cc{vHt*PK<>m15h?jtR)Jlc@ z5t?D{oNFX>grd6*n6k3A26?7BYDE}NiuE#^spd7~n&#CCNsUAaMzOVP*J6~FYf^Ua zIRbQcA~028W{7det+&J1BqkrYr87WXRj#Ta6sYJ6~(4L#pIVlp5`}~C2V_t zZ;q3y?EU-mZ0oHQmj8|H>hhdx%sSiB#Aj8^2MvcgFID!sYwtOB+VzxFSjVcA80C33 zE2jiD*uG(8GPYO`@?_&f}SLj@`^v+pgxfddTTz>fC=XajVvI<+V$lCD9u<-YEI=?a_Zpmi3Rn zpPPT?;c_{|Xw0e)Vl*UmhydGt zs4fRrgg|3K6(QCDff$Qf4I&#GkIh&}r3f?{R4HOL8c{x>R*A@gu@0BfB*Z3Wt%Ynu zE(?}00XZIAhXLJBAdO>IUC5>%!(}>hasgFjSVIUB_?XoMvJHD!vA7>O#egaetfnKv zAGLl!HeHbotLe~E7Z^H3cP;{#kp5j@2v|>+}C|2(7%Ad++#}J?)R97}yGBVm=7W z>kAZ`x0vPIyRFl!b}BJ<_Xiq&xty!8aDr3zLDg*`raRC6VlIom|9#Kbw5@WzW|2yh z-`E6PnIg4NBD7KKZ0B+51-ylBrQhz~++eV)xFY{por2Mt&eJuQ5`R8j%4eJzv~Ft2 z|Gw+k$hlv{kE|r5@~h z8`S$=@yMMga?G0?b=2F;c(}qQ9MssdfAjVZ>5yGQ{{(itHwxZ7!P8FJzkYYh^gr*d z)0UNLPJMUdh0pPfzrX~;&LKNfG-@+2973Vt07-i22}mAq67X^Kb3sZ!CTQtLjj(2D z@>%^Xh=FNo#qk9S$4^v<`^SgxPw=nbpP;JUnz2D7!b*Lkizj>M?D@-Co*i{!nR=@5`4;(sk>C%M@2VBlGtP)sZY-^`)D89r^bFGHqP8DN&eLH(y zJ9R^S)~wc`Le2@3Bc?b7Bxc5?c^Y~&D*CWw)TepX1SZ8*X?5|J6k2ew7%U0c6EOWaJ7;M%G5bkGh3EUkVPaS=46tqkP$2dIIC02M$cA=9AgHNYFyP~^%S)zD&KA|b+1P$B>s3JXDW ze}mcq2vE(0&roQq0O3MV{fJz1g4zWLu!FE+SbPmK6WQ0OW^Q06=s=*EqI=kK0)q#Aw&6N-QZUDlVPmpMTqc z=ji*tA}#v+RQhfwI!=h5u*7wPsdiG%fen02%Ep~sM~zf=9t`-h+keXc99gURI(GZ! z%WLkBajckhR%_Z z(D+9hQ?1qPw1Ua;&ka`@_f9oY*}GqN+Qv5vF*iPS^d6dHGsTAKRF+ zMn%|NU-RtL$=_9WGh_6ozq#Q%dF87FwU@7FOY=mRG#)GeQ8W4J-f!QvB43B!eEsKr zX9atOieugmv3A+_cwz z?tI&S<#IZYA%EqOh8)4wGZ*sqHi<4Vbll=mU+i#WJa9O`7ig_>5J)K z%QJ_ztXp!)cJ`dVLb(^kK|%3(;rX9ifw>?X7#soKj7(e%xN|#D84^Is&XBy%$Rxsm z$o^Qg{2j&e}MBES(K6UjN)jf0i}AV /String/ "string" +{1} "int" => /Int/ 117 +{1} "double" => /Double/ 3.14159 +{1} "boolean" => /Boolean/ true +{1} "null" => /Null/ null +{1} "array1" => /Array/ [] +{1} "array2" => /Array/ +[2] /String/ "string" +[2] /Int/ 117 +[2] /Double/ 3.14159 +[2] /Boolean/ true +[2] /Null/ null +[2] /Array/ [] + +{1} "dict1" => /Dict/ {} + diff --git a/v3_nczarr_test/ref_ut_json_parse.txt b/v3_nczarr_test/ref_ut_json_parse.txt new file mode 100644 index 0000000000..b3cfb120dd --- /dev/null +++ b/v3_nczarr_test/ref_ut_json_parse.txt @@ -0,0 +1,2 @@ +text : |{"string": "string","int": 117,"double": 3.1415926,"boolean": true,"null": null,"array1": [],"array2": ["string",117,3.1415926,true,null,[]],"dict1": {}}| +result: |{"string": "string","int": 117,"double": 3.1415926,"boolean": true,"null": null,"array1": [],"array2": ["string",117,3.1415926,true,null,[]],"dict1": {}}| diff --git a/v3_nczarr_test/ref_ut_map_create.cdl b/v3_nczarr_test/ref_ut_map_create.cdl new file mode 100644 index 0000000000..68d2ed637b --- /dev/null +++ b/v3_nczarr_test/ref_ut_map_create.cdl @@ -0,0 +1 @@ +[0] /.zgroup : (0) || diff --git a/v3_nczarr_test/ref_ut_map_readmeta.txt b/v3_nczarr_test/ref_ut_map_readmeta.txt new file mode 100644 index 0000000000..822b80270f --- /dev/null +++ b/v3_nczarr_test/ref_ut_map_readmeta.txt @@ -0,0 +1,4 @@ +/meta1/.zarray: |{ +"foo": 42, +"bar": "apples", +"baz": [1, 2, 3, 4]}| diff --git a/v3_nczarr_test/ref_ut_map_readmeta2.txt b/v3_nczarr_test/ref_ut_map_readmeta2.txt new file mode 100644 index 0000000000..ec4c5ba988 --- /dev/null +++ b/v3_nczarr_test/ref_ut_map_readmeta2.txt @@ -0,0 +1,5 @@ +/meta2/.zarray: |{ +"foo": 42, +"bar": "apples", +"baz": [1, 2, 3, 4], +"extra": 137}| diff --git a/v3_nczarr_test/ref_ut_map_search.txt b/v3_nczarr_test/ref_ut_map_search.txt new file mode 100644 index 0000000000..dd8cea3962 --- /dev/null +++ b/v3_nczarr_test/ref_ut_map_search.txt @@ -0,0 +1,4 @@ +[0] /.zgroup +[1] /data1/0 +[2] /meta1/.zarray +[3] /meta2/.zarray diff --git a/v3_nczarr_test/ref_ut_map_writedata.cdl b/v3_nczarr_test/ref_ut_map_writedata.cdl new file mode 100644 index 0000000000..6f8f27aae7 --- /dev/null +++ b/v3_nczarr_test/ref_ut_map_writedata.cdl @@ -0,0 +1,11 @@ +[0] /.zgroup : (0) || +[1] /data1/0 : (25) (int) |0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24| +[2] /meta1/.zarray : (50) |{ +"foo": 42, +"bar": "apples", +"baz": [1, 2, 3, 4]}| +[3] /meta2/.zarray : (64) |{ +"foo": 42, +"bar": "apples", +"baz": [1, 2, 3, 4], +"extra": 137}| diff --git a/v3_nczarr_test/ref_ut_map_writemeta.cdl b/v3_nczarr_test/ref_ut_map_writemeta.cdl new file mode 100644 index 0000000000..e2c72b528a --- /dev/null +++ b/v3_nczarr_test/ref_ut_map_writemeta.cdl @@ -0,0 +1,5 @@ +[0] /.zgroup : (0) || +[1] /meta1/.zarray : (50) |{ +"foo": 42, +"bar": "apples", +"baz": [1, 2, 3, 4]}| diff --git a/v3_nczarr_test/ref_ut_map_writemeta2.cdl b/v3_nczarr_test/ref_ut_map_writemeta2.cdl new file mode 100644 index 0000000000..bd0e626b18 --- /dev/null +++ b/v3_nczarr_test/ref_ut_map_writemeta2.cdl @@ -0,0 +1,10 @@ +[0] /.zgroup : (0) || +[1] /meta1/.zarray : (50) |{ +"foo": 42, +"bar": "apples", +"baz": [1, 2, 3, 4]}| +[2] /meta2/.zarray : (64) |{ +"foo": 42, +"bar": "apples", +"baz": [1, 2, 3, 4], +"extra": 137}| diff --git a/v3_nczarr_test/ref_ut_mapapi_create.cdl b/v3_nczarr_test/ref_ut_mapapi_create.cdl new file mode 100644 index 0000000000..68d2ed637b --- /dev/null +++ b/v3_nczarr_test/ref_ut_mapapi_create.cdl @@ -0,0 +1 @@ +[0] /.zgroup : (0) || diff --git a/v3_nczarr_test/ref_ut_mapapi_data.cdl b/v3_nczarr_test/ref_ut_mapapi_data.cdl new file mode 100644 index 0000000000..f644d71d5a --- /dev/null +++ b/v3_nczarr_test/ref_ut_mapapi_data.cdl @@ -0,0 +1,8 @@ +[0] /.zgroup : (50) |{ +"foo": 42, +"bar": "apples", +"baz": [1, 2, 3, 4]}| +[1] /data1 : (100) (ubyte) |0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 0, 8, 0, 0, 0, 9, 0, 0, 0, 10, 0, 0, 0, 11, 0, 0, 0, 12, 0, 0, 0, 13, 0, 0, 0, 14, 0, 0, 0, 15, 0, 0, 0, 16, 0, 0, 0, 17, 0, 0, 0, 18, 0, 0, 0, 19, 0, 0, 0, 20, 0, 0, 0, 21, 0, 0, 0, 22, 0, 0, 0, 23, 0, 0, 0, 24, 0, 0, 0| +[2] /meta1/.zarray : (34) |{ +"shape": [1,2,3], +"dtype": "<1"}| diff --git a/v3_nczarr_test/ref_ut_mapapi_meta.cdl b/v3_nczarr_test/ref_ut_mapapi_meta.cdl new file mode 100644 index 0000000000..4834000df7 --- /dev/null +++ b/v3_nczarr_test/ref_ut_mapapi_meta.cdl @@ -0,0 +1,7 @@ +[0] /.zgroup : (50) |{ +"foo": 42, +"bar": "apples", +"baz": [1, 2, 3, 4]}| +[1] /meta1/.zarray : (34) |{ +"shape": [1,2,3], +"dtype": "<1"}| diff --git a/v3_nczarr_test/ref_ut_mapapi_search.txt b/v3_nczarr_test/ref_ut_mapapi_search.txt new file mode 100644 index 0000000000..9084ce25ac --- /dev/null +++ b/v3_nczarr_test/ref_ut_mapapi_search.txt @@ -0,0 +1,3 @@ +[0] /.zgroup +[1] /data1 +[2] /meta1/.zarray diff --git a/v3_nczarr_test/ref_ut_proj.txt b/v3_nczarr_test/ref_ut_proj.txt new file mode 100644 index 0000000000..fae9a0492a --- /dev/null +++ b/v3_nczarr_test/ref_ut_proj.txt @@ -0,0 +1,13 @@ +Options: -ddim1=4 -v '4 v[dim1/2]' -s [0:4|0] +[r=0] ChunkRange{start=0 stop=2} Slice{0:4|0} +[0] Projection{id=1,chunkindex=0,first=0,last=1,limit=2,iopos=0,iocount=2,chunkslice=Slice{0:2|2},memslice=Slice{0:2|4} +[1] Projection{id=2,chunkindex=1,first=0,last=1,limit=2,iopos=2,iocount=2,chunkslice=Slice{0:2|2},memslice=Slice{2:4|4} +Options: -ddim1=6 -v '4 v[dim1/3]' -s [0:5|0] +[r=0] ChunkRange{start=0 stop=2} Slice{0:5|0} +[0] Projection{id=1,chunkindex=0,first=0,last=2,limit=3,iopos=0,iocount=3,chunkslice=Slice{0:3|3},memslice=Slice{0:3|6} +[1] Projection{id=2,chunkindex=1,first=0,last=1,limit=2,iopos=3,iocount=2,chunkslice=Slice{0:2|3},memslice=Slice{3:5|6} +Options: -ddim1=4 -ddim2=2 -v '4 v[dim1/2,dim2/2]' -s [2:3|0][0:2|0] +[r=0] ChunkRange{start=1 stop=2} Slice{2:3|0} +[0] Projection{id=1,chunkindex=1,first=0,last=0,limit=1,iopos=0,iocount=1,chunkslice=Slice{0:1|2},memslice=Slice{0:1|4} +[r=1] ChunkRange{start=0 stop=1} Slice{0:2|0} +[0] Projection{id=2,chunkindex=0,first=0,last=1,limit=2,iopos=0,iocount=2,chunkslice=Slice{0:2|2},memslice=Slice{0:2|2} diff --git a/v3_nczarr_test/ref_ut_testmap_create.cdl b/v3_nczarr_test/ref_ut_testmap_create.cdl new file mode 100644 index 0000000000..75636ee802 --- /dev/null +++ b/v3_nczarr_test/ref_ut_testmap_create.cdl @@ -0,0 +1,32 @@ +netcdf testmeta { + +group: _nczarr { + + // group attributes: + :data = "{\"zarr_format\": 2,\"nczarr_version\": \"1.0.0\"}" ; + } // group _nczarr + +group: _zgroup { + + // group attributes: + :data = "{\"zarr_format\": 2}" ; + } // group _zgroup + +group: _nczgroup { + + // group attributes: + :data = "{\"dimensions\": {},\"arrays\": [],\"groups\": []}" ; + } // group _nczgroup + +group: _nczattr { + + // group attributes: + :data = "{\"types\": {\"_NCProperties\": \"2\"}}" ; + } // group _nczattr + +group: _zattrs { + + // group attributes: + :data = "{\"_NCProperties\": \"version=\"}" ; + } // group _zattrs +} diff --git a/v3_nczarr_test/ref_whole.cdl b/v3_nczarr_test/ref_whole.cdl new file mode 100644 index 0000000000..c72e44c832 --- /dev/null +++ b/v3_nczarr_test/ref_whole.cdl @@ -0,0 +1,19 @@ +netcdf tmp_whole { +dimensions: + d0 = 8 ; + d1 = 8 ; +variables: + int v(d0, d1) ; + v:_FillValue = -1 ; +data: + + v = + _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, + _, _, _, _, 0, 1, 2, 3, + _, _, _, _, 4, 5, 6, 7, + _, _, _, _, 8, 9, 10, 11, + _, _, _, _, 12, 13, 14, 15 ; +} diff --git a/v3_nczarr_test/ref_whole.txt b/v3_nczarr_test/ref_whole.txt new file mode 100644 index 0000000000..93e36e74e0 --- /dev/null +++ b/v3_nczarr_test/ref_whole.txt @@ -0,0 +1,64 @@ +[0] 0 +[1] 1 +[2] 2 +[3] 3 +[4] 4 +[5] 5 +[6] 6 +[7] 7 +[8] 8 +[9] 9 +[10] 10 +[11] 11 +[12] 12 +[13] 13 +[14] 14 +[15] 15 +[16] 0 +[17] 0 +[18] 0 +[19] 0 +[20] 0 +[21] 0 +[22] 0 +[23] 0 +[24] 0 +[25] 0 +[26] 0 +[27] 0 +[28] 0 +[29] 0 +[30] 0 +[31] 0 +[32] 0 +[33] 0 +[34] 0 +[35] 0 +[36] 0 +[37] 0 +[38] 0 +[39] 0 +[40] 0 +[41] 0 +[42] 0 +[43] 0 +[44] 0 +[45] 0 +[46] 0 +[47] 0 +[48] 0 +[49] 0 +[50] 0 +[51] 0 +[52] 0 +[53] 0 +[54] 0 +[55] 0 +[56] 0 +[57] 0 +[58] 0 +[59] 0 +[60] 0 +[61] 0 +[62] 0 +[63] 0 diff --git a/v3_nczarr_test/ref_xarray.cdl b/v3_nczarr_test/ref_xarray.cdl index 9689897b4d..69ef9f8e8b 100644 --- a/v3_nczarr_test/ref_xarray.cdl +++ b/v3_nczarr_test/ref_xarray.cdl @@ -4,7 +4,6 @@ dimensions: y = 5 ; variables: int i(x, y) ; - i:_FillValue = -2147483647 ; data: i = diff --git a/v3_nczarr_test/ref_zarr_test_data.cdl.gz b/v3_nczarr_test/ref_zarr_test_data.cdl.gz new file mode 100644 index 0000000000000000000000000000000000000000..4f1f2d14019f267679d2137edd8fdfb846eba277 GIT binary patch literal 2716 zcmb2|=HR%#pdyWdxhOR)zACY(D83}MxFkL$u_RG1IVFeT?cLj1w>=~n9=!gc*%tS< zNpb5VnKzTVoo-$iICU$NWm{^>&ZUu`-dRt28k|)A_^qDphgL~ewa@kYZIj)&j33T~l~ne`R^6RYa@f`U3xC-z{jM7;=dby>10C zd^_lu`RLdAELrOttB)Q3AtC?x^W0(#H{m~?I z37)vpm2ZD+&SYpl7~dW*T_siXLTP5oq`y%o`R-hp{H*O?_P*GYDwBPTPS%}DIW9N< zcFXJk`3L{&-?1}!liO99F##w8mfq*R=^4w}%ik`X{ahT}&KJ`A>E*`HAsd%|fA}6{Dz-yYdM literal 0 HcmV?d00001 diff --git a/v3_nczarr_test/run_chunkcases.sh b/v3_nczarr_test/run_chunkcases.sh new file mode 100755 index 0000000000..d8f60cd62f --- /dev/null +++ b/v3_nczarr_test/run_chunkcases.sh @@ -0,0 +1,128 @@ +#!/bin/sh + +# Note that this test builds a special results. directory in +# which to run the tests, where is the process id number of +# the bash shell instance. The reason for doing this is so that +# it works correctly under Github actions. By empirical +# observation, it appears that the various workflow matrix +# elements are not running in isolation when using cmake. Rather +# they appear to be running simultaneously while sharing the build +# directory for cmake. By running the tests in a separate +# results. I can guarantee that isolation is preserved. + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. ${srcdir}/test_nczarr.sh + +set -e + +s3isolate "testdir_chunkcases" +THISDIR=`pwd` +cd $ISOPATH + +TC="${execdir}/test_chunkcases -4" +ZM="${execdir}/zmapio -t int" + +remfile() { + case "$zext" in + nc4) rm -fr $1 ;; + file) rm -fr $1 ;; + zip) rm -fr $1 ;; + s3) ;; + *) echo "no such extension: $zext" ; exit 1;; + esac +} + +remfile() { + case "$zext" in + nc4) rm -fr $1 ;; + file) rm -fr $1 ;; + zip) rm -fr $1 ;; + s3) ;; + *) echo "no such extension: $zext" ; exit 1;; + esac +} + +makefile() { + fileargs $1 + remfile $file + case "$zext" in + nc4) F=$file;; + file) F=$fileurl;; + zip) F=$fileurl;; + s3) F=$fileurl;; + *) echo "no such extension: $zext" ; exit 1;; + esac +} + +testcasesxfail() { +zext=$1 +} # testcasesxfail() + +testcasespass() { +zext=$1 +echo ""; echo "*** Test format $1" +makefile tmp_whole +rm -f tmp_whole_${zext}.txt tmp_whole_${zext}.cdl tmp_err_${zext}.txt +makefile tmp_whole +# This should succeed +$TC -d 8,8 -c 4,4 -s 4,4 -e 4,4 -OWcw $F +$TC -d 8,8 -c 4,4 -s 4,4 -e 4,4 -vn -OWr $F > tmp_whole_${zext}.txt +diff -b ${srcdir}/ref_whole.txt tmp_whole_${zext}.txt +${NCDUMP} $F > tmp_whole_${zext}.cdl +diff -b ${srcdir}/ref_whole.cdl tmp_whole_${zext}.cdl + +# Test skipping whole chunks +echo "Test chunk skipping during read" +makefile tmp_skip +rm -f tmp_skip_${zext}.txt tmp_skip_${zext}.cdl +$TC -d 6,6 -c 2,2 -s 0,0 -e 6,6 -Ocw $F +$TC -i 5,5 -p 6,6 -Or $F > tmp_skip_${zext}.txt +${NCDUMP} $F > tmp_skip_${zext}.cdl +diff -b ${srcdir}/ref_skip.txt tmp_skip_${zext}.txt +diff -b ${srcdir}/ref_skip.cdl tmp_skip_${zext}.cdl + +echo "Test chunk skipping during write" +makefile tmp_skipw +rm -f tmp_skipw_${zext}.cdl +$TC -d 6,6 -c 2,2 -i 5,5 -p 6,6 -Ocw $F +${NCDUMP} $F > tmp_skipw_${zext}.cdl +diff -b ${srcdir}/ref_skipw.cdl tmp_skipw_${zext}.cdl + +echo "Test dimlen % chunklen != 0" +makefile tmp_rem +rm -f tmp_rem_${zext}.txt tmp_rem_${zext}.cdl +$TC -d 8,8 -c 3,3 -s 0,0 -p 8,8 -Ocw $F +${NCDUMP} $F > tmp_rem_${zext}.cdl +diff -b ${srcdir}/ref_rem.cdl tmp_rem_${zext}.cdl +${NCDUMPCHUNKS} -v v $F > tmp_rem_${zext}.txt +diff -b ${srcdir}/ref_rem.dmp tmp_rem_${zext}.txt + +echo "Test rank > 2" +makefile tmp_ndims +rm -f tmp_ndims_${zext}.txt tmp_ndims_${zext}.cdl +$TC -d 8,8,8,8 -c 3,3,4,4 -s 0,0,0,0 -p 8,8,8,8 -Ocw $F +${NCDUMP} $F > tmp_ndims_${zext}.cdl +diff -b ${srcdir}/ref_ndims.cdl tmp_ndims_${zext}.cdl +${NCDUMPCHUNKS} -v v $F > tmp_ndims_${zext}.dmp +diff -b ${srcdir}/ref_ndims.dmp tmp_ndims_${zext}.dmp + +echo "Test miscellaneous 1" +makefile tmp_misc1 +rm -f tmp_misc1_${zext}.txt tmp_misc1_${zext}.cdl +$TC -d 6,12,4 -c 2,3,1 -s 0,0,0 -e 6,1,4 -Ocw $F +${NCDUMP} $F > tmp_misc1_${zext}.cdl +diff -b ${srcdir}/ref_misc1.cdl tmp_misc1_${zext}.cdl +${NCDUMPCHUNKS} -v v $F > tmp_misc1_${zext}.dmp +diff -b ${srcdir}/ref_misc1.dmp tmp_misc1_${zext}.dmp +} # testcasespass() + +testcases() { + testcasesxfail $1 + testcasespass $1 +} + +testcases file +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcases zip; fi +if test "x$FEATURE_S3TESTS" = xyes ; then testcases s3; fi diff --git a/v3_nczarr_test/run_consolidated_zarr.sh b/v3_nczarr_test/run_consolidated_zarr.sh new file mode 100755 index 0000000000..079d088442 --- /dev/null +++ b/v3_nczarr_test/run_consolidated_zarr.sh @@ -0,0 +1,68 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "$srcdir/test_nczarr.sh" + +set -e + +s3isolate "testdir_consolidated_zarr" +THISDIR=`pwd` +cd $ISOPATH + +# Force use of consolidated metadata +unset NCNOZMETADATA + +# Locate the plugin path +if ! findplugin h5blosc ; then exit 0; fi +if ! avail blosc; then exit 0; fi + +# This shell script tests support for: +# 1. consolidated zarr (noxarray) read-only +# 2. xarray read + +testcase() { + zext=$1 + echo "*** Test: consolidated zarr write then read; format=$zext" + fileargs tmp_consolidated_zarr "mode=zarr,noxarray,zmetadata,$zext" + deletemap $zext $file + ${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_consolidated_zarr_base.cdl + ${NCDUMP} $fileurl > tmp_consolidated_zarr_${zext}.cdl + diff -b ${srcdir}/ref_consolidated_zarr.cdl tmp_consolidated_zarr_${zext}.cdl + echo "*** Test: xarray zarr write then read; format=$zext" + fileargs tmp_xarray "mode=zarr,$zext" + #deletemap $zext $file + ${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_consolidated_zarr_base.cdl + ${NCDUMP} $fileurl > tmp_xarray_${zext}.cdl + diff -b ${srcdir}/ref_xarray.cdl tmp_xarray_${zext}.cdl + + echo "*** Test: consolidated zarr reading nczarr; format=$zext" + fileargs tmp_nczarr "mode=nczarr,noxarray,zmetadata,$zext" + deletemap $zext $file + ${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_whole.cdl + fileargs tmp_nczarr "mode=zarr,$zext" + ${NCDUMP} -n nczarr2zarr $fileurl > tmp_nczarr_${zext}.cdl + diff -b ${srcdir}/ref_nczarr2zarr.cdl tmp_nczarr_${zext}.cdl +} + +testcase_csl_vs_no(){ + zext=$1 + echo "*** Test: consolidated pure python zarr read; format=$zext" + deletemap $zext $file + tar -zxf $srcdir/ref_consolidated_zarr_2.18.2_python.zarr.tgz + cp -r ref_consolidated_zarr_2.18.2_python.zarr ref_consolidated_zarr_2.18.2_python.zarr.$zext + cp -r ref_consolidated_zarr_2.18.2_python.zarr ref_zarr_2.18.2_python.zarr.$zext + rm -f ref_zarr_2.18.2_python.zarr.$zext/.zmetadata + fileargs ref_consolidated_zarr_2.18.2_python.zarr "mode=zarr,${zext}" + ${NCDUMP} -n same_name $fileurl > tmp_consolidated_python_zarr_${zext}.cdl + fileargs ref_zarr_2.18.2_python.zarr "mode=zarr,$zext" + ${NCDUMP} -n same_name $fileurl > tmp_python_zarr_${zext}.cdl + rm -f dif.txt + diff -b tmp_consolidated_python_zarr_${zext}.cdl tmp_python_zarr_${zext}.cdl > diff.txt +} + +testcase file +testcase_csl_vs_no file +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi +if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi diff --git a/v3_nczarr_test/run_corrupt.sh b/v3_nczarr_test/run_corrupt.sh new file mode 100755 index 0000000000..1943f600d6 --- /dev/null +++ b/v3_nczarr_test/run_corrupt.sh @@ -0,0 +1,42 @@ +#!/bin/sh + +# Test various kinds of corrupted files + + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "${srcdir}/test_nczarr.sh" + +set -e + +s3isolate "testdir_corrupt" +THISDIR=`pwd` +cd $ISOPATH + +export NCLOGGING=WARN + +testnoshape1() { + zext=file + unzip ${srcdir}/ref_noshape.file.zip + fileargs ${ISOPATH}/ref_noshape "mode=zarr,$zext" + rm -f tmp_noshape1_${zext}.cdl + find "${ISOPATH}" + ${ZMD} -h -t int $fileurl + ${NCDUMP} $flags $fileurl > tmp_noshape1_${zext}.cdl +} + +testnoshape2() { + # Test against the original issue URL + rm -f tmp_noshape2_gs.cdl + fileurl="https://storage.googleapis.com/cmip6/CMIP6/CMIP/NASA-GISS/GISS-E2-1-G/historical/r1i1p1f1/day/tasmin/gn/v20181015/#mode=zarr,s3&aws.profile=no" + ${ZMD} -h $fileurl > tmp_noshape2_gs.zmap + diff -wb ${srcdir}/ref_cmip6.zmap tmp_noshape2_gs.zmap + # ${NCDUMP} -h $flags $fileurl > tmp_noshape2_gs.cdl -- will fail +} + +testnoshape1 +if test "x$FEATURE_S3TESTS" = xyes && test "x$FEATURE_S3_INTERNAL" = xyes ; then + # The aws-sdk-cpp driver does not support google storage + testnoshape2 +fi diff --git a/v3_nczarr_test/run_external.sh b/v3_nczarr_test/run_external.sh new file mode 100755 index 0000000000..62a9920ef5 --- /dev/null +++ b/v3_nczarr_test/run_external.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +set -e + +. "${srcdir}/test_nczarr.sh" + +s3isolate "testdir_external" +THISDIR=`pwd` +cd $ISOPATH + +# This shell script tests reading of +# publically accessible S3 zarr datasets. + +TESTCASES= +if test "x$FEATURE_BYTERANGE" = xyes && test "x$FEATURE_S3" = xyes && test "x$FP_ISCYGWIN" = x ; then +TESTCASES="${TESTCASES} OR_ABI;;;http://s3.amazonaws.com/noaa-goes16/ABI-L1b-RadF/2022/001/18/OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562.nc#mode=bytes,s3" +TESTCASES="${TESTCASES} cesmLE;blosc;;http://s3.us-west-2.amazonaws.com/ncar-cesm-lens/atm/daily/cesmLE-20C-FLNS.zarr#mode=zarr,s3" +# Test TEST_REGION defaulting and s3 inferencing +TESTCASES="${TESTCASES} wtk;;us-west-2;https://nrel-pds-wtk.s3.amazonaws.com/wtk-techno-economic/pywtk-data/met_data/0/0.nc#mode=bytes" +fi + +testcase() { +NM=`echo "$1" | cut -d';' -f1` +FILT=`echo "$1" | cut -d';' -f2` +REG=`echo "$1" | cut -d';' -f3` +URL=`echo "$1" | cut -d';' -f4` +echo "*** Test: $NM = $URL" +rm -f "tmp_external_$NM.cdl" +if test "x$FILT" != x ; then + if avail $FILT; then + if test "x$REG" != x ; then export TEST_REGION="$REG"; fi + ${NCDUMP} -h -s -n $NM $URL > "tmp_external_${NM}.cdl" + unset TEST_REGION + fi +fi +} + +if test "x$FEATURE_S3" = xyes ; then +for t in $TESTCASES ; do +testcase "$t" +done +fi diff --git a/v3_nczarr_test/run_fillonlyz.sh b/v3_nczarr_test/run_fillonlyz.sh new file mode 100755 index 0000000000..e9ab3e8b5d --- /dev/null +++ b/v3_nczarr_test/run_fillonlyz.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "${srcdir}/test_nczarr.sh" + +# This shell script tests bug reported in github issue +# https://github.com/Unidata/netcdf-c/issues/1826 + +set -e + +s3isolate "testdir_fillonlyz" +THISDIR=`pwd` +cd $ISOPATH + +echo "" +echo "*** Testing data conversions when a variable has fill value but never written" + +testcase() { +zext=$1 +fileargs tmp_fillonly +deletemap $zext $file +${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_fillonly.cdl +${execdir}/test_fillonlyz${ext} "$fileurl" +} + +testcase file +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi +if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi diff --git a/v3_nczarr_test/run_filter.sh b/v3_nczarr_test/run_filter.sh new file mode 100755 index 0000000000..20eaba526d --- /dev/null +++ b/v3_nczarr_test/run_filter.sh @@ -0,0 +1,176 @@ +#!/bin/bash + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "${srcdir}/test_nczarr.sh" + +set -e + +s3isolate "testdir_filter" +THISDIR=`pwd` +cd $ISOPATH + +testset() { +# Which test cases to exercise +testapi $1 +testng $1 +testncp $1 +testngc $1 +testmisc $1 +testmulti $1 +testrep $1 +testorder $1 +} + +# Function to extract _Filter attribute from a file +# These attributes might be platform dependent +getfilterattr() { +sed -e '/var.*:_Filter/p' -ed <$1 >$2 +} + +# Function to extract _Codecs attribute from a file +# These attributes might be platform dependent +getcodecsattr() { +sed -e '/var.*:_Codecs/p' -ed <$1 >$2 +} + +trimleft() { +sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2 +} + +# Locate the plugin path and the library names; argument order is critical + +# Find misc and capture +if ! findplugin h5misc ; then exit 0; fi +MISCLIB="${HDF5_PLUGIN_LIB}" +MISCDIR="${HDF5_PLUGIN_DIR}" +MISCPATH="${MISCDIR}/${MISCLIB}" + +# Find bzip2 and capture +if ! findplugin h5bzip2 ; then exit 0; fi +BZIP2LIB="${HDF5_PLUGIN_LIB}" +BZIP2DIR="${HDF5_PLUGIN_DIR}" +BZIP2PATH="${BZIP2DIR}/${BZIP2LIB}" + +# Verify +if ! test -f ${BZIP2path} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi +if ! test -f ${MISCPATH} ; then echo "Unable to locate ${MISCPATH}"; exit 1; fi + +# Execute the specified tests + +testapi() { +zext=$1 +echo "*** Testing dynamic filters using API for storage format $zext" +fileargs tmp_api +deletemap $zext $file +${execdir}/testfilter $fileurl +${ZMD} -h $fileurl +${NCDUMP} -s -n bzip2 $fileurl > ./tmp_api_$zext.dump +# Remove irrelevant -s output +sclean ./tmp_api_$zext.dump +diff -b -w ${srcdir}/ref_bzip2.cdl ./tmp_api_$zext.dump +echo "*** Pass: API dynamic filter for map=$zext" +} + +testmisc() { +zext=$1 +echo "*** Testing dynamic filters parameter passing for storage format $zext" +fileargs tmp_misc +deletemap $zext $file +${execdir}/testfilter_misc $fileurl +${ZMD} -h $fileurl +# Verify the parameters via ncdump +${NCDUMP} -s $fileurl > ./tmp_misc_$zext.txt +# Extract the parameters +getfilterattr ./tmp_misc_$zext.txt ./tmp_misc2_$zext.txt +rm -f ./tmp_misc_$zext.txt +trimleft ./tmp_misc2_$zext.txt ./tmp_misc_$zext.txt +rm -f ./tmp_misc2_$zext.txt +cat >./tmp_misc2_$zext.txt < ./tmp_ng_$zext.txt +# Remove irrelevant -s output +sclean ./tmp_ng_$zext.txt +diff -b -w ${srcdir}/ref_bzip2.cdl ./tmp_ng_$zext.txt +echo "*** Pass: ncgen dynamic filter for storage format $zext" +} + +testncp() { +zext=$1 +echo "*** Testing dynamic filters using nccopy for storage format $zext" +fileargs tmp_unfiltered +deletemap $zext $file +# Create our input test files +${NCGEN} -4 -lb -o $fileurl ${srcdir}/../nc_test4/ref_unfiltered.cdl +fileurl0=$fileurl +fileargs tmp_filtered +${NCCOPY} -Xf -M0 -F "/g/var,307,9,4" $fileurl0 $fileurl +${ZMD} -h $fileurl +${NCDUMP} -s -n filtered $fileurl > ./tmp_ncp_$zext.dump +# Remove irrelevant -s output +sclean ./tmp_ncp_$zext.dump +diff -b -w ${srcdir}/ref_filtered.cdl ./tmp_ncp_$zext.dump +echo " *** Pass: nccopy simple filter for storage format $zext" +} + +testngc() { +zext=$1 +echo "*** Testing dynamic filters using ncgen with -lc for storage format $zext" +fileargs tmp_ngc +deletemap $zext $file +${NCGEN} -lc -4 ${srcdir}/../nc_test4/bzip2.cdl > tmp_ngc.c +diff -b -w ${srcdir}/../nc_test4/../nc_test4/ref_bzip2.c ./tmp_ngc.c +echo "*** Pass: ncgen dynamic filter for storage format $zext" +} + +testmulti() { +zext=$1 +echo "*** Testing multiple filters for storage format $zext" +fileargs tmp_multi +deletemap $zext $file +${execdir}/testfilter_multi $fileurl +${ZMD} -h $fileurl +${NCDUMP} -hsF -n multifilter $fileurl >./tmp_multi_$zext.cdl +# Remove irrelevant -s output +sclean ./tmp_multi_$zext.cdl +diff -b -w ${srcdir}/ref_multi.cdl ./tmp_multi_$zext.cdl +echo "*** Pass: multiple filters for storage format $zext" +} + +testrep() { +zext=$1 +echo "*** Testing filter re-definition invocation for storage format $zext" +fileargs tmp_rep +deletemap $zext $file +${execdir}/testfilter_repeat $fileurl >tmp_rep_$zext.txt +diff -b -w ${srcdir}/../nc_test4/ref_filter_repeat.txt tmp_rep_$zext.txt +} + +testorder() { +zext=$1 +echo "*** Testing multiple filter order of invocation on create for storage format $zext" +fileargs tmp_order +deletemap $zext $file +${execdir}/testfilter_order create $fileurl >tmp_order_$zext.txt +diff -b -w ${srcdir}/../nc_test4/ref_filter_order_create.txt tmp_order_$zext.txt +echo "*** Testing multiple filter order of invocation on read for storage format $zext" +${execdir}/testfilter_order read $fileurl >tmp_order_rd_$zext.txt +diff -b -w ${srcdir}/../nc_test4/ref_filter_order_read.txt tmp_order_rd_$zext.txt +} + +testset file +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testset zip ; fi +if test "x$FEATURE_S3TESTS" = xyes ; then testset s3 ; fi diff --git a/v3_nczarr_test/run_filter_misc.sh b/v3_nczarr_test/run_filter_misc.sh new file mode 100755 index 0000000000..4f094c4a39 --- /dev/null +++ b/v3_nczarr_test/run_filter_misc.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +# This file is derived from ncdump/tst_filter_misc.sh +TESTNCZARR=1 + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +if test "x$TESTNCZARR" = x1; then +. ${srcdir}/test_nczarr.sh +fi + +set -e + +isolate "testdir_filter_misc" +THISDIR=`pwd` +cd $ISOPATH + +if test "x$TESTNCZARR" = x1; then +s3isolate +fi + +if test "x$TESTNCZARR" = x1; then + TFAVAIL=${execdir}/test_filter_avail + TFVLEN=${execdir}/test_filter_vlen +else + TFAVAIL=${execdir}/tst_filter_avail + TFVLEN=${execdir}/tst_filter_vlen +fi + +# Load the findplugins function +. ${builddir}/findplugin.sh +echo "findplugin.sh loaded" + +# test for deflate filter +if avail 1 ; then HAVE_DEFLATE=1; else HAVE_DEFLATE=0; fi +if test "$HAVE_DEFLAGE" = 0 ; then + echo ">>> Filter bzip2 not available; discontinuing test" + exit 0; +fi + +# Function to remove selected -s attributes from file; +# These attributes might be platform dependent +sclean() { + cat $1 \ + | sed -e '/:_IsNetcdf4/d' \ + | sed -e '/:_Endianness/d' \ + | sed -e '/_NCProperties/d' \ + | sed -e '/_SuperblockVersion/d' \ + | sed -e '/_Format/d' \ + | sed -e '/global attributes:/d' \ + | cat > $2 +} + +# Function to extract _Filter attribute from a file +# These attributes might be platform dependent +getfilterattr() { +V="$1" +sed -e '/${V}.*:_Filter/p' -ed <$2 >$3 +} + +# Function to extract _Codecs attribute from a file +# These attributes might be platform dependent +getcodecsattr() { +V="$1" +sed -e '/${V}.*:_Codecs/p' -ed <$2 >$3 +} + +trimleft() { +sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2 +} + +setfilter() { + FF="$1" + FSRC="$2" + FDST="$3" + FIH5="$4" + FICX="$5" + FFH5="$6" + FFCX="$7" + if test "x$FFH5" = x ; then FFH5="$FIH5" ; fi + if test "x$FFCX" = x ; then FFCX="$FICX" ; fi + rm -f $FDST + cat ${srcdir}/$FSRC \ + | sed -e "s/ref_any/${FF}/" \ + | sed -e "s/IH5/${FIH5}/" -e "s/FH5/${FFH5}/" \ + | sed -e "s/ICX/${FICX}/" -e "s/FCX/${FFCX}/" \ + | sed -e 's/"/\\"/g' -e 's/@/"/g' \ + | cat > $FDST +} + +# Execute the specified tests + +testavail() { + zext=$1 + if ! filteravail bzip2; then return 0; fi + ${TFAVAIL} +} + +testvlen() { + zext=$1 + ${TFVLEN} +} + +testset() { +# Which test cases to exercise + testavail $1 + testvlen $1 +} + +if test "x$TESTNCZARR" = x1 ; then + testset file + if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testset zip ; fi + if test "x$FEATURE_S3TESTS" = xyes ; then testset s3 ; fi +else + testset nc +fi diff --git a/v3_nczarr_test/run_filter_vlen.sh b/v3_nczarr_test/run_filter_vlen.sh new file mode 100755 index 0000000000..6c38ee1832 --- /dev/null +++ b/v3_nczarr_test/run_filter_vlen.sh @@ -0,0 +1,108 @@ +#!/bin/bash + +# Test derived from nc_test4/tst_filter_vlen.sh +TESTNCZARR=1 + +# Test filters on non-fixed size variables. + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +if test "x$TESTNCZARR" = x1; then +. ${srcdir}/test_nczarr.sh +fi + +set -e + +# Load the findplugins function +. ${builddir}/findplugin.sh +echo "findplugin.sh loaded" + +if ! findplugin h5deflate ; then exit 0; fi +echo "HDF5_PLUGIN_DIR=$HDF5_PLUGIN_DIR" + +isolate "testdir_filter_vlen" +THISDIR=`pwd` +cd $ISOPATH + +if test "x$TESTNCZARR" = x1; then +s3isolate +fi + +# Find deflate +if avail 1 ; then HAVE_DEFLATE=1; else HAVE_DEFLATE=0; fi + +# Function to remove selected -s attributes from file; +# These attributes might be platform dependent +sclean() { + cat $1 \ + | sed -e '/:_IsNetcdf4/d' \ + | sed -e '/:_Endianness/d' \ + | sed -e '/_NCProperties/d' \ + | sed -e '/_SuperblockVersion/d' \ + | sed -e '/_Format/d' \ + | sed -e '/global attributes:/d' \ + | cat > $2 +} + +# Function to extract _Filter attribute from a file +# These attributes might be platform dependent +getfilterattr() { +V="$1" +sed -e '/${V}.*:_Filter/p' -ed <$2 >$3 +} + +# Function to extract _Codecs attribute from a file +# These attributes might be platform dependent +getcodecsattr() { +V="$1" +sed -e '/${V}.*:_Codecs/p' -ed <$2 >$3 +} + +trimleft() { +sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2 +} + +setfilter() { + FF="$1" + FSRC="$2" + FDST="$3" + FIH5="$4" + FICX="$5" + FFH5="$6" + FFCX="$7" + if test "x$FFH5" = x ; then FFH5="$FIH5" ; fi + if test "x$FFCX" = x ; then FFCX="$FICX" ; fi + rm -f $FDST + cat ${srcdir}/$FSRC \ + | sed -e "s/ref_any/${FF}/" \ + | sed -e "s/IH5/${FIH5}/" -e "s/FH5/${FFH5}/" \ + | sed -e "s/ICX/${FICX}/" -e "s/FCX/${FFCX}/" \ + | sed -e 's/"/\\"/g' -e 's/@/"/g' \ + | cat > $FDST +} + +# Execute the specified tests + +testvlen() { + zext=$1 + if test "x$TESTNCZARR" = x1 ; then + ${execdir}/test_filter_vlen + else + ${execdir}/tst_filter_vlen + fi +} + +testset() { +# Which test cases to exercise + testvlen $1 +} + +if test "x$TESTNCZARR" = x1 ; then + testset file + if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testset zip ; fi + if test "x$FEATURE_S3TESTS" = xyes ; then testset s3 ; fi + if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup +else + testset nc +fi diff --git a/v3_nczarr_test/run_filterinstall.sh b/v3_nczarr_test/run_filterinstall.sh new file mode 100755 index 0000000000..f5cc612784 --- /dev/null +++ b/v3_nczarr_test/run_filterinstall.sh @@ -0,0 +1,124 @@ +#!/bin/bash + +# Test derived from nc_test4/tst_filterinstall.sh +TESTNCZARR=1 + +# Test the filter install +# This cannot be run as a regular test +# because installation will not have occurred + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +if test "x$TESTNCZARR" = x1; then +. ${srcdir}/test_nczarr.sh +fi + +set -e + +isolate "testdir_filterinstall" +THISDIR=`pwd` +cd $ISOPATH + +if test "x$TESTNCZARR" = x1; then +s3isolate +fi + +# Use this plugin path +export HDF5_PLUGIN_PATH="${FEATURE_PLUGIN_INSTALL_DIR}" + +# Function to remove selected -s attributes from file; +# These attributes might be platform dependent +sclean() { + cat $1 \ + | sed -e '/:_IsNetcdf4/d' \ + | sed -e '/:_Endianness/d' \ + | sed -e '/_NCProperties/d' \ + | sed -e '/_SuperblockVersion/d' \ + | sed -e '/_Format/d' \ + | sed -e '/global attributes:/d' \ + | cat > $2 +} + +# Function to extract _Filter attribute from a file +# These attributes might be platform dependent +getfilterattr() { +V="$1" +sed -e '/${V}.*:_Filter/p' -ed <$2 >$3 +} + +# Function to extract _Codecs attribute from a file +# These attributes might be platform dependent +getcodecsattr() { +V="$1" +sed -e '/${V}.*:_Codecs/p' -ed <$2 >$3 +} + +trimleft() { +sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2 +} + +setfilter() { + FF="$1" + FSRC="$2" + FDST="$3" + FIH5="$4" + FICX="$5" + FFH5="$6" + FFCX="$7" + if test "x$FFH5" = x ; then FFH5="$FIH5" ; fi + if test "x$FFCX" = x ; then FFCX="$FICX" ; fi + rm -f $FDST + cat ${srcdir}/$FSRC \ + | sed -e "s/ref_any/${FF}/" \ + | sed -e "s/IH5/${FIH5}/" -e "s/FH5/${FFH5}/" \ + | sed -e "s/ICX/${FICX}/" -e "s/FCX/${FFCX}/" \ + | sed -e 's/"/\\"/g' -e 's/@/"/g' \ + | cat > $FDST +} + +# Execute the specified tests + +runfilter() { +zext=$1 +zfilt="$2" +zparams="$3" +zcodec="$4" +echo "*** Testing processing of filter $zfilt for map $zext" +if test "x$TESTNCZARR" = x1 ; then +fileargs "tmp_filt_${zfilt}" +deletemap $zext $file +else +file="tmp_filt_${zfilt}.nc" +rm -f $file +fi +setfilter $zfilt ref_any.cdl "tmp_filt_${zfilt}.cdl" "$zparams" "$zcodec" +if test "x$TESTNCZARR" = x1 ; then +${NCGEN} -4 -lb -o $fileurl "tmp_filt_${zfilt}.cdl" +${NCDUMP} -n $zfilt -sF $fileurl > "tmp_filt_${zfilt}.tmp" +else +${NCGEN} -4 -lb -o $file "tmp_filt_${zfilt}.cdl" +${NCDUMP} -n $zfilt -sF $file > "tmp_filt_${zfilt}.tmp" +fi +sclean "tmp_filt_${zfilt}.tmp" "tmp_filt_${zfilt}.dump" +} + +testbzip2() { + zext=$1 + if ! avail bzip2; then return 0; fi + runfilter $zext bzip2 '307,9' '[{\"id\": \"bz2\",\"level\": \"9\"}]' + diff -b -w "tmp_filt_bzip2.cdl" "tmp_filt_bzip2.dump" +} + +testset() { +# Which test cases to exercise + testbzip2 $1 +} + +if test "x$TESTNCZARR" = x1 ; then + testset file + if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testset zip ; fi + if test "x$FEATURE_S3TESTS" = xyes ; then testset s3 ; fi +else + testset nc +fi diff --git a/v3_nczarr_test/run_interop.sh b/v3_nczarr_test/run_interop.sh new file mode 100755 index 0000000000..c50607b22d --- /dev/null +++ b/v3_nczarr_test/run_interop.sh @@ -0,0 +1,110 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "${srcdir}/test_nczarr.sh" + +set -x +set -e + +metaonly="-h" + +s3isolate "testdir_interop" +THISDIR=`pwd` +cd $ISOPATH + +# This shell script tests compatibility between +# this implementation and other implementations +# by means of files constructed by that other implementation + +UH="${NCZARR_S3_TEST_HOST}" +UB="${NCZARR_S3_TEST_BUCKET}" + +# Load the findplugins function +. ${builddir}/findplugin.sh +echo "findplugin.sh loaded" + +# One of the tests depends on having blosc installed +# Assume all test filters are in same plugin dir +if ! findplugin h5blosc ; then exit 0; fi + +if ! avail bzip2; then exit 0; fi +if ! avail blosc; then exit 0; fi + +testcasefile() { + echo " to Running File Testcase: $1 $2 $3" + zext=file + base=$1 + mode=$2 + fileargs ${ISOPATH}/ref_$base "mode=$mode,$zext" + rm -f tmp_${base}_${zext}.cdl + ${NCDUMP} $metaonly $fileurl > tmp_${base}_${zext}.cdl + diff -b ${srcdir}/ref_${base}.cdl tmp_${base}_${zext}.cdl +} + +testcasezip() { + echo " o Running Zip Testcase: $1 $2" + zext=zip + base=$1 + mode=$2 + fileargs ${ISOPATH}/ref_$base "mode=$mode,$zext" + rm -f tmp_${base}_${zext}.cdl + ${NCDUMP} $metaonly $flags $fileurl > tmp_${base}_${zext}.cdl + diff -b ${srcdir}/ref_${base}.cdl tmp_${base}_${zext}.cdl +} + +testcases3() { + echo -e "\to Running S3 Testcase:\t$1\t$2" + zext=s3 + base=$1 + mode=$2 + rm -f tmp_${base}_${zext}.cdl + url="https://${UH}/${UB}/${base}.zarr#mode=${mode},s3" + echo "flags: $flags" + # Dumping everything causes timeout so dump a single var + ${NCDUMP} -v "group_with_dims/var2D" $flags $url > tmp_${base}_${zext}.cdl + # Use the proper ref file + diff -b ${ISOPATH}/ref_${base}_2d.cdl tmp_${base}_${zext}.cdl + set +x +} + +testallcases() { +zext=$1 +case "$zext" in + file) + # need to unpack + unzip ref_power_901_constants.zip >> tmp_ignore.txt + mv ${ISOPATH}/ref_power_901_constants ${ISOPATH}/ref_power_901_constants.file + testcasefile power_901_constants zarr; # test xarray as default + ;; + zip) + # Move into position + testcasezip power_901_constants xarray + # Test large constant interoperability + testcasezip quotes zarr + ;; + s3) + # Read a test case created by netcdf-java zarr. + # unpack the diff baseline files + # Use gunzip because it always appears to be available + gunzip -c ${srcdir}/ref_zarr_test_data.cdl.gz > ${ISOPATH}/ref_zarr_test_data.cdl + gunzip -c ${srcdir}/ref_zarr_test_data_2d.cdl.gz > ${ISOPATH}/ref_zarr_test_data_2d.cdl + testcases3 zarr_test_data xarray + ;; + *) echo "unimplemented kind: $1" ; exit 1;; +esac +} + +# common setup + +if ! test -f ${ISOPATH}/ref_power_901_constants.zip ; then + cp -f ${srcdir}/ref_power_901_constants_orig.zip ${ISOPATH}/ref_power_901_constants.zip +fi +if ! test -f ${ISOPATH}/ref_quotes.zip ; then + cp -f ${srcdir}/ref_quotes_orig.zip ${ISOPATH}/ref_quotes.zip +fi + +testallcases file +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testallcases zip; fi +if test "x$FEATURE_S3TESTS" = xyes ; then testallcases s3; fi diff --git a/v3_nczarr_test/run_jsonconvention.sh b/v3_nczarr_test/run_jsonconvention.sh new file mode 100755 index 0000000000..5a906c07c6 --- /dev/null +++ b/v3_nczarr_test/run_jsonconvention.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "${srcdir}/test_nczarr.sh" + +set -e + +s3isolate "testdir_jsonconvention" +THISDIR=`pwd` +cd $ISOPATH + +# This shell script tests support for: +# read/write using json convention + +testcase() { +zext=$1 + +echo "*** Test: write then read using json convention" +fileargs tmp_jsonconvention "mode=nczarr,$zext" +deletemap $zext $file +${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_jsonconvention.cdl +${ZMD} -h $fileurl +${NCDUMP} $fileurl > tmp_jsonconvention_${zext}.cdl +${ZMD} -h $fileurl > tmp_jsonconvention_${zext}.txt +# Clean up extraneous changes so comparisons work +# remove '\n' from ref file before comparing +#sed -e 's|\\n||g' < ${srcdir}/ref_jsonconvention.cdl > tmp_jsonconvention_clean.cdl +sclean tmp_jsonconvention_${zext}.cdl +zmapclean tmp_jsonconvention_${zext}.txt +diff -b $srcdir/ref_jsonconvention.cdl tmp_jsonconvention_${zext}.cdl +diff -b ${srcdir}/ref_jsonconvention.zmap tmp_jsonconvention_${zext}.txt +} + +testcase file +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi +if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi diff --git a/v3_nczarr_test/run_misc.sh b/v3_nczarr_test/run_misc.sh new file mode 100755 index 0000000000..e1c68320ef --- /dev/null +++ b/v3_nczarr_test/run_misc.sh @@ -0,0 +1,63 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "${srcdir}/test_nczarr.sh" + +set -x +set -e + +s3isolate "testdir_misc" +THISDIR=`pwd` +cd $ISOPATH + +# This shell script provides a miscellaneous set of tests + +cleanup() { + resetrc +} + +# Setup the .rc files + +createrc() { + RCP="${ISOPATH}/.ncrc" + echo "Creating rc file $RCP" + echo "ZARR.DIMENSION_SEPARATOR=/" >>$RCP +} + +testcase1() { +zext=$1 +echo "*** Test: use '/' as dimension separator for write then read" +fileargs tmp_dimsep "mode=nczarr,$zext" +deletemap $zext $file +cleanup +createrc +${NCGEN} -4 -lb -o $fileurl ${abs_srcdir}/ref_misc1.cdl +${NCDUMP} -n tmp_misc1 $fileurl > tmp_misc1_$zext.cdl +diff -bw ${abs_srcdir}/ref_misc1.cdl tmp_misc1_$zext.cdl +} + +testcase2() { +zext=$1 +echo "*** Test: '/' as dimension separator creates extra groups" +fileargs tmp_extra "mode=nczarr,$zext" +deletemap $zext $file +cleanup +createrc +${NCGEN} -4 -lb -o "$fileurl" ${abs_srcdir}/ref_misc2.cdl +${NCDUMP} -n tmp_misc2 $fileurl > tmp_misc2_$zext.cdl +diff -wb ${abs_srcdir}/ref_misc2.cdl tmp_misc2_$zext.cdl +} + +testcase1 file +testcase2 file +exit 0 +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then + testcase1 zip + testcase2 zip +fi +if test "x$FEATURE_S3TESTS" = xyes ; then + testcase1 s3 + testcase2 s3 +fi diff --git a/v3_nczarr_test/run_mud.sh b/v3_nczarr_test/run_mud.sh index 9d523dd438..bc104a81d6 100755 --- a/v3_nczarr_test/run_mud.sh +++ b/v3_nczarr_test/run_mud.sh @@ -12,7 +12,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi set -e if test "x$TESTNCZARR" = x1 ; then -. ${builddir}/test_nczarr.sh +. ${srcdir}/test_nczarr.sh s3isolate "testdir_mud4" else isolate "testdir_mud4" diff --git a/v3_nczarr_test/run_nan.sh b/v3_nczarr_test/run_nan.sh index d4bd8f4429..4e0b581b17 100755 --- a/v3_nczarr_test/run_nan.sh +++ b/v3_nczarr_test/run_nan.sh @@ -8,13 +8,14 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -. "${builddir}/test_nczarr.sh" +. "${srcdir}/test_nczarr.sh" # Isolate both test and S3 s3isolate "testdir_nan" THISDIR=`pwd` cd $ISOPATH +set -x set -e # Functions @@ -37,11 +38,11 @@ for t in tst_nans ; do rm -fr ${t}.$zext rm -f tmp_${t}.dmp fileargs $t - ${NCGEN} -4 -lb -o ${fileurl} ${srcdir}/${ref}.cdl + ${NCGEN} -4 -lb -o ${fileurl} ${srcdir}/../ncdump/${ref}.cdl ${ZMD} -h -t float ${fileurl} - ${NCDUMP} ${headflag} ${specflag} -n ${ref} ${fileurl} > tmp_${t}.dmp + ${NCDUMP} ${headflag} ${specflag} ${fileurl} > tmp_${t}.dmp # compare against expected - diff -b -w ${srcdir}/${ref}.cdl ./tmp_${t}.dmp + diff -b -w ${srcdir}/../ncdump/${ref}.cdl ./tmp_${t}.dmp echo "*** SUCCEED: ${t}" done } diff --git a/v3_nczarr_test/run_nccopy5.sh b/v3_nczarr_test/run_nccopy5.sh new file mode 100755 index 0000000000..2d89abb3d0 --- /dev/null +++ b/v3_nczarr_test/run_nccopy5.sh @@ -0,0 +1,277 @@ +#!/bin/bash + +# This file is derived from ncdump/tst_nccopy5.sh +TESTNCZARR=1 + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +set -e + +if test "x$TESTNCZARR" = x1 ; then +. ${srcdir}/test_nczarr.sh +s3isolate "testdir_nccopy5" +else +isolate testdir_ncccopy5 +fi +THISDIR=`pwd` +cd $ISOPATH + +# Program to run +if test "x$TESTNCZARR" = x1 ; then + CHUNKTEST="${execdir}/test_chunking" +else + CHUNKTEST="${execdir}/tst_chunking" +fi + +# For a netCDF-4 build, test nccopy chunking rules + +echo "" + +# Trim off leading and trailing whitespace +# Also remove any and de-tabify +# usage: trim +# Leaves result in variable TRIMMED +trim() { + # trim leading whitespace and remove + TMP=`echo "$1" |tr -d '\r' | tr '\t' ' ' |sed -e 's/^[ ]*//'` + # trim trailing whitespace + TRIMMED=`echo "$TMP" | sed -e 's/[ ]*$//'` +} + +if test "x$TESTNCZARR" = x1 ; then +# NCZARR does not support contiguous storage +checkfvar() { +return +} +else +# usage: checkfvar +checkfvar() { + # Make sure that fvar was not chunked + C5FVAR=`sed -e '/fvar:_ChunkSizes/p' -e d <$1` + if test "x$C5FVAR" != x ; then + echo "***Fail: fvar was chunked" + exit 1 + fi +} +fi + +# usage: checkivar +checkivar() { + # Make sure that ivar was not chunked + C5IVAR=`sed -e '/ivar:_ChunkSizes/p' -e d <$1` + if test "x$C5IVAR" != x ; then + echo "***Fail: ivar was chunked" + exit 1 + fi +} + +# usage: verifychunkline line1 line2 +verifychunkline() { + # trim leading whitespace + trim "$1"; L1="$TRIMMED" + trim "$2"; L2="$TRIMMED" + if test "x$L1" != "x$L2" ; then + echo "chunk line mismatch |$L1| |$L2|" + exit 1; + fi +} + +# Remove any temporary files +cleanup() { + rm -f tmp_nc5.nc tmp_nc5a.nc + rm -f tmp_nc5.cdl tmp_nc5a.cdl tmp_nc5b.cdl + rm -f tmp_nc5_omit.nc tmp_nc5_omit.cdl +} + +buildfile() { +zext=$1 +index=$2 +if test "x$TESTNCZARR" = x1 ; then +fileargs "tmp_nc5_${index}_${zext}" +deletemap $zext $file +file="$fileurl" +else +file="tmp_nc5_${index}_${zext}.nc" +rm -f $file +fi +} + +testcase1() { +zext=$1 +buildfile ${zext} 1 + +rm -fr tmp1${zext}.dir +mkdir tmp1${zext}.dir +cd tmp1${zext}.dir + +# Create a simple classic input file +${CHUNKTEST} $file + +# Save a .cdl version +${NCDUMP} -n tmp_nc5_base ${file} > tmp_nc5.cdl + +echo "*** Test nccopy -c with per-variable chunking; classic->enhanced" +# This should produce same as -c dim0/,dim1/1,dim2/,dim3/1,dim4/,dim5/1,dim6/ +# But note that the chunk product is less than default, so we need to reduce it (-M) +${NCCOPY} -M1000 -c ivar:7,1,2,1,5,1,9 ${file} tmp_nc34.nc +${NCDUMP} -n tmp_nc5_base tmp_nc34.nc > tmp_nc34.cdl +# Verify that the core cdl is the same +diff tmp_nc5.cdl tmp_nc34.cdl + +# Look at the output chunking +${NCDUMP} -hs -n tmp_nc5_base tmp_nc34.nc > tmp_chunking.cdl +# extract the chunking line +TESTLINE=`sed -e '/ivar:_ChunkSizes/p' -e d tmp_nc5.cdl + +echo "*** Test nccopy -c with per-variable chunking; enhanced->enhanced" +# Use -M to ensure that chunking takes effect +${NCCOPY} -M500 -c ivar:4,1,2,1,5,2,3 $file tmp_nc44.nc +${NCDUMP} -n tmp_nc5_base tmp_nc44.nc > tmp_nc44.cdl +diff tmp_nc5.cdl tmp_nc44.cdl + +# Look at the output chunking +${NCDUMP} -hs -n tmp_nc5_base tmp_nc44.nc > tmp_chunking.cdl +# extract the chunking line +TESTLINE=`sed -e '/ivar:_ChunkSizes/p' -e d tmp_nc5.cdl + +echo "*** Test nccopy -c with FQN var name; enhanced ->enhanced" +${NCCOPY} -M500 -c /g/ivar:4,1,2,1,5,2,3 ${file} tmp_nc44.nc +${NCDUMP} -n tmp_nc5_base tmp_nc44.nc > tmp_nc44.cdl +diff tmp_nc5.cdl tmp_nc44.cdl + +# Verify chunking +${NCDUMP} -hs -n tmp_nc5_base tmp_nc44.nc > tmp_chunking.cdl +# extract the chunking line +TESTLINE=`sed -e '/ivar:_ChunkSizes/p' -e d tmp_nc5.cdl + +echo "*** Test nccopy -c with unlimited dimension; classic ->enhanced" +# Warning: make sure that nccopy does not convert small chunking to contiguous => -M +${NCCOPY} -M50 -c ivar:5,3 $file tmp_nc34.nc +${NCDUMP} -n tmp_nc5_base tmp_nc34.nc > tmp_nc34.cdl +diff tmp_nc5.cdl tmp_nc34.cdl + +# Verify chunking +${NCDUMP} -hs -n tmp_nc5_base tmp_nc34.nc > tmp_chunking.cdl +# extract the chunking line +TESTLINE=`sed -e '/ivar:_ChunkSizes/p' -e d tmp_nc5_omit.cdl + +echo "*** Test nccopy -c fvar: to suppress chunking; classic ->enhanced" +${NCCOPY} -M500 -c ivar:7,1,2,1,5,1,9 -c fvar: $file tmp_nc34_omit.nc +${NCDUMP} -n tmp_nc5_base tmp_nc34_omit.nc > tmp_nc34_omit.cdl +diff tmp_nc5_omit.cdl tmp_nc34_omit.cdl + +# Verify chunking of ivar +${NCDUMP} -hs -n tmp_nc5_omit tmp_nc34_omit.nc > tmp_chunking_omit.cdl +# extract the chunking line +TESTLINE=`sed -e '/ivar:_ChunkSizes/p' -e d ./$2 +} + +#verifychunking cdl-file ... +verifychunking() { + f=$1 + shift + for t in "$@" ; do + x=`cat $f | tr -d "[:space:]" | sed -e "/$t/p" -ed` + if test "x$x" = x ; then echo "$f: $t not found"; exit 1; fi + done +} + +testcase() { +zext=$1 +fileargs tmp +${execdir}/test_zchunks3 -e ${zext} +echo "*** Test that nccopy -c can chunk files" +${NCCOPY} -M0 tmp_chunks3.nc "$fileurl" +${NCDUMP} -sh -n tmp "$fileurl" > tmp_nccz.cdl +verifychunking tmp_nccz.cdl "ivar:_ChunkSizes=7,4,2,3,5,6,9;" "fvar:_ChunkSizes=9,6,5,3,2,4,7;" + +fileargs tmp_chunked +${execdir}/test_zchunks3 -e ${zext} +${NCCOPY} -M0 -c dim0/,dim1/1,dim2/,dim3/1,dim4/,dim5/1,dim6/ tmp_chunks3.nc "$fileurl" +${NCDUMP} -sh -n tmp "$fileurl" > tmp_chunked.cdl +verifychunking tmp_chunked.cdl "ivar:_ChunkSizes=7,1,2,1,5,1,9;" "fvar:_ChunkSizes=9,1,5,1,2,1,7;" +chunkclean tmp_nccz.cdl tmpx.cdl +chunkclean tmp_chunked.cdl tmp_chunkedx.cdl +diff tmpx.cdl tmp_chunkedx.cdl + +if x = y ; then +# Note that unchunked means that there is only one chunk +SRC="$fileurl" +fileargs tmp_unchunked +${NCCOPY} -M0 -c dim0/,dim1/,dim2/,dim3/,dim4/,dim5/,dim6/ "$SRC" "$fileurl" +${NCDUMP} -sh -n tmp "$fileurl" > tmp_unchunked.cdl +verifychunking tmp_unchunked.cdl "ivar:_ChunkSizes=7,4,2,3,5,6,9;" "fvar:_ChunkSizes=9,6,5,3,2,4,7;" +chunkclean tmp_unchunked.cdl tmp_unchunkedx.cdl +diff tmpx.cdl tmp_unchunkedx.cdl + +# Test -c / +fileargs tmp_unchunked2 +${NCCOPY} -M0 -c "//" "$SRC" "$fileurl" +${NCDUMP} -sh -n tmp "$fileurl" > tmp_unchunked2.cdl +verifychunking tmp_unchunked.cdl "ivar:_ChunkSizes=7,4,2,3,5,6,9;" "fvar:_ChunkSizes=9,6,5,3,2,4,7;" +chunkclean tmp_unchunked.cdl tmp_unchunkedx.cdl +diff tmpx.cdl tmp_unchunkedx.cdl + +echo "*** Test that nccopy -c dim/n is used " +fileargs tmp_perdimspecs +${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_perdimspecs.cdl +${NCDUMP} -n tmp_perdimspecs -hs "$fileurl" > tmp_perdimspecs.cdl +SRC=$fileurl +fileargs tmp_pds + +${NCCOPY} -M0 -4 -c "time/10,lat/15,lon/20" "$SRC" "$fileurl" +${NCDUMP} -n tmp_pds -hs "$fileurl" > tmp_pds.cdl + +STORAGE=`cat tmp_pds.cdl | sed -e "/tas:_Storage/p" -ed | tr -d "[:space:]"` +echo "STORAGE: $STORAGE" + +test "x$STORAGE" = "xtas:_Storage='chunked';" || test "x$STORAGE" = "xtas:_Storage=\"chunked\";" +CHUNKSIZES=`cat tmp_pds.cdl | sed -e "/tas:_ChunkSizes/p" -ed | tr -d "[:space:]"` +test "x$CHUNKSIZES" = "xtas:_ChunkSizes=10,15,20;" +fi +} + +testcase file +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi +if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi + +echo "*** All nccopy nczarr tests passed!" + +if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/v3_nczarr_test/run_ncgen4.sh b/v3_nczarr_test/run_ncgen4.sh new file mode 100755 index 0000000000..a19d256204 --- /dev/null +++ b/v3_nczarr_test/run_ncgen4.sh @@ -0,0 +1,79 @@ +#!/bin/sh +# Tests for ncgen4 using list of test cdl files from the cdl4 +# directory, and comparing output to expected results in the expected4 +# directory. Note that these tests are run for classic files in +# test_ncgen4_classic.sh +# Dennis Heimbigner + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "${srcdir}/test_nczarr.sh" + +# Isolate both test and S3 +s3isolate "testdir_ncgen4" +THISDIR=`pwd` +cd $ISOPATH + +set -e + +# To add a new test, +# 1. put the .cdl file in the 'ncdump/cdl' directory +# 2. put the result of running ncgen then ncdump +# into the directory 'expected' as .dmp +# 3. Add the test to the end of the TESTS variable +# 4. Add the new files into ncdump/cdl/Makefile.am +# and ncdump/expected/Makefile.am + +TESTS="\ +dimscope \ +tst_group_data \ +tst_solar_1 \ +tst_nul4 \ +" + +# Location constants +cdl="$srcdir/../ncdump/cdl" +expected="$srcdir/../ncdump/expected" + +# Functions + +# Remove fillvalue attribute since zarr generates it when hdf5 does not +fvclean() { + cat $1 \ + | sed -e '/:_FillValue/d' \ + | cat > $2 +} + +difftest() { +echo ""; echo "*** Test zext=$zext" +for t in ${TESTS} ; do + echo "*** Testing: ${t}" + # determine if we need the specflag set + # determine properties + checkprops ${t} + ref="ref_${t}" + rm -fr ${t}.$zext + rm -f tmp_${t}.dmp + fileargs $t + ${NCGEN} -4 -lb -o ${fileurl} ${cdl}/${ref}.cdl + ${NCDUMP} ${headflag} ${specflag} -n ${ref} ${fileurl} > tmp_${t}.dmp + fvclean tmp_${t}.dmp tmp_${t}.dmpx + # compare against expected + diff -b -w ${expected}/${ref}.dmp ./tmp_${t}.dmpx + echo "*** SUCCEED: ${t}" +done +} + +runtestset() { +extfor $1 +echo "*** Testing nczarr X ncgen with zmap=${zext}" +difftest +echo "*** PASSED: zext=${zext}" +} + +runtestset file +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then runtestset zip; fi +if test "x$FEATURE_S3TESTS" = xyes ; then runtestset s3; fi + +echo "*** PASSED ***" diff --git a/v3_nczarr_test/run_nczarr_fill.sh b/v3_nczarr_test/run_nczarr_fill.sh new file mode 100755 index 0000000000..3fb0f59da3 --- /dev/null +++ b/v3_nczarr_test/run_nczarr_fill.sh @@ -0,0 +1,77 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "${srcdir}/test_nczarr.sh" + +s3isolate "testdir_nczarr_fill" +THISDIR=`pwd` +cd $ISOPATH + +set -x +set -e + +echo "*** Test: Github issues #2063, #2062, #2059" + +# Note: for some of these tests, the input is actually in V2 format +# so it is necessary to invoke nccopy to convert to V3 format. + +testcase2059() { +zext=$1 +echo "*** Test: Github issue #2059" +fileargs tmp_groups_regular "mode=zarr,$zext" +deletemap $zext $file +${NCCOPY} ${srcdir}/ref_groups.h5 "$fileurl" +rm -f tmp_nczfill.cdl +${ZMD} -h "$fileurl" +${NCDUMP} -s -n tmp_groups_regular "$fileurl" > tmp_groups_regular_$zext.cdl +sclean tmp_groups_regular_$zext.cdl +diff -wb ${srcdir}/ref_groups_regular.cdl tmp_groups_regular_$zext.cdl +} + +testcase2062() { +zext=$1 +echo "*** Test: Github issue #2062" +rm -fr tmp_ref_byte.zarr +unzip ref_byte.zarr.zip >> tmp_ignore.txt +# Convert to V3 +${NCCOPY} -Xf "file://ref_byte.zarr#mode=zarr,$zext" "file://tmp_ref_byte.zarr#mode=zarr,$zext" +${ZMD} -h "file://tmp_ref_byte.zarr#mode=zarr,$zext" +${NCDUMP} -s -n ref_byte "file://tmp_ref_byte.zarr#mode=zarr,$zext" > tmp_byte_$zext.cdl +sclean tmp_byte_$zext.cdl +diff -wb ${srcdir}/ref_byte.cdl tmp_byte_$zext.cdl +rm -fr tmp_ref_byte.zarr +} + +testcase2063() { +zext=$1 +echo "*** Test: Github issue #2063" +rm -fr ref_byte_fill_value_null.zarr +unzip ref_byte_fill_value_null.zarr.zip >> tmp_ignore.txt +rm -fr tmp_nczfill.cdl +# Convert to V3 +${NCCOPY} "file://ref_byte_fill_value_null.zarr#mode=zarr,$zext" "file://tmp_ref_byte_fill_value_null.zarr#mode=zarr,$zext" +${ZMD} -h "file://tmp_ref_byte_fill_value_null.zarr#mode=zarr,$zext" +${NCDUMP} -s -n ref_byte_fill_value_null "file://tmp_ref_byte_fill_value_null.zarr#mode=zarr,$zext" > tmp_byte_fill_value_null_$zext.cdl +sclean tmp_byte_fill_value_null_$zext.cdl +diff -wb ${abs_srcdir}/ref_byte_fill_value_null.cdl tmp_byte_fill_value_null_$zext.cdl +#rm -fr tmp_nczfill.cdl tmp_ref_byte_fill_value_null.zarr +} + +if ! test -f ${ISOPATH}/ref_byte.zarr.zip ; then + cp -f ${srcdir}/ref_byte.zarr.zip ${ISOPATH}/ref_byte.zarr.zip + cp -f ${srcdir}/ref_byte_fill_value_null.zarr.zip ${ISOPATH}/ref_byte_fill_value_null.zarr.zip +fi + +testcase2062 file +testcase2063 file +if test "x$FEATURE_HDF5" = xyes ; then + testcase2059 file + if test "x$FEATURE_NCZARR_ZIP" = xyes ; then + testcase2059 zip + fi + if test "x$FEATURE_S3TESTS" = xyes ; then + testcase2059 s3 + fi +fi diff --git a/v3_nczarr_test/run_nczfilter.sh b/v3_nczarr_test/run_nczfilter.sh new file mode 100755 index 0000000000..2ce44ccd11 --- /dev/null +++ b/v3_nczarr_test/run_nczfilter.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "${srcdir}/test_nczarr.sh" + +# This shell script runs test_nczfilter.c + +set -e + +s3isolate "testdir_nczfilter" +THISDIR=`pwd` +cd $ISOPATH + +${execdir}/test_nczfilter diff --git a/v3_nczarr_test/run_newformat.sh b/v3_nczarr_test/run_newformat.sh new file mode 100755 index 0000000000..bc5691476d --- /dev/null +++ b/v3_nczarr_test/run_newformat.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "${srcdir}/test_nczarr.sh" + +set -e + +s3isolate "testdir_newformat" +THISDIR=`pwd` +cd $ISOPATH + +echo "" +echo "*** Testing backward compatibilty between nczarr meta data format V1 vs V2" + +testcaseold() { +echo "*** Test old format support" +zext=$1 +fileargs ${srcdir}/ref_oldformat +${NCDUMP} -n ref_oldformat "$fileurl" > ./tmp_oldformat.cdl +diff -w ${srcdir}/ref_oldformat.cdl ./tmp_oldformat.cdl +} + +testcasecvt() { +echo "*** Test old format to new format nczarr copy" +zext=$1 +fileargs ${srcdir}/ref_oldformat +${NCCOPY} "$fileurl" "file://tmp_newformat.file#mode=nczarr,file" +${NCDUMP} -n ref_oldformat "file://tmp_newformat.file#mode=nczarr,file" > ./tmp_newformat.cdl +diff -w ${srcdir}/ref_oldformat.cdl ./tmp_newformat.cdl +} + +testcasepure() { +echo "*** Test old format to new format pure zarr copy" +zext=$1 +fileargs ${srcdir}/ref_oldformat +${NCCOPY} "$fileurl" "file://tmp_newformat.file#mode=nczarr,file" +${NCDUMP} -n ref_oldformat "file://tmp_newformat.file#mode=zarr,file" > ./tmp_newpure.cdl +diff -w ${srcdir}/ref_newformatpure.cdl ./tmp_newpure.cdl +} + +# Do zip tests only +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then + testcaseold zip + testcasecvt zip + testcasepure zip +fi diff --git a/v3_nczarr_test/run_notzarr.sh b/v3_nczarr_test/run_notzarr.sh new file mode 100755 index 0000000000..7bc0415073 --- /dev/null +++ b/v3_nczarr_test/run_notzarr.sh @@ -0,0 +1,64 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "${srcdir}/test_nczarr.sh" + +set -e + +# Build both ISOPATH and S3ISOPATH +s3isolate "testdir_notzarr" +THISDIR=`pwd` +cd $ISOPATH + +# Test ability to detect NCZarr/Zarr files + +URL="${NCZARR_S3_TEST_HOST}/${NCZARR_S3_TEST_BUCKET}" +KEY="${S3ISOPATH}" + +sometestfailed= + +testfailed() { + if test "x$1" != "x-51" ; then + echo "*** Failed" + sometestfailed=1 + fi +} + +# Make test sets +cp ${abs_top_srcdir}/nczarr_test//ref_notzarr.tar.gz ${ISOPATH} +gunzip ref_notzarr.tar.gz +tar -xf ref_notzarr.tar +if test "x$FEATURE_S3TESTS" = xyes ; then + ${S3UTIL} -f notzarr.file/notzarr.txt -u "https://${URL}" -k "${S3ISOPATH}/notzarr.s3/notzarr.txt" upload +fi + +echo "Test empty file" +RET=`${execdir}/test_notzarr "file://empty.file#mode=zarr,file"` +testfailed "$RET" +echo "Test non-zarr file" +RET=`${execdir}/test_notzarr "file://notzarr.file#mode=zarr,file"` +testfailed "$RET" + +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then +echo "Test empty zip file" +RET=`${execdir}/test_notzarr "file://empty.zip#mode=zarr,zip"` +testfailed "$RET" +echo "Test non-zarr zip file" +RET=`${execdir}/test_notzarr "file://notzarr.zip#mode=zarr,zip"` +testfailed "$RET" +fi + +if test "x$FEATURE_S3TESTS" = xyes ; then +if test 1 = 0 ; then + # This test is NA for S3 + echo "Test empty S3 file" + KEY2="${KEY}/empty.s3" + RET=`${execdir}/test_notzarr "https://$URL${KEY2}#mode=zarr,s3"` + testfailed "$RET" +fi +echo "Test non-zarr S3 file" +RET=`${execdir}/test_notzarr "https://$URL/${ISOPATH}/notzarr.s3#mode=zarr,s3"` +testfailed "$RET" +fi diff --git a/v3_nczarr_test/run_nulls.sh b/v3_nczarr_test/run_nulls.sh new file mode 100755 index 0000000000..92e7164b20 --- /dev/null +++ b/v3_nczarr_test/run_nulls.sh @@ -0,0 +1,56 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "${srcdir}/test_nczarr.sh" + +set -x +set -e + +s3isolate "testdir_nulls" +THISDIR=`pwd` +cd $ISOPATH + +# This shell script tests support for special cases of NC_CHAR + +testcase() { +zext=$1 + +echo "*** Test: nczarr string write then read; format=$zext" +# Get pure zarr args +fileargs tmp_nulls_zarr "mode=zarr,$zext" +zarrurl="$fileurl" +zarrfile="$file" +# Get nczarr args +fileargs tmp_nulls_nczarr "mode=nczarr,$zext" +nczarrurl="$fileurl" +nczarrfile="$file" + +# setup +deletemap $zext $zarrfile +deletemap $zext $nczarrfile + +# Create alternate ref files +echo "*** create pure zarr file" +${NCGEN} -4 -b -o "$zarrurl" $srcdir/../nczarr_test/ref_nulls.cdl +echo "*** create nczarr file" +${NCGEN} -4 -b -o "$nczarrurl" $srcdir/../nczarr_test/ref_nulls.cdl + +echo "*** read purezarr" +${NCDUMP} -n ref_nulls $zarrurl > tmp_nulls_zarr_${zext}.cdl +${ZMD} -h $zarrurl > tmp_nulls_zarr_${zext}.txt +echo "*** read nczarr" +${NCDUMP} -n ref_nulls $nczarrurl > tmp_nulls_nczarr_${zext}.cdl +${ZMD} -h $nczarrurl > tmp_nulls_nczarr_${zext}.txt + +echo "*** verify zarr output" +diff -bw ${srcdir}/ref_nulls_zarr.baseline tmp_nulls_zarr_${zext}.cdl + +echo "*** verify nczarr output" +diff -bw ${srcdir}/ref_nulls_nczarr.baseline tmp_nulls_nczarr_${zext}.cdl +} + +testcase file +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi +if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi diff --git a/v3_nczarr_test/run_perf_chunks1.sh b/v3_nczarr_test/run_perf_chunks1.sh new file mode 100755 index 0000000000..e08e1417f1 --- /dev/null +++ b/v3_nczarr_test/run_perf_chunks1.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +# This shell just tests the test_chunks3 program by running it a few +# times to generate a simple test file. Then it uses ncdump -s to +# check that the output is what it should be. +# Copied from nc_perf/. + +# Russ Rew, Dennis Heimbigner + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh +. "${srcdir}/test_nczarr.sh" + +s3isolate "testdir_perf_chunks1" +THISDIR=`pwd` +cd $ISOPATH + +test1() { + FMT=$1 + DIMS=$2 + CHUNKS=$3 + ${execdir}/bm_chunks3 --format=$FMT --f=bm_chunks3 --dims="$DIMS" --chunks="$CHUNKS" +} + +testcases() { +echo "" +echo "*** Running benchmarking program bm_chunks3 for tiny test file" +test1 $1 "6,12,4" "2,3,1" +echo '*** SUCCESS!!!' +echo "" +echo "*** Testing the benchmarking program bm_chunks3 for larger variables ..." +#cachesize=10000000; cachehash=10000; cachepre=0.0 +test1 $1 "32,90,91" "8,10,13" +echo '*** SUCCESS!!!' +} + + +testcases file +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcases zip; fi +if test "x$FEATURE_S3TESTS" = xyes ; then testcases s3; fi diff --git a/v3_nczarr_test/run_purezarr.sh b/v3_nczarr_test/run_purezarr.sh new file mode 100755 index 0000000000..9d4545deb4 --- /dev/null +++ b/v3_nczarr_test/run_purezarr.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "${srcdir}/test_nczarr.sh" + +set -x +set -e + +s3isolate "testdir_purezarr" +THISDIR=`pwd` +cd $ISOPATH + +# This shell script tests support for: +# 1. pure zarr -- without xarray -- read/write +# 2. xarray read/write + +testcase() { +zext=$1 + +echo "*** Test: pure zarr write then read; format=$zext; without xarray" +fileargs tmp_purezarr "mode=zarr,noxarray,$zext" +deletemap $zext $file +${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_purezarr_base.cdl +${NCDUMP} $fileurl > tmp_purezarr_${zext}.cdl +diff -b ${srcdir}/ref_purezarr.cdl tmp_purezarr_${zext}.cdl + +echo "*** Test: xarray zarr write then read; format=$zext" +fileargs tmp_xarray "mode=zarr,$zext" +#deletemap $zext $file +${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_purezarr_base.cdl +${NCDUMP} $fileurl > tmp_xarray_${zext}.cdl +diff -b ${srcdir}/ref_xarray.cdl tmp_xarray_${zext}.cdl + +echo "*** Test: pure zarr reading nczarr; format=$zext; without xarray" +fileargs tmp_nczarr "mode=nczarr,noxarray,$zext" +deletemap $zext $file +${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_whole.cdl +fileargs tmp_nczarr "mode=zarr,$zext" +${NCDUMP} -n nczarr2zarr $fileurl > tmp_nczarr_${zext}.cdl +diff -b ${srcdir}/ref_nczarr2zarr.cdl tmp_nczarr_${zext}.cdl +} + +testcase file +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi +if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi + diff --git a/v3_nczarr_test/run_quantize.sh b/v3_nczarr_test/run_quantize.sh new file mode 100755 index 0000000000..1907c62da7 --- /dev/null +++ b/v3_nczarr_test/run_quantize.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "${srcdir}/test_nczarr.sh" + +# Construct both ISOPATH and S3ISOPATH +s3isolate "testdir_quantize" + +THISDIR=`pwd` +cd $ISOPATH + +# This shell script runs test_quantize + +set -e + +testcase() { + zext=$1 + fileargs tmp_quantize "mode=$zarr,$zext" + case "$zext" in + file) template="file://${ISOPATH}/%s.zarr#mode=zarr,$zext" ;; + zip) template="file://${ISOPATH}/%s.zip#mode=zarr,$zext" ;; + s3) template="s3://${NCZARR_S3_TEST_BUCKET}/${S3ISOPATH}/%s.zarr#mode=zarr,$zext" ;; + *) echo "unknown file type"; exit 1 ;; + esac + ${execdir}/test_quantize "$template" +} + +testcase file +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi +# There is a (currently) untraceable bug when using S3 +if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi diff --git a/v3_nczarr_test/run_scalar.sh b/v3_nczarr_test/run_scalar.sh new file mode 100755 index 0000000000..cb601e0558 --- /dev/null +++ b/v3_nczarr_test/run_scalar.sh @@ -0,0 +1,63 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "${srcdir}/test_nczarr.sh" + +set -e + +s3isolate "testdir_scalar" +THISDIR=`pwd` +cd $ISOPATH + +# This shell script tests support for the NC_STRING type + +zarrscalar() { + rm -f $2 + sed -e '/dimensions:/d' -e '/_scalar_ =/d' -e '/int v/ s|(_scalar_)||' <$1 >$2 +cat $1 > rescale_$1 +} + +testcase() { +zext=$1 + +echo "*** Test: scalar write/read" + +# Get pure zarr args +fileargs tmp_scalar_zarr "mode=zarr,$zext" +zarrurl="$fileurl" +zarrfile="$file" +# Get nczarr args +fileargs tmp_scalar_nczarr "mode=nczarr,$zext" +nczarrurl="$fileurl" +nczarrfile="$file" + +# setup +deletemap $zext $zarrfile +deletemap $zext $nczarrfile + +# Create alternate ref files +echo "*** create pure zarr file" +${NCGEN} -4 -b -o "$zarrurl" $top_srcdir/nczarr_test/ref_scalar.cdl +echo "*** create nczarr file" +${NCGEN} -4 -b -o "$nczarrurl" $top_srcdir/nczarr_test/ref_scalar.cdl + +echo "*** read purezarr" +${NCDUMP} -n ref_scalar $zarrurl > tmp_scalar_zarr_${zext}.cdl +${ZMD} -h $zarrurl > tmp_scalar_zarr_${zext}.txt +echo "*** read nczarr" +${NCDUMP} -n ref_scalar $nczarrurl > tmp_scalar_nczarr_${zext}.cdl +${ZMD} -h $nczarrurl > tmp_scalar_nczarr_${zext}.txt + +echo "*** verify" +diff -bw $top_srcdir/nczarr_test/ref_scalar_nczarr.cdl ${ISOPATH}/tmp_scalar_nczarr_${zext}.cdl + +# Fixup +zarrscalar tmp_scalar_zarr_${zext}.cdl tmp_rescale_zarr_${zext}.cdl +diff -bw $top_srcdir/nczarr_test/ref_scalar.cdl ${ISOPATH}/tmp_rescale_zarr_${zext}.cdl +} + +testcase file +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi +if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi diff --git a/v3_nczarr_test/run_specific_filters.sh b/v3_nczarr_test/run_specific_filters.sh new file mode 100755 index 0000000000..bad0623030 --- /dev/null +++ b/v3_nczarr_test/run_specific_filters.sh @@ -0,0 +1,220 @@ +#!/bin/bash + +# Test derived from nc_test4/tst_specific_filters.sh +TESTNCZARR=1 + +# Test the implementations of specific filters +# Also test nc_inq_filter_avail + +# WARNING: This file is also used to build nczarr_test/run_specific_filters.sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +set -e + +if test "x$TESTNCZARR" = x1 ; then +. ${srcdir}/test_nczarr.sh +s3isolate "testdir_specific_filters" +THISDIR=`pwd` +cd $ISOPATH +fi + +zarrfilt() { +XCODEC="[{" +if test "x$TESTNCZARR" = x1 && test "x$NCZARRFORMAT" = x3 ; then + XCODEC="${XCODEC}\"name\": \"$1\", \"configuration\": {" +else + XCODEC="${XCODEC}\"id\": \"$1\"," +fi +shift +blank= +while test 0 -lt $#; do + key=`echon "$1" | cut -d: -f1` + val=`echon "$1" | cut -d: -f2` + XCODEC="${XCODEC}${blank}\"$key\": $val" + shift + blank=", " +done +if test "x$TESTNCZARR" = x1 && test "x$NCZARRFORMAT" = x3 ; then +XCODEC="${XCODEC}}}]" +else +XCODEC="${XCODEC}}]" +fi +} + +if test "x$TESTNCZARR" = x1 ; then + BLOSCARGS="32001,0,0,0,256,5,1,1" + zarrfilt blosc clevel:5 blocksize:256 cname:\"lz4\" shuffle:1 + BLOSCCODEC="$XCODEC" +else + BLOSCARGS="32001,0,0,4,256,5,1,1" + zarrfilt blosc clevel:5 blocksize:256 cname:\"lz4\" shuffle:1 + BLOSCCODEC="$XCODEC" +fi + +# Load the findplugins function +. ${builddir}/findplugin.sh +echo "findplugin.sh loaded" + +# Locate the plugin path and the library names; argument order is critical +# Find bzip2 and capture +# Assume all test filters are in same plugin dir +if ! findplugin h5bzip2 ; then exit 0 ; fi + +# Function to remove selected -s attributes from file; +# These attributes might be platform dependent +sclean() { + cat $1 \ + | sed -e '/:_IsNetcdf4/d' \ + | sed -e '/:_Endianness/d' \ + | sed -e '/_NCProperties/d' \ + | sed -e '/_SuperblockVersion/d' \ + | sed -e '/_Format/d' \ + | sed -e '/global attributes:/d' \ + | cat > $2 +} + +# Function to extract _Filter attribute from a file +# These attributes might be platform dependent +getfilterattr() { +V="$1" +sed -e '/${V}.*:_Filter/p' -ed <$2 >$3 +} + +# Function to extract _Codecs attribute from a file +# These attributes might be platform dependent +getcodecsattr() { +V="$1" +sed -e '/${V}.*:_Codecs/p' -ed <$2 >$3 +} + +trimleft() { +sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2 +} + + +setfilter() { + FF="$1" + FSRC="$2" + FDST="$3" + FIH5="$4" + FICX="$5" + FFH5="$6" + FFCX="$7" + if test "x$FFH5" = x ; then FFH5="$FIH5" ; fi + if test "x$FFCX" = x ; then FFCX="$FICX" ; fi + rm -f $FDST + cat ${srcdir}/$FSRC \ + | sed -e "s/ref_any/${FF}/" \ + | sed -e "s/IH5/${FIH5}/" -e "s/FH5/${FFH5}/" \ + | sed -e "s/ICX/${FICX}/" -e "s/FCX/${FFCX}/" \ + | sed -e 's/"/\\"/g' -e 's/@/"/g' \ + | cat > $FDST +} + +# Execute the specified tests + +runfilter() { +zext=$1 +zfilt="$2" +zparams="$3" +zcodec="$4" +echo "*** Testing processing of filter $zfilt for map $zext" +if test "x$TESTNCZARR" = x1 ; then +fileargs "tmp_filt_${zfilt}" +deletemap $zext $file +else +file="tmp_filt_${zfilt}.nc" +rm -f $file +fi +setfilter $zfilt ref_any.cdl "tmp_filt_${zfilt}.cdl" "$zparams" "$zcodec" +if test "x$TESTNCZARR" = x1 ; then +${NCGEN} -4 -lb -o $fileurl "tmp_filt_${zfilt}.cdl" +${NCDUMP} -n $zfilt -sF $fileurl > "tmp_filt_${zfilt}.tmp" +else +${NCGEN} -4 -lb -o $file "tmp_filt_${zfilt}.cdl" +${NCDUMP} -n $zfilt -sF $file > "tmp_filt_${zfilt}.tmp" +fi +sclean "tmp_filt_${zfilt}.tmp" "tmp_filt_${zfilt}.dump" +} + +testfletcher32() { + zext=$1 + runfilter $zext fletcher32 '3' '[{\"id\": \"fletcher32\"}]' + # need to do fixup + sed -e '/_Fletcher32 = "true"/d' -e '/_Filter = 3'/d -e '/_Codecs = \"[{\"id\": \"fletcher32\"}]\"/d' \ + < tmp_filt_fletcher32.cdl > tmp_filt_fletcher32x.dump + diff -b -w "tmp_filt_fletcher32.cdl" "tmp_filt_fletcher32x.dump" +} + +testshuffle() { + zext=$1 + runfilter $zext shuffle '2' '[{\"id\": \"shuffle\",\"elementsize\": \"0\"}]' + # need to replace _Filter + sed -e '/_Shuffle = "true"/d' -e '/_Filter = 2'/d -e '/_Codecs = \"[{\"id\": \"shuffle\"}]\"/d' \ + < tmp_filt_shuffle.cdl > tmp_filt_shufflex.dump + diff -b -w "tmp_filt_shuffle.cdl" "tmp_filt_shufflex.dump" +} + +testdeflate() { + zext=$1 + if ! avail deflate; then return 0; fi + zarrfilt zlib level:9 + runfilter $zext deflate '1,9' "$XCODEC" + diff -b -w "tmp_filt_deflate.cdl" "tmp_filt_deflate.dump" +} + +testbzip2() { + zext=$1 + if ! avail bzip2; then return 0; fi + zarrfilt bz2 level:9 + runfilter $zext bzip2 '307,9' "$XCODEC" + diff -b -w "tmp_filt_bzip2.cdl" "tmp_filt_bzip2.dump" +} + +testszip() { + zext=$1 + if ! avail szip; then return 0; fi +# H5_SZIP_NN_OPTION_MASK=32; H5_SZIP_MAX_PIXELS_PER_BLOCK_IN=32 + zarrfilt szip mask:32 pixels-per-block:32 + runfilter $zext szip '4,32,32' "$XCODEC" + diff -b -w "tmp_filt_szip.cdl" "tmp_filt_szip.dump" +} + +testblosc() { + zext=$1 + if ! avail blosc; then return 0; fi + runfilter $zext blosc $BLOSCARGS "$BLOSCCODEC" + diff -b -w "tmp_filt_blosc.cdl" "tmp_filt_blosc.dump" +} + +testzstd() { + zext=$1 + if ! avail zstd; then return 0; fi + zarrfilt zstd level:1 + runfilter $zext zstd '32015,1' "$XCODEC" + diff -b -w "tmp_filt_zstd.cdl" "tmp_filt_zstd.dump" +} + +testset() { +# Which test cases to exercise +if test "x$TESTNCZARR" = x1 ; then + testfletcher32 $1 + testshuffle $1 +fi + testdeflate $1 + testszip $1 + testbzip2 $1 + testblosc $1 + testzstd $1 +} + +if test "x$TESTNCZARR" = x1 ; then + testset file + if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testset zip ; fi + if test "x$FEATURE_S3TESTS" = xyes ; then testset s3 ; fi + if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup +else + testset nc +fi diff --git a/v3_nczarr_test/run_strings.sh b/v3_nczarr_test/run_strings.sh new file mode 100755 index 0000000000..a18c72550f --- /dev/null +++ b/v3_nczarr_test/run_strings.sh @@ -0,0 +1,59 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "${srcdir}/test_nczarr.sh" + +# This shell script tests support for the NC_STRING type + +set -x +set -e + +s3isolate "testdir_strings" +THISDIR=`pwd` +cd $ISOPATH + +testcase() { +zext=$1 + +echo "*** Test: nczarr string write then read; format=$zext" +# Get pure zarr args +fileargs tmp_string_zarr "mode=zarr,$zext" +zarrurl="$fileurl" +zarrfile="$file" +# Get nczarr args +fileargs tmp_string_nczarr "mode=nczarr,$zext" +nczarrurl="$fileurl" +nczarrfile="$file" + +# setup +deletemap $zext $zarrfile +deletemap $zext $nczarrfile + +# Create alternate ref files +echo "*** create pure zarr file" +${NCGEN} -4 -b -o "$zarrurl" $srcdir/ref_string.cdl +echo "*** create nczarr file" +${NCGEN} -4 -b -o "$nczarrurl" $srcdir/ref_string.cdl + +echo "*** read purezarr" +${NCDUMP} -n ref_string $zarrurl > tmp_string_zarr_${zext}.cdl +${ZMD} -t 'string/6' $zarrurl > tmp_string_zarr_${zext}.txt +echo "*** read nczarr" +${NCDUMP} -n ref_string $nczarrurl > tmp_string_nczarr_${zext}.cdl +${ZMD} -t 'string/6' $nczarrurl > tmp_string_nczarr_${zext}.txt + +echo "*** verify zarr output" +diff -bw ${srcdir}/ref_string_zarr.baseline ${ISOPATH}/tmp_string_zarr_${zext}.cdl + +echo "*** verify nczarr output" +diff -bw ${srcdir}/ref_string_nczarr.baseline ${ISOPATH}/tmp_string_nczarr_${zext}.cdl +} + +testcase file +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi +if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi + + + diff --git a/v3_nczarr_test/run_unknown.sh b/v3_nczarr_test/run_unknown.sh new file mode 100755 index 0000000000..1112bf60d1 --- /dev/null +++ b/v3_nczarr_test/run_unknown.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +# Test derived from nc_test4/tst_unknown.sh +TESTNCZARR=1 + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +isolate "testdir_unknown" +THISDIR=`pwd` +cd $ISOPATH + +if test "x$TESTNCZARR" = x1 ; then +. ${srcdir}/test_nczarr.sh +s3isolate +fi + +set -e + +# Load the findplugins function +. ${builddir}/findplugin.sh +echo "findplugin.sh loaded" + +# Set up HDF5_PLUGIN_PATH +export HDF5_PLUGIN_PATH=${HDF5_PLUGIN_DIR} + +# Test operation with an unknown filter + +# Function to remove selected -s attributes from file; +# These attributes might be platform dependent +sclean() { + cat $1 \ + | sed -e '/:_IsNetcdf4/d' \ + | sed -e '/:_Endianness/d' \ + | sed -e '/_NCProperties/d' \ + | sed -e '/_SuperblockVersion/d' \ + | cat > $2 +} + +# Function to extract _Filter attribute from a file +# These attributes might be platform dependent +getfilterattr() { +sed -e '/var.*:_Filter/p' -ed <$1 >$2 +} + +# Function to extract _Codecs attribute from a file +# These attributes might be platform dependent +getcodecsattr() { +sed -e '/var.*:_Codecs/p' -ed <$1 >$2 +} + +trimleft() { +sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2 +} + +# Locate the plugin path and the library names; argument order is critical + +# Find noop and capture +if ! findplugin h5unknown ; then exit 0; fi +UNKNOWNDIR="${HDF5_PLUGIN_DIR}" +UNKNOWNLIB="${HDF5_PLUGIN_LIB}" +UNKNOWNFILTER="${HDF5_PLUGIN_DIR}/${UNKNOWNLIB}" + +# Getting the name is especially tricky for dylib, which puts the version before the .dylib + +# Verify +if ! test -f ${UNKNOWNFILTER} ; then echo "Unable to locate ${UNKNOWNFILTER}"; exit 1; fi + +testunk() { +zext=$1 +echo "*** Testing access to filter info when filter implementation is not available for map $zext" +if test "x$TESTNCZARR" = x1 ; then +fileargs tmp_known +deletemap $zext $file +else +file="tmp_known_${zfilt}.nc" +rm -f $file +fileurl="$file" +fi +# build .nc file using unknown +${NCGEN} -lb -4 -o $fileurl ${srcdir}/../nc_test4/unknown.cdl +# dump and clean file when filter is avail +${NCDUMP} -hs $fileurl > ./tmp_known_$zext.txt +# Remove irrelevant -s output +sclean ./tmp_known_$zext.txt tmp_known_$zext.dump +# Hide the filter +rm -fr ${UNKNOWNDIR}/save +mkdir -p ${UNKNOWNDIR}/save +# Figure out all matching libs; make sure to remove .so, so.0, etc +LSRC=`${execdir}/../ncdump/ncpathcvt -F "${UNKNOWNDIR}"` +LDST=`${execdir}/../ncdump/ncpathcvt -F ${UNKNOWNDIR}/save` +mv ${LSRC}/*unknown* ${LDST} +# Verify that the filter is no longer defined +# Try to read the data; should xfail +if ${NCDUMP} -s $fileurl > ./tmp_unk_$zext.dmp ; then +echo "*** FAIL: filter found" +found=1 +else +echo "*** XFAIL: filter not found" +found=0 +fi +# Restore the filter +mv ${LDST}/*unknown* ${LSRC} +rm -fr ${UNKNOWNDIR}/save +if test "x$found" = x1 ; then exit 1; fi +} + +if test "x$TESTNCZARR" = x1 ; then + testunk file + if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testunk zip ; fi + if test "x$FEATURE_S3TESTS" = xyes ; then testunk s3 ; fi + if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup +else +testunk nc +fi + +exit 0 diff --git a/v3_nczarr_test/run_unlim_io.sh b/v3_nczarr_test/run_unlim_io.sh new file mode 100755 index 0000000000..aec2320003 --- /dev/null +++ b/v3_nczarr_test/run_unlim_io.sh @@ -0,0 +1,107 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. ${srcdir}/test_nczarr.sh + +set -e + +s3isolate "testdir_unlim_io" +THISDIR=`pwd` +cd $ISOPATH + +TC="${execdir}/test_unlim_io -4" + +remfile() { + case "$zext" in + nc4) rm -fr $1 ;; + file) rm -fr $1 ;; + zip) rm -fr $1 ;; + s3) ;; + *) echo "no such extension: $zext" ; exit 1;; + esac +} + +remfile() { + case "$zext" in + nc4) rm -fr $1 ;; + file) rm -fr $1 ;; + zip) rm -fr $1 ;; + s3) ;; + *) echo "no such extension: $zext" ; exit 1;; + esac +} + +buildfile() { +zext=$1 +base=$2 +if test "x$TESTNCZARR" = x1 ; then +fileargs "${base}_${zext}" +deletemap $zext $file +file="$fileurl" +else +file="${base}_${zext}.nc" +rm -f $file +fi +} + +testcase1() { +zext=$1 +buildfile $zext tmp_unlim_io1 +echo ""; echo "*** Test simple use of unlimited" +rm -fr tmp_unlim_io1.nc tmp_unlim_io1.$zext tmp_unlim_io1.nc.txt tmp_unlim_io1.${zext}.txt +$TC -d 0 -c 2 -s 0 -e 1 -Ocw tmp_unlim_io1.nc +$TC -d 0 -c 2 -s 0 -e 1 -Ocw "$file" +${NCDUMP} -n tmp_unlim_io1 tmp_unlim_io1.nc >tmp_unlim_io1.nc.txt +${NCDUMP} -n tmp_unlim_io1 "$file" >tmp_unlim_io1.${zext}.txt +diff -b tmp_unlim_io1.nc.txt tmp_unlim_io1.${zext}.txt +} + +testcase2() { +zext=$1 +buildfile $zext tmp_unlim_io2 +echo ""; echo "*** Test 2-d chunking" +rm -fr tmp_unlim_io2.nc tmp_unlim_io2.$zext tmp_unlim_io2.nc.txt tmp_unlim_io2.${zext}.txt +$TC -d 0,0 -c 2,2 -s 0,0 -e 2,2 -v 17 -Ocw tmp_unlim_io2.nc +$TC -d 0,0 -c 2,2 -s 0,0 -e 2,2 -v 17 -Ocw "$file" +${NCDUMP} -n tmp_unlim_io2 tmp_unlim_io2.nc >tmp_unlim_io2.nc.txt +${NCDUMP} -n tmp_unlim_io2 "$file" >tmp_unlim_io2.${zext}.txt +diff -b tmp_unlim_io2.nc.txt tmp_unlim_io2.${zext}.txt +${NCDUMPCHUNKS} -v v "$file" +} + +testcase3() { +zext=$1 +buildfile $zext tmp_unlim_io3 +echo ""; echo "*** Test multi-chunk extension" +rm -fr tmp_unlim_io3.nc tmp_unlim_io3.$zext tmp_unlim_io3.nc.txt tmp_unlim_io3.${zext}.txt +$TC -d 0,0 -c 2,2 -s 6,6 -e 2,2 -v 17 -Ocw tmp_unlim_io3.nc +$TC -d 0,0 -c 2,2 -s 6,6 -e 2,2 -v 17 -Ocw "$file" +${NCDUMP} -n tmp_unlim_io3 tmp_unlim_io3.nc >tmp_unlim_io3.nc.txt +${NCDUMP} -n tmp_unlim_io3 "$file" >tmp_unlim_io3.${zext}.txt +diff -b tmp_unlim_io3.nc.txt tmp_unlim_io3.${zext}.txt +} + +testcase4() { +zext=$1 +buildfile $zext tmp_unlim_io4 +echo ""; echo "*** Test unlimited as second dimension" +rm -fr tmp_unlim_io4.nc tmp_unlim_io4.$zext tmp_unlim_io4.nc.txt tmp_unlim_io4.${zext}.txt +$TC -d 8,0 -c 2,2 -s 6,6 -e 2,2 -v 17 -Ocw tmp_unlim_io4.nc +$TC -d 8,0 -c 2,2 -s 6,6 -e 2,2 -v 17 -Ocw "$file" +${NCDUMP} -n tmp_unlim_io3 tmp_unlim_io4.nc >tmp_unlim_io4.nc.txt +${NCDUMP} -n tmp_unlim_io3 "$file" >tmp_unlim_io4.${zext}.txt +diff -b tmp_unlim_io4.nc.txt tmp_unlim_io4.${zext}.txt +} + +testcases() { + testcase1 $1 + testcase2 $1 + testcase3 $1 + testcase4 $1 +} + +testcases file +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcases zip; fi +if test "x$FEATURE_S3TESTS" = xyes ; then testcases s3; fi diff --git a/v3_nczarr_test/run_ut_map.sh b/v3_nczarr_test/run_ut_map.sh new file mode 100755 index 0000000000..a6023e96c4 --- /dev/null +++ b/v3_nczarr_test/run_ut_map.sh @@ -0,0 +1,118 @@ +#!/bin/sh + +export SETX=1 + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. ${srcdir}/test_nczarr.sh + +set -e + +s3isolate "testdir_utmap" +THISDIR=`pwd` +cd $ISOPATH + +# Test those map implementations where +# it is possible to look at the actual storage. +# in some cases. Note that we +# cannot easily look inside S3 storage +# except using the aws-cli, if available + +# Common +CMD="${execdir}/ut_map${ext}" + +testmapcreate() { + echo ""; echo "*** Test zmap create -k $1" + extfor "$1" + tag="map" + output="tmp_$tag.$zext" + + deletemap $1 $output + + # Create the test file + $CMD -k$1 -x create -o $output + cdl="ut_${tag}_create_${zext}.cdl" + ref="ref_ut_${tag}_create.cdl" + dumpmap $zext $output ./$cdl + diff -wb ${srcdir}/$ref ./$cdl + # delete the test file + $CMD -k$1 -x delete -f $output + rm -f $cdl + if test -f $output; then + echo "delete did not delete $output" + exit 1 + fi + # re-create the test file + $CMD -k$1 -x create -o $output +} + +testmapmeta() { + echo ""; echo "*** Test zmap read/write meta -k $1" + extfor "$1" + tag="map" + file="tmp_$tag.$zext" + + $CMD -k$1 -x writemeta -f $file + cdl="ut_${tag}_writemeta_${zext}.cdl" + ref="ref_ut_${tag}_writemeta.cdl" + dumpmap $zext $file ./$cdl + diff -wb ${srcdir}/$ref ./$cdl + + $CMD -k$1 -x writemeta2 -o ./$file + cdl="ut_${tag}_writemeta2_${zext}.cdl" + ref="ref_ut_${tag}_writemeta2.cdl" + dumpmap $zext $file ./$cdl + diff -wb ${srcdir}/$ref ./$cdl + + output="ut_${tag}_readmeta_$zext.txt" + outref="ref_ut_${tag}_readmeta.txt" + $CMD -k$1 -x readmeta -f $file > ./$output + diff -wb ${srcdir}/$outref ./$output + + output="ut_${tag}_readmeta2_$zext.txt" + outref="ref_ut_${tag}_readmeta2.txt" + $CMD -k$1 -x readmeta2 -f $file > ./$output + diff -wb ${srcdir}/$outref ./$output +} + +testmapdata() { + echo ""; echo "*** Test zmap read/write data -k $1" + extfor "$1" + tag="map" + file="tmp_$tag.$zext" + + $CMD -k$1 -x "writedata" -f $file + cdl="ut_${tag}_writedata_${zext}.cdl" + ref="ref_ut_${tag}_writedata.cdl" + dumpmap $zext $file ./$cdl + diff -wb ${srcdir}/$ref ./$cdl + # readata is verification only + $CMD -k$1 -x readdata -f $file +} + +testmapsearch() { + echo ""; echo "*** Test zmap search -k $1" + extfor "$1" + tag="map" + file="tmp_$tag.$zext" + txt=ut_${tag}_search_$zext.txt + ref=ref_ut_${tag}_search.txt + rm -f $txt + $CMD -k$1 -x "search" -f $file > $txt + diff -wb ${srcdir}/$ref ./$txt +} + +echo "" + +echo "*** Map Unit Testing" + +echo ""; echo "*** Test zmap_file" +testmapcreate file; testmapmeta file; testmapdata file; testmapsearch file + +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then +echo ""; echo "*** Test zmap_zip" +testmapcreate zip; testmapmeta zip; testmapdata zip; testmapsearch zip +fi + +exit 0 diff --git a/v3_nczarr_test/run_ut_mapapi.sh b/v3_nczarr_test/run_ut_mapapi.sh new file mode 100755 index 0000000000..2e39132349 --- /dev/null +++ b/v3_nczarr_test/run_ut_mapapi.sh @@ -0,0 +1,105 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "${srcdir}/test_nczarr.sh" + +#TR="-T10" + +set -e + +s3isolate "testdir_mapapi" +THISDIR=`pwd` +cd $ISOPATH + +# Test map implementations for consistency at the zmap API +# level. This allows testing of implementations that do not admit +# of easy examination of the actual storage. For example it is +# not easy to look inside S3 storage except using the aws-cli, if +# available. + +CMD="${execdir}/ut_mapapi${ext}" + +testmapcreate() { + echo ""; echo "*** Test zmap create -k $1" + extfor "$1" + tag=mapapi + base="tmp_$tag" + fileargs $base + + deletemap $1 $file + + # Create the test file + $CMD $TR -k$1 -x create -f $file + cdl="ut_${tag}_create_${zext}.cdl" + ref="ref_ut_${tag}_create.cdl" + ${ZMD} $fileurl > ./$cdl + diff -wb $srcdir/$ref ./$cdl + # delete the test file + $CMD $TR -k$1 -x delete -f $file + rm -f $cdl + + mapstillexists $1 + if test mapstillexists = 0 ; then exit 1; fi + # re-create the test file + $CMD -k$1 -x create -o $file +} + +testmapmeta() { + echo ""; echo "*** Test zmap read/write meta -k $1" + extfor "$1" + tag=mapapi + base="tmp_$tag" + fileargs $base + $CMD $TR -k$1 -x simplemeta -f $file + cdl="ut_${tag}_meta_${zext}.cdl" + ref="ref_ut_${tag}_meta.cdl" + ${ZMD} $fileurl > ./$cdl + diff -wb ${srcdir}/$ref ./$cdl +} + +testmapdata() { + echo ""; echo "*** Test zmap read/write data -k $1" + extfor "$1" + tag=mapapi + base="tmp_$tag" + fileargs $base + $CMD $TR -k$1 -x "simpledata" -f $file + cdl="ut_${tag}_data_${zext}.cdl" + ref="ref_ut_${tag}_data.cdl" + ${ZMD} $fileurl > ./$cdl + diff -wb ${srcdir}/$ref ./$cdl +} + +testmapsearch() { + echo ""; echo "*** Test zmap search -k $1" + extfor "$1" + tag=mapapi + base="tmp_$tag" + fileargs $base + txt=ut_${tag}_search_$zext.txt + ref=ref_ut_${tag}_search.txt + rm -f $txt + $CMD $TR -k$1 -x "search" -f $file > $txt + diff -wb ${srcdir}/$ref ./$txt +} + +main() { +echo "" +echo "*** Map Unit Testing" +echo ""; echo "*** Test zmap_file" +testmapcreate file; testmapmeta file; testmapdata file; testmapsearch file +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then + echo ""; echo "*** Test zmap_zip" + testmapcreate zip; testmapmeta zip; testmapdata zip; testmapsearch zip +fi +if test "x$FEATURE_S3TESTS" = xyes ; then + echo ""; echo "*** Test zmap_s3sdk" + export PROFILE="-p default" + testmapcreate s3; testmapmeta s3; testmapdata s3; testmapsearch s3 + s3sdkdelete "/${S3ISOPATH}" # Cleanup +fi +} + +main diff --git a/v3_nczarr_test/run_ut_misc.sh b/v3_nczarr_test/run_ut_misc.sh new file mode 100755 index 0000000000..4abbf1c0ff --- /dev/null +++ b/v3_nczarr_test/run_ut_misc.sh @@ -0,0 +1,49 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "${srcdir}/test_nczarr.sh" + +#set -e + +# Test misc functionality + +# Control which test sets are executed +# possible sets: json +TESTS=json + +# Functions + +testjson() { + file="ut_json_build.txt" + reffile="ref_${file}" + rm -f ${builddir}/$file + CMD="${execdir}/ut_json${ext}" + $CMD -x build > ${builddir}/$file + diff -wb ${srcdir}/$reffile ${builddir}/$file + file="ut_json_parse.txt" + rm -f ${builddir}/$file + reffile="ref_${file}" + $CMD -x parse > ${builddir}/$file + diff -wb ${srcdir}/$reffile ${builddir}/$file +} + +echo "" + +echo "*** Misc. Unit Testing" + +for T in $TESTS ; do +case "$T" in + +json) +echo ""; echo "*** Test zjson" +testjson +;; + +*) echo "Unknown test set: $T"; exit 1 ;; + +esac +done + +exit 0 diff --git a/v3_nczarr_test/s3util.c b/v3_nczarr_test/s3util.c new file mode 100644 index 0000000000..56fc0cda54 --- /dev/null +++ b/v3_nczarr_test/s3util.c @@ -0,0 +1,442 @@ +/* + * Copyright 2018, University Corporation for Atmospheric Research + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_GETOPT_H +#include +#endif + +#if defined(_WIN32) && !defined(__MINGW32__) +#include "XGetopt.h" +#endif + +#include "zincludes.h" +#include "ncs3sdk.h" +#include "ncpathmgr.h" +#include "nclog.h" +#include "ncuri.h" +#include "netcdf_aux.h" + +#undef NODELETE + +#undef DEBUG + +#define DATANAME "data" + +typedef enum S3op { +S3OP_NONE=0, +S3OP_LIST=1, +S3OP_CLEAR=2, +S3OP_PRINT=3, +S3OP_UPLOAD=4, +S3OP_DOWNLOAD=5, +} S3op; + +static struct S3ops { + S3op s3op; + const char* opnames[3]; +} s3ops[] = { +{S3OP_LIST,{"list","l",NULL}}, +{S3OP_CLEAR,{"clear","c",NULL}}, +{S3OP_PRINT,{"print","p",NULL}}, +{S3OP_UPLOAD,{"upload","u",NULL}}, +{S3OP_DOWNLOAD,{"download","d",NULL}}, +{S3OP_NONE,{NULL,NULL,NULL}}, +}; + +/* Command line options */ +struct Dumpptions { + int debug; + int verbose; + S3op s3op; + NCURI* url; + char* key; /* via -k flag */ + char* rootkey; /* from url | key */ + nc_type nctype; /* for printing content */ + char* filename; + char* profile; +} dumpoptions; + +struct S3SDK { + NCS3INFO s3; + void* s3client; + char* errmsg; +} s3sdk; + +/* Forward */ +static int s3list(void); +static int s3clear(void); +static int s3print(void); +static int s3upload(void); +static int s3download(void); +static nc_type typefor(const char* t); +static void printcontent(size64_t len, const char* content, nc_type nctype); + +static void +usage(void) +{ + fprintf(stderr,"usage: s3util list|print|upload|download|clear -u [-k ] [-f ]\n"); + exit(1); +} + +static S3op +decodeop(const char* name) +{ + struct S3ops* s3op = s3ops; + const char** s = NULL; + for(;s3op->opnames[0] != NULL;s3op++) { + for(s=s3op->opnames;*s;s++) { + if(strcasecmp(*s,name)==0) return s3op->s3op; + } + } + return S3OP_NONE; +} + +static int +s3setup(void) +{ + int stat = NC_NOERR; + s3sdk.s3client = NC_s3sdkcreateclient(&s3sdk.s3); + return stat; +} + +static int +s3shutdown(int deleteit) +{ + int stat = NC_NOERR; + if(deleteit) { + stat = s3clear(); + } + stat = NC_s3sdkclose(s3sdk.s3client, &s3sdk.errmsg); + return stat; +} + +int +main(int argc, char** argv) +{ + int stat = NC_NOERR; + int c; + char* tmp = NULL; + + nc_initialize(); + + memset((void*)&dumpoptions,0,sizeof(dumpoptions)); + + dumpoptions.nctype = NC_UBYTE; /* default */ + + while ((c = getopt(argc, argv, "df:hk:p:t:T:u:v")) != EOF) { + switch(c) { + case 'd': + dumpoptions.debug = 1; + break; + case 'f': + dumpoptions.filename = strdup(optarg); + break; + case 'h': + usage(); + return 0; + case 'k': { + size_t len = strlen(optarg); + dumpoptions.key = (char*)malloc(len+1+1); + if(*optarg != '/') { + fprintf(stderr,"warning: -k option does not start with '/': %s",optarg); + dumpoptions.key[0] = '/'; + memcpy(dumpoptions.key+1,optarg,len); + len++; + } else + memcpy(dumpoptions.key,optarg,strlen(optarg)); + dumpoptions.key[len] = '\0'; + } break; + case 'p': + dumpoptions.profile = strdup(optarg); + break; + case 't': + dumpoptions.nctype = typefor(optarg); + break; + case 'u': { + char* p = NC_shellUnescape(optarg); + ncuriparse(p,&dumpoptions.url); + nullfree(p); + if(dumpoptions.url == NULL) { + fprintf(stderr,"malformed -u option: %s",optarg); + stat = NC_EINVAL; + goto done; + } + } break; + case 'v': + dumpoptions.verbose = 1; + break; + case 'T': + nctracelevel(atoi(optarg)); + break; + case '?': + fprintf(stderr,"unknown option\n"); + stat = NC_EINVAL; + goto done; + } + } + + /* get command argument */ + argc -= optind; + argv += optind; + + if (argc > 1) { + int j; + fprintf(stderr, "s3util: only one command argument permitted:"); + for(j=0;j 0) { + size_t i; + /* Sort the list -- shortest first */ + NC_sortenvv(nkeys,keys); + for(i=0;i\n"); + +done: + s3shutdown(0); + NCZ_freeenvv(nkeys,keys); + return stat; +} + +static int +s3clear(void) +{ + int stat = NC_NOERR; + size_t nkeys = 0; + char** keys = NULL; + + if(s3setup()) goto done; + + if((stat = NC_s3sdklistall(s3sdk.s3client, s3sdk.s3.bucket, s3sdk.s3.rootkey, &nkeys, &keys, &s3sdk.errmsg))) goto done; + + if(nkeys > 0 && keys != NULL) { + size_t i; + /* Sort the list -- shortest first */ + NC_sortenvv(nkeys,keys); + if(dumpoptions.verbose) { + printf("deleted keys:\n"); + for(i=0;i\n"); + return; + } + + if(nctype == NC_STRING) printf("|"); + for(i=0;i 0) printf(", "); + switch(nctype) { + case NC_BYTE: printf("%d",((char*)content)[i]); break; + case NC_SHORT: printf("%d",((short*)content)[i]); break; + case NC_INT: printf("%d",((int*)content)[i]); break; + case NC_INT64: printf("%lld",((long long*)content)[i]); break; + case NC_UBYTE: printf("%u",((unsigned char*)content)[i]); break; + case NC_USHORT: printf("%u",((unsigned short*)content)[i]); break; + case NC_UINT: printf("%u",((unsigned int*)content)[i]); break; + case NC_UINT64: printf("%llu",((unsigned long long*)content)[i]); break; + case NC_FLOAT: printf("%f",((float*)content)[i]); break; + case NC_DOUBLE: printf("%lf",((double*)content)[i]); break; + case NC_STRING: putc(content[i],stdout); break; + default: abort(); + } + } + if(nctype == NC_STRING) printf("|\n"); +} + +static nc_type +typefor(const char* t) +{ + if(strcmp(t,"NC_BYTE")==0) return NC_BYTE; + else if(strcmp(t,"NC_SHORT")==0) return NC_SHORT; + else if(strcmp(t,"NC_INT")==0) return NC_INT; + else if(strcmp(t,"NC_INT64")==0) return NC_INT64; + else if(strcmp(t,"NC_UBYTE")==0) return NC_UBYTE; + else if(strcmp(t,"NC_USHORT")==0) return NC_USHORT; + else if(strcmp(t,"NC_UINT")==0) return NC_UINT; + else if(strcmp(t,"NC_UINT64")==0) return NC_UINT64; + else if(strcmp(t,"NC_FLOAT")==0) return NC_FLOAT; + else if(strcmp(t,"NC_DOUBLE")==0) return NC_DOUBLE; + else if(strcmp(t,"NC_STRING")==0) return NC_STRING; + return NC_NAT; +} diff --git a/v3_nczarr_test/test_chunkcases.c b/v3_nczarr_test/test_chunkcases.c new file mode 100644 index 0000000000..6b5f2db30a --- /dev/null +++ b/v3_nczarr_test/test_chunkcases.c @@ -0,0 +1,125 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include +#include +#include +#include "netcdf.h" +#include "ncpathmgr.h" +#include "nclist.h" + +#ifdef NETCDF_ENABLE_NCZARR +#include "zincludes.h" +#endif + +#include "test_utils.h" + +static unsigned chunkprod; +static unsigned dimprod; +static int* data = NULL; +static size_t datasize = 0; + +static int +writedata(void) +{ + int ret = NC_NOERR; + size_t i; + + for(i=0;idebug >= 1) { + fprintf(stderr,"write: dimlens=%s chunklens=%s\n", + printvector(options->rank,options->dimlens),printvector(options->rank,options->chunks)); + } + if(options->wholechunk) { + fprintf(stderr,"write var: wholechunk\n"); + if((ret = nc_put_vars(meta->ncid,meta->varid,options->start,options->edges,(ptrdiff_t*)options->stride,data))) + ERR(ret); + } else { + fprintf(stderr,"write vars: start=%s count=%s stride=%s\n", + printvector(options->rank,options->start),printvector(options->rank,options->edges),printvector(options->rank,options->stride)); + if((ret = nc_put_vars(meta->ncid,meta->varid,options->start,options->edges,(ptrdiff_t*)options->stride,data))) + ERR(ret); + } + + return 0; +} + +static int +readdata(void) +{ + int ret = NC_NOERR; + size_t i; + + memset(data,0,datasize); + + if(options->debug >= 1) + fprintf(stderr,"read: dimlens=%s chunklens=%s\n", + printvector(options->rank,options->dimlens),printvector(options->rank,options->chunks)); + fprintf(stderr,"read vars: start=%s count=%s stride=%s", + printvector(options->rank,options->start), + printvector(options->rank,options->edges), + printvector(options->rank,options->stride)); + if(options->wholechunk) + fprintf(stderr," wholechunk"); + fprintf(stderr,"\n"); + if((ret = nc_get_vars(meta->ncid,meta->varid,options->start,options->edges,(ptrdiff_t*)options->stride,data))) + ERR(ret); + + for(i=0;irank, options->start, options->stop, options->stride, options->max); + if(odom == NULL) {ret = NC_ENOMEM; goto done;} + if(options->debug > 1) + fprintf(stderr,"genodom: odom = %s\n",odom_print(odom)); + /* Iterate the odometer */ + for(i=0;odom_more(odom);odom_next(odom),i++) { + printf("[%02d] %s\n",i,(i==0?odom_print(odom):odom_printshort(odom))); + } +done: + odom_free(odom); + return ret; +} + +int +main(int argc, char** argv) +{ + int stat = NC_NOERR; + int i; + + if((stat=getoptions(&argc,&argv))) goto done; + if((stat=verifyoptions(options))) goto done; + + if((stat = getmetadata(0))) + ERR(stat); + + dimprod = 1; + chunkprod = 1; + for(i=0;irank;i++) {dimprod *= options->dimlens[i]; chunkprod *= options->chunks[i];} + + datasize = dimprod*sizeof(int); + if((data = calloc(1,datasize)) == NULL) + {fprintf(stderr,"out of memory\n"); exit(1);} + + switch (options->op) { + case Read: readdata(); break; + case Write: writedata(); break; + case Odom: genodom(); break; + default: + fprintf(stderr,"Unknown operation\n"); + exit(1); + } +done: + if(data) free(data); + cleanup(); + return 0; +} diff --git a/v3_nczarr_test/test_chunking.c b/v3_nczarr_test/test_chunking.c new file mode 100644 index 0000000000..fab26438d3 --- /dev/null +++ b/v3_nczarr_test/test_chunking.c @@ -0,0 +1,143 @@ +/* This is part of the netCDF package. Copyright 2018 University + Corporation for Atmospheric Research/Unidata. See COPYRIGHT file for + conditions of use. See www.unidata.ucar.edu for more info. + + Create a chunkable test file for nccopy to test chunking. +*/ + +/* This file is derived from ncdump/tst_chunking.c */ +#define TESTNCZARR + +#include +#include +#include "err_macros.h" + +#undef DEBUG + +static int ret = NC_NOERR; + +/* Make trackable ERR macro replacement */ +static int +lerr(int stat, const char* file, int lineno) +{ + NC_UNUSED(stat); + fflush(stdout); /* Make sure our stdout is synced with stderr. */ + err++; + fprintf(stderr, "Sorry! Unexpected result(%d), %s, line: %d\n",ret,file,lineno); + fflush(stderr); \ + return 2; \ +} +#define LERR lerr(ret,__FILE__,__LINE__) + +#define FILE_NAME "tst_chunking.nc" +#define VAR_RANK 7 +#define IVAR_NAME "ivar" +#define FVAR_NAME "fvar" +#define GRP_NAME "g" +#define UNLIM_NAME "unlimited" +#define UNLIM_SIZE 10 +#define DEFLATE_LEVEL 1 +#define NVALS 45360 /* 7 * 4 * 2 * 3 * 5 * 6 * 9 */ + +static const char *dim_names[VAR_RANK] = {"dim0", "dim1", "dim2", "dim3", "dim4", "dim5", "dim6"}; +static const size_t dim_lens[VAR_RANK] = {7, 4, 2, 3, 5, 6, 9}; + +int +main(int argc, char **argv) +{ + /* mutually exclusive command line options */ + int option_group = 0; + int option_deflate = 0; + int option_unlimited = 0; + /* file metadata */ + int mode = NC_CLOBBER; + int ncid, grpid; + int ivarid, fvarid; + int ivar_dims[VAR_RANK]; + int fvar_dims[VAR_RANK]; + int ivar_data[NVALS]; + float fvar_data[NVALS]; + size_t r, i; + char* file_name = FILE_NAME; + int unlimid; + + /* Parse command line */ + if(argc >= 2) { + file_name = argv[1]; + } + if(argc >= 3) { + if(strcmp(argv[2],"group")==0) { + option_group = 1; + mode |= NC_NETCDF4; + } else if(strcmp(argv[2],"deflate")==0) { + option_deflate = 1; + mode |= NC_NETCDF4; + } else if(strcmp(argv[2],"unlimited")==0) { + option_unlimited = 1; + } else { + fprintf(stderr,"usage: tst_chunking [ [group|deflate|unlimited]]\n"); + exit(1); + } + } + + printf("*** Creating chunkable test file %s...\n", file_name); + if(option_deflate) + printf("\toption: deflate\n"); + else if(option_unlimited) + printf("\toption: unlimited\n"); + else if(option_group) + printf("\toption: group\n"); + + if (nc_create(file_name, mode, &ncid)) LERR; + for(r = 0; r < VAR_RANK; r++) { + if (nc_def_dim(ncid, dim_names[r], dim_lens[r], &ivar_dims[r])) LERR; + fvar_dims[VAR_RANK - 1 - r] = ivar_dims[r]; + } + if(option_unlimited) { + int udims[2]; + if (nc_def_dim(ncid, UNLIM_NAME, 0, &unlimid)) LERR; + udims[0] = unlimid; + udims[1] = ivar_dims[0]; + if (nc_def_var(ncid, IVAR_NAME, NC_INT, 2, udims, &ivarid)) LERR; + } else { + if (option_group) { + if (nc_def_grp(ncid, GRP_NAME, &grpid)) LERR; + } else + grpid = ncid; + if (nc_def_var(grpid, IVAR_NAME, NC_INT, VAR_RANK, ivar_dims, &ivarid)) LERR; + if(option_deflate) { + if(nc_def_var_deflate(grpid,ivarid,NC_NOSHUFFLE, option_deflate, DEFLATE_LEVEL)) LERR; + } + } + /* fvar is unchanged */ + if (nc_def_var(ncid, FVAR_NAME, NC_FLOAT, VAR_RANK, fvar_dims, &fvarid)) LERR; + if (nc_enddef (ncid)) LERR; + + /* Fill in the data */ + if(option_unlimited) { + size_t nvals = UNLIM_SIZE * dim_lens[0]; + size_t start[2] = {0,0}; + size_t count[2]; + for(i=0;i +#include +#include + +#include "netcdf.h" +#include "nclist.h" + +#include "zincludes.h" + +#include "test_utils.h" + +#undef DEBUG + +static void +nccheck(int ret, int lineno) +{ + if(ret == NC_NOERR) return; + ncz_report(ret,lineno); +} + +#define NCCHECK(err) nccheck(err,__LINE__) + +int +main(int argc, char *argv[] ) +{ + int err, ncid, varid, dimid[1]; + size_t dimlen[1]; + float *fdat; + int *idat; + const char* filename = NULL; + const char* varname = "f"; + const char* dimname = "x"; + size_t i; + + NCCHECK(getoptions(&argc,&argv)); + if(options->op == Write) NCCHECK(verifyoptions(options)); + + filename = options->file; + + NCCHECK(err = nc_open(filename,NC_NETCDF4,&ncid)); + NCCHECK(err = nc_inq_varid(ncid, varname, &varid)); + NCCHECK(err = nc_inq_dimid(ncid, dimname, dimid)); + NCCHECK(err = nc_inq_dim(ncid, dimid[0], NULL, dimlen)); + + /* Make room for both double and floating dat */ + fdat = (float *)calloc(1,sizeof(float) * dimlen[0]); + idat = (int *)calloc(1,sizeof(int) * dimlen[0]); + + NCCHECK(err = nc_get_var_int(ncid, varid, idat)); + NCCHECK(err = nc_get_var_float(ncid, varid, fdat)); + +#ifdef DEBUG + printf("int[0..%d]:",(int)dimlen[0]); + for(i=0; i +#include +#include + +#include +#include "netcdf.h" + +/* The HDF assigned id for bzip compression */ +#define BZIP2_ID 307 +/* The compression level used in this example */ +#define BZIP2_LEVEL 9 + +#define DFALT_TESTFILE "tmp_bzip2.nc" + +/* Point at which we give up */ +#define MAXERRS 8 + +#define NDIMS 4 +#define DIMSIZE 4 +#define CHUNKSIZE 4 /* Note: not the total size of the chunk, but size wrt a dim*/ + +static size_t dimsize = DIMSIZE; +static size_t chunksize = CHUNKSIZE; +static size_t actualdims = NDIMS; + +static size_t actualproduct = 1; /* x-product over dim sizes */ +static size_t chunkproduct = 1; /* x-product over chunksizes */ + +static size_t dims[NDIMS]; +static size_t chunks[NDIMS]; + +static int nerrs = 0; + +static const char* testfile = NULL; + +static int ncid, varid; +static int dimids[NDIMS]; +static float* array = NULL; +static float* expected = NULL; +static unsigned int filterid = 0; +static unsigned int* params = NULL; + +/* Forward */ +static void init(int argc, char** argv); +static int test_bzip2(void); +static int verifychunks(void); + +#define ERRR do { \ +fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ +fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ +nerrs++;\ +} while (0) + +static int +check(int err,int line) +{ + if(err != NC_NOERR) { + fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err)); + fflush(stderr); + exit(1); + } + return NC_NOERR; +} + +#define CHECK(x) check(x,__LINE__) + +/* +Read the chunking information about the variable +and verify that it is as expected. +*/ + +static int +verifychunks(void) +{ + int i; + int store = -1; + size_t chunksizes[NDIMS]; + memset(chunksizes,0,sizeof(chunksizes)); + CHECK(nc_inq_var_chunking(ncid, varid, &store, chunksizes)); + /* Storate must be chunked, not contiguous */ + if(store != NC_CHUNKED) { + fprintf(stderr,"bad chunk store\n"); + return NC_ESTORAGE; + } + /* Chunk sizes must match our predefined set */ + for(i=0;i= MAXERRS) + break; + } + } + if(errs == 0) + printf("no data errors\n"); + if(actualproduct <= 1) + return NC_EBADDIM; + return (errs == 0 ? NC_NOERR: NC_EINVAL); +} + +/* +Create the file, write it, then re-read for comparison. +*/ +static int +test_bzip2(void) +{ + int i; + unsigned int level = BZIP2_LEVEL; + unsigned int id=0; + size_t nparams = 0; + + printf("\n*** Testing API: bzip2 compression.\n"); + + /* Clear the data array */ + memset(array,0,sizeof(float)*actualproduct); + + /* Create a file */ + CHECK(nc_create(testfile, NC_NETCDF4|NC_CLOBBER, &ncid)); + + /* Do not use fill for this file */ + CHECK(nc_set_fill(ncid, NC_NOFILL, NULL)); + + /* Define the dimensions */ + for(i=0;i 0) { + params = (unsigned int*)malloc(sizeof(unsigned int)*nparams); + if(params == NULL) + return NC_ENOMEM; + CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,params)); + } + if(filterid != BZIP2_ID) { + printf("Bzip2 id mismatch: %d\n",filterid); + return NC_EFILTER; + } + if(nparams != 1 && params != NULL && params[0] != BZIP2_LEVEL) { + printf("Compression parameter mismatch\n"); + return NC_EFILTER; + } + + /* Verify chunking */ + if(!verifychunks()) + return 0; + + /* Read the data */ + CHECK(nc_get_var_float(ncid, varid, array)); + + /* Close the file */ + CHECK(nc_close(ncid)); + return (compare() == NC_NOERR ? 0 : 1); +} + +/**************************************************/ +/* Utilities */ + +static void +init(int argc, char** argv) +{ + int i; + + /* get the testfile path */ + if(argc > 1) + testfile = argv[1]; + else + testfile = DFALT_TESTFILE; + + /* Setup various variables */ + actualproduct = 1; + chunkproduct = 1; + for(i=0;i 0?1:0); +} + diff --git a/v3_nczarr_test/test_filter_avail.c b/v3_nczarr_test/test_filter_avail.c new file mode 100644 index 0000000000..9e403fb302 --- /dev/null +++ b/v3_nczarr_test/test_filter_avail.c @@ -0,0 +1,107 @@ +#define TESTNCZARR +/* + Copyright 2018, UCAR/Unidata + See COPYRIGHT file for copying and redistribution conditions. +*/ + +#include "config.h" +#include +#include +#include + +#ifdef USE_HDF5 +#include +#endif + +#include "netcdf.h" +#include "netcdf_aux.h" +#include "netcdf_filter.h" + +#undef DEBUG + +#define MAXPARAMS 32 + +#ifdef TESTNCZARR +#define DFALT_TESTFILE "file://tmp_filter_avail.file#mode=nczarr,file" +#else +#define DFALT_TESTFILE "tmp_filter_avail.nc" +#endif + +static const char* testfile = NULL; +static int nerrs = 0; + +static int ncid; + +/* Forward */ +static int test_test1(void); +static void init(int argc, char** argv); + +#define ERRR do { \ +fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ +fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ +nerrs++;\ +} while (0) + +static int +check(int err,int line) +{ + if(err != NC_NOERR) { + fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err)); + } + return NC_NOERR; +} + +#define CHECK(x) check(x,__LINE__) + +static int +test_test1(void) +{ + int stat = NC_NOERR; + + printf("test1: bzip2 availability\n"); + CHECK(nc_create(testfile,NC_NETCDF4|NC_CLOBBER,&ncid)); + CHECK(nc_enddef(ncid)); + switch (stat = nc_inq_filter_avail(ncid,H5Z_FILTER_BZIP2)) { + case NC_NOERR: break; + case NC_ENOFILTER: break; + default: CHECK(stat); goto done; + } + if(stat == NC_ENOFILTER) { + printf("*** FAIL: filter %d not available\n",H5Z_FILTER_BZIP2); + } else { + printf("*** PASS: filter %d available\n",H5Z_FILTER_BZIP2); + } + + CHECK(nc_abort(ncid)); +done: + return stat; +} + +/**************************************************/ +/* Utilities */ + +static void +init(int argc, char** argv) +{ + /* get the testfile path */ + if(argc > 1) + testfile = argv[1]; + else + testfile = DFALT_TESTFILE; +} + +/**************************************************/ +int +main(int argc, char **argv) +{ +#ifdef USE_HDF5 +#ifdef DEBUG + H5Eprint1(stderr); + nc_set_log_level(1); +#endif +#endif + init(argc,argv); + if(test_test1() != NC_NOERR) ERRR; + exit(nerrs > 0?1:0); +} diff --git a/v3_nczarr_test/test_filter_misc.c b/v3_nczarr_test/test_filter_misc.c new file mode 100644 index 0000000000..a45ff62ff3 --- /dev/null +++ b/v3_nczarr_test/test_filter_misc.c @@ -0,0 +1,562 @@ +/* + Copyright 2018, UCAR/Unidata + See COPYRIGHT file for copying and redistribution conditions. +*/ + +#include "config.h" +#include +#include +#include + +#include +#include "netcdf.h" +#include "netcdf_aux.h" +#include "netcdf_filter.h" + +#undef TESTODDSIZE + +#undef DEBUG + +/* The C standard apparently defines all floating point constants as double; + we rely on that in this code. +*/ +#define DBLVAL 12345678.12345678 + +#define TEST_ID 32768 + +#define MAXERRS 8 + +#define MAXPARAMS 32 + +#define NPARAMS 14 + +static unsigned int baseline[NPARAMS]; + +static const char* testfile = NULL; + +#define MAXDIMS 8 + +#define DFALT_TESTFILE "tmp_misc.nc" + +#define spec "32768, -17b, 23ub, -25S, 27US, 77, 93U, 789f, 12345678.12345678d, -9223372036854775807L, 18446744073709551615UL" + +#ifdef TESTODDSIZE +#define NDIMS 1 +static size_t dimsize[NDIMS] = {4}; +static size_t chunksize[NDIMS] = {3}; +#else +#define NDIMS 4 +static size_t dimsize[NDIMS] = {4,4,4,4}; +static size_t chunksize[NDIMS] = {4,4,4,4}; +#endif + +static size_t ndims = NDIMS; + +static size_t totalproduct = 1; /* x-product over max dims */ +static size_t actualproduct = 1; /* x-product over actualdims */ +static size_t chunkproduct = 1; /* x-product over actual chunks */ + +static size_t pattern[MAXDIMS]; + +static int nerrs = 0; + +static int ncid, varid; +static int dimids[MAXDIMS]; +static size_t odom[MAXDIMS]; +static float* array = NULL; +static float* expected = NULL; + +static unsigned int filterid = 0; +static size_t nparams = 0; +static unsigned int params[MAXPARAMS]; + +/* Forward */ +static int test_test1(void); +static void init(int argc, char** argv); +static void reset(void); +static void odom_reset(void); +static int odom_more(void); +static int odom_next(void); +static int odom_offset(void); +static float expectedvalue(void); +static void verifyparams(void); + +#define ERRR do { \ +fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ +fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ +nerrs++;\ +} while (0) + +static int +check(int err,int line) +{ + if(err != NC_NOERR) { + fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err)); + } + return NC_NOERR; +} + +static void +report(const char* msg, int lineno) +{ + fprintf(stderr,"fail: line=%d %s\n",lineno,msg); + exit(1); +} + +#define CHECK(x) check(x,__LINE__) +#define REPORT(x) report(x,__LINE__) + +static int +verifychunks(void) +{ + int i; + int store = -1; + size_t localchunks[MAXDIMS]; + memset(localchunks,0,sizeof(localchunks)); + CHECK(nc_inq_var_chunking(ncid, varid, &store, localchunks)); + if(store != NC_CHUNKED) { + fprintf(stderr,"bad chunk store\n"); + return 0; + } + for(i=0;i 0) { + params = (unsigned int*)malloc(sizeof(unsigned int)*nparams); + if(params == NULL) + return NC_ENOMEM; + CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,params)); + } + if(filterid != TEST_ID) { + fprintf(stderr,"open: test id mismatch: %d\n",filterid); + free(params); + return NC_EFILTER; + } + if(nparams != NPARAMS) { + size_t i; + fprintf(stderr,"nparams mismatch\n"); + for(nerrs=0,i=0;i 0) return NC_EFILTER; + + /* Verify chunking */ + if(!verifychunks()) + return 0; + fflush(stderr); + return 1; +} + +static int +setchunking(void) +{ + int store; + + store = NC_CHUNKED; + CHECK(nc_def_var_chunking(ncid,varid,store,chunksize)); + if(!verifychunks()) + return NC_EINVAL; + return NC_NOERR; +} + +static void +fill(void) +{ + odom_reset(); + if(1) { + int i; + if(actualproduct <= 1) abort(); + for(i=0;i= MAXERRS) + break; + } + } + } else + { + odom_reset(); + while(odom_more()) { + int offset = odom_offset(); + float expect = expectedvalue(); + if(array[offset] != expect) { + fprintf(stderr,"data mismatch: array[%d]=%f expected=%f\n", + offset,array[offset],expect); + errs++; + if(errs >= MAXERRS) + break; + } + odom_next(); + } + } + + if(errs == 0) + fprintf(stderr,"no data errors\n"); + return (errs == 0); +} + +static void +showparameters(void) +{ + int i; + fprintf(stderr,"test: nparams=%ld: params=",(unsigned long)nparams); + for(i=0;i0;) { + odom[i] += 1; + if(odom[i] < dimsize[i]) break; + if(i == 0) return 0; /* leave the 0th entry if it overflows*/ + odom[i] = 0; /* reset this position*/ + } + return 1; +} + +static int +odom_offset(void) +{ + int i; + int offset = 0; + for(i=0;i 1) + testfile = argv[1]; + else + testfile = DFALT_TESTFILE; + + /* Setup various variables */ + totalproduct = 1; + actualproduct = 1; + chunkproduct = 1; + for(i=0;i 0?1:0); +} diff --git a/v3_nczarr_test/test_filter_order.c b/v3_nczarr_test/test_filter_order.c new file mode 100644 index 0000000000..953b93d09b --- /dev/null +++ b/v3_nczarr_test/test_filter_order.c @@ -0,0 +1,432 @@ +/* + Copyright 2018, UCAR/Unidata + See COPYRIGHT file for copying and redistribution conditions. +*/ + +#include "config.h" +#include +#include +#include +#include +#include + +#include "netcdf.h" +#include "netcdf_filter.h" + +#undef TESTODDSIZE + +#undef DEBUG + +#define FILTER_ID 40000 + +#define MAXERRS 8 + +#define MAXPARAMS 32 + +#define MAXDIMS 8 + +#define DFALT_TESTFILE "tmp_filter_order.nc" + +#define NPARAMS 1 +#define PARAMVAL 17 + +#define NDIMS 4 +static size_t dimsize[NDIMS] = {4,4,4,4}; +static size_t chunksize[NDIMS] = {4,4,4,4}; + +static size_t ndims = NDIMS; + +static int creating = 1; /* Default is to do filter test 1 */ +static size_t totalproduct = 1; /* x-product over max dims */ +static size_t actualproduct = 1; /* x-product over actualdims */ +static size_t chunkproduct = 1; /* x-product over actual chunks */ + +static int nerrs = 0; + +static const char* testfile = NULL; + +static int ncid, varid; +static int dimids[MAXDIMS]; +static size_t odom[MAXDIMS]; +static float* array = NULL; +static float* expected = NULL; + +/* Forward */ +static int filter_test1(void); +static void init(int argc, char** argv); +static void reset(void); +static void odom_reset(void); +static int odom_more(void); +static int odom_next(void); +static int odom_offset(void); +static float expectedvalue(void); + +#define ERRR do { \ +fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ +fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ +nerrs++;\ +} while (0) + +static int +check(int err,int line) +{ + if(err != NC_NOERR) { + fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err)); + } + return NC_NOERR; +} + +static void +report(const char* msg, int lineno) +{ + fprintf(stderr,"fail: line=%d %s\n",lineno,msg); + exit(1); +} + +#define CHECK(x) check(x,__LINE__) +#define REPORT(x) report(x,__LINE__) + +static int +verifychunks(void) +{ + int i; + int store = -1; + size_t localchunks[MAXDIMS]; + memset(localchunks,0,sizeof(localchunks)); + CHECK(nc_inq_var_chunking(ncid, varid, &store, localchunks)); + if(store != NC_CHUNKED) { + fprintf(stderr,"bad chunk store\n"); + return 0; + } + for(i=0;i= MAXERRS) + break; + } + } + } else + { + odom_reset(); + while(odom_more()) { + int offset = odom_offset(); + float expect = expectedvalue(); + if(array[offset] != expect) { + fprintf(stderr,"data mismatch: array[%d]=%f expected=%f\n", + offset,array[offset],expect); + errs++; + if(errs >= MAXERRS) + break; + } + odom_next(); + } + } + + if(errs == 0) + {printf("no data errors\n"); fflush(stdout);} + return (errs == 0); +} + +/* Test filter order on creation */ +static int +filter_test1(void) +{ + int ok = 1; + + reset(); + + printf("test1: filter order: create\n"); fflush(stdout); + create(); + setchunking(); + deffilters(); + inqfilters(); + CHECK(nc_enddef(ncid)); + + /* Fill in the array */ + fill(); + + nc_sync(ncid); + + printf("test1: compression.\n"); fflush(stdout); + /* write array */ + CHECK(nc_put_var(ncid,varid,expected)); + + printf("test1: decompression.\n"); fflush(stdout); + CHECK(nc_get_var_float(ncid, varid, array)); + ok = compare(); + + CHECK(nc_close(ncid)); + + return ok; +} + +/* Test filter order on read */ +static int +filter_test2(void) +{ + int ok = 1; + + reset(); + + printf("test2: filter order: read\n"); fflush(stdout); + + /* Fill in the array */ + fill(); + + printf("test2: decompression.\n"); fflush(stdout); + reset(); + openfile(); + inqfilters(); + + printf("test2: decompression.\n"); fflush(stdout); + CHECK(nc_get_var_float(ncid, varid, array)); + ok = compare(); + + CHECK(nc_close(ncid)); + return ok; +} + +/**************************************************/ +/* Utilities */ + +static void +reset() +{ + memset(array,0,sizeof(float)*actualproduct); +} + +static void +odom_reset(void) +{ + memset(odom,0,sizeof(odom)); +} + +static int +odom_more(void) +{ + return (odom[0] < dimsize[0]); +} + +static int +odom_next(void) +{ + for(size_t i=ndims;i-->0;) { + odom[i] += 1; + if(odom[i] < dimsize[i]) break; + if(i == 0) return 0; /* leave the 0th entry if it overflows*/ + odom[i] = 0; /* reset this position*/ + } + return 1; +} + +static int +odom_offset(void) +{ + int offset = 0; + for(size_t i=0;i 2) + testfile = argv[2]; + else + testfile = DFALT_TESTFILE; + + /* Setup various variables */ + totalproduct = 1; + actualproduct = 1; + chunkproduct = 1; + for(i=0;i 0?1:0); +} diff --git a/v3_nczarr_test/test_filter_repeat.c b/v3_nczarr_test/test_filter_repeat.c new file mode 100644 index 0000000000..609ded01c6 --- /dev/null +++ b/v3_nczarr_test/test_filter_repeat.c @@ -0,0 +1,369 @@ +/* + Copyright 2018, UCAR/Unidata + See COPYRIGHT file for copying and redistribution conditions. +*/ + +#include "config.h" +#include +#include +#include +#include +#include + +#include "netcdf.h" +#include "netcdf_filter.h" + +#undef TESTODDSIZE + +#undef DEBUG + +#define FILTER_ID 40000 + +#define MAXERRS 8 + +#define MAXPARAMS 32 + +#define MAXDIMS 8 + +#define DFALT_TESTFILE "testfilter_reg.nc" + +#define NPARAMS 1 +#define PARAMVAL 17 + +#define NDIMS 4 +static size_t dimsize[NDIMS] = {4,4,4,4}; +static size_t chunksize[NDIMS] = {4,4,4,4}; + +static size_t ndims = NDIMS; + +static size_t totalproduct = 1; /* x-product over max dims */ +static size_t actualproduct = 1; /* x-product over actualdims */ +static size_t chunkproduct = 1; /* x-product over actual chunks */ + +static int nerrs = 0; + +static char* testfile = NULL; + + +static int ncid, varid; +static int dimids[MAXDIMS]; +static size_t odom[MAXDIMS]; +static float* array = NULL; +static float* expected = NULL; + +/* Forward */ +static int filter_test1(void); +static void init(int argc, char** argv); +static void reset(void); +static void odom_reset(void); +static int odom_more(void); +static int odom_next(void); +static int odom_offset(void); +static float expectedvalue(void); + +#define ERRR do { \ +fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ +fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ +nerrs++;\ +} while (0) + +static int +check(int err,int line) +{ + if(err != NC_NOERR) { + fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err)); + } + return NC_NOERR; +} + +static void +report(const char* msg, int lineno) +{ + fprintf(stderr,"fail: line=%d %s\n",lineno,msg); + exit(1); +} + +#define CHECK(x) check(x,__LINE__) +#define REPORT(x) report(x,__LINE__) + +static int +verifychunks(void) +{ + int i; + int store = -1; + size_t localchunks[MAXDIMS]; + memset(localchunks,0,sizeof(localchunks)); + CHECK(nc_inq_var_chunking(ncid, varid, &store, localchunks)); + if(store != NC_CHUNKED) { + fprintf(stderr,"bad chunk store\n"); + return 0; + } + for(i=0;i= MAXERRS) + break; + } + } + } else + { + odom_reset(); + while(odom_more()) { + int offset = odom_offset(); + float expect = expectedvalue(); + if(array[offset] != expect) { + fprintf(stderr,"data mismatch: array[%d]=%f expected=%f\n", + offset,array[offset],expect); + errs++; + if(errs >= MAXERRS) + break; + } + odom_next(); + } + } + + if(errs == 0) + printf("no data errors\n"); + return (errs == 0); +} + +static int +filter_test1(void) +{ + int ok = 1; + unsigned int params[MAXPARAMS]; + + reset(); + + printf("test1: def filter repeat .\n"); + create(); + setchunking(); + + params[0] = 1; + params[1] = 17; + deffilter(FILTER_ID,2,params); + + params[0] = 0; + params[1] = 18; + deffilter(FILTER_ID,2,params); + + CHECK(nc_enddef(ncid)); + + /* Fill in the array */ + fill(); + + printf("test1: compression.\n"); + /* write array */ + CHECK(nc_put_var(ncid,varid,expected)); + CHECK(nc_close(ncid)); + + printf("test1: decompression.\n"); + reset(); + openfile(); + CHECK(nc_get_var_float(ncid, varid, array)); + ok = compare(); + + CHECK(nc_close(ncid)); + return ok; +} + +/**************************************************/ +/* Utilities */ + +static void +reset() +{ + memset(array,0,sizeof(float)*actualproduct); +} + +static void +odom_reset(void) +{ + memset(odom,0,sizeof(odom)); +} + +static int +odom_more(void) +{ + return (odom[0] < dimsize[0]); +} + +static int +odom_next(void) +{ + for(size_t i=ndims;i-->0;) { + odom[i] += 1; + if(odom[i] < dimsize[i]) break; + if(i == 0) return 0; /* leave the 0th entry if it overflows*/ + odom[i] = 0; /* reset this position*/ + } + return 1; +} + +static int +odom_offset(void) +{ + int offset = 0; + for(size_t i=0;i 1) + testfile = argv[1]; + else + testfile = DFALT_TESTFILE; + + /* Setup various variables */ + totalproduct = 1; + actualproduct = 1; + chunkproduct = 1; + for(i=0;i 0?1:0); +} diff --git a/v3_nczarr_test/test_filter_vlen.c b/v3_nczarr_test/test_filter_vlen.c new file mode 100644 index 0000000000..caf6761709 --- /dev/null +++ b/v3_nczarr_test/test_filter_vlen.c @@ -0,0 +1,289 @@ +/* + Copyright 2018, UCAR/Unidata + See COPYRIGHT file for copying and redistribution conditions. +*/ + +/* This test is derived from nc_test4/test_filter_vlen.c */ +#define TESTNCZARR + +#include "config.h" +#include +#include +#include + +#ifdef USE_HDF5 +#include +#endif +#include "netcdf.h" +#include "netcdf_aux.h" +#include "netcdf_filter.h" + +#undef DEBUG + +#define FILTER_ID 1 /*deflate*/ + +#define MAXERRS 8 + +#define MAXPARAMS 32 + +#define NPARAMS 14 + +static const char* testfile = NULL; + +#define MAXDIMS 8 + +#ifdef TESTNCZARR +#define DFALT_TESTFILE "file://tmp_filter_vlen.nc#mode=nczarr,file" +#else +#define DFALT_TESTFILE "tmp_filter_vlen.nc" +#endif + +#define NDIMS 4 +static size_t dimsize[NDIMS] = {4,4,4,4}; + +static size_t ndims = NDIMS; + +static size_t totalproduct = 1; /* x-product over max dims */ +static size_t actualproduct = 1; /* x-product over actualdims */ + +static int nerrs = 0; + +static int ncid, varid; +static int dimids[MAXDIMS]; +static char** array = NULL; + +/* Forward */ +static int test_test1(void); +static void init(int argc, char** argv); +static void reset(void); + +#define ERRR do { \ +fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ +fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ +nerrs++;\ +} while (0) + +static int +check(int err,int line) +{ + if(err != NC_NOERR) { + fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err)); + } + return NC_NOERR; +} +#define CHECK(x) check(x,__LINE__) + +static int +create(void) +{ + /* Create a file with one variable */ + CHECK(nc_create(testfile, NC_NETCDF4|NC_CLOBBER, &ncid)); + CHECK(nc_set_fill(ncid, NC_NOFILL, NULL)); + return NC_NOERR; +} + +static int +defvar(nc_type xtype) +{ + size_t i; + + /* Create a file with one variable-sized variable */ + for(i=0;i 1) + testfile = argv[1]; + else + testfile = DFALT_TESTFILE; + + /* Setup various variables */ + totalproduct = 1; + actualproduct = 1; + for(i=0;i 0 ? "FAILED" : "PASS")); + exit(nerrs > 0?1:0); +} diff --git a/v3_nczarr_test/test_forwardinfer.c b/v3_nczarr_test/test_forwardinfer.c new file mode 100644 index 0000000000..7be0a1f8ec --- /dev/null +++ b/v3_nczarr_test/test_forwardinfer.c @@ -0,0 +1,6 @@ +#include + +int main(int argc, char **argv) { + int ncid; + nc_create("file://foo.zarr#mode=nczarr,noxarray", 0, &ncid); +} diff --git a/v3_nczarr_test/test_grpperf.c b/v3_nczarr_test/test_grpperf.c new file mode 100644 index 0000000000..a288121421 --- /dev/null +++ b/v3_nczarr_test/test_grpperf.c @@ -0,0 +1,57 @@ +#include +#include +#include +#include "netcdf.h" + +#include "test_utils.h" + +/* This is the name of the data file we will read. */ +#define FILE_NAME "https://s3.us-east-1.amazonaws.com/unidata-zarr-test-data/"TESTSUBTREE"/testfile.zarr#mode=zarr,s3" +#define GROUPNAME "g1" +#define VARNAME "v" + +#define NX 1 +#define NY 1 + +/* Handle errors by printing an error message and exiting with a + * non-zero status. */ +#define ERRCODE 2 +#define ERR(e) {printf("Error: %s\n", nc_strerror(e)); exit(ERRCODE);} + +int +main() +{ + /* There will be netCDF IDs for the file, each group, and each + * variable. */ + int ncid, varid1, grp1id; + + uint8_t data_in[3 * NX * NY]; + + /* Loop indexes, and error handling. */ + int retval; + + /* Open the file. NC_NOWRITE tells netCDF we want read-only access + * to the file.*/ + if ((retval = nc_open(FILE_NAME, NC_NOWRITE, &ncid))) + ERR(retval); + + /* Get the group ids of our group. */ + if ((retval = nc_inq_ncid(ncid, GROUPNAME, &grp1id))) + ERR(retval); + + /* Get the varid of the uint8 data variable, based on its name, in + * grp1. */ + if ((retval = nc_inq_varid(grp1id, VARNAME, &varid1))) + ERR(retval); + + /* Read the data. */ + if ((retval = nc_get_var_ubyte(grp1id, varid1, &data_in[0]))) + ERR(retval); + + /* Close the file, freeing all resources. */ + if ((retval = nc_close(ncid))) + ERR(retval); + + printf("*** SUCCESS reading example file %s!\n", FILE_NAME); + return 0; +} diff --git a/v3_nczarr_test/test_h5_endians.c b/v3_nczarr_test/test_h5_endians.c new file mode 100644 index 0000000000..3d8b2cc90f --- /dev/null +++ b/v3_nczarr_test/test_h5_endians.c @@ -0,0 +1,340 @@ +/*! Test for NCF-331. Added May 11, 2015. + * Copyright 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, + * 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, + * 2015, 2016, 2017, 2018 + * University Corporation for Atmospheric Research/Unidata. + * + * See \ref copyright file for more info. + * See the following links for more information: + * + * o Issue on GitHub: https://github.com/Unidata/netcdf-c/issues/112 + * o Issue in JIRA: https://bugtracking.unidata.ucar.edu/browse/NCF-331 + * + * Test contributed by Jeff Whitaker + */ + +/* This test is derived from nc_test4/tst_h5_endians.c */ +#define TESTNCZARR + +#include +#include +#include "nc_logging.h" + +#undef BE_DEBUG + +#ifdef TESTNCZARR +#include "test_utils.h" +#define FILE_NAME_NC "file://tmp_h5_endians.file#mode=nczarr,file" +#else +#define FILE_NAME_NC "tst_h5_endians.nc" +#endif + +#define NDIM 10 +#define NLON 20 +#define DIM_NAME "x" +#define DIM_LEN 4 +#define GRP_NAME "grp" +#define LE_FLOAT_VARNAME "fl_le" +#define BE_FLOAT_VARNAME "fl_be" +#define LE_INT_VARNAME "int_le" +#define BE_INT_VARNAME "int_be" +#define LE_DBL_VARNAME "dbl_le" +#define BE_DBL_VARNAME "dbl_be" + +#if defined BE_DEBUG || defined TESTNCZARR +static float +f32swap(float x) +{ + union { + unsigned char bytes[4]; + float f; + } u; + unsigned char c; + u.f = x; + c = u.bytes[0]; u.bytes[0] = u.bytes[3]; u.bytes[3] = c; + c = u.bytes[1]; u.bytes[1] = u.bytes[2]; u.bytes[2] = c; + return u.f; +} + +static double +f64swap(double x) +{ + union { + unsigned char bytes[8]; + double d; + } u; + unsigned char c; + u.d = x; + c = u.bytes[0]; u.bytes[0] = u.bytes[7]; u.bytes[7] = c; + c = u.bytes[1]; u.bytes[1] = u.bytes[6]; u.bytes[6] = c; + c = u.bytes[2]; u.bytes[2] = u.bytes[5]; u.bytes[5] = c; + c = u.bytes[3]; u.bytes[3] = u.bytes[4]; u.bytes[4] = c; + return u.d; +} +#endif + +int main() { + + int ncid, dimid; + int le_float_varid; + int be_float_varid; + int le_int_varid; + int be_int_varid; + int le_dbl_varid; + int be_dbl_varid; + int ed; + int failures = 0; + int retval; + + printf("* Checking that endianness is properly read from file.\n"); + printf("** Generating test files.\n"); + /* + * 1. Create a netcdf file with endianness as desired. + */ + { + + printf("*** Creating a file via netcdf API: %s.\n",FILE_NAME_NC); + if ((retval = nc_create(FILE_NAME_NC, NC_NETCDF4 | NC_CLOBBER, &ncid))) + return retval; + + if ((retval = nc_def_dim(ncid, DIM_NAME, NDIM, &dimid))) + return retval; + + /* Little-Endian Float */ + if ((retval = nc_def_var(ncid, LE_FLOAT_VARNAME, NC_FLOAT, 1, &dimid, &le_float_varid))) + return retval; + if ((retval = nc_def_var_endian(ncid, le_float_varid, NC_ENDIAN_LITTLE))) + return retval; + + /* Big-Endian Float */ + if ((retval = nc_def_var(ncid, BE_FLOAT_VARNAME, NC_FLOAT, 1, &dimid, &be_float_varid))) + return retval; + if ((retval = nc_def_var_endian(ncid, be_float_varid, NC_ENDIAN_BIG))) + return retval; + + /* Little-Endian Int */ + if ((retval = nc_def_var(ncid, LE_INT_VARNAME, NC_INT, 1, &dimid, &le_int_varid))) + return retval; + if ((retval = nc_def_var_endian(ncid, le_int_varid, NC_ENDIAN_LITTLE))) + return retval; + + /* Big-Endian Int */ + if ((retval = nc_def_var(ncid, BE_INT_VARNAME, NC_INT, 1, &dimid, &be_int_varid))) + return retval; + if ((retval = nc_def_var_endian(ncid, be_int_varid, NC_ENDIAN_BIG))) + return retval; + + /* Little-Endian Double */ + if ((retval = nc_def_var(ncid, LE_DBL_VARNAME, NC_DOUBLE, 1, &dimid, &le_dbl_varid))) + return retval; + if ((retval = nc_def_var_endian(ncid, le_dbl_varid, NC_ENDIAN_LITTLE))) + return retval; + + /* Big-Endian Double */ + if ((retval = nc_def_var(ncid, BE_DBL_VARNAME, NC_DOUBLE, 1, &dimid, &be_dbl_varid))) + return retval; + if ((retval = nc_def_var_endian(ncid, be_dbl_varid, NC_ENDIAN_BIG))) + return retval; + + + if ((retval = nc_close(ncid))) + return retval; + } + + /* + * 2. Reopen netcdf-generated file, check to see if the endianness attribute + * exists. + */ + printf("** Checking test files.\n"); + { + ncid = 0; + le_float_varid = 0; + be_float_varid = 0; + le_int_varid = 0; + be_int_varid = 0; + le_dbl_varid = 0; + be_dbl_varid = 0; + + printf("*** %s\n",FILE_NAME_NC); + if ((retval = nc_open(FILE_NAME_NC, NC_NETCDF4 | NC_NOWRITE, &ncid))) + return retval; + + if ((retval = nc_inq_varid(ncid,LE_FLOAT_VARNAME,&le_float_varid))) + return retval; + if ((retval = nc_inq_varid(ncid,BE_FLOAT_VARNAME,&be_float_varid))) + return retval; + if ((retval = nc_inq_varid(ncid,LE_INT_VARNAME,&le_int_varid))) + return retval; + if ((retval = nc_inq_varid(ncid,BE_INT_VARNAME,&be_int_varid))) + return retval; + if ((retval = nc_inq_varid(ncid,LE_DBL_VARNAME,&le_dbl_varid))) + return retval; + if ((retval = nc_inq_varid(ncid,BE_DBL_VARNAME,&be_dbl_varid))) + return retval; + + printf("\tLittle-Endian Float...\t"); + if ((retval = nc_inq_var_endian(ncid,le_float_varid,&ed))) + return retval; + if(ed == NC_ENDIAN_LITTLE) printf("passed\n"); else {printf("failed\n"); failures++;} + + printf("\tBig-Endian Float...\t"); + if ((retval = nc_inq_var_endian(ncid,be_float_varid,&ed))) + return retval; + if(ed == NC_ENDIAN_BIG) printf("passed\n"); else {printf("failed\n"); failures++;} + + printf("\tLittle-Endian Int...\t"); + if ((retval = nc_inq_var_endian(ncid,le_int_varid,&ed))) + return retval; + if(ed == NC_ENDIAN_LITTLE) printf("passed\n"); else {printf("failed\n"); failures++;} + + printf("\tBig-Endian Int...\t"); + if ((retval = nc_inq_var_endian(ncid,be_int_varid,&ed))) + return retval; + if(ed == NC_ENDIAN_BIG) printf("passed\n"); else {printf("failed\n"); failures++;} + + printf("\tLittle-Endian Double...\t"); + if ((retval = nc_inq_var_endian(ncid,le_dbl_varid,&ed))) + return retval; + if(ed == NC_ENDIAN_LITTLE) printf("passed\n"); else {printf("failed\n"); failures++;} + + printf("\tBig-Endian Double...\t"); + if ((retval = nc_inq_var_endian(ncid,be_dbl_varid,&ed))) + return retval; + if(ed == NC_ENDIAN_BIG) printf("passed\n"); else {printf("failed\n"); failures++;} + + if ((retval = nc_close(ncid))) + return retval; + } + + /* + * 3. Reopen netcdf-generated file, write data and reread. + */ + printf("** Read/Write test files.\n"); + { + ncid = 0; + le_float_varid = 0; + be_float_varid = 0; + le_int_varid = 0; + be_int_varid = 0; + le_dbl_varid = 0; + be_dbl_varid = 0; + + printf("*** %s\n",FILE_NAME_NC); + if ((retval = nc_open(FILE_NAME_NC, NC_NETCDF4 | NC_WRITE, &ncid))) + return retval; + + if ((retval = nc_inq_varid(ncid,LE_FLOAT_VARNAME,&le_float_varid))) + return retval; + if ((retval = nc_inq_varid(ncid,BE_FLOAT_VARNAME,&be_float_varid))) + return retval; + if ((retval = nc_inq_varid(ncid,LE_INT_VARNAME,&le_int_varid))) + return retval; + if ((retval = nc_inq_varid(ncid,BE_INT_VARNAME,&be_int_varid))) + return retval; + if ((retval = nc_inq_varid(ncid,LE_DBL_VARNAME,&le_dbl_varid))) + return retval; + if ((retval = nc_inq_varid(ncid,BE_DBL_VARNAME,&be_dbl_varid))) + return retval; + + { + int i, failed; + int idata_in[NDIM]; + float fdata_in[NDIM]; + double ddata_in[NDIM]; + int idata_le_out[NDIM]; + float fdata_le_out[NDIM]; + double ddata_le_out[NDIM]; + int idata_be_out[NDIM]; +#if defined BE_DEBUG || defined TESTNCZARR + float fdata_be_out[NDIM]; + double ddata_be_out[NDIM]; +#endif + + /* Setup data in/out */ + for(i=0;i +#include +#include + +#include +#include "netcdf.h" +#include "netcdf_filter.h" + +/* The HDF assigned id for bzip compression */ +#define BZIP2_ID 307 +/* The compression level used in this example */ +#define BZIP2_LEVEL 9 + +#define DEFLATE_LEVEL 2 + +#define NOOP_ID 40000 + +#define NFILTERS 3 + +#define DFALT_TESTFILE "tmp_multifilter.nc" + +/* Point at which we give up */ +#define MAXERRS 8 + +#define NDIMS 4 +#define DIMSIZE 4 +#define CHUNKSIZE 4 /* Note: not the total size of the chunk, but size wrt a dim*/ + +static const char* testfile = NULL; + +static size_t dimsize = DIMSIZE; +static size_t chunksize = CHUNKSIZE; +static size_t actualdims = NDIMS; + +static size_t actualproduct = 1; /* x-product over dim sizes */ +static size_t chunkproduct = 1; /* x-product over chunksizes */ + +static size_t dims[NDIMS]; +static size_t chunks[NDIMS]; + +static int nerrs = 0; + +static int ncid, varid; +static int dimids[NDIMS]; +static float* array = NULL; +static float* expected = NULL; + +/* Forward */ +static void init(int argc, char** argv); +static int test_multi(void); +static int verifychunks(void); + +#define ERRR do { \ +fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ +fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ +nerrs++;\ +} while (0) + +static int +check(int err,int line) +{ + if(err != NC_NOERR) { + fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err)); + fflush(stderr); + exit(1); + } + return NC_NOERR; +} + +#define CHECK(x) check(x,__LINE__) + +/* +Read the chunking information about the variable +and verify that it is as expected. +*/ + +static int +verifychunks(void) +{ + int i; + int store = -1; + size_t chunksizes[NDIMS]; + memset(chunksizes,0,sizeof(chunksizes)); + CHECK(nc_inq_var_chunking(ncid, varid, &store, chunksizes)); + /* Storate must be chunked, not contiguous */ + if(store != NC_CHUNKED) { + fprintf(stderr,"bad chunk store\n"); + return NC_ESTORAGE; + } + /* Chunk sizes must match our predefined set */ + for(i=0;i= MAXERRS) + break; + } + } + if(errs == 0) + printf("no data errors\n"); + if(actualproduct <= 1) + return NC_EBADDIM; + return (errs == 0 ? NC_NOERR: NC_EINVAL); +} + +int +verifyfilters(int ncid, int varid) +{ + size_t nparams; + size_t nfilters; + unsigned int filterids[NFILTERS]; + unsigned int params[2]; + + /* Read back the compression info and verify it */ + CHECK(nc_inq_var_filter_ids(ncid,varid,&nfilters,filterids)); + if(nfilters != NFILTERS) { + fprintf(stderr,"Fail: nfilters mismatch: expected=%d actual=%u\n",NFILTERS,(unsigned)nfilters); + return NC_EINVAL; + } + if(filterids[0] != BZIP2_ID + || filterids[1] != H5Z_FILTER_DEFLATE + || filterids[2] != NOOP_ID + ) { + fprintf(stderr,"Fail: filter id mismatch: actual/expected={%u/%u,%u/%u,%u/%u}\n", + filterids[0],BZIP2_ID, + filterids[1],H5Z_FILTER_DEFLATE, + filterids[2],NOOP_ID); + return NC_EINVAL; + } + /* Get level for each filter */ + CHECK(nc_inq_var_filter_info(ncid,varid,BZIP2_ID,&nparams,params)); + if(nparams != 1) { + fprintf(stderr,"Fail: nparams mismatch: id=%u expected=1 actual=%u\n",filterids[0],(unsigned)nparams); + return NC_EINVAL; + } + if(params[0] != BZIP2_LEVEL) { + fprintf(stderr,"Fail: parameter mismatch: expected=%u actual=%u\n",BZIP2_LEVEL,params[0]); + return NC_EINVAL; + } + CHECK(nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_DEFLATE,&nparams,params)); + if(nparams != 1) { + fprintf(stderr,"Fail: nparams mismatch: id=%u expected=1 actual=%u\n",filterids[1],(unsigned)nparams); + return NC_EINVAL; + } + if(params[0] != DEFLATE_LEVEL) { + fprintf(stderr,"Fail: parameter mismatch: expected=%u actual=%u\n",BZIP2_LEVEL,params[0]); + return NC_EINVAL; + } + CHECK(nc_inq_var_filter_info(ncid,varid,NOOP_ID,&nparams,params)); + if(nparams != 0) { + fprintf(stderr,"Fail: parameter mismatch: id=%u nparams: expected=0 actual=%u\n",NOOP_ID,(unsigned)nparams); + return NC_EINVAL; + } + return NC_NOERR; +} + +/* +Create the file, write it, then re-read for comparison. +*/ +static int +test_multi(void) +{ + int i; + unsigned int params[2]; + + printf("\n*** Testing Multi-filter application: filter set = bzip2 deflate noop"); + printf("\n"); + + /* Clear the data array */ + memset(array,0,sizeof(float)*actualproduct); + + /* Create a file */ + CHECK(nc_create(testfile, NC_NETCDF4|NC_CLOBBER, &ncid)); + + /* Do not use fill for this file */ + CHECK(nc_set_fill(ncid, NC_NOFILL, NULL)); + + /* Define the dimensions */ + for(i=0;i 1) + testfile = argv[1]; + else + testfile = DFALT_TESTFILE; + + /* Setup various variables */ + actualproduct = 1; + chunkproduct = 1; + for(i=0;i 0?1:0); +} + diff --git a/v3_nczarr_test/test_nczarr.sh b/v3_nczarr_test/test_nczarr.sh new file mode 100644 index 0000000000..381b2794c8 --- /dev/null +++ b/v3_nczarr_test/test_nczarr.sh @@ -0,0 +1,218 @@ +#!/bin/sh + +# This file must kept in sync with +# nczarr_test/test_nczarr.sh +# and +# v3_nczarr_test/test_nczarr.sh + +# Load only once +if test "x$TEST_NCZARR_SH" = x ; then +export TEST_NCZARR_SH=1 + +if test "x$SETX" != x; then set -x; fi + +# Figure out which cloud repo to use +if test "x$NCZARR_S3_TEST_HOST" = x ; then +# export NCZARR_S3_TEST_HOST=stratus.ucar.edu + export NCZARR_S3_TEST_HOST=s3.us-east-1.amazonaws.com +fi +if test "x$NCZARR_S3_TEST_BUCKET" = x ; then + export NCZARR_S3_TEST_BUCKET="${S3TESTBUCKET}" +fi +export NCZARR_S3_TEST_URL="https://${NCZARR_S3_TEST_HOST}/${NCZARR_S3_TEST_BUCKET}" + +# TAG for zarr format to use; uses the environment variable NCZARRFORMAT +if test "x${NCZARRFORMAT}" = x3 ; then + export ZDF="_v3" +else + export ZDF="" +fi + +# Fix execdir +NCZARRDIR="${execdir}/../nczarr_test" + +ZMD="${NCZARRDIR}/${DL}zmapio" +S3UTIL="${NCZARRDIR}/${DL}s3util" +ZS3PARSE="${NCZARRDIR}/${DL}zs3parse" +NCDUMPCHUNKS="${NCZARRDIR}/${DL}ncdumpchunks" +ZHEX="${NCZARRDIR}/${DL}zhex" +ZISJSON="${NCZARRDIR}/${DL}zisjson" + +# Check settings +checksetting() { +if test -f ${TOPBUILDDIR}/libnetcdf.settings ; then + local PATTERN + PATTERN="${1}:[ ]*yes" + if grep "$PATTERN" <${TOPBUILDDIR}/libnetcdf.settings ; then + HAVE_SETTING=1 + else + unset HAVE_SETTING + fi +fi +} + +checkprops() { + specflag= + headflag= + isxfail= + # determine if this is an xfailtest + for t in ${XFAILTESTS} ; do + if test "x${t}" = "x${x}" ; then isxfail=1; fi + done + for t in ${SPECTESTS} ; do + if test "x${t}" = "x${f}" ; then specflag="-s"; fi + done + for t in ${HEADTESTS} ; do + if test "x${t}" = "x${f}" ; then headflag="-h"; fi + done +} + +extfor() { + case "$1" in + file) zext="file" ;; + zip) zext="zip" ;; + s3) zext="s3" ;; + *) echo "unknown kind: $1" ; exit 1;; + esac +} + +deletemap() { + case "$1" in + file) rm -fr $2;; + zip) rm -f $2;; + s3) S3KEY=`${ZS3PARSE} -k $2`; s3sdkdelete $S3KEY ;; + *) echo "unknown kind: $1" ; exit 1;; + esac +} + +mapstillexists() { + mapstillexists=0 + if "${ZMD} $fileurl" &> /dev/null ; then + echo "delete failed: $1" + mapstillexists=1 + fi +} + +fileargs() { + f="$1" + frag="$2" + if test "x$frag" = x ; then frag="mode=nczarr,$zext" ; fi + case "$zext" in + s3) + S3PATH="${NCZARR_S3_TEST_URL}/${S3ISOPATH}" + fileurl="${S3PATH}/${f}#${frag}" + file=$fileurl + S3HOST=`${ZS3PARSE} -h $S3PATH` + S3BUCKET=`${ZS3PARSE} -b $S3PATH` + S3PREFIX=`${ZS3PARSE} -k $S3PATH` + ;; + *) + file="${f}.$zext" + fileurl="file://${f}.$zext#${frag}" + ;; + esac +} + +dumpmap() { + zext=$1 + zbase=`basename $2 ".$zext"` + fileargs $zbase + ${ZMD} -t int -x objdump $fileurl > $3 +} + +# Function to remove selected -s attributes from file; +# These attributes might be platform dependent +sclean() { +sed -i.bak -e '/:_IsNetcdf4/d' $1 +sed -i.bak -e '/:_Endianness/d' $1 +sed -i.bak -e '/_NCProperties/d' $1 +sed -i.bak -e '/_SuperblockVersion/d' $1 +} + +# s3clean plus remove additional lines +scleanplus() { +sclean $1 +sed -i.bak -e '/_Format/d' $1 +sed -i.bak -e '/_global attributes:/d' $1 +} + +# Function to rewrite selected key values in a zmapio output. +# because these values might be platform dependent +zmapclean() { +sed -i.bak -e 's|^\([^(]*\)([0-9][0-9]*)|\1()|' $1 +sed -i.bak -e 's/"_NCProperties":[ ]*"version=\([0-9]\),[^"]*"/"_NCProperties": "version=\1,netcdf=0.0.0,nczarr=0.0.0"/g' $1 +sed -i.bak -e 's/"_nczarr_superblock":[ ]*{[^}]*}/"_nczarr_superblock": {"version": "0.0.0", "format": 2}/g' $1 +sed -i.bak -e 's/"_nczarr_superblock":[ ]*{[^}]*}/"_nczarr_superblock": {"version": "0.0.0", "format": 2}/g' $1 +} + +# Make sure execdir and srcdir absolute paths are available +WD=`pwd` +cd $srcdir ; abs_srcdir=`pwd` ; cd $WD +cd $execdir ; abs_execdir=`pwd` ; cd $WD + +# Clear out any existing .rc files +WD=`pwd` +if test "x$NCAUTH_HOMETEST" != x ; then RCHOME=1; fi + +# Set plugin path + +if test "x$FP_USEPLUGINS" = xyes; then +# Load the findplugins function +. ${builddir}/findplugin.sh +echo "findplugin.sh loaded" + +# Locate the plugin path and the library names; argument order is critical +# Find misc in order to determine HDF5_PLUGIN+PATH. +# Assume all test filters are in same plugin dir +if ! findplugin h5misc ; then exit 0; fi + +echo "final HDF5_PLUGIN_DIR=${HDF5_PLUGIN_DIR}" +export HDF5_PLUGIN_PATH="${HDF5_PLUGIN_DIR}" +fi # USEPLUGINS + +resetrc() { + if test "x$RCHOME" = x1 ; then + rm -f ${HOME}/.dodsrc ${HOME}/.daprc ${HOME}/.ncrc + fi + rm -f ${WD}/.dodsrc ${WD}/.daprc ${WD}/.ncrc + unset NCRCENV_IGNORE + unset NCRCENV_RC + unset DAPRCFILE +} + +s3sdkdelete() { +if test -f ${S3UTIL} ; then + ${S3UTIL} ${PROFILE} -u "${NCZARR_S3_TEST_URL}" -k "$1" clear +elif which aws ; then + aws s3api delete-object --endpoint-url=https://${NCZARR_S3_TEST_HOST} --bucket=${NCZARR_S3_TEST_BUCKET} --key="/${S3ISOPATH}/$1" +else + echo "**** Could not delete ${NCZAR_S3_TEST_URL}" +fi +} + +s3sdkcleanup() { +if test -f ${S3UTIL} ; then + ${S3UTIL} ${PROFILE} -u "${NCZARR_S3_TEST_URL}" -k "$1" clear +elif which aws ; then + aws s3api delete-object --endpoint-url=https://${NCZARR_S3_TEST_HOST} --bucket=${NCZARR_S3_TEST_BUCKET} --key="/${S3ISOPATH}/$1" +else + echo "**** Could not delete ${NCZAR_S3_TEST_URL}" +fi +} + +# Create an isolation path for S3; build on the isolation directory +s3isolate() { + if test "x${S3ISOPATH}" = x ; then + if test "x${ISOPATH}" = x ; then isolate "$1"; fi + # Need isolation path to include the test directory + BNAME=`basename $srcdir` + S3ISODIR="${BNAME}_${TESTUID}/${ISODIR}" + S3ISOPATH="${S3TESTSUBTREE}/${S3ISODIR}" + fi +} + +GDBB="gdb -batch -ex r -ex bt -ex q --args" + +resetrc + +fi #TEST_NCZARR_SH diff --git a/v3_nczarr_test/test_nczarr_utils.h b/v3_nczarr_test/test_nczarr_utils.h new file mode 100644 index 0000000000..72bae34b60 --- /dev/null +++ b/v3_nczarr_test/test_nczarr_utils.h @@ -0,0 +1,167 @@ +/* This is part of the netCDF package. + Copyright 2018 University Corporation for Atmospheric Research/Unidata + See COPYRIGHT file for conditions of use. + + @Author Dennis Heimbigner +*/ + +/********************************************************************* + * Copyright 2018, UCAR/Unidata + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + *********************************************************************/ +/* $Id: main.c,v 1.33 2010/05/26 21:43:36 dmh Exp $ */ +/* $Header: /upc/share/CVS/netcdf-3/ncgen/main.c,v 1.33 2010/05/26 21:43:36 dmh Exp $ */ + +#ifndef TEST_NCZARR_UTILS_H +#define TEST_NCZARR_UTILS_H + +#include "config.h" +#include +#ifdef HAVE_GETOPT_H +#include +#endif + +#if defined(_WIN32) && !defined(__MINGW32__) +#include "XGetopt.h" +#endif + +#include "nc_tests.h" +#include "err_macros.h" +#include "ncbytes.h" +#include "zincludes.h" + +#ifndef nulldup + #define nulldup(x) ((x)?strdup(x):(x)) +#endif + +static char* progname = NULL; + +struct ITOptions { + NCZM_IMPL impl; + char* path; + char* cloud; + char* otherfragments; +} itoptions = {NCZM_UNDEF, NULL, NULL, NULL}; + +static void +test_usage(void) +{ + fprintf(stderr,"usage: [-e ]\n"); + exit(1); +} + +/* strip off leading path; result is malloc'd */ +static char * +ubasename(char *logident) +{ + char* sep; + + sep = strrchr(logident,'/'); +#ifdef MSDOS + if(sep == NULL) sep = strrchr(logident,'\\'); +#endif + if(sep == NULL) return logident; + sep++; /* skip past the separator */ + return sep; +} + +static void +setimpl(const char* name) +{ + if(strcasecmp(name,"s3")==0) itoptions.impl = NCZM_S3; + else if(strcasecmp(name,"file")==0) itoptions.impl = NCZM_FILE; + else if(strcasecmp(name,"zip")==0) itoptions.impl = NCZM_ZIP; + else test_usage(); +} + +static const char* +implname(void) +{ + switch (itoptions.impl) { + case NCZM_S3: return "s3"; + case NCZM_FILE: return "file"; + case NCZM_ZIP: return "zip"; + default: test_usage(); + } + return NULL; +} + +static void +buildpath(const char* target,NCZM_IMPL impl) +{ + NCbytes* buf = ncbytesnew(); + NC_UNUSED(impl); + switch(itoptions.impl) { + case NCZM_ZIP: + case NCZM_FILE: + ncbytescat(buf,"file://"); + ncbytescat(buf,target); + ncbytescat(buf,"."); + ncbytescat(buf,implname()); + ncbytescat(buf,"#mode=nczarr"); + ncbytescat(buf,","); + ncbytescat(buf,implname()); + break; + case NCZM_S3: + ncbytescat(buf,itoptions.cloud); + if(itoptions.cloud[strlen(itoptions.cloud)-1] != '/') + ncbytescat(buf,"/"); + ncbytescat(buf,target); + ncbytescat(buf,"#mode=nczarr"); + ncbytescat(buf,","); + ncbytescat(buf,implname()); + break; + default: test_usage(); + } + if(itoptions.otherfragments != NULL) { + ncbytescat(buf,","); + ncbytescat(buf,itoptions.otherfragments); + } + itoptions.path = ncbytesextract(buf); + ncbytesfree(buf); +} + +void +processoptions(int argc, char** argv, const char* base_file_name) +{ + int c; + + if(argc == 1) test_usage(); + progname = nulldup(ubasename(argv[0])); + + while ((c = getopt(argc, argv, "e:c:F:")) != EOF) + switch(c) { + case 'e': /* zmap choice */ + setimpl(optarg); + break; + case 'c': /* cloud appliance url prefix*/ + itoptions.cloud = strdup(optarg); + break; + case 'F': /* fragments */ + itoptions.otherfragments = strdup(optarg); + break; + case '?': + test_usage(); + break; + } + + argc -= optind; + argv += optind; + + if(itoptions.impl == NCZM_UNDEF) itoptions.impl = NCZM_FILE; + if(itoptions.impl == NCZM_S3 && itoptions.cloud == NULL) test_usage(); + + buildpath(base_file_name,itoptions.impl); + +} + +void +clearoptions(void) +{ + nullfree(itoptions.path); + nullfree(itoptions.cloud); + nullfree(itoptions.otherfragments); + nullfree(progname); +} + +#endif /*TEST_NCZARR_UTILS_H*/ diff --git a/v3_nczarr_test/test_nczfilter.c b/v3_nczarr_test/test_nczfilter.c new file mode 100644 index 0000000000..8e1e987c1a --- /dev/null +++ b/v3_nczarr_test/test_nczfilter.c @@ -0,0 +1,67 @@ +/* This is part of the netCDF package. + Copyright 2018 University Corporation for Atmospheric Research/Unidata + See COPYRIGHT file for conditions of use. + + Test nczarr filter loading + Author: Dennis Heimbigner +*/ + +#include +#include +#include + +#include "netcdf.h" +#include "netcdf_filter.h" + +#define ERR(r) {fprintf(stderr,"fail: line %d: (%d) %s\n",__LINE__,(r),nc_strerror((r)));} + +#define FILENAME "file://tmp_nczfilter.nc#mode=nczarr,file" + +#define FILTERID 1 + +int +main(int argc, char **argv) +{ + int ret = NC_NOERR; + int ncid; + int dimid; + int varid; + size_t chunksizes[1] = {4}; + unsigned params[1] = {9}; + char* furl = NULL; + int data[4] = {17,18,19,20}; + size_t nfilters; + unsigned int filterids[8]; + size_t nparams; + unsigned inqparams[8]; + + if(argc == 1) + furl = FILENAME; + else + furl = argv[1]; + + if ((ret=nc_create(furl, NC_NETCDF4, &ncid))) ERR(ret); + if ((ret=nc_def_dim(ncid, "d", 4, &dimid))) ERR(ret); + if ((ret=nc_def_var(ncid, "v", NC_INT, 1, &dimid, &varid))) ERR(ret); + if((ret=nc_def_var_chunking(ncid,varid,NC_CHUNKED,chunksizes))) ERR(ret); + if((ret=nc_def_var_filter(ncid,varid,FILTERID,1,params))) ERR(ret); + if((ret=nc_put_var(ncid,varid,data))) ERR(ret); + if ((ret=nc_close(ncid))) ERR(ret); + + if ((ret=nc_open(furl, 0, &ncid))) ERR(ret); + if ((ret=nc_inq_varid(ncid, "v", &varid))) ERR(ret); + + if((ret=nc_inq_var_filter_ids(ncid, varid, &nfilters, filterids))) ERR(ret); + if(nfilters != 1) ERR(NC_EFILTER); + if(filterids[0] != FILTERID) ERR(NC_EFILTER); + printf("nfilters=%u filterids[0]=%u\n",(unsigned)nfilters,(unsigned)filterids[0]); + if((ret=nc_inq_var_filter_info(ncid, varid, filterids[0], &nparams, inqparams))) ERR(ret); + if(nparams != 1) ERR(NC_EFILTER); + if(inqparams[0] != params[0]) ERR(NC_EFILTER); + printf("nparams=%u params[0]=%u\n",(unsigned)nparams,(unsigned)inqparams[0]); + + if ((ret=nc_close(ncid))) ERR(ret); + + nc_finalize(); + return 0; +} diff --git a/v3_nczarr_test/test_notzarr.c b/v3_nczarr_test/test_notzarr.c new file mode 100644 index 0000000000..b025b4de2a --- /dev/null +++ b/v3_nczarr_test/test_notzarr.c @@ -0,0 +1,31 @@ +/* This is part of the netCDF package. + Copyright 2018 University Corporation for Atmospheric Research/Unidata + See COPYRIGHT file for conditions of use. + + Test nczarr filter loading + Author: Dennis Heimbigner +*/ + +#include +#include +#include + +#include "netcdf.h" + +#define ERR(r) {fprintf(stderr,"fail: line %d: (%d) %s\n",__LINE__,(r),nc_strerror((r)));} + +int +main(int argc, char **argv) +{ + int ret = NC_NOERR; + int ncid; + + if(argc < 2) { + fprintf(stderr,"Usage: tst_notzarr \n"); + exit(1); + } + ret = nc_open(argv[1],NC_NETCDF4,&ncid); + printf("%d",ret); + if(ret == NC_NOERR) nc_close(ncid); + exit(0); +} diff --git a/v3_nczarr_test/test_put_vars_two_unlim_dim.c b/v3_nczarr_test/test_put_vars_two_unlim_dim.c new file mode 100644 index 0000000000..3bcae9415d --- /dev/null +++ b/v3_nczarr_test/test_put_vars_two_unlim_dim.c @@ -0,0 +1,93 @@ +/*! \file + +Copyright 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, +2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, +2015, 2016, 2017, 2018 +University Corporation for Atmospheric Research/Unidata. + +See \ref copyright file for more info. + +*/ + +/* + * Test contributed in support of netCDF issue + * https://github.com/Unidata/netcdf-c/issues/160 + */ + +/* Test derived from nc_test4/tst_put_vars_two_unlim_dim.c */ +#define TESTNCZARR + +#include +#include "netcdf.h" + +#include + +#ifdef TESTNCZARR +#include "test_utils.h" +#define FILE_NAME "file://tmp_put_vars_two_unlim_dim.file#mode=nczarr,file" +#else +#define FILE_NAME "tst_put_vars_two_unlim_dim.nc" +#endif + +int +main(int argc, char* argv[]) +{ + int ret; + int ncid; + int dim1id, dim2id; + int var1id, var2id; + size_t start = 0; + size_t count = 5; + double vals[] = { 1.0, 2.0, 3.0, 4.0, 5.0 }; + + NC_UNUSED(argc); + NC_UNUSED(argv); + + if ((ret = nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid))) { + printf("nc_create(...): error code = %d\n", ret); + return -1; + } + + if ((ret = nc_def_dim(ncid, "dim1", NC_UNLIMITED, &dim1id))) { + printf("nc_def_dim(...\"dim1\"...): error code = %d\n", ret); + nc_close(ncid); + return -1; + } + + if ((ret = nc_def_dim(ncid, "dim2", NC_UNLIMITED, &dim2id))) { + printf("nc_def_dim(...\"dim1\"...): error code = %d\n", ret); + nc_close(ncid); + return -1; + } + + if ((ret = nc_def_var(ncid, "var1", NC_DOUBLE, 1, &dim1id, &var1id))) { + printf("nc_def_var(...\"var1\"...): error code = %d\n", ret); + nc_close(ncid); + return -1; + } + + if ((ret = nc_def_var(ncid, "var2", NC_DOUBLE, 1, &dim2id, &var2id))) { + printf("nc_def_var(...\"var2\"...): error code = %d\n", ret); + nc_close(ncid); + return -1; + } + + if ((ret = nc_put_vars_double(ncid, var1id, &start, &count, NULL, &vals[0]))) { + printf("nc_put_var_double(...var1id...): error code = %d\n", ret); + nc_close(ncid); + return -1; + } + + if ((ret = nc_put_vars_double(ncid, var2id, &start, &count, NULL, &vals[0]))) { + printf("nc_put_var_double(...var2id...): error code = %d\n", ret); + nc_close(ncid); + return -1; + } + + if ((ret = nc_close(ncid))) { + printf("nc_close(...): error code = %d\n", ret); + return -1; + } + + return 0; +} diff --git a/v3_nczarr_test/test_quantize.c b/v3_nczarr_test/test_quantize.c new file mode 100644 index 0000000000..2f521ed0f7 --- /dev/null +++ b/v3_nczarr_test/test_quantize.c @@ -0,0 +1,1392 @@ +/* This is part of the netCDF package. + Copyright 2021 University Corporation for Atmospheric Research/Unidata + See COPYRIGHT file for conditions of use. + + Test quantization of netcdf-4 variables. Quantization is the + zeroing-out of bits in float or double data beyond a desired + precision. + + Ed Hartnett, 8/19/21 + Dennis Heimbigner, 1/16/22 +*/ + +/* This test is derived from nc_test4/test_quantize.c */ +#define TESTNCZARR + +#include /* Define fabs(), powf(), round() */ +#include +#include "err_macros.h" +#include "netcdf.h" + +#define TEST "tst_quantize" +#define FILE_NAME "tst_quantize.nc" +#define NDIM1 1 +#define DIM_NAME_1 "meters_along_canal" +#define DIM_LEN_3 3 +#define DIM_LEN_1 1 +#define DIM_LEN_5 5 +#define DIM_LEN_8 8 +#define VAR_NAME_1 "Amsterdam_houseboat_location" +#define VAR_NAME_2 "Amsterdam_street_noise_decibels" +#define NSD_3 3 +#define NSD_9 9 + +/* This var used to help print a float in hex. */ +char pf_str[20]; + +/* This struct allows us to treat float as uint32_t + * types. */ +union FU { + float f; + uint32_t u; +}; + +/* This struct allows us to treat double points as uint64_t + * types. */ +union DU { + double d; + uint64_t u; +}; + +/* This function prints a float as hex. */ +char * +pf(float myf) +{ + union { + float f; + uint32_t u; + } fu; + fu.f = myf; + snprintf(pf_str, sizeof(pf_str), "0x%x", fu.u); + return pf_str; +} + +/* This function prints a double as hex. */ +char * +pd(double myd) +{ + union { + double d; + uint64_t u; + } du; + du.d = myd; + snprintf(pf_str, sizeof(pf_str), "0x%llx", (unsigned long long)du.u); + return pf_str; +} + +int +main(int argc, char **argv) +{ +#ifdef TESTNCZARR + const char* template = NULL; + char file_url[4096]; + + if(argc == 1) + {fprintf(stderr,"usage: test_quantize \n"); exit(1);} + + template = argv[1]; + + snprintf(file_url,sizeof(file_url),template,FILE_NAME); +fprintf(stderr,"URL=%s\n",file_url); + +#undef FILE_NAME +#define FILE_NAME file_url +#endif + +#define NUM_MODE_TESTS 2 +#define NUM_QUANTIZE_MODES 3 + int mode = NC_NETCDF4|NC_CLOBBER; + int m; + int q, quantize_mode[NUM_QUANTIZE_MODES] = {NC_QUANTIZE_BITGROOM, NC_QUANTIZE_GRANULARBR, + NC_QUANTIZE_BITROUND}; + + printf("\n*** Testing netcdf-4 variable quantization functions.\n"); + for (m = 0; m < NUM_MODE_TESTS; m++) + { + if (!m) + printf("**** testing with NC_NETCDF4...\n"); + else + { + printf("**** testing with NC_NETCDF4|NC_CLASSIC_MODEL...\n"); + mode |= NC_CLASSIC_MODEL; + } + + printf("\t**** testing quantization setting and error conditions...\n"); + { + int ncid, dimid, varid1, varid2; + int quantize_mode_in, nsd_in; + + for (q = 0; q < NUM_QUANTIZE_MODES; q++) + { + printf("\t\t**** testing quantize algorithm %d...\n", quantize_mode[q]); +#ifndef TESTNCZARR + /* Create a netcdf classic file with one var. Attempt + * quantization. It will not work. */ + if (nc_create(FILE_NAME, NC_CLOBBER, &ncid)) ERR; + if (nc_def_dim(ncid, DIM_NAME_1, DIM_LEN_3, &dimid)) ERR; + if (nc_def_var(ncid, VAR_NAME_1, NC_FLOAT, NDIM1, &dimid, &varid1)) ERR; + if (nc_def_var_quantize(ncid, varid1, quantize_mode[q], NSD_3) != NC_ENOTNC4) ERR; + if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in) != NC_ENOTNC4) ERR; + if (nc_close(ncid)) ERR; +#endif + + /* Create a netcdf-4 file with two vars. Attempt + * quantization. It will work, eventually... */ + if (nc_create(FILE_NAME, mode, &ncid)) ERR; + if (nc_def_dim(ncid, DIM_NAME_1, DIM_LEN_3, &dimid)) ERR; + if (nc_def_var(ncid, VAR_NAME_1, NC_FLOAT, NDIM1, &dimid, &varid1)) ERR; + if (nc_def_var(ncid, VAR_NAME_2, NC_DOUBLE, NDIM1, &dimid, &varid2)) ERR; + + /* Bad varid. */ + if (nc_def_var_quantize(ncid, NC_GLOBAL, quantize_mode[q], NSD_3) != NC_EGLOBAL) ERR; + if (nc_def_var_quantize(ncid, varid2 + 1, quantize_mode[q], NSD_3) != NC_ENOTVAR) ERR; + /* Invalid values. */ + if (nc_def_var_quantize(ncid, varid1, NUM_QUANTIZE_MODES + 1, NSD_3) != NC_EINVAL) ERR; + if (nc_def_var_quantize(ncid, varid1, quantize_mode[q], -1) != NC_EINVAL) ERR; + if (quantize_mode[q] == NC_QUANTIZE_BITROUND) + { + if (nc_def_var_quantize(ncid, varid1, quantize_mode[q], NC_QUANTIZE_MAX_FLOAT_NSB + 1) != NC_EINVAL) ERR; + } + else + { + if (nc_def_var_quantize(ncid, varid1, quantize_mode[q], NC_QUANTIZE_MAX_FLOAT_NSD + 1) != NC_EINVAL) ERR; + } + if (nc_def_var_quantize(ncid, varid2, NUM_QUANTIZE_MODES + 1, 3) != NC_EINVAL) ERR; + if (nc_def_var_quantize(ncid, varid2, quantize_mode[q], -1) != NC_EINVAL) ERR; + if (quantize_mode[q] == NC_QUANTIZE_BITROUND) + { + if (nc_def_var_quantize(ncid, varid2, quantize_mode[q], NC_QUANTIZE_MAX_DOUBLE_NSB + 1) != NC_EINVAL) ERR; + } + else + { + if (nc_def_var_quantize(ncid, varid2, quantize_mode[q], NC_QUANTIZE_MAX_DOUBLE_NSD + 1) != NC_EINVAL) ERR; + } + if (nc_def_var_quantize(ncid, varid2, quantize_mode[q], 0) != NC_EINVAL) ERR; + + /* This will work. */ + if (nc_def_var_quantize(ncid, varid1, quantize_mode[q], NSD_3)) ERR; + if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in)) ERR; + if (quantize_mode_in != quantize_mode[q]) ERR; + if (nsd_in != NSD_3) ERR; + + /* Wait, I changed my mind! Let's turn off quantization. */ + if (nc_def_var_quantize(ncid, varid1, NC_NOQUANTIZE, 0)) ERR; + if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in)) ERR; + if (quantize_mode_in != NC_NOQUANTIZE) ERR; + if (nsd_in != 0) ERR; + + /* Changed my mind again, turn it on. */ + if (nc_def_var_quantize(ncid, varid1, quantize_mode[q], NSD_3)) ERR; + + /* I changed my mind again! Turn it off! */ + if (nc_def_var_quantize(ncid, varid1, NC_NOQUANTIZE, 0)) ERR; + if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in)) ERR; + if (quantize_mode_in != NC_NOQUANTIZE) ERR; + if (nsd_in != 0) ERR; + + /* Changed my mind again, turn it on. */ + if (nc_def_var_quantize(ncid, varid1, quantize_mode[q], NSD_3)) ERR; + + /* This also will work for double. */ + if (nc_def_var_quantize(ncid, varid2, quantize_mode[q], NSD_9)) ERR; + if (nc_inq_var_quantize(ncid, varid2, &quantize_mode_in, &nsd_in)) ERR; + if (quantize_mode_in != quantize_mode[q]) ERR; + if (nsd_in != NSD_9) ERR; + + /* End define mode. */ + if (nc_enddef(ncid)) ERR; + + /* This will not work, it's too late! */ + if (nc_def_var_quantize(ncid, varid1, quantize_mode[q], NSD_3) != NC_ELATEDEF) ERR; + + /* Close the file. */ + if (nc_close(ncid)) ERR; + + /* Open the file and check. */ + if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; + /* Don't assume the varid !!! */ + if (nc_inq_varid(ncid, VAR_NAME_1, &varid1)) ERR; + if (nc_inq_varid(ncid, VAR_NAME_2, &varid2)) ERR; + if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in)) ERR; + if (quantize_mode_in != quantize_mode[q]) ERR; + if (nsd_in != NSD_3) ERR; + if (nc_inq_var_quantize(ncid, varid2, &quantize_mode_in, &nsd_in)) ERR; + if (quantize_mode_in != quantize_mode[q]) ERR; + if (nsd_in != NSD_9) ERR; + if (nc_close(ncid)) ERR; + } + } + SUMMARIZE_ERR; + +#define NX_BIG 100 +#define NY_BIG 100 +#define NTYPES 9 +#define VAR_NAME "Amsterdam_coffeeshop_location" +#define X_NAME "distance_from_center" +#define Y_NAME "distance_along_canal" +#define NDIM2 2 + + printf("\t**** testing quantization handling of non-floats...\n"); + { + for (q = 0; q < NUM_QUANTIZE_MODES; q++) + { + int ncid; + int dimid[NDIM2]; + int varid; + int nsd_in, quantize_mode_in; + int nsd_out = 3; + char file_name[NC_MAX_FILENAME + 1]; + int xtype[NTYPES] = {NC_CHAR, NC_SHORT, NC_INT, NC_BYTE, NC_UBYTE, + NC_USHORT, NC_UINT, NC_INT64, NC_UINT64}; + int t; + + printf("\t\t**** testing quantize algorithm %d...\n", quantize_mode[q]); + for (t = 0; t < NTYPES; t++) + { + snprintf(file_name, sizeof(file_name), "%s_quantize_%d_type_%d.nc", TEST, quantize_mode[q], xtype[t]); +#ifdef TESTNCZARR + { + char url[NC_MAX_FILENAME + 1]; + snprintf(url,sizeof(url),template,file_name); + strcpy(file_name,url); + } +#endif + /* Create file. */ + if (nc_create(file_name, NC_NETCDF4, &ncid)) ERR; + if (nc_def_dim(ncid, X_NAME, NX_BIG, &dimid[0])) ERR; + if (nc_def_dim(ncid, Y_NAME, NY_BIG, &dimid[1])) ERR; + if (nc_def_var(ncid, VAR_NAME, xtype[t], NDIM2, dimid, &varid)) ERR; + + /* Quantzie returns NC_EINVAL because this is not + * an NC_FLOAT or NC_DOULBE. */ + if (nc_def_var_quantize(ncid, varid, quantize_mode[q], nsd_out) != NC_EINVAL) ERR; + if (nc_close(ncid)) ERR; + + /* Check file. */ + { + if (nc_open(file_name, NC_NETCDF4, &ncid)) ERR; + if (nc_inq_varid(ncid,VAR_NAME,&varid)) ERR; + if (nc_inq_var_quantize(ncid, varid, &quantize_mode_in, &nsd_in)) + ERR; + if (quantize_mode_in) ERR; + if (nc_close(ncid)) ERR; + } + } + } + } + SUMMARIZE_ERR; + printf("\t**** testing quantization of scalars...\n"); + { + for (q = 0; q < NUM_QUANTIZE_MODES; q++) + { + int ncid, varid1, varid2; + int quantize_mode_in, nsd_in; + float float_data[DIM_LEN_1] = {1.1111111f}; + double double_data[DIM_LEN_1] = {1.111111111111}; + + printf("\t\t**** testing quantize algorithm %d...\n", quantize_mode[q]); + + /* Create a netcdf-4 file with two scalar vars. */ + if (nc_create(FILE_NAME, mode, &ncid)) ERR; + if (nc_def_var(ncid, VAR_NAME_1, NC_FLOAT, 0, NULL, &varid1)) ERR; + if (nc_def_var(ncid, VAR_NAME_2, NC_DOUBLE, 0, NULL, &varid2)) ERR; + + /* Turn on quantize for both vars. */ + if (nc_def_var_quantize(ncid, varid1, quantize_mode[q], NSD_3)) ERR; + if (nc_def_var_quantize(ncid, varid2, quantize_mode[q], NSD_3)) ERR; + + /* For classic mode, we must call enddef. */ + if (m) + if (nc_enddef(ncid)) ERR; + + /* Write some data. */ + if (nc_put_var_float(ncid, varid1, float_data)) ERR; + if (nc_put_var_double(ncid, varid2, double_data)) ERR; + + /* Close the file. */ + if (nc_close(ncid)) ERR; + + { + float float_in; + double double_in; + union FU fin; + int nsd_att_in; + union DU dfin; + union FU fout; + union DU dfout; + + NC_UNUSED(fout); NC_UNUSED(dfout); + + /* Open the file and check metadata. */ + if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; + if (nc_inq_varid(ncid, VAR_NAME_1, &varid1)) ERR; + if (nc_inq_varid(ncid, VAR_NAME_2, &varid2)) ERR; + if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in)) ERR; + if (quantize_mode_in != quantize_mode[q] || nsd_in != NSD_3) ERR; + if (nc_inq_var_quantize(ncid, varid2, &quantize_mode_in, &nsd_in)) ERR; + if (quantize_mode_in != quantize_mode[q] || nsd_in != NSD_3) ERR; + + /* Each var now has an attribute describing the quantize settings. */ + switch (quantize_mode[q]) + { + case NC_QUANTIZE_BITGROOM: + if (nc_get_att_int(ncid, 0, NC_QUANTIZE_BITGROOM_ATT_NAME, &nsd_att_in)) ERR; + if (nsd_att_in != NSD_3) ERR; + if (nc_get_att_int(ncid, 1, NC_QUANTIZE_BITGROOM_ATT_NAME, &nsd_att_in)) ERR; + if (nsd_att_in != NSD_3) ERR; + break; + case NC_QUANTIZE_GRANULARBR: + if (nc_get_att_int(ncid, 0, NC_QUANTIZE_GRANULARBR_ATT_NAME, &nsd_att_in)) ERR; + if (nsd_att_in != NSD_3) ERR; + if (nc_get_att_int(ncid, 1, NC_QUANTIZE_GRANULARBR_ATT_NAME, &nsd_att_in)) ERR; + if (nsd_att_in != NSD_3) ERR; + break; + case NC_QUANTIZE_BITROUND: + if (nc_get_att_int(ncid, 0, NC_QUANTIZE_BITROUND_ATT_NAME, &nsd_att_in)) ERR; + if (nsd_att_in != NSD_3) ERR; + if (nc_get_att_int(ncid, 1, NC_QUANTIZE_BITROUND_ATT_NAME, &nsd_att_in)) ERR; + if (nsd_att_in != NSD_3) ERR; + break; + default: + ERR; + } + + /* Check the data. */ + if (nc_get_var(ncid, varid1, &float_in)) ERR; + if (nc_get_var(ncid, varid2, &double_in)) ERR; +#if 0 + fout.f = float_data[0]; + dfout.d = double_data[0]; +#endif + fin.f = float_in; + dfin.d = double_in; +#if 0 + printf ("\nfloat_data: %10f : 0x%x float_data_in: %10f : 0x%x\n", + float_data[0], fout.u, float_data[0], fin.u); + printf ("\ndouble_data: %15g : 0x%16lx double_data_in: %15g : 0x%lx\n", + double_data[0], dfout.u, double_data[0], dfin.u); + +#endif + /* Check the results, slightly different for each quantize algorithm. */ + switch (quantize_mode[q]) + { + case NC_QUANTIZE_BITGROOM: + if (fin.u != 0x3f8e3000) ERR; + if (dfin.u != 0x3ff1c60000000000) ERR; + break; + case NC_QUANTIZE_GRANULARBR: + if (fin.u != 0x3f8e0000) ERR; + if (dfin.u != 0x3ff1c00000000000) ERR; + break; + case NC_QUANTIZE_BITROUND: + if (fin.u != 0x3f900000) ERR; + if (dfin.u != 0x3ff2000000000000) ERR; + break; + default: + ERR; + } + + /* Close the file again. */ + if (nc_close(ncid)) ERR; + } + } + } + SUMMARIZE_ERR; + printf("\t**** testing quantization of one value...\n"); + { + for (q = 0; q < NUM_QUANTIZE_MODES; q++) + { + int ncid, dimid, varid1, varid2; + int quantize_mode_in, nsd_in; + float float_data[DIM_LEN_1] = {1.1111111f}; + double double_data[DIM_LEN_1] = {1.111111111111}; + + printf("\t\t**** testing quantize algorithm %d...\n", quantize_mode[q]); + + /* Create a netcdf-4 file with two vars. */ + if (nc_create(FILE_NAME, mode, &ncid)) ERR; + if (nc_def_dim(ncid, DIM_NAME_1, DIM_LEN_1, &dimid)) ERR; + if (nc_def_var(ncid, VAR_NAME_1, NC_FLOAT, NDIM1, &dimid, &varid1)) ERR; + if (nc_def_var(ncid, VAR_NAME_2, NC_DOUBLE, NDIM1, &dimid, &varid2)) ERR; + + /* Turn on quantize for both vars. */ + if (nc_def_var_quantize(ncid, varid1, quantize_mode[q], NSD_3)) ERR; + if (nc_def_var_quantize(ncid, varid2, quantize_mode[q], NSD_3)) ERR; + + /* For classic mode, we must call enddef. */ + if (m) + if (nc_enddef(ncid)) ERR; + + /* Write some data. */ + if (nc_put_var_float(ncid, varid1, float_data)) ERR; + if (nc_put_var_double(ncid, varid2, double_data)) ERR; + + /* Close the file. */ + if (nc_close(ncid)) ERR; + + { + float float_in; + double double_in; + union FU fin; + union DU dfin; + union FU fout; + union DU dfout; + + NC_UNUSED(fout); NC_UNUSED(dfout); + + /* Open the file and check metadata. */ + if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_1,&varid1)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_2,&varid2)) ERR; + if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in)) ERR; + if (quantize_mode_in != quantize_mode[q] || nsd_in != NSD_3) ERR; + if (nc_inq_var_quantize(ncid, varid2, &quantize_mode_in, &nsd_in)) ERR; + if (quantize_mode_in != quantize_mode[q] || nsd_in != NSD_3) ERR; + + /* Check the data. */ + if (nc_get_var(ncid, varid1, &float_in)) ERR; + if (nc_get_var(ncid, varid2, &double_in)) ERR; +#if 0 + fout.f = float_data[0]; + dfout.d = double_data[0]; +#endif + fin.f = float_in; + dfin.d = double_in; +#if 0 + printf ("\nfloat_data: %10f : 0x%x float_data_in: %10f : 0x%x\n", + float_data[0], fout.u, float_data[0], fin.u); + printf ("\ndouble_data: %15g : 0x%16lx double_data_in: %15g : 0x%lx\n", + double_data[0], dfout.u, double_data[0], dfin.u); +#endif + /* Check the results, slightly different for each quantize algorithm. */ + switch (quantize_mode[q]) + { + case NC_QUANTIZE_BITGROOM: + if (fin.u != 0x3f8e3000) ERR; + if (dfin.u != 0x3ff1c60000000000) ERR; + break; + case NC_QUANTIZE_GRANULARBR: + if (fin.u != 0x3f8e0000) ERR; + if (dfin.u != 0x3ff1c00000000000) ERR; + break; + case NC_QUANTIZE_BITROUND: + if (fin.u != 0x3f900000) ERR; + if (dfin.u != 0x3ff2000000000000) ERR; + break; + default: + ERR; + } + + /* Close the file again. */ + if (nc_close(ncid)) ERR; + } + } + } + SUMMARIZE_ERR; + printf("\t**** testing more quantization values...\n"); + { + for (q = 0; q < NUM_QUANTIZE_MODES; q++) + { + int ncid, dimid, varid1, varid2; + int quantize_mode_in, nsd_in; + float float_data[DIM_LEN_5] = {1.11111111f, 1.0f, 9.99999999f, 12345.67f, .1234567f}; + double double_data[DIM_LEN_5] = {1.1111111, 1.0, 9.999999999, 1234567890.12345, 123456789012345.0}; + int x; + + printf("\t\t**** testing quantize algorithm %d...\n", quantize_mode[q]); + + /* Create a netcdf-4 file with two vars. */ + if (nc_create(FILE_NAME, mode, &ncid)) ERR; + if (nc_def_dim(ncid, DIM_NAME_1, DIM_LEN_5, &dimid)) ERR; + if (nc_def_var(ncid, VAR_NAME_1, NC_FLOAT, NDIM1, &dimid, &varid1)) ERR; + if (nc_def_var(ncid, VAR_NAME_2, NC_DOUBLE, NDIM1, &dimid, &varid2)) ERR; + + /* Turn on quantize for both vars. */ + if (nc_def_var_quantize(ncid, varid1, quantize_mode[q], NSD_3)) ERR; + if (nc_def_var_quantize(ncid, varid2, quantize_mode[q], NSD_3)) ERR; + + /* For classic mode, we must call enddef. */ + if (m) + if (nc_enddef(ncid)) ERR; + + /* Write some data. */ + if (nc_put_var_float(ncid, varid1, float_data)) ERR; + if (nc_put_var_double(ncid, varid2, double_data)) ERR; + + /* Close the file. */ + if (nc_close(ncid)) ERR; + + { + float float_in[DIM_LEN_5]; + double double_in[DIM_LEN_5]; + union FU { + float f; + uint32_t u; + }; + + union FU fin; + union FU xpect[NUM_QUANTIZE_MODES][DIM_LEN_5]; + union DU dfin; + union FU fout; + union DU dfout; + union DU double_xpect[NUM_QUANTIZE_MODES][DIM_LEN_5]; + + NC_UNUSED(fout); NC_UNUSED(dfout); + + switch (quantize_mode[q]) + { + case NC_QUANTIZE_BITGROOM: + xpect[0][0].u = 0x3f8e3000; + xpect[0][1].u = 0x3f800fff; + xpect[0][2].u = 0x41200000; + xpect[0][3].u = 0x4640efff; + xpect[0][4].u = 0x3dfcd000; + double_xpect[0][0].u = 0x3ff1c60000000000; + double_xpect[0][1].u = 0x3ff001ffffffffff; + double_xpect[0][2].u = 0x4023fe0000000000; + double_xpect[0][3].u = 0x41d265ffffffffff; + double_xpect[0][4].u = 0x42dc120000000000; + break; + case NC_QUANTIZE_GRANULARBR: + xpect[1][0].u = 0x3f8e0000; + xpect[1][1].u = 0x3f800000; + xpect[1][2].u = 0x41200000; + xpect[1][3].u = 0x46410000; + xpect[1][4].u = 0x3dfc0000; + double_xpect[1][0].u = 0x3ff1c00000000000; + double_xpect[1][1].u = 0x3ff0000000000000; + double_xpect[1][2].u = 0x4024000000000000; + double_xpect[1][3].u = 0x41d2600000000000; + double_xpect[1][4].u = 0x42dc200000000000; + break; + case NC_QUANTIZE_BITROUND: + xpect[2][0].u = 0x3f900000; + xpect[2][1].u = 0x3f800000; + xpect[2][2].u = 0x41200000; + xpect[2][3].u = 0x46400000; + xpect[2][4].u = 0x3e000000; + double_xpect[2][0].u = 0x3ff2000000000000; + double_xpect[2][1].u = 0x3ff0000000000000; + double_xpect[2][2].u = 0x4024000000000000; + double_xpect[2][3].u = 0x41d2000000000000; + double_xpect[2][4].u = 0x42dc000000000000; + break; + default: + ERR; + } + + /* Open the file and check metadata. */ + if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_1,&varid1)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_2,&varid2)) ERR; + if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in)) ERR; + if (quantize_mode_in != quantize_mode[q] || nsd_in != NSD_3) ERR; + if (nc_inq_var_quantize(ncid, varid2, &quantize_mode_in, &nsd_in)) ERR; + if (quantize_mode_in != quantize_mode[q] || nsd_in != NSD_3) ERR; + + /* Check the data. */ + if (nc_get_var(ncid, varid1, float_in)) ERR; + if (nc_get_var(ncid, varid2, double_in)) ERR; +#if 0 + printf("\n"); +#endif + for (x = 0; x < DIM_LEN_5; x++) + { + fin.f = float_in[x]; +#if 0 + fout.f = float_data[x]; + printf ("float_data: %10f : 0x%x float_data_in: %10f : 0x%x\n", + float_data[x], fout.u, float_data[x], fin.u); +#endif + if (fin.u != xpect[q][x].u) ERR; + dfin.d = double_in[x]; +#if 0 + dfout.d = double_data[x]; + printf("double_data: %15g : 0x%16lx double_data_in: %15g : 0x%16lx\n", + double_data[x], dfout.u, double_data[x], dfin.u); +#endif + if (dfin.u != double_xpect[q][x].u) ERR; + } + + /* Close the file again. */ + if (nc_close(ncid)) ERR; + } + } + } + SUMMARIZE_ERR; + printf("\t**** testing quantization of one value with type conversion...\n"); + { + for (q = 0; q < NUM_QUANTIZE_MODES; q++) + { + int ncid, dimid, varid1, varid2; + int quantize_mode_in, nsd_in; + float float_data[DIM_LEN_1] = {1.1111111f}; + double double_data[DIM_LEN_1] = {1.111111111111}; + + printf("\t\t**** testing quantize algorithm %d...\n", quantize_mode[q]); + + /* Create a netcdf-4 file with two vars. */ + if (nc_create(FILE_NAME, mode, &ncid)) ERR; + if (nc_def_dim(ncid, DIM_NAME_1, DIM_LEN_1, &dimid)) ERR; + if (nc_def_var(ncid, VAR_NAME_1, NC_FLOAT, NDIM1, &dimid, &varid1)) ERR; + if (nc_def_var(ncid, VAR_NAME_2, NC_DOUBLE, NDIM1, &dimid, &varid2)) ERR; + + /* Turn on quantize for both vars. */ + if (nc_def_var_quantize(ncid, varid1, quantize_mode[q], NSD_3)) ERR; + if (nc_def_var_quantize(ncid, varid2, quantize_mode[q], NSD_3)) ERR; + + /* For classic mode, we must call enddef. */ + if (m) + if (nc_enddef(ncid)) ERR; + + /* Write some double data to float var. */ + if (nc_put_var_double(ncid, varid1, double_data)) ERR; + + /* Write some float data to double var. */ + if (nc_put_var_float(ncid, varid2, float_data)) ERR; + + /* Close the file. */ + if (nc_close(ncid)) ERR; + + { + float float_in; + double double_in; + union FU fin; + union DU dfin; + union FU fout; + union DU dfout; + int nsd_att_in; + + NC_UNUSED(fout); NC_UNUSED(dfout); + + /* Open the file and check metadata. */ + if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_1,&varid1)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_2,&varid2)) ERR; + if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in)) ERR; + if (quantize_mode_in != quantize_mode[q] || nsd_in != NSD_3) ERR; + if (nc_inq_var_quantize(ncid, varid2, &quantize_mode_in, &nsd_in)) ERR; + if (quantize_mode_in != quantize_mode[q] || nsd_in != NSD_3) ERR; + + /* Each var now has an attribute describing the quantize settings. */ + switch (quantize_mode[q]) + { + case NC_QUANTIZE_BITGROOM: + if (nc_get_att_int(ncid, 0, NC_QUANTIZE_BITGROOM_ATT_NAME, &nsd_att_in)) ERR; + if (nsd_att_in != NSD_3) ERR; + if (nc_get_att_int(ncid, 1, NC_QUANTIZE_BITGROOM_ATT_NAME, &nsd_att_in)) ERR; + if (nsd_att_in != NSD_3) ERR; + break; + case NC_QUANTIZE_GRANULARBR: + if (nc_get_att_int(ncid, 0, NC_QUANTIZE_GRANULARBR_ATT_NAME, &nsd_att_in)) ERR; + if (nsd_att_in != NSD_3) ERR; + if (nc_get_att_int(ncid, 1, NC_QUANTIZE_GRANULARBR_ATT_NAME, &nsd_att_in)) ERR; + if (nsd_att_in != NSD_3) ERR; + break; + case NC_QUANTIZE_BITROUND: + if (nc_get_att_int(ncid, 0, NC_QUANTIZE_BITROUND_ATT_NAME, &nsd_att_in)) ERR; + if (nsd_att_in != NSD_3) ERR; + if (nc_get_att_int(ncid, 1, NC_QUANTIZE_BITROUND_ATT_NAME, &nsd_att_in)) ERR; + if (nsd_att_in != NSD_3) ERR; + break; + default: + ERR; + } + + /* Check the data. */ + if (nc_get_var(ncid, varid1, &float_in)) ERR; + if (nc_get_var(ncid, varid2, &double_in)) ERR; + fin.f = float_in; + dfin.d = double_in; +#if 0 + fout.f = (float)double_data[0]; + dfout.d = float_data[0]; + printf ("\ndouble_data: %15g : 0x%x float_data_in: %10f : 0x%x\n", + double_data[0], fout.u, float_in, fin.u); + printf ("\nfloat_data: %15g : 0x%16lx double_data_in: %15g : 0x%lx\n", + float_data[0], dfout.u, double_in, dfin.u); + +#endif + switch (quantize_mode[q]) + { + case NC_QUANTIZE_BITGROOM: + if (fin.u != 0x3f8e3000) ERR; + if (dfin.u != 0x3ff1c60000000000) ERR; + break; + case NC_QUANTIZE_GRANULARBR: + if (fin.u != 0x3f8e0000) ERR; + if (dfin.u != 0x3ff1c00000000000) ERR; + break; + case NC_QUANTIZE_BITROUND: + if (fin.u !=0x3f900000 ) ERR; + if (dfin.u != 0x3ff2000000000000) ERR; + break; + default: + ERR; + } + + /* Close the file again. */ + if (nc_close(ncid)) ERR; + } + } + } + SUMMARIZE_ERR; + printf("\t**** testing more quantization values with type conversion...\n"); + { + for (q = 0; q < NUM_QUANTIZE_MODES; q++) + { + int ncid, dimid, varid1, varid2; + int quantize_mode_in, nsd_in; + float float_data[DIM_LEN_5] = {1.11111111f, 1.0, 9.99999999f, 12345.67f, .1234567f}; + double double_data[DIM_LEN_5] = {1.1111111, 1.0, 9.999999999, 1234567890.12345, 123456789012345.0}; + int x; + + printf("\t\t**** testing quantize algorithm %d...\n", quantize_mode[q]); + + /* Create a netcdf-4 file with two vars. */ + if (nc_create(FILE_NAME, mode, &ncid)) ERR; + if (nc_def_dim(ncid, DIM_NAME_1, DIM_LEN_5, &dimid)) ERR; + if (nc_def_var(ncid, VAR_NAME_1, NC_FLOAT, NDIM1, &dimid, &varid1)) ERR; + if (nc_def_var(ncid, VAR_NAME_2, NC_DOUBLE, NDIM1, &dimid, &varid2)) ERR; + + /* Turn on quantize for both vars. */ + if (nc_def_var_quantize(ncid, varid1, quantize_mode[q], NSD_3)) ERR; + if (nc_def_var_quantize(ncid, varid2, quantize_mode[q], NSD_3)) ERR; + + /* For classic mode, we must call enddef. */ + if (m) + if (nc_enddef(ncid)) ERR; + + /* Write some data. */ + if (nc_put_var_double(ncid, varid1, double_data)) ERR; + if (nc_put_var_float(ncid, varid2, float_data)) ERR; + + /* Close the file. */ + if (nc_close(ncid)) ERR; + + { + float float_in[DIM_LEN_5]; + double double_in[DIM_LEN_5]; + union FU { + float f; + uint32_t u; + }; + + union FU fin; + union FU xpect[NUM_QUANTIZE_MODES][DIM_LEN_5]; + union DU dfin; + union FU fout; + union DU dfout; + union DU double_xpect[NUM_QUANTIZE_MODES][DIM_LEN_5]; + + NC_UNUSED(fout); NC_UNUSED(dfout); + + switch (quantize_mode[q]) + { + case NC_QUANTIZE_BITGROOM: + xpect[0][0].u = 0x3f8e3000; + xpect[0][1].u = 0x3f800fff; + xpect[0][2].u = 0x41200000; + xpect[0][3].u = 0x4e932fff; + xpect[0][4].u = 0x56e09000; + double_xpect[0][0].u = 0x3ff1c60000000000; + double_xpect[0][1].u = 0x3ff001ffffffffff; + double_xpect[0][2].u = 0x4024000000000000; + double_xpect[0][3].u = 0x40c81dffffffffff; + double_xpect[0][4].u = 0x3fbf9a0000000000; + break; + case NC_QUANTIZE_GRANULARBR: + xpect[1][0].u = 0x3f8e0000; + xpect[1][1].u =0x3f800000 ; + xpect[1][2].u = 0x41200000; + xpect[1][3].u = 0x4e930000; + xpect[1][4].u = 0x56e10000; + double_xpect[1][0].u = 0x3ff1c00000000000; + double_xpect[1][1].u = 0x3ff0000000000000; + double_xpect[1][2].u = 0x4024000000000000; + double_xpect[1][3].u = 0x40c8200000000000; + double_xpect[1][4].u = 0x3fbf800000000000; + break; + case NC_QUANTIZE_BITROUND: + xpect[2][0].u = 0x3f900000; + xpect[2][1].u = 0x3f800000; + xpect[2][2].u = 0x41200000; + xpect[2][3].u = 0x4e900000; + xpect[2][4].u = 0x56e00000; + double_xpect[2][0].u = 0x3ff2000000000000; + double_xpect[2][1].u = 0x3ff0000000000000; + double_xpect[2][2].u = 0x4024000000000000; + double_xpect[2][3].u = 0x40c8000000000000; + double_xpect[2][4].u = 0x3fc0000000000000; + break; + default: + ERR; + } + + /* Open the file and check metadata. */ + if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_1,&varid1)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_2,&varid2)) ERR; + if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in)) ERR; + if (quantize_mode_in != quantize_mode[q] || nsd_in != NSD_3) ERR; + if (nc_inq_var_quantize(ncid, varid2, &quantize_mode_in, &nsd_in)) ERR; + if (quantize_mode_in != quantize_mode[q] || nsd_in != NSD_3) ERR; + + /* Check the data. */ + if (nc_get_var(ncid, varid1, float_in)) ERR; + if (nc_get_var(ncid, varid2, double_in)) ERR; +#if 0 + printf("\n"); +#endif + for (x = 0; x < DIM_LEN_5; x++) + { + fin.f = float_in[x]; + dfin.d = double_in[x]; +#if 0 + fout.f = float_data[x]; + printf ("float_data: %10f : 0x%x float_data_in: %10f : 0x%x\n", + float_data[x], fout.u, float_data[x], fin.u); + dfout.d = double_data[x]; + printf("double_data: %15g : 0x%16lx double_data_in: %15g : 0x%16lx\n", + double_data[x], dfout.u, double_data[x], dfin.u); +#endif + if (fin.u != xpect[q][x].u) ERR; + if (dfin.u != double_xpect[q][x].u) ERR; + } + + /* Close the file again. */ + if (nc_close(ncid)) ERR; + } + } + } + SUMMARIZE_ERR; + printf("\t**** testing more quantization values with default fill values...\n"); + { + for (q = 0; q < NUM_QUANTIZE_MODES; q++) + { + int ncid, dimid, varid1, varid2; + int quantize_mode_in, nsd_in; + float float_data[DIM_LEN_5] = {1.11111111f, NC_FILL_FLOAT, 9.99999999f, 12345.67f, NC_FILL_FLOAT}; + double double_data[DIM_LEN_5] = {1.1111111, NC_FILL_DOUBLE, 9.999999999, 1234567890.12345, NC_FILL_DOUBLE}; + int x; + + printf("\t\t**** testing quantize algorithm %d...\n", quantize_mode[q]); + + /* Create a netcdf-4 file with two vars. */ + if (nc_create(FILE_NAME, mode, &ncid)) ERR; + if (nc_def_dim(ncid, DIM_NAME_1, DIM_LEN_5, &dimid)) ERR; + if (nc_def_var(ncid, VAR_NAME_1, NC_FLOAT, NDIM1, &dimid, &varid1)) ERR; + if (nc_def_var(ncid, VAR_NAME_2, NC_DOUBLE, NDIM1, &dimid, &varid2)) ERR; + + /* Turn on quantize for both vars. */ + if (nc_def_var_quantize(ncid, varid1, quantize_mode[q], NSD_3)) ERR; + if (nc_def_var_quantize(ncid, varid2, quantize_mode[q], NSD_3)) ERR; + + /* For classic mode, we must call enddef. */ + if (m) + if (nc_enddef(ncid)) ERR; + + /* Write some data. */ + if (nc_put_var_float(ncid, varid1, float_data)) ERR; + if (nc_put_var_double(ncid, varid2, double_data)) ERR; + + /* Close the file. */ + if (nc_close(ncid)) ERR; + + { + float float_in[DIM_LEN_5]; + double double_in[DIM_LEN_5]; + union FU { + float f; + uint32_t u; + }; + + union FU fin; + union FU xpect[NUM_QUANTIZE_MODES][DIM_LEN_5]; + union DU dfin; + union FU fout; + union DU dfout; + union DU double_xpect[NUM_QUANTIZE_MODES][DIM_LEN_5]; + + NC_UNUSED(fout); NC_UNUSED(dfout); + + switch (quantize_mode[q]) + { + case NC_QUANTIZE_BITGROOM: + xpect[0][0].u = 0x3f8e3000; + xpect[0][1].u = 0x7cf00000; + xpect[0][2].u = 0x41200000; + xpect[0][3].u = 0x4640efff; + xpect[0][4].u = 0x7cf00000; + double_xpect[0][0].u = 0x3ff1c60000000000; + double_xpect[0][1].u = 0x479e000000000000; + double_xpect[0][2].u = 0x4023fe0000000000; + double_xpect[0][3].u = 0x41d265ffffffffff; + double_xpect[0][4].u = 0x479e000000000000; + break; + case NC_QUANTIZE_GRANULARBR: + xpect[1][0].u = 0x3f8e0000; + xpect[1][1].u = 0x7cf00000; + xpect[1][2].u = 0x41200000; + xpect[1][3].u = 0x46410000; + xpect[1][4].u = 0x7cf00000; + double_xpect[1][0].u = 0x3ff1c00000000000; + double_xpect[1][1].u = 0x479e000000000000; + double_xpect[1][2].u = 0x4024000000000000; + double_xpect[1][3].u = 0x41d2600000000000; + double_xpect[1][4].u = 0x479e000000000000; + break; + case NC_QUANTIZE_BITROUND: + xpect[2][0].u = 0x3f900000; + xpect[2][1].u = 0x7cf00000; + xpect[2][2].u = 0x41200000; + xpect[2][3].u = 0x46400000; + xpect[2][4].u = 0x7cf00000; + double_xpect[2][0].u = 0x3ff2000000000000; + double_xpect[2][1].u = 0x479e000000000000; + double_xpect[2][2].u = 0x4024000000000000; + double_xpect[2][3].u = 0x41d2000000000000; + double_xpect[2][4].u = 0x479e000000000000; + break; + default: + ERR; + } + + /* Open the file and check metadata. */ + if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_1,&varid1)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_2,&varid2)) ERR; + if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in)) ERR; + if (quantize_mode_in != quantize_mode[q] || nsd_in != NSD_3) ERR; + if (nc_inq_var_quantize(ncid, varid2, &quantize_mode_in, &nsd_in)) ERR; + if (quantize_mode_in != quantize_mode[q] || nsd_in != NSD_3) ERR; + + /* Check the data. */ + if (nc_get_var(ncid, varid1, float_in)) ERR; + if (nc_get_var(ncid, varid2, double_in)) ERR; +#if 0 + printf("\n"); +#endif + for (x = 0; x < DIM_LEN_5; x++) + { + fin.f = float_in[x]; + dfin.d = double_in[x]; +#if 0 + fout.f = float_data[x]; + dfout.d = double_data[x]; + printf ("float_data: %10f : 0x%x float_data_in: %10f : 0x%x\n", + float_data[x], fout.u, float_data[x], fin.u); + printf("double_data: %15g : 0x%16lx double_data_in: %15g : 0x%16lx\n", + double_data[x], dfout.u, double_data[x], dfin.u); +#endif + if (fin.u != xpect[q][x].u) ERR; + if (dfin.u != double_xpect[q][x].u) ERR; + } + + /* Close the file again. */ + if (nc_close(ncid)) ERR; + } + } + } + SUMMARIZE_ERR; + printf("\t**** testing more quantization values with custom fill values...\n"); + { +#define CUSTOM_FILL_FLOAT 99.99999f +#define CUSTOM_FILL_DOUBLE -99999.99999 + for (q = 0; q < NUM_QUANTIZE_MODES; q++) + { + int ncid, dimid, varid1, varid2; + int quantize_mode_in, nsd_in; + float float_data[DIM_LEN_5] = {1.11111111f, CUSTOM_FILL_FLOAT, 9.99999999f, 12345.67f, CUSTOM_FILL_FLOAT}; + double double_data[DIM_LEN_5] = {1.1111111, CUSTOM_FILL_DOUBLE, 9.999999999, 1234567890.12345, CUSTOM_FILL_DOUBLE}; + float custom_fill_float = CUSTOM_FILL_FLOAT; + double custom_fill_double = CUSTOM_FILL_DOUBLE; + int x; + + printf("\t\t**** testing quantize algorithm %d...\n", quantize_mode[q]); + + /* Create a netcdf-4 file with two vars. */ + if (nc_create(FILE_NAME, mode, &ncid)) ERR; + if (nc_def_dim(ncid, DIM_NAME_1, DIM_LEN_5, &dimid)) ERR; + if (nc_def_var(ncid, VAR_NAME_1, NC_FLOAT, NDIM1, &dimid, &varid1)) ERR; + if (nc_put_att_float(ncid, varid1, NC_FillValue, NC_FLOAT, 1, &custom_fill_float)) ERR; + if (nc_def_var(ncid, VAR_NAME_2, NC_DOUBLE, NDIM1, &dimid, &varid2)) ERR; + if (nc_put_att_double(ncid, varid2, NC_FillValue, NC_DOUBLE, 1, &custom_fill_double)) ERR; + + /* Turn on quantize for both vars. */ + if (nc_def_var_quantize(ncid, varid1, quantize_mode[q], NSD_3)) ERR; + if (nc_def_var_quantize(ncid, varid2, quantize_mode[q], NSD_3)) ERR; + + /* For classic mode, we must call enddef. */ + if (m) + if (nc_enddef(ncid)) ERR; + + /* Write some data. */ + if (nc_put_var_float(ncid, varid1, float_data)) ERR; + if (nc_put_var_double(ncid, varid2, double_data)) ERR; + + /* Close the file. */ + if (nc_close(ncid)) ERR; + + { + float float_in[DIM_LEN_5]; + double double_in[DIM_LEN_5]; + union FU { + float f; + uint32_t u; + }; + + union FU fin; + union FU xpect[NUM_QUANTIZE_MODES][DIM_LEN_5]; + union DU dfin; + union FU fout; + union DU dfout; + union DU double_xpect[NUM_QUANTIZE_MODES][DIM_LEN_5]; + + NC_UNUSED(fout); NC_UNUSED(dfout); + + switch (quantize_mode[q]) + { + case NC_QUANTIZE_BITGROOM: + xpect[0][0].u = 0x3f8e3000; + xpect[0][1].u = 0x42c7ffff; + xpect[0][2].u = 0x41200000; + xpect[0][3].u = 0x4640efff; + xpect[0][4].u = 0x42c7ffff; + double_xpect[0][0].u = 0x3ff1c60000000000; + double_xpect[0][1].u = 0xc0f869fffff583a5; + double_xpect[0][2].u = 0x4023fe0000000000; + double_xpect[0][3].u = 0x41d265ffffffffff; + double_xpect[0][4].u = 0xc0f869fffff583a5; + break; + case NC_QUANTIZE_GRANULARBR: + xpect[1][0].u = 0x3f8e0000; + xpect[1][1].u = 0x42c7ffff; + xpect[1][2].u = 0x41200000; + xpect[1][3].u = 0x46410000; + xpect[1][4].u = 0x42c7ffff; + double_xpect[1][0].u = 0x3ff1c00000000000; + double_xpect[1][1].u = 0xc0f869fffff583a5; + double_xpect[1][2].u = 0x4024000000000000; + double_xpect[1][3].u = 0x41d2600000000000; + double_xpect[1][4].u = 0xc0f869fffff583a5; + break; + case NC_QUANTIZE_BITROUND: + xpect[2][0].u = 0x3f900000; + xpect[2][1].u = 0x42c7ffff; + xpect[2][2].u = 0x41200000; + xpect[2][3].u = 0x46400000; + xpect[2][4].u = 0x42c7ffff; + double_xpect[2][0].u = 0x3ff2000000000000; + double_xpect[2][1].u = 0xc0f869fffff583a5; + double_xpect[2][2].u = 0x4024000000000000; + double_xpect[2][3].u = 0x41d2000000000000; + double_xpect[2][4].u = 0xc0f869fffff583a5; + break; + default: + ERR; + } + + /* Open the file and check metadata. */ + if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_1,&varid1)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_2,&varid2)) ERR; + if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in)) ERR; + if (quantize_mode_in != quantize_mode[q] || nsd_in != NSD_3) ERR; + if (nc_inq_var_quantize(ncid, varid2, &quantize_mode_in, &nsd_in)) ERR; + if (quantize_mode_in != quantize_mode[q] || nsd_in != NSD_3) ERR; + + /* Check the data. */ + if (nc_get_var(ncid, varid1, float_in)) ERR; + if (nc_get_var(ncid, varid2, double_in)) ERR; +#if 0 + printf("\n"); +#endif + for (x = 0; x < DIM_LEN_5; x++) + { + fin.f = float_in[x]; + dfin.d = double_in[x]; +#if 0 + fout.f = float_data[x]; + dfout.d = double_data[x]; + printf ("float_data: %10f : 0x%x float_data_in: %10f : 0x%x\n", + float_data[x], fout.u, float_data[x], fin.u); + printf("double_data: %15g : 0x%16lx double_data_in: %15g : 0x%16lx\n", + double_data[x], dfout.u, double_data[x], dfin.u); +#endif + if (fin.u != xpect[q][x].u) ERR; + if (dfin.u != double_xpect[q][x].u) ERR; + } + + /* Close the file again. */ + if (nc_close(ncid)) ERR; + } + } + } + SUMMARIZE_ERR; + printf("\t*** Checking BitGroom values with type conversion between ints and floats...\n"); + { + for (q = 0; q < NUM_QUANTIZE_MODES; q++) + { + int ncid; + int dimid; + int varid1, varid2; + unsigned char uc = 99; + signed char sc = -99; + unsigned short us = 9999; + signed short ss = -9999; + unsigned int ui = 9999999; + signed int si = -9999999; + unsigned long long int ull = 999999999; + signed long long int sll = -999999999; + size_t index; + + printf("\t\t**** testing quantize algorithm %d...\n", quantize_mode[q]); + + /* Create file. */ + if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR; + + /* Create dims. */ + if (nc_def_dim(ncid, X_NAME, DIM_LEN_8, &dimid)) ERR; + + /* Create the variables. */ + if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIM1, &dimid, &varid1)) ERR; + if (nc_def_var(ncid, VAR_NAME_2, NC_DOUBLE, NDIM1, &dimid, &varid2)) ERR; + + /* Set up quantization. */ + if (nc_def_var_quantize(ncid, varid1, quantize_mode[q], NSD_3)) ERR; + if (nc_def_var_quantize(ncid, varid2, quantize_mode[q], NSD_3)) ERR; + + /* For classic mode, we must call enddef. */ + if (m) + if (nc_enddef(ncid)) ERR; + + /* Write data. */ + index = 0; + if (nc_put_var1_uchar(ncid, varid1, &index, &uc)) ERR; + if (nc_put_var1_uchar(ncid, varid2, &index, &uc)) ERR; + index = 1; + if (nc_put_var1_schar(ncid, varid1, &index, &sc)) ERR; + if (nc_put_var1_schar(ncid, varid2, &index, &sc)) ERR; + index = 2; + if (nc_put_var1_ushort(ncid, varid1, &index, &us)) ERR; + if (nc_put_var1_ushort(ncid, varid2, &index, &us)) ERR; + index = 3; + if (nc_put_var1_short(ncid, varid1, &index, &ss)) ERR; + if (nc_put_var1_short(ncid, varid2, &index, &ss)) ERR; + index = 4; + if (nc_put_var1_uint(ncid, varid1, &index, &ui)) ERR; + if (nc_put_var1_uint(ncid, varid2, &index, &ui)) ERR; + index = 5; + if (nc_put_var1_int(ncid, varid1, &index, &si)) ERR; + if (nc_put_var1_int(ncid, varid2, &index, &si)) ERR; + index = 6; + if (nc_put_var1_ulonglong(ncid, varid1, &index, &ull)) ERR; + if (nc_put_var1_ulonglong(ncid, varid2, &index, &ull)) ERR; + index = 7; + if (nc_put_var1_longlong(ncid, varid1, &index, &sll)) ERR; + if (nc_put_var1_longlong(ncid, varid2, &index, &sll)) ERR; + + /* Close the file. */ + if (nc_close(ncid)) ERR; + + { + float float_data_in[DIM_LEN_8]; + double double_data_in[DIM_LEN_8]; + int x; + + /* Now reopen the file and check. */ + if (nc_open(FILE_NAME, NC_NETCDF4, &ncid)) ERR; + if (nc_inq_varid(ncid,VAR_NAME,&varid1)) ERR; + if (nc_inq_varid(ncid,VAR_NAME_2,&varid2)) ERR; + + /* Read the data. */ + if (nc_get_var_float(ncid, varid1, float_data_in)) ERR; + if (nc_get_var_double(ncid, varid2, double_data_in)) ERR; + + union FU xpect[NUM_QUANTIZE_MODES][DIM_LEN_8]; + union DU double_xpect[NUM_QUANTIZE_MODES][DIM_LEN_8]; + + /* This test comes up with different answers to this than + * the corresponding bitgroom filter test, but that's + * OK. In netcdf-c quantization is applied as the data are + * written by the user, but in HDF5 filters, the bitgroom + * filter is applied to all data values as they are + * written to disk. See + * https://github.com/ccr/ccr/issues/194 for a full + * explanation. */ + switch (quantize_mode[q]) + { + case NC_QUANTIZE_BITGROOM: + xpect[0][0].u = 0x42c60000; + xpect[0][1].u = 0xc2c60000; + xpect[0][2].u = 0x461c3000; + xpect[0][3].u = 0xc61c3000; + xpect[0][4].u = 0x4b189000; + xpect[0][5].u = 0xcb189000; + xpect[0][6].u = 0x4e6e6000; + xpect[0][7].u = 0xce6e6000; + double_xpect[0][0].u = 0x4058c00000000000; + double_xpect[0][1].u = 0xc058c00000000000; + double_xpect[0][2].u = 0x40c3860000000000; + double_xpect[0][3].u = 0xc0c3860000000000; + double_xpect[0][4].u = 0x4163120000000000; + double_xpect[0][5].u = 0xc163120000000000; + double_xpect[0][6].u = 0x41cdcc0000000000; + double_xpect[0][7].u = 0xc1cdcc0000000000; + break; + case NC_QUANTIZE_GRANULARBR: + xpect[1][0].u = 0x42c60000; + xpect[1][1].u = 0xc2c60000; + xpect[1][2].u = 0x461c4000; + xpect[1][3].u = 0xc61c4000; + xpect[1][4].u = 0x4b18a000; + xpect[1][5].u = 0xcb18a000; + xpect[1][6].u = 0x4e6e6000; + xpect[1][7].u = 0xce6e6000; + double_xpect[1][0].u = 0x4058c00000000000; + double_xpect[1][1].u = 0xc058c00000000000; + double_xpect[1][2].u = 0x40c3880000000000; + double_xpect[1][3].u = 0xc0c3880000000000; + double_xpect[1][4].u = 0x4163140000000000; + double_xpect[1][5].u = 0xc163140000000000; + double_xpect[1][6].u = 0x41cdcc0000000000; + double_xpect[1][7].u = 0xc1cdcc0000000000; + break; + case NC_QUANTIZE_BITROUND: + xpect[2][0].u = 0x42c00000; + xpect[2][1].u = 0xc2c00000; + xpect[2][2].u = 0x46200000; + xpect[2][3].u = 0xc6200000; + xpect[2][4].u = 0x4b200000; + xpect[2][5].u = 0xcb200000; + xpect[2][6].u = 0x4e700000; + xpect[2][7].u = 0xce700000; + double_xpect[2][0].u = 0x4058000000000000; + double_xpect[2][1].u = 0xc058000000000000; + double_xpect[2][2].u = 0x40c4000000000000; + double_xpect[2][3].u = 0xc0c4000000000000; + double_xpect[2][4].u = 0x4164000000000000; + double_xpect[2][5].u = 0xc164000000000000; + double_xpect[2][6].u = 0x41ce000000000000; + double_xpect[2][7].u = 0xc1ce000000000000; + break; + default: + ERR; + } + + for (x = 0; x < DIM_LEN_8; x++) + { + union FU fin; + union DU dfin; + fin.f = float_data_in[x]; + dfin.d = double_data_in[x]; +#if 0 + printf ("%d float_data_in : %08.8f : 0x%x expected %08.8f : 0x%x\n", + x, float_data_in[x], fin.u, xpect[q][x].f, xpect[q][x].u); + printf ("%d double_data_in : %15g : 0x%lx expected %15g : 0x%lx\n", + x, double_data_in[x], dfin.u, double_xpect[q][x].d, double_xpect[q][x].u); +#endif + if (fin.u != xpect[q][x].u) ERR; + if (dfin.u != double_xpect[q][x].u) ERR; + } + + /* Close the file. */ + if (nc_close(ncid)) ERR; + } + } + } + SUMMARIZE_ERR; + printf("\t**** Nice, simple example of using BitGroom plus zlib..."); + { +#define DIM_LEN_SIMPLE 100 +#define EPSILON .1 + int ncid; + int dimid; + int varid1, varid2; + float *float_data; + double *double_data; + int i; + + /* Set up some data to write. */ + if (!(float_data = malloc(DIM_LEN_SIMPLE * sizeof(float)))) + ERR; + if (!(double_data = malloc(DIM_LEN_SIMPLE * sizeof(double)))) + ERR; + for (i = 0; i < DIM_LEN_SIMPLE; i++) + { + float_data[i] = 1.5f * (float)i; + double_data[i] = 1.5 * (double)i; + } + + /* Create the file. */ + if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR; + + /* Add one dimension. */ + if (nc_def_dim(ncid, "dim1", DIM_LEN_SIMPLE, &dimid)) ERR; + + /* Create two variables, one float, one double. Quantization + * may only be applied to floating point data. */ + if (nc_def_var(ncid, "var1", NC_FLOAT, NDIM1, &dimid, &varid1)) ERR; + if (nc_def_var(ncid, "var2", NC_DOUBLE, NDIM1, &dimid, &varid2)) ERR; + + /* Set up quantization. This will not make the data any + * smaller, unless compression is also turned on. In this + * case, we will set 3 significant digits. */ + if (nc_def_var_quantize(ncid, varid1, NC_QUANTIZE_BITGROOM, NSD_3)) ERR; + if (nc_def_var_quantize(ncid, varid2, NC_QUANTIZE_BITGROOM, NSD_3)) ERR; + +#ifdef TESTNCZARR +#ifdef NETCDF_ENABLE_NCZARR_FILTERS + /* Set up zlib compression. This will work better because the + * data are quantized, yielding a smaller output file. We will + * set compression level to 1, which is usually the best + * choice. */ + if (nc_def_var_deflate(ncid, varid1, 0, 1, 1)) ERR; +#endif +#endif + /* For classic mode, we must call enddef. */ + if (m) + if (nc_enddef(ncid)) ERR; + + /* Write the data. */ + if (nc_put_var_float(ncid, varid1, float_data)) ERR; + if (nc_put_var_double(ncid, varid2, double_data)) ERR; + + /* Close the file. */ + if (nc_close(ncid)) ERR; + + /* Check the resulting file for correctness. */ + { + float float_data_in[DIM_LEN_SIMPLE]; + double double_data_in[DIM_LEN_SIMPLE]; + + /* Now reopen the file and check. */ + if (nc_open(FILE_NAME, NC_NETCDF4, &ncid)) ERR; + if (nc_inq_varid(ncid,"var1",&varid1)) ERR; + if (nc_inq_varid(ncid,"var2",&varid2)) ERR; + + /* Read the data. */ + if (nc_get_var_float(ncid, varid1, float_data_in)) ERR; + if (nc_get_var_double(ncid, varid2, double_data_in)) ERR; + + for (i = 0; i < DIM_LEN_SIMPLE; i++) + { + if (fabs(float_data_in[i] - float_data[i]) > EPSILON) + ERR; + if (fabs(double_data_in[i] - double_data[i]) > EPSILON) + ERR; + } + + /* Close the file. */ + if (nc_close(ncid)) ERR; + } + + /* Free resources. */ + free(float_data); + free(double_data); + } + SUMMARIZE_ERR; + } + FINAL_RESULTS; +} diff --git a/v3_nczarr_test/test_readcaching.c b/v3_nczarr_test/test_readcaching.c new file mode 100644 index 0000000000..29002b283e --- /dev/null +++ b/v3_nczarr_test/test_readcaching.c @@ -0,0 +1,84 @@ +#include +#include +#include +#include + +#if defined(__APPLE__) && defined(__MACH__) +#include +#endif + +#include "netcdf.h" + +#define filename "file://tmp_cachetest.zarr#mode=zarr,file" +#define chunkSize (size_t)(1<<17) /* 128k */ +#define numCells (size_t)(50 * chunkSize) +#define numSteps (size_t)360 + +static float var[numCells]; + +size_t getPeakRSS(void) +{ + struct rusage rusage; + getrusage( RUSAGE_SELF, &rusage ); +#if defined(__APPLE__) && defined(__MACH__) + return (size_t)rusage.ru_maxrss; +#else + return (size_t)(rusage.ru_maxrss * 1024L); +#endif +} + +static void +nce(int istat) +{ + if (istat != NC_NOERR) + { + fprintf(stderr, "%s\n", nc_strerror(istat)); + exit(-1); + } +} + +int +main(void) +{ + printf("read: chunkSize=%zu, numCells=%zu, numSteps=%zu, filename=%s\n", chunkSize, numCells, numSteps, filename); + + int ncId; + nce(nc_open(filename, NC_NOWRITE, &ncId)); + + int varId; + nce(nc_inq_varid(ncId, "var", &varId)); + + size_t size, nelems; + float preemption; + nce(nc_get_var_chunk_cache(ncId, varId, &size, &nelems, &preemption)); + printf("default chunk cache: size=%zu, nelems=%zu, preemption=%g\n", size, nelems, preemption); + size = 4 * numCells; // one float field at one time step + nelems = 1000; + preemption = 0.5; + nce(nc_set_var_chunk_cache(ncId, varId, size, nelems, preemption)); + printf("set chunk cache: size=%zu, nelems=%zu, preemption=%g\n", size, nelems, preemption); + + { + for (size_t i = 0; i < numCells; ++i) var[i] = 0.0f; + for (size_t i = 0; i < numSteps; ++i) + { + size_t start[2], count[2]; + start[0] = i; start[1] = 0; + count[0] = 1; count[1] = numCells; + nce(nc_get_vara_float(ncId, varId, start, count, var)); + } + } + + nce(nc_close(ncId)); + + { + size_t mbused = getPeakRSS() / (1024 * 1024); + printf("Max mem: %zu MB\n", mbused); + if(mbused > 100) { + fprintf(stderr,"*** Failed: used: %luMB expected: < 100MB\n",mbused); + return (1); + } + } + + return 0; +} diff --git a/v3_nczarr_test/test_unlim_io.c b/v3_nczarr_test/test_unlim_io.c new file mode 100644 index 0000000000..9542bee90f --- /dev/null +++ b/v3_nczarr_test/test_unlim_io.c @@ -0,0 +1,125 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include +#include +#include +#include +#include "netcdf.h" +#include "ncpathmgr.h" +#include "nclist.h" + +#ifdef NETCDF_ENABLE_NCZARR +#include "zincludes.h" +#endif + +#include "test_utils.h" + +#define NDATA MAX_DATA +static int data[NDATA]; + +static size_t chunkprod; +static size_t dimprod; +static size_t datasize = 0; + +static int +writedata(void) +{ + int ret = NC_NOERR; + int i; + + for(i=0;idata == 0x7fffffff ? i: options->data); + + if(options->debug >= 1) { + fprintf(stderr,"write: dimlens=%s chunklens=%s\n", + printvector(options->rank,options->dimlens),printvector(options->rank,options->chunks)); + } + fprintf(stderr,"write vars: start=%s edges=%s stride=%s\n", + printvector(options->rank,options->start),printvector(options->rank,options->edges),printvector(options->rank,options->stride)); + if((ret = nc_put_vars(meta->ncid,meta->varid,options->start,options->edges,(ptrdiff_t*)options->stride,data))) + ERR(ret); + return 0; +} + +static int +readdata(void) +{ + int ret = NC_NOERR; + size_t i; + + memset(data,0,datasize); + + if(options->debug >= 1) + fprintf(stderr,"read: dimlens=%s chunklens=%s\n", + printvector(options->rank,options->dimlens),printvector(options->rank,options->chunks)); + fprintf(stderr,"read vars: start=%s edges=%s stride=%s", + printvector(options->rank,options->start), + printvector(options->rank,options->edges), + printvector(options->rank,options->stride)); + fprintf(stderr,"\n"); + if((ret = nc_get_vars(meta->ncid,meta->varid,options->start,options->edges,(ptrdiff_t*)options->stride,data))) + ERR(ret); + + for(i=0;irank, options->start, options->stop, options->stride, options->max); + if(odom == NULL) {ret = NC_ENOMEM; goto done;} + if(options->debug > 1) + fprintf(stderr,"genodom: odom = %s\n",odom_print(odom)); + /* Iterate the odometer */ + for(i=0;odom_more(odom);odom_next(odom),i++) { + printf("[%02d] %s\n",i,(i==0?odom_print(odom):odom_printshort(odom))); + } +done: + odom_free(odom); + return ret; +} +#endif + +int +main(int argc, char** argv) +{ + int stat = NC_NOERR; + size_t i; + + if((stat=getoptions(&argc,&argv))) goto done; + if((stat=verifyoptions(options))) goto done; + + if(meta->ncid == 0) { + fprintf(stderr,"File not found: %s\n",options->file); + ERR(NC_EACCESS); + } + + if(options->create == Create) { + if((stat = getmetadata(1))) ERR(stat); + if(meta->ncid && (stat = nc_close(meta->ncid))) ERR(stat); + } + if((stat = getmetadata(0))) + ERR(stat); + + dimprod = 1; + chunkprod = 1; + for(i=0;irank;i++) {dimprod *= options->dimlens[i]; chunkprod *= options->chunks[i];} + + datasize = dimprod*sizeof(int); + + switch (options->op) { + case Read: readdata(); break; + case Write: writedata(); break; + default: break; + } + if(meta->ncid) {if((stat = nc_close(meta->ncid))) ERR(stat);} + +done: + cleanup(); + return 0; +} diff --git a/v3_nczarr_test/test_unlim_vars.c b/v3_nczarr_test/test_unlim_vars.c new file mode 100644 index 0000000000..3e3dc1d6d7 --- /dev/null +++ b/v3_nczarr_test/test_unlim_vars.c @@ -0,0 +1,293 @@ +/* This is part of the netCDF package. + Copyright 2020 University Corporation for Atmospheric Research/Unidata + See COPYRIGHT file for conditions of use. + + Test netcdf-4 variables with unlimited dimensions. + + Ed Hartnett +*/ + +/* Test derived from nc_test4/tst_unlim_vars.c */ +#define TESTNCZARR + +#include +#include +#include "err_macros.h" + +#ifdef TESTNCZARR +#define FILE_NAME "file://tmp_unlim_vars.file#mode=nczarr,file" +#else +#define FILE_NAME "tst_unlim_vars.nc" +#endif + +#define SFC_TEMP_NAME "surface_temperature" +#define LAT_NAME "lat" +#define LAT_LEN 2 +#define LON_NAME "lon" +#define LON_LEN 3 +#define TIME_NAME "time" +#define NUM_TIMESTEPS 4 +#define NDIM3 3 +#define NDIM2 2 + +int +main(int argc, char **argv) +{ + int stat = NC_NOERR; + + NC_UNUSED(argc); + NC_UNUSED(argv); + + printf("\n*** Testing netcdf-4 variables with unlimited dimensions.\n"); + printf("*** Testing file with one var, one unlim dim..."); + { + int ncid, sfc_tempid; + float data_out[NUM_TIMESTEPS][LAT_LEN][LON_LEN], data_in[NUM_TIMESTEPS][LAT_LEN][LON_LEN]; + int lat, lon, time; + int dimids[NDIM3]; + nc_type xtype_in; + int ndims_in, dimids_in[10], natts_in, nvars_in, unlimdimid_in; + size_t len_in; + char name_in[NC_MAX_NAME+1]; + size_t start[NDIM3], count[NDIM3]; + int d; + + + /* Set up phony data. */ + for (time = 0; time < NUM_TIMESTEPS; time++) + for (lat = 0; lat < LAT_LEN; lat++) + for (lon = 0; lon < LON_LEN; lon++) + data_out[time][lat][lon] = 25.5f + (float)(lat + lon + time); + + /* Create a file with a 3D surface temp variable. */ + if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR; + + /* Create three dims, one unlimited. */ + if (nc_def_dim(ncid, TIME_NAME, NC_UNLIMITED, &dimids[0])) ERR; + if (nc_def_dim(ncid, LAT_NAME, LAT_LEN, &dimids[1])) ERR; + if (nc_def_dim(ncid, LON_NAME, LON_LEN, &dimids[2])) ERR; + + /* Define a var. */ + for (d = 0; d < NDIM3; d++) + dimids[d] = d; + if (nc_def_var(ncid, SFC_TEMP_NAME, NC_FLOAT, NDIM3, dimids, &sfc_tempid)) ERR; + + /* Check some metadata. */ + if (nc_inq(ncid, &ndims_in, &nvars_in, &natts_in, &unlimdimid_in)) ERR; + if (ndims_in != 3 || nvars_in != 1 || natts_in != 0 || unlimdimid_in != 0) ERR; + if (nc_inq_var(ncid, 0, name_in, &xtype_in, &ndims_in, dimids_in, + &natts_in)) ERR; + if (strcmp(name_in, SFC_TEMP_NAME) || xtype_in != NC_FLOAT || + ndims_in != 3 || natts_in != 0) ERR; + for (d = 0; d < NDIM3; d++) + if (dimids_in[d] != dimids[d]) ERR; + if (nc_inq_dim(ncid, 0, name_in, &len_in)) ERR; + if (len_in != 0 || strcmp(name_in, TIME_NAME)) ERR; + if (nc_inq_dim(ncid, 1, name_in, &len_in)) ERR; + if (len_in != LAT_LEN || strcmp(name_in, LAT_NAME)) ERR; + if (nc_inq_dim(ncid, 2, name_in, &len_in)) ERR; + if (len_in != LON_LEN || strcmp(name_in, LON_NAME)) ERR; + + { + size_t chunksizes[3]; + int i,storage; + fprintf(stderr,">>> chunks ="); + if (nc_inq_var_chunking(ncid, sfc_tempid, &storage, chunksizes)) ERR; + for(i=0;i<3;i++) fprintf(stderr," %llu", (unsigned long long)chunksizes[i]); + fprintf(stderr," ; storage=%d\n", storage); + } + + if (nc_close(ncid)) ERR; + + if ((stat=nc_open(FILE_NAME, 0, &ncid))) + ERR; + + /* Check metadata. */ + if (nc_inq(ncid, &ndims_in, &nvars_in, &natts_in, &unlimdimid_in)) ERR; + if (ndims_in != 3 || nvars_in != 1 || natts_in != 0 || unlimdimid_in != 0) ERR; + if (nc_inq_var(ncid, 0, name_in, &xtype_in, &ndims_in, dimids_in, + &natts_in)) ERR; + if (strcmp(name_in, SFC_TEMP_NAME) || xtype_in != NC_FLOAT || + ndims_in != 3 || natts_in != 0) ERR; /* _FillValue is defined att */ + for (d = 0; d < NDIM3; d++) + if (dimids_in[d] != dimids[d]) ERR; + if (nc_inq_dim(ncid, 0, name_in, &len_in)) ERR; + if (len_in != 0 || strcmp(name_in, TIME_NAME)) ERR; + if (nc_inq_dim(ncid, 1, name_in, &len_in)) ERR; + if (len_in != LAT_LEN || strcmp(name_in, LAT_NAME)) ERR; + if (nc_inq_dim(ncid, 2, name_in, &len_in)) ERR; + if (len_in != LON_LEN || strcmp(name_in, LON_NAME)) ERR; + + if (nc_close(ncid)) ERR; + + if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; + + /* Check metadata. */ + if (nc_inq(ncid, &ndims_in, &nvars_in, &natts_in, &unlimdimid_in)) ERR; + if (ndims_in != 3 || nvars_in != 1 || natts_in != 0 || unlimdimid_in != 0) ERR; /* _NCProperty */ + if (nc_inq_var(ncid, 0, name_in, &xtype_in, &ndims_in, dimids_in, + &natts_in)) ERR; + if (strcmp(name_in, SFC_TEMP_NAME) || xtype_in != NC_FLOAT || + ndims_in != 3 || natts_in != 0) ERR; + for (d = 0; d < NDIM3; d++) + if (dimids_in[d] != dimids[d]) ERR; + if (nc_inq_dim(ncid, 0, name_in, &len_in)) ERR; + if (len_in != 0 || strcmp(name_in, TIME_NAME)) ERR; + if (nc_inq_dim(ncid, 1, name_in, &len_in)) ERR; + if (len_in != LAT_LEN || strcmp(name_in, LAT_NAME)) ERR; + if (nc_inq_dim(ncid, 2, name_in, &len_in)) ERR; + if (len_in != LON_LEN || strcmp(name_in, LON_NAME)) ERR; + if (nc_close(ncid)) ERR; + + /* Write some data to the var.*/ + if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; + for (d = 0; d < NDIM3; d++) + start[d] = 0; + count[0] = NUM_TIMESTEPS; + count[1] = LAT_LEN; + count[2] = LON_LEN; + if (nc_put_vara_float(ncid, 0, start, count, (float *)data_out)) ERR; + + /* Check metadata. */ + if (nc_inq(ncid, &ndims_in, &nvars_in, &natts_in, &unlimdimid_in)) ERR; + if (ndims_in != 3 || nvars_in != 1 || natts_in != 0 || unlimdimid_in != 0) ERR; + if (nc_inq_var(ncid, 0, name_in, &xtype_in, &ndims_in, dimids_in, + &natts_in)) ERR; + if (strcmp(name_in, SFC_TEMP_NAME) || xtype_in != NC_FLOAT || + ndims_in != 3 || natts_in != 0) ERR; + for (d = 0; d < NDIM3; d++) + if (dimids_in[d] != dimids[d]) ERR; + if (nc_inq_dim(ncid, 0, name_in, &len_in)) ERR; + if (len_in != NUM_TIMESTEPS || strcmp(name_in, TIME_NAME)) ERR; + if (nc_inq_dim(ncid, 1, name_in, &len_in)) ERR; + if (len_in != LAT_LEN || strcmp(name_in, LAT_NAME)) ERR; + if (nc_inq_dim(ncid, 2, name_in, &len_in)) ERR; + if (len_in != LON_LEN || strcmp(name_in, LON_NAME)) ERR; + + if (nc_close(ncid)) ERR; + + if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; + + /* Check data. */ + if (nc_get_vara_float(ncid, 0, start, count, (float *)data_in)) ERR; + for (time = 0; time < NUM_TIMESTEPS; time++) + for (lat = 0; lat < LAT_LEN; lat++) + for (lon = 0; lon < LON_LEN; lon++) + if (data_in[time][lat][lon] != data_out[time][lat][lon]) ERR; + + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; + printf("*** Testing netcdf-4 variable 2 unlimited dimensions..."); + { +#define DIM0_NAME "Speed" +#define DIM1_NAME "Height" +#define VAR_NAME "Superman" +#define VAR1_NAME "Black_Widow" + int ncid, varid, varid1, dimid[NDIM2]; + int data = TEST_VAL_42; + int data_in; + size_t index[NDIM2] = {1, 1}; + char name_in[NC_MAX_NAME + 1]; + size_t len_in; + + /* Create a file with 2 unlimited dims. */ + if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR; + if (nc_def_dim(ncid, DIM0_NAME, NC_UNLIMITED, &dimid[0])) ERR; + if (nc_def_dim(ncid, DIM1_NAME, NC_UNLIMITED, &dimid[1])) ERR; + if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM2, dimid, &varid)) ERR; + if (nc_def_var(ncid, VAR1_NAME, NC_INT, NDIM2, dimid, &varid1)) ERR; + if (nc_close(ncid)) ERR; + + /* Check the file. */ + if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR; + if (nc_inq_dim(ncid, 0, name_in, &len_in)) ERR; + if (strcmp(name_in, DIM0_NAME) || len_in != 0) ERR; + if (nc_inq_dim(ncid, 1, name_in, &len_in)) ERR; + if (strcmp(name_in, DIM1_NAME) || len_in != 0) ERR; + if (nc_close(ncid)) ERR; + + /* Reopen the file and add data. */ + if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; + if (nc_put_var1_int(ncid, 0, index, &data)) ERR; + if (nc_inq_dim(ncid, 0, NULL, &len_in)) ERR; + if (len_in != 2) ERR; + if (nc_inq_dim(ncid, 1, NULL, &len_in)) ERR; + if (len_in != 2) ERR; + if (nc_get_var1_int(ncid, 0, index, &data_in)) ERR; + if (data_in != data) ERR; + if (nc_get_var1_int(ncid, 1, index, &data_in)) ERR; + if (data_in != NC_FILL_INT) ERR; + if (nc_close(ncid)) ERR; + + } + SUMMARIZE_ERR; + printf("*** Testing netcdf-4 variable 3 unlimited dimensions..."); + { +#define D3_DIM0_NAME "Gadgets" +#define D3_DIM1_NAME "Brains" +#define D3_DIM2_NAME "Money" +#define D3_VAR_NAME "Batman" +#define D3_VAR1_NAME "Aquaman" + int ncid, varid, dimid[NDIM3]; + int data = TEST_VAL_42; + int data_in; + size_t start[NDIM3] = {1, 1, 1}, count[NDIM3] = {1, 1, 1}; + size_t index[NDIM3] = {0, 0, 1}; + char name_in[NC_MAX_NAME + 1]; + size_t len_in; + + /* Create a file with 2 unlimited dims. */ + if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR; + if (nc_def_dim(ncid, D3_DIM0_NAME, NC_UNLIMITED, &dimid[0])) ERR; + if (nc_def_dim(ncid, D3_DIM1_NAME, NC_UNLIMITED, &dimid[1])) ERR; + if (nc_def_dim(ncid, D3_DIM2_NAME, NC_UNLIMITED, &dimid[2])) ERR; + if (nc_def_var(ncid, D3_VAR_NAME, NC_INT, NDIM3, dimid, &varid)) ERR; + if (nc_def_var(ncid, D3_VAR1_NAME, NC_INT, NDIM3, dimid, &varid)) ERR; + if (nc_close(ncid)) ERR; + + /* Check the file. */ + if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR; + if (nc_inq_dim(ncid, 0, name_in, &len_in)) ERR; + if (strcmp(name_in, D3_DIM0_NAME) || len_in != 0) ERR; + if (nc_inq_dim(ncid, 1, name_in, &len_in)) ERR; + if (strcmp(name_in, D3_DIM1_NAME) || len_in != 0) ERR; + if (nc_inq_dim(ncid, 2, name_in, &len_in)) ERR; + if (strcmp(name_in, D3_DIM2_NAME) || len_in != 0) ERR; + if (nc_close(ncid)) ERR; + + /* Reopen the file and add data. */ + if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; + if (nc_put_vara_int(ncid, 0, start, count, &data)) ERR; + if (nc_put_var1_int(ncid, 1, index, &data)) ERR; + if (nc_inq_dim(ncid, 0, NULL, &len_in)) ERR; + if (len_in != 2) ERR; + if (nc_inq_dim(ncid, 1, NULL, &len_in)) ERR; + if (len_in != 2) ERR; + if (nc_inq_dim(ncid, 2, NULL, &len_in)) ERR; + if (len_in != 2) ERR; + if (nc_get_vara_int(ncid, 0, start, count, &data_in)) ERR; + if (data_in != data) ERR; + if (nc_get_vara_int(ncid, 1, start, count, &data_in)) ERR; + if (data_in != NC_FILL_INT) ERR; + if (nc_close(ncid)) ERR; + + /* Reopen file and check again. */ + if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR; + if (nc_inq_dim(ncid, 0, NULL, &len_in)) ERR; + if (len_in != 2) ERR; + if (nc_inq_dim(ncid, 1, NULL, &len_in)) ERR; + if (len_in != 2) ERR; + if (nc_inq_dim(ncid, 2, NULL, &len_in)) ERR; + if (len_in != 2) ERR; + if (nc_get_vara_int(ncid, 0, start, count, &data_in)) ERR; + if (data_in != data) ERR; + if (nc_get_vara_int(ncid, 1, start, count, &data_in)) ERR; + if (data_in != NC_FILL_INT) ERR; + if (nc_get_var1_int(ncid, 1, index, &data_in)) ERR; + if (data_in != data) ERR; + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; + FINAL_RESULTS; +} diff --git a/v3_nczarr_test/test_unlimited.c b/v3_nczarr_test/test_unlimited.c new file mode 100644 index 0000000000..cc6d486dff --- /dev/null +++ b/v3_nczarr_test/test_unlimited.c @@ -0,0 +1,196 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include +#include +#include +#include "netcdf.h" +#include "ncpathmgr.h" +#include "nclist.h" + +#ifdef USE_HDF5 +#include +#include +#endif + +#ifdef NETCDF_ENABLE_NCZARR +#include "zincludes.h" +#endif + + +#include "test_utils.h" + +static unsigned chunkprod; +static unsigned dimprod; +static int* data = NULL; +static size_t datasize = 0; + +static void +usage(void) +{ + fprintf(stderr,"usage: tst_unlimited " +" [-3|-4]" +" [-c int,int...]" +" [-d int,int...]" +" [-e int,int...]" +" [-f int,int...]" +" [-m int,int...]" +" [-n int]" +" [-p int,int...]" +" [-s int,int...]" +" [-D int]" +" [-O x|r|w|W|o]" +" [-T ]" +" [-X opt|wd]" +" \n" +"where:\n" +"-3|-4 -- mode: netcdf-3 vs netcdf-4\n" +"-c -- chunksizes\n" +"-d -- dimension sizes: 0 => unlimited\n" +"-e -- edge vector\n" +"-f -- start (aka first) vector\n" +"-m -- max vector\n" +"-n -- rank\n" +"-p -- stop vector\n" +"-s -- stride vector\n" +"-D -- debug level\n" +"-O -- operation: o=>odometer x=>extend r=>read w=>write W=>wholechunk\n" +"-T -- trace level\n" +"-X -- extra options: opt=>optimize wd=>walk debug\n" +" \n"); +} + +static int +writedata(void) +{ + int ret = NC_NOERR; + size_t i; + + /* Make sure data > 0 so we can distinguish written from unwritten */ + for(i=0;idebug >= 1) { + fprintf(stderr,"write: dimlens=%s chunklens=%s\n", + printvector(options->rank,options->dimlens),printvector(options->rank,options->chunks)); + } + { + fprintf(stderr,"write vars: start=%s count=%s stride=%s\n", + printvector(options->rank,options->start),printvector(options->rank,options->count),printvector(options->rank,options->stride)); + if((ret = nc_put_vars(meta->ncid,meta->varid,options->start,options->count,(ptrdiff_t*)options->stride,data))) + ERR(ret); + } + + return 0; +} + +static int +readdata(void) +{ + int ret = NC_NOERR; + size_t i; + + memset(data,0,datasize); + + if(options->debug >= 1) + fprintf(stderr,"read: dimlens=%s chunklens=%s\n", + printvector(options->rank,options->dimlens),printvector(options->rank,options->chunks)); + fprintf(stderr,"read vars: start=%s count=%s stride=%s", + printvector(options->rank,options->start), + printvector(options->rank,options->count), + printvector(options->rank,options->stride)); + fprintf(stderr,"\n"); + if((ret = nc_get_vars(meta->ncid,meta->varid,options->start,options->count,(ptrdiff_t*)options->stride,data))) + ERR(ret); + + for(i=0;i 0 so we can distinguish written from unwritten */ + for(i=0;idebug >= 1) { + fprintf(stderr,"write: dimlens=%s chunklens=%s\n", + printvector(options->rank,options->dimlens),printvector(options->rank,options->chunks)); + } + { + fprintf(stderr,"write vars: start=%s count=%s stride=%s\n", + printvector(options->rank,options->start),printvector(options->rank,options->count),printvector(options->rank,options->stride)); + if((ret = nc_put_vars(meta->ncid,meta->varid,options->start,options->count,(ptrdiff_t*)options->stride,data))) + ERR(ret); + } + + return 0; +} + +static int +genodom(void) +{ + int i,ret = NC_NOERR; + Odometer* odom = odom_new(options->rank, options->start, options->stop, options->stride, options->max); + if(odom == NULL) {ret = NC_ENOMEM; goto done;} + if(options->debug > 1) + fprintf(stderr,"genodom: odom = %s\n",odom_print(odom)); + /* Iterate the odometer */ + for(i=0;odom_more(odom);odom_next(odom),i++) { + printf("[%02d] %s\n",i,(i==0?odom_print(odom):odom_printshort(odom))); + } +done: + odom_free(odom); + return ret; +} + +int +main(int argc, char** argv) +{ + int stat = NC_NOERR; + int i; + + if((stat=getoptions(&argc,&argv))) goto done; + + switch (options->op) { + case Read: + if((stat = getmetadata(0))) + ERR(stat); + if (argc == 0) {fprintf(stderr, "no input file specified\n");exit(1);} + break; + case Write: + if((stat = getmetadata(1))) + ERR(stat); + if (argc == 0) {fprintf(stderr, "no output file specified\n");exit(1);} + break; + default: + break; /* do not need a file */ + } + + dimprod = 1; + chunkprod = 1; + for(i=0;irank;i++) {dimprod *= options->dimlens[i]; chunkprod *= options->chunks[i];} + + datasize = dimprod*sizeof(int); + if((data = calloc(1,datasize)) == NULL) + {fprintf(stderr,"out of memory\n"); exit(1);} + + switch (options->op) { + case Read: readdata(); break; + case Write: writedata(); break; + case Odom: genodom(); break; + case Extend: extenddata(); break; + default: + fprintf(stderr,"Unknown operation\n"); + exit(1); + } +done: + if(data) free(data); + cleanup(); + return 0; +} diff --git a/v3_nczarr_test/test_utils.c b/v3_nczarr_test/test_utils.c new file mode 100644 index 0000000000..abeb454ee7 --- /dev/null +++ b/v3_nczarr_test/test_utils.c @@ -0,0 +1,653 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_GETOPT_H +#include +#endif + +#if defined(_WIN32) && !defined(__MINGW32__) +#include "XGetopt.h" +#endif + +#ifdef USE_HDF5 +#include +#include +#endif + +#include "test_utils.h" + +Options* options = NULL; +Metadata* meta = NULL; + +NClist* capture = NULL; + +void +usage(int err) +{ + if(err) { + fprintf(stderr,"error: (%d) %s\n",err,nc_strerror(err)); + } + fprintf(stderr,"usage:"); + fprintf(stderr," -d [,*]"); + fprintf(stderr," -c [,*]"); + fprintf(stderr," -s [,*]"); + fprintf(stderr," -e [,*]"); + fprintf(stderr," -i [,*]"); + fprintf(stderr," -v [,*]"); + fprintf(stderr," -x c|r|w"); + fprintf(stderr," "); + fprintf(stderr,"\n"); + fflush(stderr); + exit(1); +} + +static void +CHECKRANK(size_t r) +{ + if(options->rank == 0) + options->rank = r; + else if(r != options->rank) { + fprintf(stderr,"FAIL: options->rank mismatch\n"); + exit(1); + } +} + +int +getoptions(int* argcp, char*** argvp) +{ + int ret = NC_NOERR; + size_t i; + int c; + const char* p; + + /* initialize */ + if(options == NULL) { + if((options = calloc(1,sizeof(Options))) == NULL) + {ret = NC_ENOMEM; goto done;} + } + /* Set defaults */ + options->mode = 0; /* classic netcdf-3 */ + + while ((c = getopt(*argcp, *argvp, "34c:d:e:hi:m:n:p:s:v:D:O:T:X:")) != EOF) { + switch(c) { + case '3': + options->mode = 0; + break; + case '4': + options->mode = NC_NETCDF4; + break; + case 'c': + CHECKRANK(parsevector(optarg,options->chunks)); + options->flags |= HAS_CHUNKS; + break; + case 'd': + CHECKRANK(parsevector(optarg,options->dimlens)); + options->flags |= HAS_DIMLENS; + break; + case 'e': + CHECKRANK(parsevector(optarg,options->edges)); + options->flags |= HAS_EDGES; + break; + case 'h': usage(0); break; + case 'i': + CHECKRANK(parsevector(optarg,options->stride)); + options->flags |= HAS_STRIDE; + break; + case 'm': + CHECKRANK(parsevector(optarg,options->max)); + options->flags |= HAS_MAX; + break; + case 'n': + CHECKRANK((size_t)atoi(optarg)); + break; + case 'p': + CHECKRANK(parsevector(optarg,options->stop)); + options->flags |= HAS_STOP; + break; + case 's': + CHECKRANK(parsevector(optarg,options->start)); + options->flags |= HAS_START; + break; + case 'v': + if(strcmp(optarg,"n")==0) + options->data = 0x7fffffff; + else + options->data = atoi(optarg); + options->flags |= HAS_DATA; + break; + case 'D': + options->debug = (unsigned)atoi(optarg); + break; + case 'O': + for(p=optarg;*p;p++) { + switch (*p) { + case 'c': options->create = Create; break; + case 'r': options->op = Read; break; + case 'w': options->op = Write; break; + case 'x': options->op = Extend; break; + case 'o': options->op = Odom; break; + case 'W': options->wholechunk = 1; break; + default: fprintf(stderr,"Unknown operation '%c'\n",*p); exit(1); + } + } break; + case 'T': + nctracelevel(atoi(optarg)); + break; + case 'X': + if(strcmp(optarg,"opt")==0) { + options->optimize = 1; + } else if(strncmp(optarg,"wd",2)==0) { + options->wdebug = atoi(optarg+2); + } + break; + case '?': + fprintf(stderr,"unknown option\n"); + exit(1); + } + } + + /* get file argument */ + *argcp -= optind; + *argvp += optind; + + if(*argcp > 0) { + char* p = NC_shellUnescape((*argvp)[0]); + strcpy(options->file,filenamefor(p)); + nullfree(p); + } + + /* Figure out the FORMATX for this file */ + if(options->file[0]) { + NCURI* uri = NULL; + ncuriparse(options->file,&uri); + if(uri == NULL) { /* not a url */ + switch (options->mode) { + default: /* fall thru to default */ + case 0: options->formatx = NC_FORMATX_NC3; break; + case NC_NETCDF4: options->formatx = NC_FORMATX_NC4; break; + } + } else { + options->formatx = NC_FORMATX_NCZARR; /* assume */ + ncurifree(uri); + } + } + if(options->debug) { + const char* fmt = "unknown"; + switch(options->formatx) { + case NC_FORMATX_NC3: fmt = "NC3"; break; + case NC_FORMATX_NC4: fmt = "NC4"; break; + case NC_FORMATX_NCZARR: fmt = "NCZARR"; break; + default: break; + } + fprintf(stderr,"Formatx: %s\n",fmt); + } + +#ifndef _WIN32 + if(options->wdebug) { + char s[64]; + snprintf(s,sizeof(s),"%d",options->wdebug); + setenv("NCZ_WDEBUG",s,1); + } + if(options->optimize) { + unsetenv("NCZ_NOOPTIMIZE"); + } else { + setenv("NCZ_NOOPTIMIZE","1",1); + } +#endif + + /* Default some vectors */ + if(!(options->flags & HAS_STRIDE)) { + for(i=0;istride[i] = 1;} + options->flags |= HAS_STRIDE; + } + + /* Computed Defaults */ + if((options->flags & HAS_EDGES) && (options->flags & HAS_STOP)) { + fprintf(stderr,"cannot specify both edges and stop\n"); + ERR(NC_EINVAL); + } + if( !(options->flags & HAS_EDGES) + && (options->flags & HAS_DIMLENS) + && (options->flags & HAS_STRIDE)) { + for(i=0;irank;i++) + options->edges[i] = (options->dimlens[i]+options->stride[i]-1)/options->stride[i]; + options->flags |= HAS_EDGES; + } + if( !(options->flags & HAS_STOP) + && (options->flags & HAS_START) + && (options->flags & HAS_EDGES) + && (options->flags & HAS_STRIDE)) { + for(i=0;irank;i++) + options->stop[i] = options->start[i] + (options->edges[i] * options->stride[i]); + options->flags |= HAS_STOP; + } + if( !(options->flags & HAS_EDGES) + && (options->flags & HAS_STRIDE) + && (options->flags & HAS_STOP)) { + for(i=0;irank;i++) + options->edges[i] = ((options->stop[i]+(options->stride[i]-1)) / (options->stride[i])); + options->flags |= HAS_EDGES; + } + + if( !(options->flags & HAS_MAX) + && (options->flags & HAS_STOP)) { + for(i=0;imax[i] = options->stop[i];} + options->flags |= HAS_MAX; + } + + if(options->create == Create) { + if((ret=getmetadata(1))) + ERR(ret); + if(meta->ncid && (ret = nc_close(meta->ncid))) + ERR(ret); + } + + if(options->debug) { +#ifdef USE_HDF5 + H5Eset_auto2(H5E_DEFAULT,(H5E_auto2_t)H5Eprint1,stderr); +#endif + } + +done: + return ret; +} + +int +verifyoptions(Options* options) +{ + int ret = NC_NOERR; + /* Check that we have or can compute relevant values */ + if(options->create == Create) { + if(!(options->flags & HAS_DIMLENS)) { + fprintf(stderr,"dimlens option not computable\n"); + ERR(NC_EINVAL); + } + if(!(options->flags & HAS_CHUNKS)) { + fprintf(stderr,"chunklens option not computable\n"); + ERR(NC_EINVAL); + } + if(!(options->flags & HAS_STRIDE)) { + fprintf(stderr,"stride option not computable\n"); + ERR(NC_EINVAL); + } + } + if(options->op == Read || options->op == Write) { + if(!(options->flags & HAS_STOP)) { + fprintf(stderr,"stop option not computable\n"); + ERR(NC_EINVAL); + } + if(!(options->flags & HAS_EDGES)) { + fprintf(stderr,"edges option not computable\n"); + ERR(NC_EINVAL); + } + if(!(options->flags & HAS_MAX)) { + fprintf(stderr,"max option not computable\n"); + ERR(NC_EINVAL); + } + } + return ret; +} + +int +getmetadata(int create) +{ + int ret = NC_NOERR; + char dname[NC_MAX_NAME]; + size_t i; + + if(meta == NULL) { + if((meta = calloc(1,sizeof(Metadata)))==NULL) + {ret = NC_ENOMEM; goto done;} + /* Non-zero defaults */ + meta->fill = -1; + } + + if(create) { + if((options->flags & (HAS_DIMLENS | HAS_CHUNKS)) != (HAS_DIMLENS | HAS_CHUNKS)) { + fprintf(stderr,"dimlens or chunks not specified\n"); + ret = NC_EINVAL; + goto done; + } + if((ret = nc_create(options->file,options->mode|NC_CLOBBER,&meta->ncid))) goto done; + for(i=0;irank;i++) { + snprintf(dname,sizeof(dname),"d%zu",i); + if(options->dimlens[i] == 0) + ret = nc_def_dim(meta->ncid,dname,NC_UNLIMITED,&meta->dimids[i]); + else + ret = nc_def_dim(meta->ncid,dname,options->dimlens[i],&meta->dimids[i]); + if(ret) goto done; + } + if((ret = nc_def_var(meta->ncid,"v",NC_INT,options->rank,meta->dimids,&meta->varid))) goto done; + if((ret = nc_def_var_fill(meta->ncid,meta->varid,0,&meta->fill))) goto done; + if(options->formatx == NC_FORMATX_NC4 || options->formatx == NC_FORMATX_NCZARR) { + if((ret = nc_def_var_chunking(meta->ncid,meta->varid,NC_CHUNKED,options->chunks))) goto done; + } + if((ret = nc_enddef(meta->ncid))) goto done; + } else {/*Open*/ + if((ret = nc_open(options->file,options->mode|NC_WRITE,&meta->ncid))) goto done; + for(i=0;irank;i++) { + snprintf(dname,sizeof(dname),"d%zu",i); + if((ret = nc_inq_dimid(meta->ncid,dname,&meta->dimids[i]))) goto done; + if((ret = nc_inq_dimlen(meta->ncid,meta->dimids[i],&options->dimlens[i]))) goto done; + } + options->flags |= HAS_DIMLENS; + if((ret = nc_inq_varid(meta->ncid,"v",&meta->varid))) goto done; + if(options->formatx == NC_FORMATX_NC4 || options->formatx == NC_FORMATX_NCZARR) { + int storage = -1; + /* Get chunk sizes also */ + if((ret = nc_inq_var_chunking(meta->ncid,meta->varid,&storage,options->chunks))) goto done; + if(storage != NC_CHUNKED) {ret = NC_EBADCHUNK; goto done;} + options->flags |= HAS_CHUNKS; + } + } + +done: + return ret; +} + +void +cleanup(void) +{ + if(meta) { + if(meta->ncid) {nc_close(meta->ncid); meta->ncid = 0;} + } + nclistfreeall(capture); + nullfree(meta); + nullfree(options); +} + +size_t +parsevector(const char* s0, size_t* vec) +{ + char* s = strdup(s0); + char* p = NULL; + size_t i, done; + + if(s0 == NULL || vec == NULL) abort(); + + for(done=0,p=s,i=0;!done;) { + char* q; + q = p; + p = strchr(q,','); + if(p == NULL) {p = q+strlen(q); done=1;} + *p++ = '\0'; + if(strcasecmp(q,"u")==0) + vec[i++] = 0; /* signals unlimited */ + else + vec[i++] = (size_t)atol(q); + } + if(s) free(s); + return i; +} + +size_t +parsedata(const char* s0, int* data) +{ + char* s = strdup(s0); + char* p = NULL; + size_t i, done; + + + if(s0 == NULL || data == NULL) abort(); + + for(done=0,p=s,i=0;!done;) { + char* q; + q = p; + p = strchr(q,','); + if(p == NULL) {p = q+strlen(q); done=1;} + *p++ = '\0'; + data[i++] = atoi(q); + } + if(s) free(s); + return i; +} + +const char* +printvector(size_t rank, const size_t* vec) +{ + size64_t v64[NC_MAX_VAR_DIMS]; + size_t r; + for(r=0;r 0) strcat(s,","); + strcat(s,e); + } + if(capture == NULL) capture = nclistnew(); + ss = strdup(s); + nclistpush(capture,ss); + return ss; +} + +Odometer* +odom_new(size_t rank, const size_t* start, const size_t* stop, const size_t* stride, const size_t* max) +{ + size_t i; + Odometer* odom = NULL; + if((odom = calloc(1,sizeof(Odometer))) == NULL) + return NULL; + odom->rank = rank; + for(i=0;istart[i] = start[i]; + odom->stop[i] = stop[i]; + odom->stride[i] = stride[i]; + odom->max[i] = (max?max[i]:stop[i]); + odom->edges[i] = (odom->stop[i]+odom->stride[i]-1)/odom->stride[i]; + odom->index[i] = 0; + } + return odom; +} + +void +odom_free(Odometer* odom) +{ + if(odom) free(odom); +} + +int +odom_more(Odometer* odom) +{ + return (odom->index[0] < odom->stop[0]); +} + +int +odom_next(Odometer* odom) +{ + int i; + for(i=odom->rank-1;i>=0;i--) { + odom->index[i] += odom->stride[i]; + if(odom->index[i] < odom->stop[i]) break; + if(i == 0) return 0; /* leave the 0th entry if it overflows */ + odom->index[i] = odom->start[i]; /* reset this position */ + } + return 1; +} + +/* Get the value of the odometer */ +size_t* +odom_indices(Odometer* odom) +{ + return odom->index; +} + +size_t +odom_offset(Odometer* odom) +{ + size_t offset; + size_t i; + + offset = 0; + for(i=0;irank;i++) { + offset *= odom->max[i]; + offset += odom->index[i]; + } + return offset; +} + +const char* +odom_print1(Odometer* odom, int isshort) +{ + static char s[4096]; + static char tmp[4096]; + const char* sv; + + s[0] = '\0'; + strcat(s,"{"); + if(!isshort) { + snprintf(tmp,sizeof(tmp),"rank=%u",(unsigned)odom->rank); strcat(s,tmp); + strcat(s," start=("); sv = printvector(odom->rank,odom->start); strcat(s,sv); strcat(s,")"); + strcat(s," stop=("); sv = printvector(odom->rank,odom->stop); strcat(s,sv); strcat(s,")"); + strcat(s," stride=("); sv = printvector(odom->rank,odom->stride); strcat(s,sv); strcat(s,")"); + strcat(s," max=("); sv = printvector(odom->rank,odom->max); strcat(s,sv); strcat(s,")"); + strcat(s," edges=("); sv = printvector(odom->rank,odom->edges); strcat(s,sv); strcat(s,")"); + } + snprintf(tmp,sizeof(tmp)," offset=%u",(unsigned)odom_offset(odom)); strcat(s,tmp); + strcat(s," indices=("); sv = printvector(odom->rank,odom->index); strcat(s,sv); strcat(s,")"); + strcat(s,"}"); + return s; +} + +const char* +odom_print(Odometer* odom) +{ + return odom_print1(odom,0); +} + +const char* +odom_printshort(Odometer* odom) +{ + return odom_print1(odom,1); +} + +static const char* urlexts[] = {"nzf", "zip", "nz4", NULL}; + +nc_type +gettype(const char* name) +{ + if(strcasecmp(name,"byte")==0) return NC_BYTE; + if(strcasecmp(name,"ubyte")==0) return NC_UBYTE; + if(strcasecmp(name,"short")==0) return NC_SHORT; + if(strcasecmp(name,"ushort")==0) return NC_USHORT; + if(strcasecmp(name,"int")==0) return NC_INT; + if(strcasecmp(name,"uint")==0) return NC_UINT; + if(strcasecmp(name,"int64")==0) return NC_INT64; + if(strcasecmp(name,"uint64")==0) return NC_UINT64; + if(strcasecmp(name,"float")==0) return NC_FLOAT; + if(strcasecmp(name,"double")==0) return NC_DOUBLE; + return NC_NAT; +} + +size_t +gettypesize(nc_type t) +{ + switch (t) { + case NC_BYTE: return sizeof(char); + case NC_UBYTE: return sizeof(unsigned char); + case NC_SHORT: return sizeof(short); + case NC_USHORT: return sizeof(unsigned short); + case NC_INT: return sizeof(int); + case NC_UINT: return sizeof(unsigned int); + case NC_INT64: return sizeof(long long int); + case NC_UINT64: return sizeof(unsigned long long int); + case NC_FLOAT: return sizeof(float); + case NC_DOUBLE: return sizeof(double); + default: break; + } + return 0; +} + +const char* +gettypename(nc_type t) +{ + switch (t) { + case NC_BYTE: return "byte"; + case NC_UBYTE: return "ubyte"; + case NC_SHORT: return "short"; + case NC_USHORT: return "ushort"; + case NC_INT: return "int"; + case NC_UINT: return "uint"; + case NC_INT64: return "int64"; + case NC_UINT64: return "uint64"; + case NC_FLOAT: return "float"; + case NC_DOUBLE: return "double"; + default: break; + } + return NULL; +} + +const char* +filenamefor(const char* f0) +{ + static char result[4096]; + const char** extp; + char* p; + + strcpy(result,f0); /* default */ + if(nc__testurl(f0,NULL)) goto done; + /* Not a URL */ + p = strrchr(f0,'.'); /* look at the extension, if any */ + if(p == NULL) goto done; /* No extension */ + p++; + for(extp=urlexts;*extp;extp++) { + if(strcmp(p,*extp)==0) break; + } + if(*extp == NULL) goto done; /* not found */ + /* Assemble the url */ + strcpy(result,"file://"); + strcat(result,f0); /* core path */ + strcat(result,"#mode=nczarr,"); + strcat(result,*extp); +done: + return result; +} + +static char s3testurl[8192]; +static char* s3testurlp = NULL; + +const char* +ncz_gets3testurl(void) +{ + char* s; + if(s3testurlp == NULL) { + s3testurl[0] = '\0'; + strcat(s3testurl,"https://"); + s = getenv("NCZARR_S3_TEST_HOST"); + if(s == NULL) s = "stratus.ucar.edu"; + strcat(s3testurl,s); + strcat(s3testurl,"/"); + s = getenv("NCZARR_S3_TEST_BUCKET"); + if(s == NULL) s = "unidata-netcdf-zarr-testing"; + strcat(s3testurl,s); + s3testurlp = s3testurl; + } + return s3testurlp; +} + +void +ncz_report(int err, int lineno) +{ + fprintf(stderr,"Error: %d: %s\n", lineno, nc_strerror(err)); + exit(1); +} + diff --git a/v3_nczarr_test/test_utils.h b/v3_nczarr_test/test_utils.h new file mode 100644 index 0000000000..6f18ce737f --- /dev/null +++ b/v3_nczarr_test/test_utils.h @@ -0,0 +1,99 @@ +#ifndef TST_UTILS_H +#define TST_UTILS_H + +#include "netcdf.h" + +typedef enum Op {None=0, Create=1, Read=2, Write=3, Wholechunk=4, Odom=5, Extend=6} Op; + +/* Bit mask of defined options; powers of 2*/ +#define HAS_DIMLENS (1<<0) +#define HAS_CHUNKS (1<<1) +#define HAS_STRIDE (1<<2) +#define HAS_START (1<<3) +#define HAS_STOP (1<<4) +#define HAS_EDGES (1<<5) +#define HAS_MAX (1<<6) +#define HAS_DATA (1<<7) + +#define MAX_DATA 4096 + +/* Options */ + +typedef struct Options { + unsigned debug; + int wdebug; + int optimize; + int wholechunk; + Op create; + Op op; + int mode; + int formatx; + size_t rank; + char file[1024]; + unsigned flags; + size_t dimlens[NC_MAX_VAR_DIMS]; + size_t chunks[NC_MAX_VAR_DIMS]; + size_t start[NC_MAX_VAR_DIMS]; + size_t edges[NC_MAX_VAR_DIMS]; + size_t stride[NC_MAX_VAR_DIMS]; + size_t stop[NC_MAX_VAR_DIMS]; + size_t max[NC_MAX_VAR_DIMS]; + int data; +} Options; + +typedef struct Metadata { + int ncid; + int varid; + int dimids[NC_MAX_VAR_DIMS]; + int fill; +} Metadata; + +typedef struct Odometer { + size_t rank; /*rank */ + size_t start[NC_MAX_VAR_DIMS]; + size_t edges[NC_MAX_VAR_DIMS]; + size_t stride[NC_MAX_VAR_DIMS]; + size_t stop[NC_MAX_VAR_DIMS]; + size_t max[NC_MAX_VAR_DIMS]; /* max size of ith index */ + size_t index[NC_MAX_VAR_DIMS]; /* current value of the odometer*/ +} Odometer; + +extern void usage(int); + +EXTERNL Odometer* odom_new(size_t rank, const size_t* start, const size_t* stop, const size_t* stride, const size_t* max); +EXTERNL void odom_free(Odometer* odom); +EXTERNL int odom_more(Odometer* odom); +EXTERNL int odom_next(Odometer* odom); +EXTERNL size_t* odom_indices(Odometer* odom); +EXTERNL size_t odom_offset(Odometer* odom); +EXTERNL const char* odom_print1(Odometer* odom, int isshort); +EXTERNL const char* odom_print(Odometer* odom); +EXTERNL const char* odom_printshort(Odometer* odom); + +EXTERNL size_t parsevector(const char* s0, size_t* vec); +EXTERNL size_t parsedata(const char* s0, int* data); +EXTERNL const char* filenamefor(const char* f0); +EXTERNL const char* printvector(size_t rank, const size_t* vec); +EXTERNL const char* printvector64(size_t rank, const size64_t* vec); + +EXTERNL int getoptions(int* argcp, char*** argvp); +EXTERNL int verifyoptions(Options*); +EXTERNL int getmetadata(int create); +EXTERNL void cleanup(void); + +EXTERNL nc_type gettype(const char* name); +EXTERNL size_t gettypesize(nc_type t); +EXTERNL const char* gettypename(nc_type t); + +EXTERNL int nc__testurl(const char*,char**); + +EXTERNL const char* ncz_gets3testurl(void); + +EXTERNL Options* options; +EXTERNL Metadata* meta; + +EXTERNL void ncz_report(int err, int line); + +#define ERR(e) ncz_report(e,__LINE__) + +#endif /*TST_UTILS_H*/ diff --git a/v3_nczarr_test/test_writecaching.c b/v3_nczarr_test/test_writecaching.c new file mode 100644 index 0000000000..d4238185f8 --- /dev/null +++ b/v3_nczarr_test/test_writecaching.c @@ -0,0 +1,99 @@ +#include +#include +#include +#include +#if defined(__APPLE__) && defined(__MACH__) +#include +#endif + +#include "netcdf.h" + +#define filename "file://tmp_cachetest.zarr#mode=zarr,file" +#define chunkSize ((size_t)(1<<17)) /* 128k */ +#define numCells ((size_t)(50 * chunkSize)) +#define numSteps ((size_t)360) + +static float var[numCells]; + +size_t getPeakRSS(void) +{ + struct rusage rusage; + getrusage( RUSAGE_SELF, &rusage ); +#if defined(__APPLE__) && defined(__MACH__) + return (size_t)rusage.ru_maxrss; +#else + return (size_t)(rusage.ru_maxrss * 1024L); +#endif +} + +static void +nce(int istat) +{ + if (istat != NC_NOERR) + { + fprintf(stderr, "%s\n", nc_strerror(istat)); + exit(-1); + } +} + +int +main(void) +{ + printf("write: chunkSize=%zu, numCells=%zu, numSteps=%zu, filename=%s\n", chunkSize, numCells, numSteps, filename); + + int ncId; + nce(nc_create(filename, NC_CLOBBER | NC_NETCDF4, &ncId)); + int oldfill; + nce(nc_set_fill(ncId, NC_NOFILL, &oldfill)); + + int dims[2]; + nce(nc_def_dim(ncId, "time", numSteps, &dims[0])); + nce(nc_def_dim(ncId, "cells", numCells, &dims[1])); + + int varId; + nce(nc_def_var(ncId, "var", NC_FLOAT, 2, dims, &varId)); + + size_t chunks[2] = {1, chunkSize}; + nc_def_var_chunking(ncId, varId, NC_CHUNKED, chunks); + + int shuffle = 0, deflate = 1, level = 3; + nce(nc_def_var_deflate(ncId, varId, shuffle, deflate, level)); + + size_t size, nelems; + float preemption; + nce(nc_get_var_chunk_cache(ncId, varId, &size, &nelems, &preemption)); + printf("default chunk cache: size=%zu, nelems=%zu, preemption=%g\n", size, nelems, preemption); + size = 4 * numCells; // one float field at one time step + nelems = 1000; + preemption = 0.5; + nce(nc_set_var_chunk_cache(ncId, varId, size, nelems, preemption)); + printf("set chunk cache: size=%zu, nelems=%zu, preemption=%g\n", size, nelems, preemption); + + nce(nc_enddef(ncId)); + + { + for (size_t i = 0; i < numCells; ++i) var[i] = 0.0f; + for (size_t i = 0; i < numSteps; ++i) + { + size_t start[2], count[2]; + start[0] = i; start[1] = 0; + count[0] = 1; count[1] = numCells; + nce(nc_put_vara_float(ncId, varId, start, count, var)); + } + } + + nce(nc_close(ncId)); + + { + size_t mbused = getPeakRSS() / (1024 * 1024); + printf("Max mem: %zu MB\n", mbused); + if(mbused > 128) { + printf("*** Failed: used: %luMB expected: < 128\n",mbused); + return (1); + } else { + printf("*** Passed: used: %luMB (< 128)\n",mbused); + return 0; + } + } + +} diff --git a/v3_nczarr_test/test_zchunks.c b/v3_nczarr_test/test_zchunks.c new file mode 100644 index 0000000000..af49308a7a --- /dev/null +++ b/v3_nczarr_test/test_zchunks.c @@ -0,0 +1,391 @@ +/* This is part of the netCDF package. + Copyright 2018 University Corporation for Atmospheric Research/Unidata + See COPYRIGHT file for conditions of use. + + Test netcdf-4 variables. + Ed Hartnett, Dennis Heimbigner + modified version of nc_test4/tst_chunks.c +*/ + +#include "ut_includes.h" +#include "test_nczarr_utils.h" +#include + +#define DEBUGNOFILL +#undef PRINT_DEFAULT_CHUNKSIZE_TABLE + +#undef FILTERS + +#define FILE_NAME "tst_chunks" +#define NDIMS1 1 +#define D_SMALL "small_dim" +#define D_SMALL_LEN 16 +#define D_MEDIUM "medium_dim" +#define D_MEDIUM_LEN 65546 +#define D_LARGE "large_dim" +#define D_LARGE_LEN 1048586 +#define V_SMALL "small_var" +#define V_MEDIUM "medium_var" +#define V_LARGE "large_var" + +int +main(int argc, char **argv) +{ + processoptions(argc,argv,FILE_NAME); + + printf("\n*** Testing netcdf-4 variable chunking.\n"); + printf("**** testing that fixed vars with forced chunking end up being chunked, with good sizes..."); + { + + int ncid; + int nvars, ndims, ngatts, unlimdimid; + int storage; + int ndims_in, natts_in, dimids_in; + int small_dimid, medium_dimid, large_dimid; + int small_varid, medium_varid, large_varid; + char var_name_in[NC_MAX_NAME + 1]; + size_t chunksize_in[NDIMS1]; + nc_type xtype_in; + + /* Create a netcdf-4 file with three dimensions. */ + if (nc_create(itoptions.path, NC_NETCDF4, &ncid)) ERR; + if (nc_def_dim(ncid, D_SMALL, D_SMALL_LEN, &small_dimid)) ERR; + if (nc_def_dim(ncid, D_MEDIUM, D_MEDIUM_LEN, &medium_dimid)) ERR; + if (nc_def_dim(ncid, D_LARGE, D_LARGE_LEN, &large_dimid)) ERR; + + /* Add three vars, with forced chunking. */ + if (nc_def_var(ncid, V_SMALL, NC_INT64, NDIMS1, &small_dimid, &small_varid)) ERR; + if (nc_def_var(ncid, V_MEDIUM, NC_INT64, NDIMS1, &medium_dimid, &medium_varid)) ERR; + if (nc_def_var(ncid, V_LARGE, NC_INT64, NDIMS1, &large_dimid, &large_varid)) ERR; +#ifdef FILTERS + if (nc_def_var_deflate(ncid, small_varid, 0, 1, 4)) ERR; + if (nc_def_var_deflate(ncid, medium_varid, 1, 0, 0)) ERR; + if (nc_def_var_fletcher32(ncid, large_varid, 1)) ERR; +#else + if(nc_def_var_chunking(ncid,small_varid,NC_CHUNKED,NULL)) ERR; + if(nc_def_var_chunking(ncid,medium_varid,NC_CHUNKED,NULL)) ERR; + if(nc_def_var_chunking(ncid,large_varid,NC_CHUNKED,NULL)) ERR; +#endif + if (nc_close(ncid)) ERR; + + /* Open the file and check. */ + if (nc_open(itoptions.path, NC_WRITE, &ncid)) ERR; + if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR; + if (nvars != 3 || ndims != 3 || ngatts != 0 || unlimdimid != -1) ERR; + if (nc_inq_var(ncid, 0, var_name_in, &xtype_in, &ndims_in, &dimids_in, &natts_in)) ERR; + if (strcmp(var_name_in, V_SMALL) != 0 || xtype_in != NC_INT64 || ndims_in != 1 || + natts_in != 0) ERR; + + /* Make sure chunking sizes are what we expect. */ + if (nc_inq_var_chunking(ncid, small_varid, &storage, chunksize_in)) ERR; + if (storage || chunksize_in[0] != D_SMALL_LEN) ERR; + if (nc_inq_var_chunking(ncid, medium_varid, &storage, chunksize_in)) ERR; + if (storage || chunksize_in[0] * sizeof(long long) > DEFAULT_CHUNK_SIZE) ERR; + if (nc_inq_var_chunking(ncid, large_varid, &storage, chunksize_in)) ERR; + if (storage || chunksize_in[0] * sizeof(long long) > DEFAULT_CHUNK_SIZE) ERR; + + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; + + printf("**** testing default chunksizes..."); + { + int nvars, ndims, ngatts, unlimdimid; + int storage; +#define NUM_DIM 4 +#define NUM_TYPE 2 + int ncid; + size_t dim_len[NUM_DIM] = {1, 100, 1000, 2000}; + size_t chunksize_in[NUM_DIM]; + int type_id[NUM_TYPE] = {NC_BYTE, NC_INT}; + int dimid[NUM_DIM], varid[NUM_TYPE]; + char dim_name[NC_MAX_NAME + 1], var_name[NC_MAX_NAME + 1]; + int d, t; + + /* Create a netcdf-4 file with NUM_DIM dimensions. */ + if (nc_create(itoptions.path, NC_NETCDF4, &ncid)) ERR; +#ifdef DEBUGNOFILL + if(nc_set_fill(ncid,NC_NOFILL,&d)) ERR; +#endif + + for (d = 0; d < NUM_DIM; d++) + { + snprintf(dim_name, sizeof(dim_name), "dim_%zu", dim_len[d]); +#ifdef PRINT_DEFAULT_CHUNKSIZE_TABLE + printf("creating dim[%d] %s = %d\n", d, dim_name, dim_len[d]); +#endif + if (nc_def_dim(ncid, dim_name, dim_len[d], &dimid[d])) ERR; + } + + for (t = 0; t < NUM_TYPE; t++) + { + snprintf(var_name, sizeof(var_name), "var_%d", type_id[t]); + if (nc_def_var(ncid, var_name, type_id[t], NUM_DIM, dimid, &varid[t])) ERR; + if (nc_inq_var_chunking(ncid, varid[t], &storage, chunksize_in)) ERR; +#ifdef PRINT_DEFAULT_CHUNKSIZE_TABLE + printf("chunksizes for %d x %d x %d x %d var %s: %d x %d x %d x %d (=%d)\n", + dim_len[0], dim_len[1], dim_len[2], dim_len[3], + var_name, + (int)chunksize_in[0], (int)chunksize_in[1], (int)chunksize_in[2], + (int)chunksize_in[3], + (int)(chunksize_in[0] * chunksize_in[1] * chunksize_in[2] * chunksize_in[3])); +#endif + } + + if (nc_close(ncid)) ERR; + + /* Open the file and check. */ + if (nc_open(itoptions.path, NC_WRITE, &ncid)) ERR; + if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR; + if (nvars != NUM_TYPE || ndims != NUM_DIM || ngatts != 0 || unlimdimid == 0) ERR; + + + for (t = 0; t < NUM_TYPE; t++) + { + snprintf(var_name, sizeof(var_name), "var_%d", type_id[t]); + if (nc_inq_var_chunking(ncid, varid[t], &storage, chunksize_in)) ERR; + if (storage) ERR; +#ifdef PRINT_DEFAULT_CHUNKSIZE_TABLE + printf("chunksizes for %d x %d x %d x %d var: %d x %d x %d x %d (=%d)\n", + dim_len[0], dim_len[1], dim_len[2], dim_len[3], + (int)chunksize_in[0], (int)chunksize_in[1], (int)chunksize_in[2], + (int)chunksize_in[3], + (int)(chunksize_in[0] * chunksize_in[1] * chunksize_in[2] * chunksize_in[3])); +#endif + } + + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; + + printf("**** testing that chunking works on classic mode files..."); + { +#define D_SMALL_LEN2 66 + int ncid; + int nvars, ndims, ngatts, unlimdimid; + int storage; + int ndims_in, natts_in, dimids_in; + int small_dimid, medium_dimid, large_dimid; + int small_varid, medium_varid, large_varid; + char var_name_in[NC_MAX_NAME + 1]; + size_t chunks[1], chunksize_in; + nc_type xtype_in; + + /* Create a netcdf-4 file with three dimensions. */ + if (nc_create(itoptions.path, NC_NETCDF4, &ncid)) ERR; + if (nc_def_dim(ncid, D_SMALL, D_SMALL_LEN2, &small_dimid)) ERR; + if (nc_def_dim(ncid, D_MEDIUM, D_MEDIUM_LEN, &medium_dimid)) ERR; + if (nc_def_dim(ncid, D_LARGE, D_LARGE_LEN, &large_dimid)) ERR; + + /* Add three vars. */ + if (nc_def_var(ncid, V_SMALL, NC_INT64, NDIMS1, &small_dimid, &small_varid)) ERR; + if (nc_def_var_chunking(ncid, small_varid, 1, NULL)) ERR; + + if (nc_def_var(ncid, V_MEDIUM, NC_INT64, NDIMS1, &medium_dimid, &medium_varid)) ERR; + chunks[0] = D_MEDIUM_LEN / 100; + if (nc_def_var_chunking(ncid, medium_varid, 0, chunks)) ERR; +#ifdef FILTERS + if (nc_def_var_deflate(ncid, medium_varid, 1, 0, 0)) ERR; +#endif + if (nc_def_var(ncid, V_LARGE, NC_INT64, NDIMS1, &large_dimid, &large_varid)) ERR; + chunks[0] = D_LARGE_LEN / 1000; + if (nc_def_var_chunking(ncid, large_varid, 0, chunks)) ERR; +#ifdef FILTERS + if (nc_def_var_fletcher32(ncid, large_varid, 1)) ERR; +#endif + if (nc_close(ncid)) ERR; + + /* Open the file and check. */ + if (nc_open(itoptions.path, NC_WRITE, &ncid)) ERR; + if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR; + if (nvars != 3 || ndims != 3 || ngatts != 0 || unlimdimid != -1) ERR; + if (nc_inq_var(ncid, 0, var_name_in, &xtype_in, &ndims_in, &dimids_in, &natts_in)) ERR; + if (strcmp(var_name_in, V_SMALL) != 0 || xtype_in != NC_INT64 || ndims_in != 1 || + natts_in != 0) ERR; + + /* Make sure chunking settings are what we expect. */ + if (nc_inq_var_chunking(ncid, small_varid, &storage, &chunksize_in)) ERR; + if (storage != NC_CHUNKED) ERR; + if (nc_inq_var_chunking(ncid, medium_varid, &storage, &chunksize_in)) ERR; + if (storage || chunksize_in != D_MEDIUM_LEN / 100) ERR; + if (nc_inq_var_chunking(ncid, large_varid, &storage, &chunksize_in)) ERR; + if (storage || chunksize_in != D_LARGE_LEN / 1000) ERR; + + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; + printf("**** testing many chunking variables..."); + { +#define NDIMS_3 3 +#define NUM_PLANS 30 +#define D_SNEAKINESS "sneakiness" +#define D_SNEAKINESS_LEN 5 +#define D_CLEVERNESS "clevernesss" +#define D_CLEVERNESS_LEN 3 +#define D_EFFECTIVENESS "effectiveness" +#define D_EFFECTIVENESS_LEN 2 + + int ncid, dimids[NDIMS_3], varid[NUM_PLANS]; + size_t chunksize[NDIMS_3] = {D_SNEAKINESS_LEN, D_CLEVERNESS_LEN, + D_EFFECTIVENESS_LEN}; + char plan_name[NC_MAX_NAME + 1]; + int storage; + size_t chunksize_in[NDIMS_3]; + int i, j; + + /* Create a netcdf-4 file with three dimensions. */ + if (nc_create(itoptions.path, NC_NETCDF4, &ncid)) ERR; + if (nc_def_dim(ncid, D_SNEAKINESS, D_SNEAKINESS_LEN, &dimids[0])) ERR; + if (nc_def_dim(ncid, D_CLEVERNESS, D_CLEVERNESS_LEN, &dimids[1])) ERR; + if (nc_def_dim(ncid, D_EFFECTIVENESS, D_EFFECTIVENESS_LEN, &dimids[2])) ERR; + + /* Oh that tricky Cardinal Richelieu, he had many plans! */ + for (i = 0; i < NUM_PLANS; i++) + { + snprintf(plan_name, sizeof(plan_name), "Richelieu_sneaky_plan_%d", i); + if (nc_def_var(ncid, plan_name, i % (NC_STRING - 1) + 1, NDIMS_3, + dimids, &varid[i])) ERR; + if (nc_def_var_chunking(ncid, varid[i], 0, chunksize)) ERR; + } + + /* Check the chunking. */ + for (i = 0; i < NUM_PLANS; i++) + { + if (nc_inq_var_chunking(ncid, varid[i], &storage, chunksize_in)) ERR; + { + for (j = 0; j < NDIMS_3; j++) + if (chunksize_in[j] != chunksize[j]) ERR; + } + } + if (nc_close(ncid)) ERR; + + /* Open the file and check. */ + if (nc_open(itoptions.path, NC_WRITE, &ncid)) ERR; + /* Check the chunking. */ + for (i = 0; i < NUM_PLANS; i++) + { + if (nc_inq_var_chunking(ncid, varid[i], &storage, chunksize_in)) ERR; + { + for (j = 0; j < NDIMS_3; j++) + if (chunksize_in[j] != chunksize[j]) ERR; + } + } + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; + printf("**** testing that too large chunksizes fail..."); + { +#define D_SMALL_LEN2 66 + int stat = NC_NOERR; + int ncid; + int small_dimid; + int small_varid; + size_t chunks[1]; + + /* Create a netcdf-4 file with three dimensions. */ + if (nc_create(itoptions.path, NC_NETCDF4, &ncid)) ERR; + if (nc_def_dim(ncid, D_SMALL, D_SMALL_LEN2, &small_dimid)) ERR; + + /* Add one var. */ + if (nc_def_var(ncid, V_SMALL, NC_INT64, NDIMS1, &small_dimid, &small_varid)) ERR; + + /* Attempt to set too large chunksizes */ + chunks[0] = D_SMALL_LEN2 + 1; + stat = nc_def_var_chunking(ncid, small_varid, NC_CHUNKED, chunks); + if(stat != NC_EBADCHUNK) { + printf("Return code is '%s', expected NC_BADCHUNK",nc_strerror(stat)); + ERR; + } + /* try again with proper chunksize */ + chunks[0] = D_SMALL_LEN2; + stat = nc_def_var_chunking(ncid, small_varid, NC_CHUNKED, chunks); + if(stat != NC_NOERR) { + printf("Return code is '%s', expected NC_NOERR",nc_strerror(stat)); + ERR; + } + if (nc_abort(ncid)) ERR; + } + SUMMARIZE_ERR; + printf("**** testing cache size smaller than chunk size..."); + { +#define NDIM2 2 +#define DIM_X_LEN 10000 +#define DIM_Y_LEN 10000 +#define DIM_NAME_X_CACHE_CHUNK "Height" +#define DIM_NAME_Y_CACHE_CHUNK "Width" +#define VAR_NAME_CACHE_CHUNK "House_Size" +#define VAR_NAME_CACHE_CHUNK_2 "Boat_Size" +#define VAR_NAME_CACHE_CHUNK_3 "Deck_Size" + + int ncid; + int dimid[NDIM2]; + int varid, varid2, varid3; + size_t chunks[NDIM2] = {100, 100}; + size_t chunks_big[NDIM2] = {DIM_X_LEN, DIM_Y_LEN}; + size_t chunks_in[NDIM2]; + int storage; + size_t cache_size = 16; + size_t cache_nelems = 1; + float cache_preemption = 0.5; + size_t cache_size_in; + size_t cache_nelems_in; + float cache_preemption_in; + + /* Create a netcdf-4 file with two dimensions. */ + if (nc_create(itoptions.path, NC_NETCDF4, &ncid)) ERR; + if (nc_def_dim(ncid, DIM_NAME_X_CACHE_CHUNK, DIM_X_LEN, &dimid[0])) ERR; + if (nc_def_dim(ncid, DIM_NAME_Y_CACHE_CHUNK, DIM_Y_LEN, &dimid[1])) ERR; + + /* Add vars. */ + if (nc_def_var(ncid, VAR_NAME_CACHE_CHUNK, NC_INT64, NDIM2, dimid, &varid)) ERR; + if (nc_def_var(ncid, VAR_NAME_CACHE_CHUNK_2, NC_INT64, NDIM2, dimid, &varid2)) ERR; + if (nc_def_var(ncid, VAR_NAME_CACHE_CHUNK_3, NC_INT64, NDIM2, dimid, &varid3)) ERR; + + /* Set the var cache to something arbitrary but small */ + if (nc_set_var_chunk_cache(ncid, varid, cache_size, cache_nelems, + cache_preemption)) ERR; + + /* Set the chunking. */ + if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, chunks)) ERR; + if (nc_inq_var_chunking(ncid, varid, &storage, chunks_in)) ERR; + if (storage || chunks_in[0] != chunks[0] || chunks_in[1] != chunks[1]) ERR; + if (nc_def_var_chunking(ncid, varid2, NC_CHUNKED, chunks)) ERR; + if (nc_inq_var_chunking(ncid, varid2, &storage, chunks_in)) ERR; + if (storage || chunks_in[0] != chunks[0] || chunks_in[1] != chunks[1]) ERR; + if (nc_def_var_chunking(ncid, varid3, NC_CHUNKED, chunks_big)) ERR; + if (nc_inq_var_chunking(ncid, varid3, &storage, chunks_in)) ERR; + if (storage || chunks_in[0] != chunks_big[0] || chunks_in[1] != chunks_big[1]) ERR; + + /* Get the var cache values. */ + if (nc_get_var_chunk_cache(ncid, varid, &cache_size_in, &cache_nelems_in, + &cache_preemption_in)) ERR; + if (cache_size_in != cache_size || cache_nelems_in != cache_nelems || + cache_preemption_in != cache_preemption) ERR; + if (nc_get_var_chunk_cache(ncid, varid2, &cache_size_in, &cache_nelems_in, + &cache_preemption_in)) ERR; + if (cache_size_in != DEFAULT_CHUNK_CACHE_SIZE) ERR; + +#if 0 + /* Inapplicable to zarr */ + /* The cache_size has been increased due to larger chunksizes + * for varid3. */ + if (nc_get_var_chunk_cache(ncid, varid3, &cache_size_in, &cache_nelems_in, + &cache_preemption_in)) ERR; + if (cache_nelems_in != CHUNK_CACHE_NELEMS || + cache_preemption_in != CHUNK_CACHE_PREEMPTION) ERR; + /* printf("cache_size_in %ld\n", cache_size_in); */ +#endif + + /* Close the file. */ + if (nc_close(ncid)) ERR; + + /* Reopen the file. */ + if (nc_open(itoptions.path, NC_NOWRITE, &ncid)) ERR; + + /* Close the file. */ + if (nc_close(ncid)) ERR; + } + clearoptions(); + SUMMARIZE_ERR; + FINAL_RESULTS; +} diff --git a/v3_nczarr_test/test_zchunks2.c b/v3_nczarr_test/test_zchunks2.c new file mode 100644 index 0000000000..2ad80cf02e --- /dev/null +++ b/v3_nczarr_test/test_zchunks2.c @@ -0,0 +1,462 @@ +/* This is part of the netCDF package. + Copyright 2018 University Corporation for Atmospheric Research/Unidata + See COPYRIGHT file for conditions of use. + + Test netcdf-4 chunking. + Ed Hartnett, Dennis Heimbigner + modified version of nc_test4/tst_chunks2.c +*/ + +#include "ut_includes.h" +#include "test_nczarr_utils.h" +#include + +#define FILE_NAME "tst_chunks2" +#define MAX_WASTE 25.0 +#define NUM_RANDOM_TESTS 3 +#define NDIMS3 3 + +/* Calculate the waste of the chunking. A waste of 10% means the + * chunked data is 10% larger then the unchunked data. */ +static int +calculate_waste(int ndims, size_t *dimlen, size_t *chunksize, float *waste) +{ + int d; + float chunked = 1, unchunked = 1; + size_t *num_chunks; + + assert(waste && dimlen && chunksize && ndims); + if (!(num_chunks = calloc((size_t)ndims, sizeof(size_t)))) ERR; + +#ifdef PRINT_CHUNK_WASTE_REPORT + printf("\n"); +#endif + /* Caclulate the total space taken up by the chunked data. */ + for (d = 0; d < ndims; d++) + { + /* How many chunks along this dimension are required to hold all the data? */ + for (num_chunks[d] = 0; (num_chunks[d] * chunksize[d]) < (dimlen[d] ? dimlen[d] : 1); + num_chunks[d]++) + ; + chunked *= (float)(num_chunks[d] * chunksize[d]); + } + + /* Calculate the minimum space required for this data + * (i.e. unchunked) or one record of it. */ + for (d = 0; d < ndims; d++) + unchunked *= (dimlen[d] ? (float)dimlen[d] : 1); + +#ifdef PRINT_CHUNK_WASTE_REPORT + printf("size for unchunked %g elements; size for chunked %g elements\n", + unchunked, chunked); +#endif + + /* Percent of the chunked file that is wasted space. */ + *waste = ((chunked - unchunked) / chunked) * 100.0f; + +#ifdef PRINT_CHUNK_WASTE_REPORT + printf("\ndimlen\tchunksize\tnum_chunks\n"); + size_t chunk_size = 1; + for (d = 0; d < ndims; d++) + { + printf("%ld\t%ld\t\t%ld\n", (long int)dimlen[d], (long int)chunksize[d], + (long int)num_chunks[d]); + chunk_size *= chunksize[d]; + } + printf("size of chunk: %ld elements; wasted space: %2.2f percent\n", + (long int)chunk_size, *waste); +#endif + + free(num_chunks); + return 0; +} + +int +main(int argc, char **argv) +{ + processoptions(argc,argv,FILE_NAME); + + printf("\n*** Testing netcdf-4 variable chunking.\n"); + printf("**** testing default chunksizes..."); + { +#define NDIMS3 3 +#define NUM_VARS 1 +#define Y_NAME "y" +#define X_NAME "x" +#define Z_NAME "z" +#define VAR_NAME_JOE "joe" +#define XDIM_LEN 2 +#define YDIM_LEN 5 +#define ZDIM_LEN 3000 + + int varid, ncid, dims[NDIMS3], dims_in[NDIMS3]; + int ndims, nvars, ngatts, unlimdimid, natts; + char name_in[NC_MAX_NAME + 1]; + nc_type type_in; + size_t len_in[NDIMS3]; + int storage = 0; + size_t chunksizes[NDIMS3]; + float waste = 0; + + /* Create a file with 3D var, turn on chunking, but don't provide chunksizes. */ + if (nc_create(itoptions.path, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; + if (nc_def_dim(ncid, X_NAME, XDIM_LEN, &dims[0])) ERR; + if (nc_def_dim(ncid, Y_NAME, YDIM_LEN, &dims[1])) ERR; + if (nc_def_dim(ncid, Z_NAME, ZDIM_LEN, &dims[2])) ERR; + if (nc_def_var(ncid, VAR_NAME_JOE, NC_FLOAT, NDIMS3, dims, &varid)) ERR; + if (nc_def_var_chunking(ncid, 0, NC_CHUNKED, NULL)) ERR; + + /* Check it out. */ + if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR; + if (nvars != NUM_VARS || ndims != NDIMS3 || ngatts != 0 || unlimdimid != -1) ERR; + if (nc_inq_var(ncid, 0, name_in, &type_in, &ndims, dims_in, &natts)) ERR; + if (strcmp(name_in, VAR_NAME_JOE) != 0 || type_in != NC_FLOAT || ndims != NDIMS3 || + dims_in[0] != dims[0] || dims_in[1] != dims[1] || dims_in[2] != dims[2] || natts != 0) ERR; + if (nc_inq_dim(ncid, 0, name_in, &len_in[0])) ERR; + if (strcmp(name_in, X_NAME) != 0 || len_in[0] != XDIM_LEN) ERR; + if (nc_inq_dim(ncid, 1, name_in, &len_in[1])) ERR; + if (strcmp(name_in, Y_NAME) != 0 || len_in[1] != YDIM_LEN) ERR; + if (nc_inq_dim(ncid, 2, name_in, &len_in[2])) ERR; + if (strcmp(name_in, Z_NAME) != 0 || len_in[2] != ZDIM_LEN) ERR; + if (nc_inq_var_chunking(ncid, 0, &storage, chunksizes)) ERR; + if (storage != NC_CHUNKED) ERR; + if (nc_close(ncid)) ERR; + + /* Open the file and check again. */ + if (nc_open(itoptions.path, NC_WRITE, &ncid)) ERR; + if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR; + if (nvars != NUM_VARS || ndims != NDIMS3 || ngatts != 0 || unlimdimid != -1) ERR; + if (nc_inq_var(ncid, 0, name_in, &type_in, &ndims, dims_in, &natts)) ERR; + if (strcmp(name_in, VAR_NAME_JOE) != 0 || type_in != NC_FLOAT || ndims != NDIMS3 || + dims_in[0] != dims[0] || dims_in[1] != dims[1] || dims_in[2] != dims[2] || natts != 0) ERR; + if (nc_inq_dim(ncid, 0, name_in, &len_in[0])) ERR; + if (strcmp(name_in, X_NAME) != 0 || len_in[0] != XDIM_LEN) ERR; + if (nc_inq_dim(ncid, 1, name_in, &len_in[1])) ERR; + if (strcmp(name_in, Y_NAME) != 0 || len_in[1] != YDIM_LEN) ERR; + if (nc_inq_dim(ncid, 2, name_in, &len_in[2])) ERR; + if (strcmp(name_in, Z_NAME) != 0 || len_in[2] != ZDIM_LEN) ERR; + if (nc_inq_var_chunking(ncid, 0, &storage, chunksizes)) ERR; + if (storage != NC_CHUNKED) ERR; + if (calculate_waste(NDIMS3, len_in, chunksizes, &waste)) ERR; + /*if (waste > MAX_WASTE) ERR;*/ + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; + printf("**** testing default chunksizes some more for a 3D var..."); + { +#define NDIMS3 3 +#define VAR_NAME "op-amp" + + int varid, ncid; + int dimids[NDIMS3]; + size_t dim_len[NDIMS3] = {1, 11, 152750}; + + int storage = 0; + size_t chunksizes[NDIMS3]; + int d; + char dim_name[NC_MAX_NAME + 1]; + float waste; + + if (nc_create(itoptions.path, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; + + /* Create a few dimensions. */ + for (d = 0; d < NDIMS3; d++) + { + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); + if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; + } + + /* Define a var with these dimensions, and turn on chunking. */ + if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR; + if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR; + + /* Check how default chunking worked. */ + if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR; + if (storage != NC_CHUNKED) ERR; + if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR; +/* if (waste > MAX_WASTE) ERR;*/ + + if (nc_close(ncid)) ERR; + + /* Open the file and check. */ + if (nc_open(itoptions.path, NC_WRITE, &ncid)) ERR; + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; + printf("**** testing default chunksizes even more for a 3D var..."); + { + int varid, ncid; + int dimids[NDIMS3]; + size_t dim_len[NDIMS3] = {1804289383, 846930886, 1681692777}; + + int storage = 0; + size_t chunksizes[NDIMS3]; + int d; + char dim_name[NC_MAX_NAME + 1]; + float waste; + + if (nc_create(itoptions.path, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; + + /* Create a few dimensions. */ + for (d = 0; d < NDIMS3; d++) + { + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); + if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; + } + + /* Define a var with these dimensions, and turn on chunking. */ + if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR; + if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR; + + /* Check how default chunking worked. */ + if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR; + if (storage != NC_CHUNKED) ERR; + if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR; +/* if (waste > MAX_WASTE) ERR;*/ + + if (nc_close(ncid)) ERR; + + /* Open the file and check. */ + if (nc_open(itoptions.path, NC_WRITE, &ncid)) ERR; + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; + printf("**** testing default chunksizes even even more for a 3D var..."); + { + int varid, ncid; + int dimids[NDIMS3]; + size_t dim_len[NDIMS3] = {1714636915, 1957747793, 424238335}; + + int storage = 0; + size_t chunksizes[NDIMS3]; + int d; + char dim_name[NC_MAX_NAME + 1]; + float waste; + + if (nc_create(itoptions.path, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; + + /* Create a few dimensions. */ + for (d = 0; d < NDIMS3; d++) + { + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); + if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; + } + + /* Define a var with these dimensions, and turn on chunking. */ + if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR; + if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR; + + /* Check how default chunking worked. */ + if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR; + if (storage != NC_CHUNKED) ERR; + if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR; +/* if (waste > MAX_WASTE) ERR;*/ + + if (nc_close(ncid)) ERR; + + /* Open the file and check. */ + if (nc_open(itoptions.path, NC_WRITE, &ncid)) ERR; + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; + printf("**** testing default chunksizes some more for a 3D var..."); + { +#define NDIMS3 3 +#define VAR_NAME "op-amp" + + int varid, ncid; + int dimids[NDIMS3]; + size_t dim_len[NDIMS3] = {1967513926, 1365180540, 426}; + + int storage = 0; + size_t chunksizes[NDIMS3]; + int d; + char dim_name[NC_MAX_NAME + 1]; + float waste; + + if (nc_create(itoptions.path, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; + + /* Create a few dimensions. */ + for (d = 0; d < NDIMS3; d++) + { + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); + if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; + } + + /* Define a var with these dimensions, and turn on chunking. */ + if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR; + if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR; + + /* Check how default chunking worked. */ + if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR; + if (storage != NC_CHUNKED) ERR; + if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR; +/* if (waste > MAX_WASTE) ERR;*/ + + if (nc_close(ncid)) ERR; + + /* Open the file and check. */ + if (nc_open(itoptions.path, NC_WRITE, &ncid)) ERR; + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; + printf("**** testing default chunksizes for very large 3D var..."); + { +#define NDIMS3 3 + + int varid, ncid; + int dimids[NDIMS3]; + size_t dim_len[NDIMS3] = {1804289383, 846930886, 1681692777}; + + int storage = 0; + size_t chunksizes[NDIMS3]; + int d; + char dim_name[NC_MAX_NAME + 1]; + float waste; + + if (nc_create(itoptions.path, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; + + /* Create a few dimensions. */ + for (d = 0; d < NDIMS3; d++) + { + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); + if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; + } + + /* Define a var with these dimensions, and turn on chunking. */ + if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR; + if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR; + + /* Check how default chunking worked. */ + if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR; + if (storage != NC_CHUNKED) ERR; + if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR; +/* if (waste > MAX_WASTE) ERR;*/ + + if (nc_close(ncid)) ERR; + + /* Open the file and check. */ + if (nc_open(itoptions.path, NC_WRITE, &ncid)) ERR; + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; + printf("**** testing default chunksizes some randomly sized 3D vars..."); + { +#define NDIMS3 3 + + int varid, ncid; + int dimids[NDIMS3]; + size_t dim_len[NDIMS3]; + int storage = 0; + size_t chunksizes[NDIMS3]; + int t; + char dim_name[NC_MAX_NAME + 1]; + float waste; + + for (t = 0; t < NUM_RANDOM_TESTS; t++) + { + if (nc_create(itoptions.path, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; + + /* Create a few dimensions. */ + for (size_t d = 0; d < NDIMS3; d++) + { + dim_len[d] = (size_t)rand(); + snprintf(dim_name, sizeof(dim_name), "dim_%zu", d); + if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; + } + + /* Define a var with these dimensions, and turn on chunking. */ + if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR; + if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR; + + /* Check how well default chunking worked. */ + if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR; + if (storage != NC_CHUNKED) ERR; + if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR; + if (waste > MAX_WASTE) ERR; + + if (nc_close(ncid)) ERR; + } + } + SUMMARIZE_ERR; + printf("**** testing default chunksizes some randomly sized 3D vars, with one small dimension..."); + { + int varid, ncid; + int dimids[NDIMS3]; + size_t dim_len[NDIMS3]; + int storage = 0; + size_t chunksizes[NDIMS3]; + int d, t; + char dim_name[NC_MAX_NAME + 1]; + float waste; + + for (t = 0; t < NUM_RANDOM_TESTS; t++) + { + if (nc_create(itoptions.path, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; + + dim_len[0] = (size_t)rand(); + dim_len[1] = (size_t)rand(); + dim_len[2] = (size_t)rand() % 1000; + /* Create a few dimensions. */ + for (d = 0; d < NDIMS3; d++) + { + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); + if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; + } + + /* Define a var with these dimensions, and turn on chunking. */ + if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR; + if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR; + + /* Check how well default chunking worked. */ + if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR; + if (storage != NC_CHUNKED) ERR; + if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR; + if (waste > MAX_WASTE) ERR; + + if (nc_close(ncid)) ERR; + } + } + SUMMARIZE_ERR; + printf("**** testing default chunksizes some randomly sized 3D vars, with two small dimensions..."); + { + int varid, ncid; + int dimids[NDIMS3]; + size_t dim_len[NDIMS3]; + int storage = 0; + size_t chunksizes[NDIMS3]; + int d, t; + char dim_name[NC_MAX_NAME + 1]; + float waste; + + for (t = 0; t < NUM_RANDOM_TESTS; t++) + { + if (nc_create(itoptions.path, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; + + dim_len[0] = (size_t)rand(); + dim_len[1] = (size_t)rand() % 1000; + dim_len[2] = (size_t)rand() % 1000; + /* Create a few dimensions. */ + for (d = 0; d < NDIMS3; d++) + { + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); + if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; + } + + /* Define a var with these dimensions, and turn on chunking. */ + if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR; + if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR; + + /* Check how well default chunking worked. */ + if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR; + if (storage != NC_CHUNKED) ERR; + if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR; + if (waste > MAX_WASTE) ERR; + + if (nc_close(ncid)) ERR; + } + } + clearoptions(); + SUMMARIZE_ERR; + FINAL_RESULTS; +} diff --git a/v3_nczarr_test/test_zchunks3.c b/v3_nczarr_test/test_zchunks3.c new file mode 100644 index 0000000000..699e278210 --- /dev/null +++ b/v3_nczarr_test/test_zchunks3.c @@ -0,0 +1,84 @@ +/* This is part of the netCDF package. Copyright 2018 University + Corporation for Atmospheric Research/Unidata. See COPYRIGHT file for + conditions of use. See www.unidata.ucar.edu for more info. + + Create a chunkable test file for nccopy to test chunking. +*/ + +#include "ut_includes.h" +#include "test_nczarr_utils.h" + +#undef DEBUG + +static int ret = NC_NOERR; +#define FILE_NAME "tmp_chunks3.nc" + +#define VAR_RANK 7 +#define IVAR_NAME "ivar" +#define FVAR_NAME "fvar" +#define NVALS 45360 /* 7 * 4 * 2 * 3 * 5 * 6 * 9 */ + +/* Make trackable ERR macro replacement */ +static int +lerr(int stat, const char* file, int lineno) +{ + NC_UNUSED(stat); + fflush(stdout); /* Make sure our stdout is synced with stderr. */ + err++; + fprintf(stderr, "Sorry! Unexpected result(%d), %s, line: %d\n",ret,file,lineno); + fflush(stderr); \ + return 2; \ +} +#define LERR lerr(ret,__FILE__,__LINE__) + +static const char *dim_names[VAR_RANK] = {"dim0", "dim1", "dim2", "dim3", "dim4", "dim5", "dim6"}; +static const size_t dim_lens[VAR_RANK] = {7, 4, 2, 3, 5, 6, 9}; + +int +main(int argc, char** argv) +{ + /* file metadata */ + int mode = NC_CLOBBER|NC_NETCDF4; + int ncid; + int ivarid, fvarid; + int ivar_dims[VAR_RANK]; + int fvar_dims[VAR_RANK]; + int ivar_data[NVALS]; + float fvar_data[NVALS]; + int r, i; + char* file_name = FILE_NAME; + + NC_UNUSED(argc); + NC_UNUSED(argv); + + printf("*** Creating chunkable test file %s...\n", file_name); + + if ((ret=nc_create(FILE_NAME, mode, &ncid))) LERR; + for(r = 0; r < VAR_RANK; r++) { + if ((ret=nc_def_dim(ncid, dim_names[r], dim_lens[r], &ivar_dims[r]))) LERR; + fvar_dims[VAR_RANK - 1 - r] = ivar_dims[r]; + } + if ((ret=nc_def_var(ncid, IVAR_NAME, NC_INT, VAR_RANK, ivar_dims, &ivarid))) LERR; + + /* fvar is unchanged */ + if ((ret=nc_def_var(ncid, FVAR_NAME, NC_FLOAT, VAR_RANK, fvar_dims, &fvarid))) LERR; + if ((ret=nc_enddef (ncid))) LERR; + + /* Fill in the data */ + for(i=0; i < NVALS; i++) { + ivar_data[i] = i; + } + if ((ret=nc_put_var(ncid, ivarid, ivar_data))) LERR; + + /* fvar is unchanged */ + for(i=0; i < NVALS; i++) { + fvar_data[i] = (float)(NVALS - i); + } + if ((ret=nc_put_var(ncid, fvarid, fvar_data))) LERR; + + if ((ret=nc_close(ncid))) LERR; + + SUMMARIZE_ERR; + FINAL_RESULTS; + +} diff --git a/v3_nczarr_test/testfilter.c b/v3_nczarr_test/testfilter.c new file mode 100644 index 0000000000..f68773cc17 --- /dev/null +++ b/v3_nczarr_test/testfilter.c @@ -0,0 +1,312 @@ +/* + Copyright 2018, UCAR/Unidata + See COPYRIGHT file for copying and redistribution conditions. +*/ + +/*! \file +Example program for write then read of a variable using bzip2 compression. + +\ingroup tutorial + +This is an example which +creates a file with a variable that is compressed using bzip2. +Then it reads that file and verifies that it returned the correct +uncompressed data. + +The meta-data (.cdl) for the created file is as follows: +\code +netcdf bzip2 { +dimensions: + dim0 = 4 ; + dim1 = 4 ; + dim2 = 4 ; + dim3 = 4 ; +variables: + float var(dim0, dim1, dim2, dim3) ; + var:_Storage = "chunked" ; + var:_ChunkSizes = 4, 4, 4, 4 ; + var:_Filter = "307,9" ; + var:_NoFill = "true" ; +data: + + var = + 0, 1, 2, 3, + 4, 5, 6, 7, + ... + 252, 253, 254, 255 ; +} +\endcode +*/ + +#include +#include +#include + +#include "netcdf.h" + +/* The HDF assigned id for bzip compression */ +#define BZIP2_ID 307 +/* The compression level used in this example */ +#define BZIP2_LEVEL 9 + +#define DFALT_TESTFILE "tmp_bzip2.nc" + +/* Point at which we give up */ +#define MAXERRS 8 + +#define NDIMS 4 +#define DIMSIZE 4 +#define CHUNKSIZE 4 /* Note: not the total size of the chunk, but size wrt a dim*/ + +static size_t dimsize = DIMSIZE; +static size_t chunksize = CHUNKSIZE; +static size_t actualdims = NDIMS; + +static size_t actualproduct = 1; /* x-product over dim sizes */ +static size_t chunkproduct = 1; /* x-product over chunksizes */ + +static size_t dims[NDIMS]; +static size_t chunks[NDIMS]; + +static int nerrs = 0; + +static const char* testfile = NULL; + +static int ncid, varid; +static int dimids[NDIMS]; +static float* array = NULL; +static float* expected = NULL; +static unsigned int filterid = 0; +static unsigned int* params = NULL; + +/* Forward */ +static void init(int argc, char** argv); +static int test_bzip2(void); +static int verifychunks(void); + +#define ERRR do { \ +fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ +fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ +nerrs++;\ +} while (0) + +static int +check(int err,int line) +{ + if(err != NC_NOERR) { + fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err)); + fflush(stderr); + exit(1); + } + return NC_NOERR; +} + +#define CHECK(x) check(x,__LINE__) + +/* +Read the chunking information about the variable +and verify that it is as expected. +*/ + +static int +verifychunks(void) +{ + size_t i; + int store = -1; + size_t chunksizes[NDIMS]; + memset(chunksizes,0,sizeof(chunksizes)); + CHECK(nc_inq_var_chunking(ncid, varid, &store, chunksizes)); + /* Storate must be chunked, not contiguous */ + if(store != NC_CHUNKED) { + fprintf(stderr,"bad chunk store\n"); + return NC_ESTORAGE; + } + /* Chunk sizes must match our predefined set */ + for(i=0;i= MAXERRS) + break; + } + } + if(errs == 0) + printf("no data errors\n"); + if(actualproduct <= 1) + return NC_EBADDIM; + return (errs == 0 ? NC_NOERR: NC_EINVAL); +} + +/* +Create the file, write it, then re-read for comparison. +*/ +static int +test_bzip2(void) +{ + size_t i; + unsigned int level = BZIP2_LEVEL; + unsigned int id=0; + size_t nparams = 0; + + printf("\n*** Testing API: bzip2 compression.\n"); + + /* Clear the data array */ + memset(array,0,sizeof(float)*actualproduct); + + /* Create a file */ + CHECK(nc_create(testfile, NC_NETCDF4|NC_CLOBBER, &ncid)); + + /* Do not use fill for this file */ + CHECK(nc_set_fill(ncid, NC_NOFILL, NULL)); + + /* Define the dimensions */ + for(i=0;i 0) { + params = (unsigned int*)malloc(sizeof(unsigned int)*nparams); + if(params == NULL) + return NC_ENOMEM; + CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,params)); + } + if(filterid != BZIP2_ID) { + printf("Bzip2 id mismatch: %d\n",filterid); + return NC_EFILTER; + } + if(nparams != 1 && params != NULL && params[0] != BZIP2_LEVEL) { + printf("Compression parameter mismatch\n"); + return NC_EFILTER; + } + + /* Verify chunking */ + if(!verifychunks()) + return 0; + + /* Read the data */ + CHECK(nc_get_var_float(ncid, varid, array)); + + /* Close the file */ + CHECK(nc_close(ncid)); + return (compare() == NC_NOERR ? 0 : 1); +} + +/**************************************************/ +/* Utilities */ + +static void +init(int argc, char** argv) +{ + size_t i; + + /* get the testfile path */ + if(argc > 1) + testfile = argv[1]; + else + testfile = DFALT_TESTFILE; + + /* Setup various variables */ + actualproduct = 1; + chunkproduct = 1; + for(i=0;i 0?1:0); +} + diff --git a/v3_nczarr_test/testfilter_misc.c b/v3_nczarr_test/testfilter_misc.c new file mode 100644 index 0000000000..6985509445 --- /dev/null +++ b/v3_nczarr_test/testfilter_misc.c @@ -0,0 +1,591 @@ +/* + Copyright 2018, UCAR/Unidata + See COPYRIGHT file for copying and redistribution conditions. +*/ + +#include "config.h" +#include +#include +#include + +#ifdef USE_HDF5 +#include +#endif + +#include "netcdf.h" +#include "netcdf_aux.h" +#include "netcdf_filter.h" + +#include "h5misc.h" /* from plugins dir */ + +#undef TESTODDSIZE + +#undef DEBUG + +/* The C standard apparently defines all floating point constants as double; + we rely on that in this code. +*/ +#define DBLVAL 12345678.12345678 + +#define TEST_ID 32768 + +#define MAXERRS 8 + +#define MAXPARAMS 32 + +static unsigned int baselineparams[NPARAMS]; + +static const char* testfile = NULL; + +#define MAXDIMS 8 + +#define DFALT_TESTFILE "tmp_misc.nc" + +#ifdef TESTODDSIZE +#define NDIMS 1 +static size_t dimsize[NDIMS] = {4}; +static size_t chunksize[NDIMS] = {3}; +#else +#define NDIMS 4 +static size_t dimsize[NDIMS] = {4,4,4,4}; +static size_t chunksize[NDIMS] = {4,4,4,4}; +#endif + +static size_t ndims = NDIMS; + +static size_t totalproduct = 1; /* x-product over max dims */ +static size_t actualproduct = 1; /* x-product over actualdims */ +static size_t chunkproduct = 1; /* x-product over actual chunks */ + +static size_t pattern[MAXDIMS]; + +static int nerrs = 0; + +static int ncid, varid; +static int dimids[MAXDIMS]; +static size_t odom[MAXDIMS]; +static float* array = NULL; +static float* expected = NULL; + +static unsigned int filterid = 0; +static size_t nparams = 0; +static unsigned int params[MAXPARAMS]; + +/* Test values: must match plugins/H5Zmisc.c */ +struct All spec = { +(char)-17, /* signed byte */ +(unsigned char)23, /* unsigned byte */ +(signed short)-25, /* signed short */ +(unsigned short)27U, /* unsigned short */ +77, /* signed int */ +93U, /* unsigned int */ +789.0f, /* float */ +-9223372036854775807LL, /* signed int64 */ +18446744073709551615ULL,/* unsigned int64 */ +(double)12345678.12345678/* double */ +}; + +/* Forward */ +static int test_test1(void); +static void init(int argc, char** argv); +static void reset(void); +static void odom_reset(void); +static int odom_more(void); +static int odom_next(void); +static int odom_offset(void); +static float expectedvalue(void); +static void verifyparams(void); + +#define ERRR do { \ +fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ +fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ +nerrs++;\ +} while (0) + +static int +check(int err,int line) +{ + if(err != NC_NOERR) { + fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err)); + } + return NC_NOERR; +} + +static void +report(const char* msg, int lineno) +{ + fprintf(stderr,"fail: line=%d %s\n",lineno,msg); + exit(1); +} + +#define CHECK(x) check(x,__LINE__) +#define REPORT(x) report(x,__LINE__) + +static int +verifychunks(void) +{ + size_t i; + int store = -1; + size_t localchunks[MAXDIMS]; + memset(localchunks,0,sizeof(localchunks)); + CHECK(nc_inq_var_chunking(ncid, varid, &store, localchunks)); + if(store != NC_CHUNKED) { + fprintf(stderr,"bad chunk store\n"); + return 0; + } + for(i=0;i 0) { + params = (unsigned int*)malloc(sizeof(unsigned int)*nparams); + if(params == NULL) + return NC_ENOMEM; + CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,params)); + } + if(filterid != TEST_ID) { + fprintf(stderr,"open: test id mismatch: %d\n",filterid); + free(params); + return NC_EFILTER; + } + if(nparams != NPARAMS) { + size_t i; + fprintf(stderr,"nparams mismatch\n"); + for(nerrs=0,i=0;i 0) return NC_EFILTER; + + /* Verify chunking */ + if(!verifychunks()) + return 0; + fflush(stderr); + return 1; +} + +static void +fill(void) +{ + +#ifdef NOODOM + { + int i; + if(actualproduct <= 1) abort(); + for(i=0;i= MAXERRS) + break; + } + } + } +#else + { + odom_reset(); + while(odom_more()) { + int offset = odom_offset(); + float expect = expectedvalue(); + if(array[offset] != expect) { + fprintf(stderr,"data mismatch: array[%d]=%f expected=%f\n", + offset,array[offset],expect); + errs++; + if(errs >= MAXERRS) + break; + } + odom_next(); + } + } +#endif + + if(errs == 0) + fprintf(stderr,"no data errors\n"); + return (errs == 0); +} + +static void +showparameters(void) +{ + size_t i; + fprintf(stderr,"test: nparams=%ld: params=",(unsigned long)nparams); + for(i=0;i=0;i--) { + odom[i] += 1; + if(odom[i] < dimsize[i]) break; + if(i == 0) return 0; /* leave the 0th entry if it overflows*/ + odom[i] = 0; /* reset this position*/ + } + return 1; +} + +static int +odom_offset(void) +{ + size_t i; + size_t offset = 0; + for(i=0;i 1) + testfile = argv[1]; + else + testfile = DFALT_TESTFILE; + + /* Setup various variables */ + totalproduct = 1; + actualproduct = 1; + chunkproduct = 1; + for(i=0;i 0?1:0); +} diff --git a/v3_nczarr_test/testfilter_multi.c b/v3_nczarr_test/testfilter_multi.c new file mode 100644 index 0000000000..361905f45e --- /dev/null +++ b/v3_nczarr_test/testfilter_multi.c @@ -0,0 +1,331 @@ +/* + Copyright 2018, UCAR/Unidata + See COPYRIGHT file for copying and redistribution conditions. +*/ + +/*! \file +Test support for multiple filters per variable +*/ + +#include "config.h" +#include +#include +#include + +#ifdef USE_HDF5 +#include +#endif + +#include "netcdf.h" +#include "netcdf_filter.h" + +/* The HDF assigned id for bzip compression */ +#define BZIP2_ID 307 +/* The compression level used in this example */ +#define BZIP2_LEVEL 9 + +#define DEFLATE_LEVEL 2 + +#define NOOP_ID 40000 + +#define NFILTERS 3 + +#define DFALT_TESTFILE "tmp_multifilter.nc" + +/* Point at which we give up */ +#define MAXERRS 8 + +#define NDIMS 4 +#define DIMSIZE 4 +#define CHUNKSIZE 4 /* Note: not the total size of the chunk, but size wrt a dim*/ + +static const char* testfile = NULL; + +static size_t dimsize = DIMSIZE; +static size_t chunksize = CHUNKSIZE; +static size_t actualdims = NDIMS; + +static size_t actualproduct = 1; /* x-product over dim sizes */ +static size_t chunkproduct = 1; /* x-product over chunksizes */ + +static size_t dims[NDIMS]; +static size_t chunks[NDIMS]; + +static int nerrs = 0; + +static int ncid, varid; +static int dimids[NDIMS]; +static float* array = NULL; +static float* expected = NULL; + +/* Forward */ +static void init(int argc, char** argv); +static int test_multi(void); +static int verifychunks(void); + +#define ERRR do { \ +fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ +fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ +nerrs++;\ +} while (0) + +static int +check(int err,int line) +{ + if(err != NC_NOERR) { + fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err)); + fflush(stderr); + exit(1); + } + return NC_NOERR; +} + +#define CHECK(x) check(x,__LINE__) + +/* +Read the chunking information about the variable +and verify that it is as expected. +*/ + +static int +verifychunks(void) +{ + size_t i; + int store = -1; + size_t chunksizes[NDIMS]; + memset(chunksizes,0,sizeof(chunksizes)); + CHECK(nc_inq_var_chunking(ncid, varid, &store, chunksizes)); + /* Storate must be chunked, not contiguous */ + if(store != NC_CHUNKED) { + fprintf(stderr,"bad chunk store\n"); + return NC_ESTORAGE; + } + /* Chunk sizes must match our predefined set */ + for(i=0;i= MAXERRS) + break; + } + } + if(errs == 0) + printf("no data errors\n"); + if(actualproduct <= 1) + return NC_EBADDIM; + return (errs == 0 ? NC_NOERR: NC_EINVAL); +} + +int +verifyfilters(int ncid, int varid) +{ + size_t nparams; + size_t nfilters; + unsigned int filterids[NFILTERS]; + unsigned int params[2]; + + /* Read back the compression info and verify it */ + CHECK(nc_inq_var_filter_ids(ncid,varid,&nfilters,filterids)); + if(nfilters != NFILTERS) { + fprintf(stderr,"Fail: nfilters mismatch: expected=%d actual=%u\n",NFILTERS,(unsigned)nfilters); + return NC_EINVAL; + } + if(filterids[0] != BZIP2_ID + || filterids[1] != H5Z_FILTER_DEFLATE + || filterids[2] != NOOP_ID + ) { + fprintf(stderr,"Fail: filter id mismatch: actual/expected={%u/%u,%u/%u,%u/%u}\n", + filterids[0],BZIP2_ID, + filterids[1],H5Z_FILTER_DEFLATE, + filterids[2],NOOP_ID); + return NC_EINVAL; + } + /* Get level for each filter */ + CHECK(nc_inq_var_filter_info(ncid,varid,BZIP2_ID,&nparams,params)); + if(nparams != 1) { + fprintf(stderr,"Fail: nparams mismatch: id=%u expected=1 actual=%u\n",filterids[0],(unsigned)nparams); + return NC_EINVAL; + } + if(params[0] != BZIP2_LEVEL) { + fprintf(stderr,"Fail: parameter mismatch: expected=%u actual=%u\n",BZIP2_LEVEL,params[0]); + return NC_EINVAL; + } + CHECK(nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_DEFLATE,&nparams,params)); + if(nparams != 1) { + fprintf(stderr,"Fail: nparams mismatch: id=%u expected=1 actual=%u\n",filterids[1],(unsigned)nparams); + return NC_EINVAL; + } + if(params[0] != DEFLATE_LEVEL) { + fprintf(stderr,"Fail: parameter mismatch: expected=%u actual=%u\n",BZIP2_LEVEL,params[0]); + return NC_EINVAL; + } + CHECK(nc_inq_var_filter_info(ncid,varid,NOOP_ID,&nparams,params)); + if(nparams != 0) { + fprintf(stderr,"Fail: parameter mismatch: id=%u nparams: expected=0 actual=%u\n",NOOP_ID,(unsigned)nparams); + return NC_EINVAL; + } + return NC_NOERR; +} + +/* +Create the file, write it, then re-read for comparison. +*/ +static int +test_multi(void) +{ + size_t i; + unsigned int params[2]; + + printf("\n*** Testing Multi-filter application: filter set = bzip2 deflate noop"); + printf("\n"); + + /* Clear the data array */ + memset(array,0,sizeof(float)*actualproduct); + + /* Create a file */ + CHECK(nc_create(testfile, NC_NETCDF4|NC_CLOBBER, &ncid)); + + /* Do not use fill for this file */ + CHECK(nc_set_fill(ncid, NC_NOFILL, NULL)); + + /* Define the dimensions */ + for(i=0;i 1) + testfile = argv[1]; + else + testfile = DFALT_TESTFILE; + + /* Setup various variables */ + actualproduct = 1; + chunkproduct = 1; + for(i=0;i 0?1:0); +} + diff --git a/v3_nczarr_test/testfilter_order.c b/v3_nczarr_test/testfilter_order.c new file mode 100644 index 0000000000..5c37496a31 --- /dev/null +++ b/v3_nczarr_test/testfilter_order.c @@ -0,0 +1,441 @@ +/* + Copyright 2018, UCAR/Unidata + See COPYRIGHT file for copying and redistribution conditions. +*/ + +#include "config.h" +#include +#include +#include +#include +#include + +#include "netcdf.h" +#include "netcdf_filter.h" + +#undef TESTODDSIZE + +#undef DEBUG + +#define FILTER_ID 40000 + +#define MAXERRS 8 + +#define MAXPARAMS 32 + +#define MAXDIMS 8 + +#define DFALT_TESTFILE "tmp_filter_order.nc" + +#define NPARAMS 1 +#define PARAMVAL 17 + +#define NDIMS 4 +static size_t dimsize[NDIMS] = {4,4,4,4}; +static size_t chunksize[NDIMS] = {4,4,4,4}; + +static size_t ndims = NDIMS; + +static int creating = 1; /* Default is to do filter test 1 */ +static size_t totalproduct = 1; /* x-product over max dims */ +static size_t actualproduct = 1; /* x-product over actualdims */ +static size_t chunkproduct = 1; /* x-product over actual chunks */ + +static int nerrs = 0; + +static const char* testfile = NULL; + +static int ncid, varid; +static int dimids[MAXDIMS]; +static size_t odom[MAXDIMS]; +static float* array = NULL; +static float* expected = NULL; + +/* Forward */ +static int filter_test1(void); +static void init(int argc, char** argv); +static void reset(void); +static void odom_reset(void); +static int odom_more(void); +static int odom_next(void); +static size_t odom_offset(void); +static float expectedvalue(void); + +#define ERRR do { \ +fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ +fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ +nerrs++;\ +} while (0) + +static int +check(int err,int line) +{ + if(err != NC_NOERR) { + fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err)); + } + return NC_NOERR; +} + +static void +report(const char* msg, int lineno) +{ + fprintf(stderr,"fail: line=%d %s\n",lineno,msg); + exit(1); +} + +#define CHECK(x) check(x,__LINE__) +#define REPORT(x) report(x,__LINE__) + +static int +verifychunks(void) +{ + size_t i; + int store = -1; + size_t localchunks[MAXDIMS]; + memset(localchunks,0,sizeof(localchunks)); + CHECK(nc_inq_var_chunking(ncid, varid, &store, localchunks)); + if(store != NC_CHUNKED) { + fprintf(stderr,"bad chunk store\n"); + return 0; + } + for(i=0;i= MAXERRS) + break; + } + } + } else + { + odom_reset(); + while(odom_more()) { + size_t offset = odom_offset(); + float expect = expectedvalue(); + if(array[offset] != expect) { + fprintf(stderr,"data mismatch: array[%zu]=%f expected=%f\n", + offset,array[offset],expect); + errs++; + if(errs >= MAXERRS) + break; + } + odom_next(); + } + } + + if(errs == 0) + {printf("no data errors\n"); fflush(stdout);} + return (errs == 0); +} + +/* Test filter order on creation */ +static int +filter_test1(void) +{ + int ok = 1; + + reset(); + + printf("test1: filter order: create\n"); fflush(stdout); + create(); + setchunking(); + deffilters(); + inqfilters(); + CHECK(nc_enddef(ncid)); + + /* Fill in the array */ + fill(); + + nc_sync(ncid); + + printf("test1: compression.\n"); fflush(stdout); + /* write array */ + CHECK(nc_put_var(ncid,varid,expected)); + + printf("test1: decompression.\n"); fflush(stdout); + CHECK(nc_get_var_float(ncid, varid, array)); + ok = compare(); + + CHECK(nc_close(ncid)); + + return ok; +} + +/* Test filter order on read */ +static int +filter_test2(void) +{ + int ok = 1; + + reset(); + + printf("test2: filter order: read\n"); fflush(stdout); + + /* Fill in the array */ + fill(); + + printf("test2: decompression.\n"); fflush(stdout); + reset(); + openfile(); + inqfilters(); + + printf("test2: decompression.\n"); fflush(stdout); + CHECK(nc_get_var_float(ncid, varid, array)); + ok = compare(); + + CHECK(nc_close(ncid)); + return ok; +} + +/**************************************************/ +/* Utilities */ + +static void +reset() +{ + memset(array,0,sizeof(float)*actualproduct); +} + +static void +odom_reset(void) +{ + memset(odom,0,sizeof(odom)); +} + +static int +odom_more(void) +{ + return (odom[0] < dimsize[0]); +} + +static int +odom_next(void) +{ + int i; /* do not make unsigned */ + for(i=ndims-1;i>=0;i--) { + odom[i] += 1; + if(odom[i] < dimsize[i]) break; + if(i == 0) return 0; /* leave the 0th entry if it overflows*/ + odom[i] = 0; /* reset this position*/ + } + return 1; +} + +static size_t +odom_offset(void) +{ + size_t i; + size_t offset = 0; + for(i=0;i 2) + testfile = argv[2]; + else + testfile = DFALT_TESTFILE; + + /* Setup various variables */ + totalproduct = 1; + actualproduct = 1; + chunkproduct = 1; + for(i=0;i 0?1:0); +} diff --git a/v3_nczarr_test/testfilter_repeat.c b/v3_nczarr_test/testfilter_repeat.c new file mode 100644 index 0000000000..851958287a --- /dev/null +++ b/v3_nczarr_test/testfilter_repeat.c @@ -0,0 +1,378 @@ +/* + Copyright 2018, UCAR/Unidata + See COPYRIGHT file for copying and redistribution conditions. +*/ + +#include "config.h" +#include +#include +#include +#include +#include + +#include "netcdf.h" +#include "netcdf_filter.h" + +#undef TESTODDSIZE + +#undef DEBUG + +#define FILTER_ID 40000 + +#define MAXERRS 8 + +#define MAXPARAMS 32 + +#define MAXDIMS 8 + +#define DFALT_TESTFILE "testfilter_reg.nc" + +#define NPARAMS 1 +#define PARAMVAL 17 + +#define NDIMS 4 +static size_t dimsize[NDIMS] = {4,4,4,4}; +static size_t chunksize[NDIMS] = {4,4,4,4}; + +static size_t ndims = NDIMS; + +static size_t totalproduct = 1; /* x-product over max dims */ +static size_t actualproduct = 1; /* x-product over actualdims */ +static size_t chunkproduct = 1; /* x-product over actual chunks */ + +static int nerrs = 0; + +static char* testfile = NULL; + + +static int ncid, varid; +static int dimids[MAXDIMS]; +static size_t odom[MAXDIMS]; +static float* array = NULL; +static float* expected = NULL; + +/* Forward */ +static int filter_test1(void); +static void init(int argc, char** argv); +static void reset(void); +static void odom_reset(void); +static int odom_more(void); +static int odom_next(void); +static size_t odom_offset(void); +static float expectedvalue(void); + +#define ERRR do { \ +fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ +fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ +nerrs++;\ +} while (0) + +static int +check(int err,int line) +{ + if(err != NC_NOERR) { + fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err)); + } + return NC_NOERR; +} + +static void +report(const char* msg, int lineno) +{ + fprintf(stderr,"fail: line=%d %s\n",lineno,msg); + exit(1); +} + +#define CHECK(x) check(x,__LINE__) +#define REPORT(x) report(x,__LINE__) + +static int +verifychunks(void) +{ + size_t i; + int store = -1; + size_t localchunks[MAXDIMS]; + memset(localchunks,0,sizeof(localchunks)); + CHECK(nc_inq_var_chunking(ncid, varid, &store, localchunks)); + if(store != NC_CHUNKED) { + fprintf(stderr,"bad chunk store\n"); + return 0; + } + for(i=0;i= MAXERRS) + break; + } + } + } else + { + odom_reset(); + while(odom_more()) { + size_t offset = odom_offset(); + float expect = expectedvalue(); + if(array[offset] != expect) { + fprintf(stderr,"data mismatch: array[%zu]=%f expected=%f\n", + offset,array[offset],expect); + errs++; + if(errs >= MAXERRS) + break; + } + odom_next(); + } + } + + if(errs == 0) + printf("no data errors\n"); + return (errs == 0); +} + +static int +filter_test1(void) +{ + int ok = 1; + unsigned int params[MAXPARAMS]; + + reset(); + + printf("test1: def filter repeat .\n"); + create(); + setchunking(); + + params[0] = 1; + params[1] = 17; + deffilter(FILTER_ID,2,params); + + params[0] = 0; + params[1] = 18; + deffilter(FILTER_ID,2,params); + + CHECK(nc_enddef(ncid)); + + /* Fill in the array */ + fill(); + + printf("test1: compression.\n"); + /* write array */ + CHECK(nc_put_var(ncid,varid,expected)); + CHECK(nc_close(ncid)); + + printf("test1: decompression.\n"); + reset(); + openfile(); + CHECK(nc_get_var_float(ncid, varid, array)); + ok = compare(); + + CHECK(nc_close(ncid)); + return ok; +} + +/**************************************************/ +/* Utilities */ + +static void +reset() +{ + memset(array,0,sizeof(float)*actualproduct); +} + +static void +odom_reset(void) +{ + memset(odom,0,sizeof(odom)); +} + +static int +odom_more(void) +{ + return (odom[0] < dimsize[0]); +} + +static int +odom_next(void) +{ + int i; /* do not make unsigned */ + for(i=ndims-1;i>=0;i--) { + odom[i] += 1; + if(odom[i] < dimsize[i]) break; + if(i == 0) return 0; /* leave the 0th entry if it overflows*/ + odom[i] = 0; /* reset this position*/ + } + return 1; +} + +static size_t +odom_offset(void) +{ + size_t i; + size_t offset = 0; + for(i=0;i 1) + testfile = argv[1]; + else + testfile = DFALT_TESTFILE; + + /* Setup various variables */ + totalproduct = 1; + actualproduct = 1; + chunkproduct = 1; + for(i=0;i 0?1:0); +} diff --git a/v3_nczarr_test/timer_utils.c b/v3_nczarr_test/timer_utils.c new file mode 100644 index 0000000000..0b948b2a96 --- /dev/null +++ b/v3_nczarr_test/timer_utils.c @@ -0,0 +1,170 @@ +/********************************************************************* + * Copyright 2020, UCAR/Unidata + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + *********************************************************************/ + +#include "config.h" +#include +#include +#include +#include +#ifdef HAVE_TIME_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_SYS_RESOURCE_H +#include +#endif +#ifdef _WIN32 +#include +#endif + +#include "timer_utils.h" + +#undef DEBUG + +static int NCT_initialized = 0; + +#ifdef _WIN32 +LARGE_INTEGER frequency; +LARGE_INTEGER starttime; + +void +NCT_inittimer(void) +{ + if(NCT_initialized) return; +#ifdef DEBUG + fprintf(stderr,"timer mechanism: QueryPerformanceCounter\n"); +#endif + LARGE_INTEGER li; + (void)QueryPerformanceFrequency(&frequency); + QueryPerformanceCounter(&starttime); +#ifdef DEBUG +fprintf(stderr,"frequency=%lld starttime=%lld\n",frequency.QuadPart,starttime.QuadPart); +#endif + NCT_initialized = 1; +} +#else +void +NCT_inittimer(void) +{ + if(NCT_initialized) return; +#ifdef DEBUG +#if defined HAVE_CLOCK_GETTIME + fprintf(stderr,"timer mechanism: clock_gettime\n"); +#elif defined HAVE_GETTIMEOFDAY + fprintf(stderr,"timer mechanism: gettimeofday\n"); +#elif defined HAVE_GETRUSAGE + fprintf(stderr,"timer mechanism: getrusage\n"); +#else + fprintf(stderr,"timer mechanism: Unknown\n"); +#endif +#endif /*DEBUG*/ + NCT_initialized = 1; +} +#endif + +void +NCT_marktime(Nanotime* nt) +{ +#ifdef _WIN32 + LARGE_INTEGER endtime; + QueryPerformanceCounter(&endtime); + nt->tv_sec = endtime.QuadPart / 1000000000; + nt->tv_nsec = endtime.QuadPart % 1000000000; +#ifdef DEBUG +fprintf(stderr,"endtime=%lld\n",endtime.QuadPart); +#endif +#endif + +#ifndef _WIN32 +/* Pick one */ +#ifdef HAVE_CLOCK_GETTIME + clockid_t clk_id = CLOCK_MONOTONIC; + struct timespec t; + clock_gettime(clk_id,&t); + nt->tv_sec = (long long)t.tv_sec; + nt->tv_nsec = (long long)t.tv_nsec; +#elif defined HAVE_GETTIMEOFDAY + struct timeval tp; + gettimeofday(&tp, NULL); + nt->tv_sec = (long long)tp.tv_sec; + nt->tv_nsec = 1000*(long long)tp.tv_usec; +# elif defined HAVE_GETRUSAGE + struct rusage ru; + getrusage(RUSAGE_SELF, &ru); + nt->tv_sec = (long long)(ru.ru_utime.tv_sec + ru.ru_stime.tv_sec); + nt->tv_nsec = (long long)(1000*(ru.ru_utime.tv_usec + ru.ru_stime.tv_usec)); +#endif +#endif +} + +void +NCT_elapsedtime(Nanotime* nt0, Nanotime* nt1, Nanotime* delta) +{ + long long nsec[2]; + long long deltansec; + + nsec[0] = nt0->tv_nsec+(1000000000 * nt0->tv_sec); + nsec[1] = nt1->tv_nsec+(1000000000 * nt1->tv_sec); + + deltansec = nsec[1] - nsec[0]; + delta->tv_nsec = deltansec % 1000000000; + delta->tv_sec = deltansec / 1000000000; +#ifdef DEBUG +fprintf(stderr,"delta=(%lld,%lld)\n",delta->tv_sec,delta->tv_nsec); +#endif +} + +int +NCT_reporttime(unsigned nelems, Nanotime* times, struct TimeRange range, const char* tag) +{ + Nanotime delta; + long long nsec,avg; + double dnsec,dsec,davg; + + NCT_elapsedtime(×[0],×[1],&delta); + nsec = NCT_nanoseconds(delta); + avg = nsec / nelems; +#ifdef DEBUG +fprintf(stderr,"nsec=%lld avg=%lld\n",nsec,avg); +#endif + dnsec = (double)nsec; + dsec = dnsec / 1000000000.0; +#ifdef DEBUG +fprintf(stderr,"dsec=%g dnsec=%g\n",dsec,dnsec); +#endif + davg = (dnsec/nelems); + fprintf(stderr,"\t%s:\t%8.6lf sec",tag,dsec); + fprintf(stderr," avg=%5.2lf nsec\n",davg); +#ifdef DEBUG + fprintf(stderr,"range: min=%lld max=%lld\n",range.min,range.max); +#endif + if(!NCT_rangetest(avg,range)) { + fprintf(stderr,"*** WARNING: unexpectedly large timing values%s\n",tag); + } + return 1; +} + +long long +NCT_nanoseconds(Nanotime time) +{ + return (time.tv_sec * 1000000000) + time.tv_nsec; +} + +/* Provide a time range tester */ +int +NCT_rangetest(long long nsec, struct TimeRange range) +{ + if(nsec < range.min) { + fprintf(stderr,"range: time=%lld < min=%lld\n",nsec,range.min); + return 0; + } + if(nsec > range.max) { + fprintf(stderr,"range: time=%lld > max=%lld\n",nsec,range.max); + return 0; + } + return 1; +} diff --git a/v3_nczarr_test/timer_utils.h b/v3_nczarr_test/timer_utils.h new file mode 100644 index 0000000000..13869a2204 --- /dev/null +++ b/v3_nczarr_test/timer_utils.h @@ -0,0 +1,30 @@ +/********************************************************************* + * Copyright 2020, UCAR/Unidata + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + *********************************************************************/ + +#ifndef TIMER_UTILS_H +#define TIMER_UTILS_H 1 + +#include "netcdf.h" + +/* Wrap time interval computations */ +typedef struct Nanotime { + long long tv_sec; /* seconds */ + long long tv_nsec; /* nanoseconds */ +} Nanotime; + + +struct TimeRange { + long long min; + long long max; +}; + +extern void NCT_inittimer(void); +extern void NCT_marktime(Nanotime* nt); +extern void NCT_elapsedtime(Nanotime* nt0, Nanotime* nt1, Nanotime* delta); +extern int NCT_reporttime(unsigned count, Nanotime* times, struct TimeRange, const char* tag); +extern long long NCT_nanoseconds(Nanotime time); +extern int NCT_rangetest(long long nsecs, struct TimeRange range); +#endif /*TIMER_UTILS_H*/ + diff --git a/v3_nczarr_test/tst_multifilter.c b/v3_nczarr_test/tst_multifilter.c new file mode 100644 index 0000000000..ad516ee61d --- /dev/null +++ b/v3_nczarr_test/tst_multifilter.c @@ -0,0 +1,328 @@ +/* + Copyright 2018, UCAR/Unidata + See COPYRIGHT file for copying and redistribution conditions. +*/ + +/*! \file +Test support for multiple filters per variable +*/ + +#include "config.h" +#include +#include +#include + +#include +#include "netcdf.h" +#include "netcdf_filter.h" + +/* The HDF assigned id for bzip compression */ +#define BZIP2_ID 307 +/* The compression level used in this example */ +#define BZIP2_LEVEL 9 + +#define DEFLATE_LEVEL 2 + +#define NOOP_ID 40000 + +#define NFILTERS 3 + +#define DFALT_TESTFILE "tmp_multifilter.nc" + +/* Point at which we give up */ +#define MAXERRS 8 + +#define NDIMS 4 +#define DIMSIZE 4 +#define CHUNKSIZE 4 /* Note: not the total size of the chunk, but size wrt a dim*/ + +static const char* testfile = NULL; + +static size_t dimsize = DIMSIZE; +static size_t chunksize = CHUNKSIZE; +static size_t actualdims = NDIMS; + +static size_t actualproduct = 1; /* x-product over dim sizes */ +static size_t chunkproduct = 1; /* x-product over chunksizes */ + +static size_t dims[NDIMS]; +static size_t chunks[NDIMS]; + +static int nerrs = 0; + +static int ncid, varid; +static int dimids[NDIMS]; +static float* array = NULL; +static float* expected = NULL; + +/* Forward */ +static void init(int argc, char** argv); +static int test_multi(void); +static int verifychunks(void); + +#define ERRR do { \ +fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ +fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ +nerrs++;\ +} while (0) + +static int +check(int err,int line) +{ + if(err != NC_NOERR) { + fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err)); + fflush(stderr); + exit(1); + } + return NC_NOERR; +} + +#define CHECK(x) check(x,__LINE__) + +/* +Read the chunking information about the variable +and verify that it is as expected. +*/ + +static int +verifychunks(void) +{ + int i; + int store = -1; + size_t chunksizes[NDIMS]; + memset(chunksizes,0,sizeof(chunksizes)); + CHECK(nc_inq_var_chunking(ncid, varid, &store, chunksizes)); + /* Storate must be chunked, not contiguous */ + if(store != NC_CHUNKED) { + fprintf(stderr,"bad chunk store\n"); + return NC_ESTORAGE; + } + /* Chunk sizes must match our predefined set */ + for(i=0;i= MAXERRS) + break; + } + } + if(errs == 0) + printf("no data errors\n"); + if(actualproduct <= 1) + return NC_EBADDIM; + return (errs == 0 ? NC_NOERR: NC_EINVAL); +} + +int +verifyfilters(int ncid, int varid) +{ + size_t nparams; + size_t nfilters; + unsigned int filterids[NFILTERS]; + unsigned int params[2]; + + /* Read back the compression info and verify it */ + CHECK(nc_inq_var_filter_ids(ncid,varid,&nfilters,filterids)); + if(nfilters != NFILTERS) { + fprintf(stderr,"Fail: nfilters mismatch: expected=%d actual=%u\n",NFILTERS,(unsigned)nfilters); + return NC_EINVAL; + } + if(filterids[0] != BZIP2_ID + || filterids[1] != H5Z_FILTER_DEFLATE + || filterids[2] != NOOP_ID + ) { + fprintf(stderr,"Fail: filter id mismatch: actual/expected={%u/%u,%u/%u,%u/%u}\n", + filterids[0],BZIP2_ID, + filterids[1],H5Z_FILTER_DEFLATE, + filterids[2],NOOP_ID); + return NC_EINVAL; + } + /* Get level for each filter */ + CHECK(nc_inq_var_filter_info(ncid,varid,BZIP2_ID,&nparams,params)); + if(nparams != 1) { + fprintf(stderr,"Fail: nparams mismatch: id=%u expected=1 actual=%u\n",filterids[0],(unsigned)nparams); + return NC_EINVAL; + } + if(params[0] != BZIP2_LEVEL) { + fprintf(stderr,"Fail: parameter mismatch: expected=%u actual=%u\n",BZIP2_LEVEL,params[0]); + return NC_EINVAL; + } + CHECK(nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_DEFLATE,&nparams,params)); + if(nparams != 1) { + fprintf(stderr,"Fail: nparams mismatch: id=%u expected=1 actual=%u\n",filterids[1],(unsigned)nparams); + return NC_EINVAL; + } + if(params[0] != DEFLATE_LEVEL) { + fprintf(stderr,"Fail: parameter mismatch: expected=%u actual=%u\n",BZIP2_LEVEL,params[0]); + return NC_EINVAL; + } + CHECK(nc_inq_var_filter_info(ncid,varid,NOOP_ID,&nparams,params)); + if(nparams != 0) { + fprintf(stderr,"Fail: parameter mismatch: id=%u nparams: expected=0 actual=%u\n",NOOP_ID,(unsigned)nparams); + return NC_EINVAL; + } + return NC_NOERR; +} + +/* +Create the file, write it, then re-read for comparison. +*/ +static int +test_multi(void) +{ + int i; + unsigned int params[2]; + + printf("\n*** Testing Multi-filter application: filter set = bzip2 deflate noop"); + printf("\n"); + + /* Clear the data array */ + memset(array,0,sizeof(float)*actualproduct); + + /* Create a file */ + CHECK(nc_create(testfile, NC_NETCDF4|NC_CLOBBER, &ncid)); + + /* Do not use fill for this file */ + CHECK(nc_set_fill(ncid, NC_NOFILL, NULL)); + + /* Define the dimensions */ + for(i=0;i 1) + testfile = argv[1]; + else + testfile = DFALT_TESTFILE; + + /* Setup various variables */ + actualproduct = 1; + chunkproduct = 1; + for(i=0;i 0?1:0); +} + diff --git a/v3_nczarr_test/tst_pure_awssdk.cpp b/v3_nczarr_test/tst_pure_awssdk.cpp new file mode 100644 index 0000000000..3c9feb81a5 --- /dev/null +++ b/v3_nczarr_test/tst_pure_awssdk.cpp @@ -0,0 +1,19 @@ +#include +#include +int main(int argc, char** argv) +{ + Aws::SDKOptions options; + + std::cout << "Running a pure-aws test instantiation to test the aws-cpp-sdk install.\nA failure may manifest as a hang.\n\n"; + + std::cout << "\t* Testing InitAPI()\n"; + options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; + Aws::InitAPI(options); + std::cout << "\t\t* Passed.\n"; + + std::cout << "\t* Testing ShutdownAPI()\n"; + Aws::ShutdownAPI(options); + std::cout << "\t\t* Passed.\n\nFinished.\n\n"; + + return 0; +} \ No newline at end of file diff --git a/v3_nczarr_test/ut_chunking.c b/v3_nczarr_test/ut_chunking.c new file mode 100644 index 0000000000..c0aa7140d0 --- /dev/null +++ b/v3_nczarr_test/ut_chunking.c @@ -0,0 +1,93 @@ +/* + * Copyright 2018, University Corporation for Atmospheric Research + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + */ + +#include "ut_includes.h" + +void ut_chunk_print(int sort, ...); + +/** +Test computation of applying a slice to a sequence of chunks +*/ + +int +main(int argc, char** argv) +{ + int stat = NC_NOERR; + NCZSliceProjections slpv[NC_MAX_VAR_DIMS]; + NCZOdometer* odom = NULL; + Vardef* var = NULL; + struct Common common; + + /* Initialize */ + memset(&slpv,0,sizeof(slpv)); + + if((stat = ut_init(argc, argv, &utoptions))) goto done; + + var = nclistget(utoptions.vardefs,0); + + /* Fill in parts of common */ + fillcommon(&common,var); + + /* Set the printer */ + zutester.tests = UTEST_RANGE; + zutester.print = ut_chunk_print; + zutest = &zutester; + + if((stat = NCZ_projectslices(&common, utoptions.slices, &odom))) + goto done; + +#if 0 + { + int i; + size64_t* mem = (size64_t*)printer.output; + for(i=0;i +#endif + +#include "zmap.h" + +#include "netcdf.h" +#include "nclist.h" +#include "ncbytes.h" +#include "ncuri.h" +#include "ncpathmgr.h" + +#include "zincludes.h" +#include "ut_test.h" +#include "ut_util.h" + +#endif /*UT_INCLUDES_H*/ diff --git a/v3_nczarr_test/ut_json.c b/v3_nczarr_test/ut_json.c new file mode 100644 index 0000000000..e591466b8b --- /dev/null +++ b/v3_nczarr_test/ut_json.c @@ -0,0 +1,284 @@ +/* + * Copyright 2018, University Corporation for Atmospheric Research + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + */ + +#include "ut_includes.h" + +#undef DEBUG + +typedef enum Cmds { + cmd_none = 0, + cmd_build = 1, + cmd_parse = 2, +} Cmds; + +/* Forward */ +static int testbuild(void); +static int testparse(void); + +static void dump(NCjson* json); +static void dumpR(NCjson* json, int depth); +static char* sortname(int sort); + +struct Test tests[] = { +{"build", testbuild}, +{"parse", testparse}, +{NULL, NULL} +}; + +typedef struct NCJ { + NCjson* ncj_string; + NCjson* ncj_int; + NCjson* ncj_double; + NCjson* ncj_boolean; + NCjson* ncj_null; + NCjson* ncj_array1; + NCjson* ncj_array2; + NCjson* ncj_dict1; + NCjson* ncj_dict2; +} NCJ; + +int +main(int argc, char** argv) +{ + int stat = NC_NOERR; + + if((stat = ut_init(argc, argv, &utoptions))) goto done; + if((stat = runtests((const char**)utoptions.cmds,tests))) goto done; + +done: + ut_final(); + if(stat) usage(stat); + return 0; +} + +/* Build a reasonably complex json structure */ +static int +build(NCJ* ncj) +{ + int stat = NC_NOERR; + NCjson* clone = NULL; + + memset(ncj,0,sizeof(NCJ)); + + /* Build instances of primitives */ + if((stat = NCJnew(NCJ_STRING,&ncj->ncj_string))) goto done; + NCJsetstring(ncj->ncj_string, strdup("string")); + if((stat = NCJnew(NCJ_INT,&ncj->ncj_int))) goto done; + NCJsetstring(ncj->ncj_int, strdup("117")); + if((stat = NCJnew(NCJ_DOUBLE,&ncj->ncj_double))) goto done; + NCJsetstring(ncj->ncj_double, strdup("3.1415926")); + if((stat = NCJnew(NCJ_BOOLEAN,&ncj->ncj_boolean))) goto done; + NCJsetstring(ncj->ncj_boolean, strdup("true")); + if((stat = NCJnew(NCJ_NULL,&ncj->ncj_null))) goto done; + + /* Create an empty array */ + if((stat = NCJnew(NCJ_ARRAY,&ncj->ncj_array1))) goto done; + + /* Create a filled array */ + if((stat = NCJnew(NCJ_ARRAY,&ncj->ncj_array2))) goto done; + NCJclone(ncj->ncj_string,&clone); + if((stat = NCJappend(ncj->ncj_array2,clone))) goto done; + NCJclone(ncj->ncj_int,&clone); + if((stat = NCJappend(ncj->ncj_array2,clone))) goto done; + NCJclone(ncj->ncj_double,&clone); + if((stat = NCJappend(ncj->ncj_array2,clone))) goto done; + NCJclone(ncj->ncj_boolean,&clone); + if((stat = NCJappend(ncj->ncj_array2,clone))) goto done; + NCJclone(ncj->ncj_null,&clone); + if((stat = NCJappend(ncj->ncj_array2,clone))) goto done; + NCJclone(ncj->ncj_array1,&clone); + if((stat = NCJappend(ncj->ncj_array2,clone))) goto done; + + /* Create an empty dict */ + if((stat = NCJnew(NCJ_DICT,&ncj->ncj_dict1))) goto done; + + /* Create a filled dict */ + if((stat = NCJnew(NCJ_DICT,&ncj->ncj_dict2))) goto done; + NCJclone(ncj->ncj_string,&clone); + if((stat = NCJinsert(ncj->ncj_dict2,"string",clone))) goto done; + NCJclone(ncj->ncj_int,&clone); + if((stat = NCJinsert(ncj->ncj_dict2,"int",clone))) goto done; + NCJclone(ncj->ncj_double,&clone); + if((stat = NCJinsert(ncj->ncj_dict2,"double",clone))) goto done; + NCJclone(ncj->ncj_boolean,&clone); + if((stat = NCJinsert(ncj->ncj_dict2,"boolean",clone))) goto done; + NCJclone(ncj->ncj_null,&clone); + if((stat = NCJinsert(ncj->ncj_dict2,"null",clone))) goto done; + NCJclone(ncj->ncj_array1,&clone); + if((stat = NCJinsert(ncj->ncj_dict2,"array1",clone))) goto done; + NCJclone(ncj->ncj_array2,&clone); + if((stat = NCJinsert(ncj->ncj_dict2,"array2",clone))) goto done; + NCJclone(ncj->ncj_dict1,&clone); + if((stat = NCJinsert(ncj->ncj_dict2,"dict1",clone))) goto done; + +done: + return THROW(stat); +} +static void +clear(NCJ* ncj) +{ + NCJreclaim(ncj->ncj_array1); + NCJreclaim(ncj->ncj_array2); + NCJreclaim(ncj->ncj_dict1); + NCJreclaim(ncj->ncj_dict2); + NCJreclaim(ncj->ncj_string); + NCJreclaim(ncj->ncj_int); + NCJreclaim(ncj->ncj_double); + NCJreclaim(ncj->ncj_boolean); + NCJreclaim(ncj->ncj_null); +} + +/* Create test netcdf4 file via netcdf.h API*/ +static int +testbuild(void) +{ + int stat = NC_NOERR; + NCJ ncj; + + /* Build */ + if((stat = build(&ncj))) goto done; + + /* Primitives */ + dump(ncj.ncj_string); + dump(ncj.ncj_int); + dump(ncj.ncj_double); + dump(ncj.ncj_boolean); + dump(ncj.ncj_null); + + /* Empty array */ + dump(ncj.ncj_array1); + + /* Filled array */ + dump(ncj.ncj_array2); + + /* Empty dict */ + dump(ncj.ncj_dict1); + + /* Filled dict */ + dump(ncj.ncj_dict2); + +done: + clear(&ncj); + return THROW(stat); +} + +/* Test the parser */ +static int +testparse(void) +{ + NCJ ncj; + int stat = NC_NOERR; + char* text = NULL; + char* result = NULL; + NCjson* json = NULL; + + /* Build */ + if((stat = build(&ncj))) goto done; + + if((stat = NCJunparse(ncj.ncj_dict2,0,&text))) goto done; + + if((stat = NCJparse(text,0,&json))) goto done; + + if((stat = NCJunparse(json,0,&result))) goto done; + + printf("text : |%s|\nresult: |%s|\n",text,result); + +done: + nullfree(text); + nullfree(result); + NCJreclaim(json); + clear(&ncj); + return stat; +} + +static void +dump(NCjson* json) +{ + dumpR(json,0); + fflush(stdout); + fflush(stderr); +} + +static void +dumpR(NCjson* json, int depth) +{ + int ok, count; + size_t i; + long long int64v; + double float64v; + + printf("/%s/ ",sortname(NCJsort(json))); + switch(NCJsort(json)) { + case NCJ_STRING: printf("\"%s\"",NCJstring(json)); break; + case NCJ_INT: + ok = sscanf(NCJstring(json),"%lld%n",&int64v,&count); + if(ok != 1 || ((size_t)count) != strlen(NCJstring(json))) goto fail; + printf("%lld",int64v); + break; + case NCJ_DOUBLE: + ok = sscanf(NCJstring(json),"%lg%n",&float64v,&count); + if(ok != 1 || ((size_t)count) != strlen(NCJstring(json))) goto fail; + printf("%lg",float64v); + break; + case NCJ_BOOLEAN: + if(strcasecmp(NCJstring(json),"true") != 0 + && strcasecmp(NCJstring(json),"false") != 0) goto fail; + printf("%s",NCJstring(json)); + break; + case NCJ_NULL: + printf("null"); + break; + case NCJ_DICT: + if(NCJdictlength(json) == 0) { + printf("{}"); + } else { + printf("\n"); + for(i=0;i ",NCJstring(j)); + if(i >= NCJdictlength(json)) {/* malformed */ + printf(""); + } else + dumpR((NCjson*)NCJdictvalue(json,i),depth+1); + } + } + break; + case NCJ_ARRAY: + if(NCJarraylength(json) == 0) { + printf("[]"); + } else { + printf("\n"); + for(i=0;i totallen) + last = totallen; + count = last - start; + if((stat = nczmap_read(map, path, start, count, &data1p[start]))) + goto done; + } + + /* Validate */ + for(i=0;i + +#undef DEBUG + +#define META1 "/meta1" +#define META2 "/meta2" +#define DATA1 "/data1" +#define DATA1LEN 25 + +#define PASS 1 +#define FAIL 0 +#define XFAIL -1 + +static const char* metadata1 = "{\n\"foo\": 42,\n\"bar\": \"apples\",\n\"baz\": [1, 2, 3, 4]}"; +static const char* metaarray1 = "{\n\"shape\": [1,2,3],\n\"dtype\": \"<1\"}"; + +static char* url = NULL; +static NCZM_IMPL impl = NCZM_UNDEF; +static char* keyprefix = NULL; /* Hold, e.g. S3 bucket name */ + +/* Forward */ +static void title(const char*); +static int report(int pf, const char* op, NCZMAP*); +static char* makekey(const char* key); + +static int simplecreate(void); +static int simpledelete(void); +static int simplemeta(void); +static int simpledata(void); +static int search(void); + +struct Test tests[] = { +{"create",simplecreate}, +{"delete",simpledelete}, +{"simplemeta", simplemeta}, +{"simpledata", simpledata}, +{"search", search}, +{NULL,NULL} +}; + +int +main(int argc, char** argv) +{ + int stat = NC_NOERR; + char* tmp = NULL; + + if((stat = ut_init(argc, argv, &utoptions))) goto done; + if(utoptions.file == NULL && utoptions.output != NULL) utoptions.file = strdup(utoptions.output); + if(utoptions.output == NULL && utoptions.file != NULL)utoptions.output = strdup(utoptions.file); + + /* Canonicalize */ + if((stat = NCpathcanonical(utoptions.file,&tmp))) goto done; + free(utoptions.file); + utoptions.file = tmp; + if((stat = NCpathcanonical(utoptions.output,&tmp))) goto done; + free(utoptions.output); + utoptions.output = tmp; + + impl = kind2impl(utoptions.kind); + url = makeurl(utoptions.file,impl,&utoptions); + + if((stat = runtests((const char**)utoptions.cmds,tests))) goto done; + +done: + nullfree(tmp); + nullfree(url); url = NULL; + nullfree(keyprefix); + ut_final(); + if(stat) usage(THROW(stat)); + return 0; +} + +/* Do a simple create */ +static int +simplecreate(void) +{ + int stat = NC_NOERR; + NCZMAP* map = NULL; + char* truekey = NULL; + + title(__func__); + + if((stat = nczmap_truncate(impl,url))) + goto done; + report(PASS,"truncate",map); + + switch(stat = nczmap_create(impl,url,0,0,NULL,&map)) { + case NC_EOBJECT: break; /* already exists */ + case NC_NOERR: break; /*created*/ + default: goto done; + } + + printf("Pass: create: create: %s\n",url); + + truekey = makekey(Z2METAROOT); + if((stat = nczmap_write(map, truekey, 0, NULL))) + goto done; + printf("Pass: create: defineobj: %s\n",truekey); + + /* Do not delete */ + if((stat = nczmap_close(map,0))) + goto done; + map = NULL; + printf("Pass: create: close\n"); + + /* Reopen and see if exists */ + if((stat = nczmap_open(impl,url,0,0,NULL,&map))) + goto done; + printf("Pass: create: open: %s\n",url); + + if((stat = nczmap_exists(map,truekey))) + goto done; + printf("Pass: create: exists: %s\n",truekey); + + /* close again */ + if((stat = nczmap_close(map,0))) + goto done; + map = NULL; + printf("Pass: create: close\n"); + +done: + nullfree(truekey); + return THROW(stat); +} + +/* Do a simple delete of previously created file */ +static int +simpledelete(void) +{ + int stat = NC_NOERR; + NCZMAP* map = NULL; + + title(__func__); + + switch ((stat = nczmap_open(impl,url,0,0,NULL,&map))) { + case NC_NOERR: + report(PASS,"open",map); + break; + default: + {report(FAIL,"open",map); goto done;} + } + /* Delete dataset while closing */ + if((stat = nczmap_close(map,1))) goto done; + map = NULL; + report(PASS,"close: delete",map); + + switch ((stat = nczmap_open(impl,url,0,0,NULL,&map))) { + case NC_NOERR: + report(FAIL,"open",map); + break; + case NC_ENOOBJECT: + report(XFAIL,"open",map); + stat = NC_NOERR; + break; + default: abort(); + } + +done: + return THROW(stat); +} + +static int +simplemeta(void) +{ + int stat = NC_NOERR; + NCZMAP* map = NULL; + char* key = NULL; + char* truekey = NULL; + size64_t size = 0; + char* content = NULL; + + title(__func__); + + if((stat = nczmap_open(impl,url,NC_WRITE,0,NULL,&map))) + goto done; + report(PASS,"open",map); + + /* Make sure .nczarr exists (from simplecreate) */ + truekey = makekey(Z2METAROOT); + if((stat = nczmap_exists(map,truekey))) + goto done; + report(PASS,".nczarr: exists",map); + free(truekey); truekey = NULL; + + if((stat=nczm_concat(META1,Z2ARRAY,&key))) + goto done; + truekey = makekey(key); + nullfree(key); key = NULL; + if((stat = nczmap_write(map, truekey, 0, NULL))) + goto done; + report(PASS,".zarray: def",map); + free(truekey); truekey = NULL; + + truekey = makekey(Z2METAROOT); + if((stat = nczmap_write(map, truekey, strlen(metadata1), metadata1))) + goto done; + report(PASS,".nczarr: writemetadata",map); + free(truekey); truekey = NULL; + + if((stat=nczm_concat(META1,Z2ARRAY,&key))) + goto done; + truekey = makekey(key); + free(key); key = NULL; + + if((stat = nczmap_write(map, truekey, strlen(metaarray1), metaarray1))) + goto done; + report(PASS,".zarray: writemetaarray1",map); + free(truekey); truekey = NULL; + + if((stat = nczmap_close(map,0))) + goto done; + map = NULL; + report(PASS,"close",map); + + if((stat = nczmap_open(impl,url,0,0,NULL,&map))) + goto done; + report(PASS,"re-open",map); + + /* Read previously written */ + truekey = makekey(Z2METAROOT); + if((stat = nczmap_exists(map, truekey))) + goto done; + report(PASS,".nczarr: exists",map); + if((stat = nczmap_len(map, truekey, &size))) + goto done; + report(PASS,".nczarr: len",map); + if(size != strlen(metadata1)) + report(FAIL,".nczarr: len verify",map); + if((content = calloc(1,strlen(metadata1)+1))==NULL) + {stat = NC_ENOMEM; goto done;} + if((stat = nczmap_read(map, truekey, 0, strlen(metadata1), content))) + goto done; + report(PASS,".nczarr: readmetadata",map); + free(truekey); truekey = NULL; + if(memcmp(content,metadata1,size)!=0) + report(FAIL,".nczarr: content verify",map); + else report(PASS,".nczarr: content verify",map); + nullfree(content); content = NULL; + + if((stat=nczm_concat(META1,Z2ARRAY,&key))) + goto done; + truekey = makekey(key); + nullfree(key); key = NULL; + if((stat = nczmap_exists(map, truekey))) + goto done; + report(PASS,".zarray: exists",map); + if((stat = nczmap_len(map, truekey, &size))) + goto done; + report(PASS,".zarray: len",map); + if(size != strlen(metaarray1)) + report(FAIL,".zarray: len verify",map); + content = calloc(1,strlen(metaarray1)+1); + if((stat = nczmap_read(map, truekey, 0, strlen(metaarray1), content))) + goto done; + report(PASS,".zarray: readmeta",map); + free(truekey); truekey = NULL; + if(memcmp(content,metaarray1,size)!=0) + report(FAIL,".zarray: content verify",map); + else + report(PASS,".zarray:content verify",map); + nullfree(content); content = NULL; + + if((stat = nczmap_close(map,0))) + goto done; + map = NULL; + report(PASS,"close",map); + +done: + if(map) nczmap_close(map,0); + nullfree(content); + nullfree(truekey); + nullfree(key); + return THROW(stat); +} + +static int +simpledata(void) +{ + int stat = NC_NOERR; + NCZMAP* map = NULL; + char* truekey = NULL; + int data1[DATA1LEN]; + int readdata[DATA1LEN]; + int i; + size64_t totallen, size; + char* data1p = (char*)&data1[0]; /* byte level version of data1 */ + + title(__func__); + + /* Create the data */ + for(i=0;iprotocol,"file")==0) return; + + segments = nclistnew(); + nczm_split_delim(uri->path,'/',segments); + /* Extract the first two segments */ + if(nclistlength(segments) < 1) return; /* not enough to qualify */ + /* Remove the bucket */ + { char* s = nclistremove(segments,0); + nullfree(s); /* do not nest because arg is eval'd twice */ + } + nczm_join(segments,&keyprefix); + nclistfreeall(segments); + ncurifree(uri); +} +#endif + +static char* +makekey(const char* key) +{ + char* truekey = NULL; + nczm_concat(keyprefix,key,&truekey); + return truekey; +} + +static void +title(const char* func) +{ + printf("testing: %s:\n",func); + fflush(stdout); +} + +static int +report(int pf, const char* op, NCZMAP* map) +{ + const char* result; + switch (pf) { + case PASS: result = "Pass"; break; + case XFAIL: result = "XFail"; break; + case FAIL: default: result = "Fail"; break; + } + fprintf(stderr,"%s: %s\n",result,op); + fflush(stderr); + if(pf == FAIL) { + if(map) (void)nczmap_close(map,0); + exit(1); + } + return NC_NOERR; +} diff --git a/v3_nczarr_test/ut_projections.c b/v3_nczarr_test/ut_projections.c new file mode 100644 index 0000000000..8ee0dba780 --- /dev/null +++ b/v3_nczarr_test/ut_projections.c @@ -0,0 +1,137 @@ +/* + * Copyright 2018, University Corporation for Atmospheric Research + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + */ + +#include "ut_includes.h" + +void ut_chunk_test(int sort, ...); + +/** +Test computation of applying a slice to a sequence of chunks +*/ + +int +main(int argc, char** argv) +{ + int i,r,stat = NC_NOERR; + Vardef* var = NULL; + struct Common common; + NCZSliceProjections slpv[NC_MAX_VAR_DIMS]; + NCZChunkRange ncrv[NC_MAX_VAR_DIMS]; + + /* Initialize */ + memset(&slpv,0,sizeof(slpv)); + memset(&common,0,sizeof(common)); + + if((stat = ut_init(argc, argv, &utoptions))) goto done; + + /* printer off for these tests */ + zutester.tests = 0; + zutester.print = NULL; + zutest = &zutester; + + printoptions(&utoptions); + + var = nclistget(utoptions.vardefs,0); + + fillcommon(&common,var); + + /* Compute chunk ranges */ + if((stat = NCZ_compute_chunk_ranges(&common,utoptions.slices,ncrv))) + goto done; + + if((stat=NCZ_compute_all_slice_projections( + &common, + utoptions.slices, + ncrv, + slpv))) goto done; + + /* Dump Results */ + for(r=0;rrank;r++) { + NCZSliceProjections* slp = &slpv[r]; + if(r != slp->r) usage(NC_EINTERNAL); + printf("[r=%d] %s %s\n",r,nczprint_chunkrange(slp->range),nczprint_slice(utoptions.slices[r])); + for(i=0;icount;i++) { + NCZProjection* proj = &slp->projections[i]; + printf("[%d] %s\n",i,nczprint_projection(*proj)); + } + } + + /* Cleanup */ + NCZ_clearsliceprojections(var->rank,slpv); + +#if 0 + /* Compute corresponding slice projections */ + for(r=0;rrank;r++) { + if((stat = NCZ_compute_per_slice_projections( + r, + &utoptions.slices[r], + &ncrv[r], + var->dimsizes[r], + var->chunksizes[r], + &slpv[r]))) goto done; + } + + /* Dump Results */ + for(r=0;rrank;r++) { + NCZSliceProjections* slp = &slpv[r]; + char *sr, *sl; + if(r != slp->r) usage(NC_EINTERNAL); + sr = nczprint_chunkrange(slp->range); + sl = nczprint_slice(utoptions.slices[r]); + printf("[r=%d] %s %s\n",r,sr,sl); + nullfree(sr); nullfree(sl); + for(i=0;icount;i++) { + NCZProjection* proj = &slp->projections[i]; + printf("[%d] %s\n",i,nczprint_projection(*proj)); + } + } + /* Cleanup */ + NCZ_clearsliceprojections(var->rank,slpv); +#endif + +done: + fflush(stdout); + nczprint_reclaim(); + ut_final(); + if(stat) usage(stat); + return 0; +} + +void +ut_chunk_test(int sort,...) +{ + int i; + va_list ap; +#if 0 + struct Common* common = NULL; +#endif + int rank; /* variable rank */ + NCZSlice* slices = NULL; /* the complete set of slices |slices| == R*/ + size64_t* chunksizes = NULL; /* the chunk length corresponding to the dimensions */ + NCZChunkRange* ranges = NULL; /* computed chunk ranges */ + + va_start(ap,sort); + + switch (sort) { + default: break; /* ignore */ + case UTEST_RANGE: /* () */ + rank = va_arg(ap,size64_t); + slices = va_arg(ap,NCZSlice*); + chunksizes = va_arg(ap,size64_t*); + ranges = va_arg(ap,NCZChunkRange*); + printf("[r=%d] Chunksizes: %s\n",rank,nczprint_vector(rank,chunksizes)); + printf("Slices: "); + for(i=0;i +#endif + +#if defined(_WIN32) && !defined(__MINGW32__) +#include "XGetopt.h" +#endif + +#ifdef UTTESST +struct ZUTEST zutester; +#endif + +struct UTOptions utoptions; + +/*Forward*/ +static void canonicalfile(char** fp); + +void +usage(int err) +{ + if(err) { + fprintf(stderr,"error: (%d) %s\n",err,nc_strerror(err)); + } + fprintf(stderr,"usage:"); + fprintf(stderr," -D/*debug*/"); + fprintf(stderr," -x"); + fprintf(stderr," -f"); + fprintf(stderr," -o"); + fprintf(stderr," -k"); + fprintf(stderr," -d="); + fprintf(stderr," -vvar()"); + fprintf(stderr," -s"); + fprintf(stderr," -W,..."); + fprintf(stderr,"\n"); + fflush(stderr); + exit(1); +} + +int +ut_init(int argc, char** argv, struct UTOptions * options) +{ + int stat = NC_NOERR; + int c; + Dimdef* dimdef = NULL; + Vardef* vardef = NULL; + + nc_initialize(); + + if(options != NULL) { + options->dimdefs = nclistnew(); + options->vardefs = nclistnew(); + while ((c = getopt(argc, argv, "T:Dx:f:o:p:k:d:v:s:W:")) != EOF) { + switch(c) { + case 'T': + nctracelevel(atoi(optarg)); + break; + case 'D': + options->debug = 1; + break; + case 'x': /*execute*/ + if(parsestringvector(optarg,0,&options->cmds) <= 0) usage(THROW(0)); + break; + case 'f': + options->file = strdup(optarg); + break; + case 'o': + options->output = strdup(optarg); + break; + case 'p': + options->profile = strdup(optarg); + break; + case 'k': /*implementation*/ + options->kind = strdup(optarg); + break; + case 'd': /*dimdef*/ + if((stat=parsedimdef(optarg,&dimdef))) usage(THROW(stat)); + nclistpush(options->dimdefs,dimdef); + dimdef = NULL; + break; + case 'v': /*vardef*/ + if((stat=parsevardef(optarg,options->dimdefs,&vardef))) usage(THROW(stat)); + nclistpush(options->vardefs,vardef); + vardef = NULL; + break; + case 's': /*slices*/ + if((stat=parseslices(optarg,&options->nslices,options->slices))) usage(THROW(stat)); + break; + case '?': + fprintf(stderr,"unknown option: '%c'\n",c); + stat = NC_EINVAL; + goto done; + } + } + } + + canonicalfile(&options->file); + canonicalfile(&options->output); + +done: + return THROW(stat); +} + +void +ut_final(void) +{ + nc_finalize(); +} + +#if 0 +static void +getpathcwd(char** cwdp) +{ + char buf[4096]; + (void)NCgetcwd(buf,sizeof(buf)); + if(cwdp) *cwdp = strdup(buf); +} +#endif + +static void +canonicalfile(char** fp) +{ + size_t len; + char* f = NULL; + char* abspath = NULL; + NCURI* uri = NULL; +#ifdef _WIN32 + int fwin32=0, cwd32=0; +#endif + + if(fp == NULL || *fp == NULL) return; + f = *fp; + len = strlen(f); + if(len <= 1) return; + ncuriparse(f,&uri); + if(uri != NULL) {ncurifree(uri); return;} /* its a url */ + +#if 1 + abspath = NCpathabsolute(f); +#else + if(f[0] == '/' || f[0] == '\\' || hasdriveletter(f)) + return; /* its already absolute */ +#ifdef _WIN32 + for(p=f;*p;p++) {if(*p == '\\') {*p = '/';}} +#endif + if(len >= 2 && memcmp(f,"./",2)==0) { + offset = 1; /* leave the '/' */ + } else if(len >= 3 && memcmp(f,"../",3)==0) { + offset = 2; + } else + offset = 0; + getpathcwd(&cwd); + len2 = strlen(cwd); +#ifdef _WIN32 + for(cwd32=0,p=cwd;*p;p++) {if(*p == '\\') {*p = '/'; cwd32 = 1;}} +#endif + if(offset == 2) { + p = strrchr(cwd,'/'); + /* remove last segment including the preceding '/' */ + if(p == NULL) {cwd[0] = '\0';} else {*p = '\0';} + } + len2 = (len-offset)+strlen(cwd); + if(offset == 0) len2++; /* need to add '/' */ + abspath = (char*)malloc(len2+1); + abspath[0] = '\0'; + strlcat(abspath,cwd,len2+1); + if(offset == 0) strlcat(abspath,"/",len2+1); + strlcat(abspath,f+offset,len2+1); +#ifdef _WIN32 + if(fwin32) + for(p=abspath;*p;p++) {if(*p == '/') {*p = '\\';}} +#endif + nullfree(cwd); +#endif + nullfree(f); +fprintf(stderr,"canonicalfile: %s\n",abspath); + *fp = abspath; +} + +void +nccheck(int stat, int line) +{ + if(stat) { + fprintf(stderr,"%d: %s\n",line,nc_strerror(stat)); + fflush(stderr); + exit(1); + } +} + +char* +makeurl(const char* file, NCZM_IMPL impl, struct UTOptions* options) +{ + char* url = NULL; + NCbytes* buf = ncbytesnew(); + NCURI* uri = NULL; + const char* kind = impl2kind(impl); + char* urlpath = NULL; + char* p; + + if(file && strlen(file) > 0) { + switch (impl) { + case NCZM_FILE: + case NCZM_ZIP: + /* Massage file to make it usable as URL path */ + urlpath = strdup(file); + for(p=urlpath;*p;p++) {if(*p == '\\') *p = '/';} + ncbytescat(buf,"file://"); + ncbytescat(buf,urlpath); + nullfree(urlpath); urlpath = NULL; + ncbytescat(buf,"#mode=nczarr"); /* => use default file: format */ + ncbytescat(buf,","); + ncbytescat(buf,kind); + break; + case NCZM_S3: + /* Assume that we have a complete url */ + if(ncuriparse(file,&uri)) return NULL; + if(options->profile) { + const char* profile = ncurifragmentlookup(uri,"aws.profile"); + if(profile == NULL) { + ncurisetfragmentkey(uri,"aws.profile",options->profile); + /* rebuild the url */ + file = (const char*)ncuribuild(uri,NULL,NULL,NCURIALL); /* BAD but simple */ + } + } + ncbytescat(buf,file); + break; + default: abort(); + } + url = ncbytesextract(buf); + } + ncurifree(uri); + ncbytesfree(buf); + fprintf(stderr,"url=|%s|\n",url); + fflush(stderr); + return url; +} + +struct Test* +findtest(const char* cmd, struct Test* tests) +{ + struct Test* t = NULL; + for(t=tests;t->cmd;t++) { + if(strcasecmp(t->cmd,cmd)==0) return t; + } + return NULL; +} + +int +runtests(const char** cmds, struct Test* tests) +{ + int stat = NC_NOERR; + struct Test* test = NULL; + const char** cmd = NULL; + if(cmds == NULL) return THROW(NC_EINVAL); + for(cmd=cmds;*cmd;cmd++) { + for(test=tests;test->cmd;test++) { + if(strcmp(test->cmd,*cmd)==0) { + if(test->cmd == NULL) return THROW(NC_EINVAL); + if((stat=test->test())) goto done; /* Execute */ + } + } + } +done: + return THROW(stat); +} diff --git a/v3_nczarr_test/ut_test.h b/v3_nczarr_test/ut_test.h new file mode 100644 index 0000000000..c2610c92c7 --- /dev/null +++ b/v3_nczarr_test/ut_test.h @@ -0,0 +1,78 @@ +/* + * Copyright 2018, University Corporation for Atmospheric Research + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + */ + +#ifndef ZTEST_H +#define ZTEST_H + +#include "nclist.h" + +typedef struct Dimdef { + char* name; + size64_t size; + int isunlimited; +} Dimdef; + +typedef struct Vardef { + char* name; + nc_type typeid; + size_t typesize; + size_t rank; + Dimdef* dimrefs[NC_MAX_VAR_DIMS]; + size64_t dimsizes[NC_MAX_VAR_DIMS]; + size64_t chunksizes[NC_MAX_VAR_DIMS]; +} Vardef; + +/* Expose functions for unit tests */ +typedef struct NCZ_UT_PRINTER { + int printsort; + void (*printer)(struct NCZ_UT_PRINTER*); + /* Union of all fields */ + size_t rank; + size64_t count; + size64_t offset; + size64_t* indices; + size64_t* vector; + void** pvector; + NCZOdometer* odom; + void* output; + size_t used; +} NCZ_UT_PRINTER; + +/* Arguments from command line */ +struct UTOptions { + int debug; + char** cmds; + char* file; + char* output; + char* kind; + char* profile; + NCZChunkRange ranges[NC_MAX_VAR_DIMS]; + int nslices; + NCZSlice slices[NC_MAX_VAR_DIMS]; + NClist* dimdefs; /*List */ + NClist* vardefs; /*List */ + int* idata; +}; + +struct Test { + char* cmd; + int (*test)(void); +}; + +EXTERNL struct UTOptions utoptions; + +#define NCCHECK(expr) nccheck((expr),__LINE__) + +EXTERNL void usage(int err); +EXTERNL int ut_init(int argc, char** argv, struct UTOptions* test); +EXTERNL void ut_final(void); + +EXTERNL void nccheck(int stat, int line); +EXTERNL char* makeurl(const char* file, NCZM_IMPL, struct UTOptions*); +//extern int setup(int argc, char** argv); +EXTERNL struct Test* findtest(const char* cmd, struct Test* tests); +EXTERNL int runtests(const char** cmds, struct Test* tests); + +#endif /*ZTEST_H*/ diff --git a/v3_nczarr_test/ut_util.c b/v3_nczarr_test/ut_util.c new file mode 100644 index 0000000000..ab348def7b --- /dev/null +++ b/v3_nczarr_test/ut_util.c @@ -0,0 +1,537 @@ +/* + * Copyright 2018, University Corporation for Atmospheric Research + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + */ + +#include "ut_includes.h" +#include + +#undef DEBUG + +#define OPEN "[{(" +#define CLOSE "]})" + +#define LPAREN '(' +#define RPAREN ')' +#define BLANK ' ' + +NCbytes* buf = NULL; + +static Dimdef* finddim(const char* name, NClist* defs); + +#if 0 +static void +ranktest(size_t rank, char c, int count) +{ + if(rank != count) { + fprintf(stderr,"Option '%c': rank mismatch: rank=%d count=%d\n", + c,rank,count); + exit(1); + } +} +#endif + +int +parseslices(const char* s0, int* nslicesp, NCZSlice* slices) +{ + int count,nchars,nslices,i; + const char* s = NULL; + unsigned long start,stop,stride; + + /* First, compute number of slices */ + for(s=s0,nslices=0;*s;s++) { + if(*s == '[') nslices++; + } + + if(nslices > NC_MAX_VAR_DIMS) return THROW(NC_EINVAL); /* too many */ + if(nslicesp) *nslicesp = nslices; + + /* Extract the slices */ + for(i=0,s=s0;*s;s+=nchars,i++) { + NCZSlice* sl = &slices[i]; + /* Try 3-element slice first */ + stride = 1; /* default */ + nchars = -1; + count = sscanf(s,"[%lu:%lu]%n",&start,&stop,&nchars); + if(nchars == -1) { + nchars = -1; + count = sscanf(s,"[%lu:%lu:%lu]%n",&start,&stop,&stride,&nchars); + if(count != 3) return THROW(NC_EINVAL); + } + sl->start = start; + sl->stop = stop; + sl->stride = stride; + } + return NC_NOERR; +} + +int +parsedimdef(const char* s0, Dimdef** defp) +{ + int nchars; + const char* s = NULL; + Dimdef* def = NULL; + unsigned l; + const char* p; + ptrdiff_t count; + + if((def = calloc(1,sizeof(struct Dimdef)))==NULL) + return THROW(NC_ENOMEM); + + /* Extract */ + s = s0; + if((p = strchr(s,'=')) == NULL) abort(); + if((count = (p - s)) == 0) return THROW(NC_EINVAL); + def->name = malloc((size_t)count+1); + memcpy(def->name,s,(size_t)count); + def->name[count] = '\0'; + s = p+1; + sscanf(s,"%u%n",&l,&nchars); + if(nchars == -1) return NC_EINVAL; + def->size = (size_t)l; + if(def->size == 0) def->isunlimited = 1; + s += nchars; + if(*s != '\0') return NC_EINVAL; + if(defp) *defp = def; + return NC_NOERR; +} + +int +parsevardef(const char* s0, NClist* dimdefs, Vardef** varp) +{ + size_t count; + const char* s = NULL; + Vardef* vd = NULL; + const char* p; + ptrdiff_t len; + char name[NC_MAX_NAME]; + + if((vd = calloc(1,sizeof(Vardef)))==NULL) + return THROW(NC_ENOMEM); + + s=s0; + + /* Scan for the end of type name */ + p = strchr(s,BLANK); + if(p == NULL) return THROW(NC_EINVAL); + len = (p - s); + if(len == 0) return THROW(NC_EINVAL); + memcpy(name,s,(size_t)len); + name[len] = '\0'; + vd->typeid = ut_typeforname(name); + vd->typesize = ut_typesize(vd->typeid); + while(*p == BLANK) p++; + s = p; + + /* Scan for the end of var name */ + p = strchr(s,LPAREN); + if(p == NULL) return THROW(NC_EINVAL); + len = (p - s); + if(len == 0) return THROW(NC_EINVAL); + memcpy(name,s,(size_t)len); + name[len] = '\0'; + vd->name = strdup(name); + /* parse a vector of dimnames and chunksizes and convert */ + s = p; + if(*s == LPAREN) { + char** names = NULL; + char* p; + s++; + count = parsestringvector(s,RPAREN,&names); + if(count >= NC_MAX_VAR_DIMS) return THROW(NC_EINVAL); + vd->rank = count; + if(vd->rank > 0) { + size_t j; + for(j=0;jrank;j++) { + Dimdef* dimref = NULL; + /* Split on / to get chunksize */ + p = strchr(names[j],'/'); + if(p) *p++ = '\0'; + if((dimref = finddim(names[j],dimdefs)) == NULL) + return THROW(NC_EINVAL); + vd->dimrefs[j] = dimref; + vd->dimsizes[j] = dimref->size; + if(p == NULL) + vd->chunksizes[j] = dimref->size; + else { + unsigned long l; + sscanf(p,"%lu",&l); + vd->chunksizes[j] = (size_t)l; + } + } + /* Skip past the trailing rparen */ + if((p = strchr(s,RPAREN)) == NULL) abort(); + p++; + } + freestringvec(names); + } + if(varp) *varp = vd; + return NC_NOERR; +} + +size_t +parsestringvector(const char* s0, int stopchar, char*** namesp) +{ + size_t nelems,i; + const char* s; + char** names = NULL; + + /* First, compute number of elements */ + for(s=s0,nelems=1;*s;s++) {if(*s == ',') nelems++; if(*s == stopchar) break;} + if(nelems == 0) return 0; + names = calloc((size_t)nelems+1,sizeof(char*)); + for(s=s0,i=0;iname); + nullfree(dd); + } +} + +void +freevardefs(NClist* defs) +{ + size_t i; + for(i=0;iname); + nullfree(vd); + } +} + +void +freeranges(NCZChunkRange* ranges) +{ + NC_UNUSED(ranges); +} + +void +freeslices(NCZSlice* slices) +{ + NC_UNUSED(slices); +} + +void +freestringvec(char** vec) +{ + if(vec != NULL) { + char** p; + for(p=vec;*p;p++) free(*p); + } + nullfree(vec); +} + +void +freeprojvector(size_t rank, NCZProjection** vec) +{ + if(vec != NULL) { + size_t r; + for(r=0;r 0) ncbytescat(buf,","); + snprintf(value,sizeof(value),"%lu",(unsigned long)vec[i]); + ncbytescat(buf,value); + } + ncbytescat(buf,")"); + result = ncbytesextract(buf); + ncbytesfree(buf); + return result; +} +#endif /*0*/ + +/**************************************************/ +size_t +ut_typesize(nc_type t) +{ + switch (t) { + case NC_BYTE: case NC_UBYTE: return 1; + case NC_SHORT: case NC_USHORT: return 2; + case NC_INT: case NC_UINT: return 4; + case NC_INT64: case NC_UINT64: return 8; + case NC_FLOAT: return 4; + case NC_DOUBLE: return 8; + default: usage(THROW(NC_EINVAL)); + } + return 0; +} + +nc_type +ut_typeforname(const char* tname) +{ + if(strcasecmp("byte",tname)==0) return NC_BYTE; + if(strcasecmp("ubyte",tname)==0) return NC_UBYTE; + if(strcasecmp("short",tname)==0) return NC_SHORT; + if(strcasecmp("ushort",tname)==0) return NC_USHORT; + if(strcasecmp("int",tname)==0) return NC_INT; + if(strcasecmp("uint",tname)==0) return NC_UINT; + if(strcasecmp("int64",tname)==0) return NC_INT64; + if(strcasecmp("uint64",tname)==0) return NC_UINT64; + if(strcasecmp("float",tname)==0) return NC_FLOAT; + if(strcasecmp("double",tname)==0) return NC_DOUBLE; + usage(THROW(NC_EINVAL)); + return NC_NAT; +} + +static Dimdef* +finddim(const char* name, NClist* defs) +{ + size_t i; + for(i=0;iname,name) == 0) + return dd; + } + return NULL; +} + +NCZM_IMPL +kind2impl(const char* kind) +{ + if(strcasecmp("s3",kind)==0) return NCZM_S3; + else if(strcasecmp("file",kind)==0) return NCZM_FILE; + else if(strcasecmp("zip",kind)==0) return NCZM_ZIP; + else return NCZM_UNDEF; +} + +const char* +impl2kind(NCZM_IMPL impl) +{ + switch (impl) { + case NCZM_S3: return "s3"; + case NCZM_GS3: return "gs3"; + case NCZM_FILE: return "file"; + case NCZM_ZIP: return "zip"; + case NCZM_UNDEF: break; + } + return NULL; +} + +/* Goal: Given a set of per-dimension indices, + compute the corresponding linear position. +*/ +size64_t +computelinearoffset(int R, const size64_t* indices, const size64_t* max, size64_t* productp) +{ + size64_t offset, product; + int i; + + offset = 0; product = 1; + for(i=0;idebug); + printf(" file=|%s|",opts->file); + printf(" output=|%s|",opts->output); +#endif + if(opts->kind) + printf(" kind=%s",opts->kind); + if(opts->cmds) { + printf(" cmds="); + for(i=0,p=opts->cmds;*p;p++,i++) + printf("%s%s",(i==0?"(":","),*p); + printf(")"); + } + + for(i=0;idimdefs);i++) { + struct Dimdef* dd = (struct Dimdef*)nclistget(opts->dimdefs,i); + printf(" -d%s=%llu",dd->name,dd->size); + } + + for(i=0;ivardefs);i++) { + size_t j; + struct Vardef* vd = (struct Vardef*)nclistget(opts->vardefs,i); + printf(" -v '%d %s[",vd->typeid,vd->name); + for(j=0;jrank;j++) { + Dimdef* vdd = vd->dimrefs[j]; + if(j > 0) printf(","); + printf("%s/%llu",vdd->name,vd->chunksizes[j]); + } + printf("]'"); + } + + printf(" -s "); + for(i=0;i<(size_t)opts->nslices;i++) { + NCZSlice* sl = &opts->slices[i]; + printf("%s",nczprint_slicex(*sl,1)); + } + printf("\n"); +} + +int +hasdriveletter(const char* f) +{ + if(f == NULL || *f == '\0' || strlen(f) < 3) return 0; + if(f[1] != ':') return 0; + if(f[2] != '/' && f[2] != '\\') return 0; + if((f[0] < 'z' && f[0] >= 'a') || (f[0] < 'Z' && f[0] >= 'A')) + return 1; + return 0; +} + +/* bubble sort a list of strings */ +void +ut_sortlist(NClist* l) +{ + int switched; + size_t i; + + if(nclistlength(l) <= 1) return; + do { + switched = 0; + for(i=0;i 0) { + nclistset(l,i,ith1); + nclistset(l,i+1,ith); + switched = 1; + } + } + } while(switched); +#ifdef DEBUG +for(i=0;itypesize = sizeof(int); + if(var != NULL) { + common->rank = var->rank; + memcpy(common->dimlens,var->dimsizes,sizeof(size64_t)*(size_t)common->rank); + memcpy(common->chunklens,var->chunksizes,sizeof(size64_t)*(size_t)common->rank); + memcpy(common->memshape,common->dimlens,sizeof(size64_t)*(size_t)common->rank); /* fake it */ + } +} + +#if 0 +static int +searchR(NCZMAP* map, int depth, const char* prefix0, NClist* objects) +{ + int i,stat = NC_NOERR; + NClist* matches = nclistnew(); + char prefix[4096]; /* only ok because we know testdata */ + size_t prefixlen; + + nclistpush(objects,strdup(prefix0)); + + prefix[0] = '\0'; + strlcat(prefix,prefix0,sizeof(prefix)); + prefixlen = strlen(prefix); + + /* get next level object keys **below** the prefix: should have form: */ + switch (stat = nczmap_search(map, prefix, matches)) { + case NC_NOERR: break; + case NC_ENOTFOUND: stat = NC_NOERR; break;/* prefix is not a dir */ + default: goto done; + } + reportx(PASS,prefix,"search",map); + + /* recurse */ + for(i=0;i +extern int parseslices(const char* s0, int* nslicesp, NCZSlice* slices); +extern int parsedimdef(const char* s0, Dimdef** defp); +extern int parsevardef(const char* s0, NClist* dimdefs, Vardef** varp); +extern size_t parsestringvector(const char* s0, int stopchar, char*** namesp); +extern void freedimdefs(NClist* defs); +extern void freevardefs(NClist* defs); +extern void freeranges(NCZChunkRange* ranges); +extern void freeslices(NCZSlice* slices); +extern void freestringvec(char** vec); +extern void freeprojvector(size_t rank, NCZProjection** vec); +extern size_t ut_typesize(nc_type t); +extern nc_type ut_typeforname(const char* tname); +extern NCZM_IMPL kind2impl(const char* kind); +extern const char* impl2kind(NCZM_IMPL impl); +extern size64_t computelinearoffset(int R, const size64_t* indices, const size64_t* max, size64_t* productp); +extern void slices2vector(size_t rank, NCZSlice* slices, size64_t** startp, size64_t** stopp, size64_t** stridep, size64_t** maxp); +extern void printoptions(struct UTOptions* opts); +extern int hasdriveletter(const char* f); +extern void ut_sortlist(NClist* l); +extern void fillcommon(struct Common* common, Vardef* var); +extern int ut_search(NCZMAP* map, const char* prefix, NClist* objects); + +#endif /*UT_UTIL_H*/ diff --git a/v3_nczarr_test/v3manifest.am b/v3_nczarr_test/v3manifest.am index cd717f6910..e84b001fc3 100644 --- a/v3_nczarr_test/v3manifest.am +++ b/v3_nczarr_test/v3manifest.am @@ -8,7 +8,7 @@ TESTFILES_NCZARR_SH = test_nczarr.sh run_chunkcases.sh run_corrupt.sh run_extern # Program files that are copies of same files from nczarr_test TESTFILES_NCZARR_C = bm_chunks3.c bm_utils.c bm_utils.h bm_timer.h test_chunkcases.c test_chunking.c test_fillonlyz.c test_filter_avail.c test_filter.c testfilter.c test_filter_misc.c testfilter_misc.c testfilter_multi.c test_filter_order.c testfilter_order.c test_filter_repeat.c testfilter_repeat.c test_filter_vlen.c test_forwardinfer.c test_grpperf.c test_h5_endians.c test_multifilter.c test_nczfilter.c test_notzarr.c test_put_vars_two_unlim_dim.c test_quantize.c test_readcaching.c test_unlim_io.c test_unlimited.c test_unlim_vars.c test_utils.c test_writecaching.c test_zchunks2.c test_zchunks3.c test_zchunks.c tst_multifilter.c timer_utils.c timer_utils.h ut_chunking.c ut_json.c ut_mapapi.c ut_map.c ut_projections.c ut_test.c ut_util.c test_nczarr_utils.h test_utils.h ut_includes.h ut_projtest.h ut_test.h ut_util.h -TESTDATA_NCZARR = ref_nulls_nczarr.baseline ref_zarr_test_data.cdl.gz ref_avail1.cdl ref_byte.cdl ref_byte_fill_value_null.cdl ref_fillonly.cdl ref_misc1.cdl ref_ndims.cdl ref_newformatpure.cdl ref_nulls.cdl ref_oldformat.cdl ref_perdimspecs.cdl ref_power_901_constants.cdl ref_purezarr_base.cdl ref_quotes.cdl ref_rem.cdl ref_scalar.cdl ref_skip.cdl ref_skipw.cdl ref_string.cdl ref_t_meta_dim1.cdl ref_t_meta_var1.cdl ref_ut_mapapi_create.cdl ref_ut_mapapi_data.cdl ref_ut_mapapi_meta.cdl ref_ut_map_create.cdl ref_ut_map_writedata.cdl ref_ut_map_writemeta.cdl ref_ut_map_writemeta2.cdl ref_ut_testmap_create.cdl ref_whole.cdl ref_avail1.dmp ref_misc1.dmp ref_ndims.dmp ref_rem.dmp ref_noshape.file.zip ref_groups.h5 ref_notzarr.tar.gz ref_avail1.txt ref_skip.txt ref_ut_json_build.txt ref_ut_json_parse.txt ref_ut_mapapi_search.txt ref_ut_map_readmeta.txt ref_ut_map_readmeta2.txt ref_ut_map_search.txt ref_ut_proj.txt ref_whole.txt ref_byte.zarr.zip ref_byte_fill_value_null.zarr.zip ref_oldformat.zip ref_power_901_constants_orig.zip ref_quotes_orig.zip ref_cmip6.zmap +TESTDATA_NCZARR = ref_nulls_nczarr.baseline ref_zarr_test_data.cdl.gz ref_avail1.cdl ref_byte.cdl ref_byte_fill_value_null.cdl ref_fillonly.cdl ref_misc1.cdl ref_ndims.cdl ref_newformatpure.cdl ref_oldformat.cdl ref_perdimspecs.cdl ref_power_901_constants.cdl ref_purezarr_base.cdl ref_quotes.cdl ref_rem.cdl ref_scalar.cdl ref_skip.cdl ref_skipw.cdl ref_string.cdl ref_t_meta_dim1.cdl ref_t_meta_var1.cdl ref_ut_mapapi_create.cdl ref_ut_mapapi_data.cdl ref_ut_mapapi_meta.cdl ref_ut_map_writedata.cdl ref_ut_map_writemeta.cdl ref_ut_map_writemeta2.cdl ref_ut_testmap_create.cdl ref_whole.cdl ref_avail1.dmp ref_misc1.dmp ref_ndims.dmp ref_rem.dmp ref_noshape.file.zip ref_groups.h5 ref_avail1.txt ref_skip.txt ref_ut_json_parse.txt ref_ut_mapapi_search.txt ref_ut_map_readmeta.txt ref_ut_map_readmeta2.txt ref_ut_map_search.txt ref_ut_proj.txt ref_whole.txt ref_byte.zarr.zip ref_byte_fill_value_null.zarr.zip ref_oldformat.zip ref_power_901_constants_orig.zip ref_quotes_orig.zip ref_cmip6.zmap diff --git a/v3_nczarr_test/zhex.c b/v3_nczarr_test/zhex.c new file mode 100644 index 0000000000..97f0f141e3 --- /dev/null +++ b/v3_nczarr_test/zhex.c @@ -0,0 +1,38 @@ +/* + * Copyright 2018, University Corporation for Atmospheric Research + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + */ + +#include "config.h" + +#include "stdlib.h" +#include "stdio.h" +#include + +#ifdef HAVE_UNISTD_H +#include +#endif + +#undef DEBUG + +int +main(int argc, char** argv) +{ + unsigned char c; + FILE* f = NULL; + + if(argc > 1) { + /* use argv[1] as input */ + f = fopen(argv[1],"r"); + if(f == NULL) {fprintf(stderr,"No such file: %s\n",argv[1]); exit(1);} + } else + f = stdin; + + for(;;) { + size_t ret = fread(&c, 1, 1, f); + if(ret != 1) break; + printf("%.2hhx", c); + } + if(f != stdin) fclose(f); + return 0; +} diff --git a/v3_nczarr_test/zisjson.c b/v3_nczarr_test/zisjson.c new file mode 100644 index 0000000000..93db2abcee --- /dev/null +++ b/v3_nczarr_test/zisjson.c @@ -0,0 +1,152 @@ +/* + * Copyright 2018, University Corporation for Atmospheric Research + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + */ + +/* Parse input to see if it looks like json. + Output 1 or 0. +*/ + + +#include +#ifdef HAVE_UNISTD_H +#include +#endif + +#include "stdlib.h" +#include "stdio.h" +#include "string.h" + +#ifdef HAVE_UNISTD_H +#include +#endif + +#if defined(_WIN32) && !defined(__MINGW32__) +#include "XGetopt.h" +#else +#include +#endif + +#include "netcdf.h" +#include "nclist.h" +#include "ncjson.h" + +#define MAXREAD 8192 + +/* Command line options */ +struct Jsonpptions { + int trace; +} jsonoptions; + +static const char* +sortname(int thesort) +{ + switch(thesort) { + default: break; + case NCJ_INT: return "NCJ_INT"; + case NCJ_DOUBLE: return "NCJ_DOUBLE"; + case NCJ_BOOLEAN: return "NCJ_BOOLEAN"; + case NCJ_STRING: return "NCJ_STRING"; + case NCJ_DICT: return "NCJ_DICT"; + case NCJ_ARRAY: return "NCJ_ARRAY"; + case NCJ_NULL: return "NCJ_NULL"; + } + return "?"; +} + + +static void +jsontrace(NCjson* json, int depth) +{ + size_t i; + if(json == NULL) goto done; + printf("[%d] sort=%s",depth,sortname(NCJsort(json))); + switch(NCJsort(json)) { + case NCJ_INT: + case NCJ_DOUBLE: + case NCJ_BOOLEAN: + case NCJ_STRING: + printf(" string=|%s|\n",NCJstring(json)); + break; + case NCJ_NULL: + printf("\n"); + break; + case NCJ_ARRAY: + printf("\n"); + for(i=0;i 1) { + fprintf(stderr, "zisjson: only one input file argument permitted\n"); + exit(1); + } + if (argc == 0) + f = stdin; + else { + /* use argv[0] as input */ + f = fopen(argv[0],"r"); + if(f == NULL) {fprintf(stderr,"No such file: %s\n",argv[1]); exit(1);} + } + + /* Read json from stdin */ + for(i=0;;i++) { + unsigned char c; + size_t red = fread(&c, 1, 1, f); + if(red != 1) break; + if(i < MAXREAD) text[i] = (char)c; + } + if(i >= MAXREAD) { + fprintf(stderr,"Input too long\n"); + exit(1); + } + text[i] = '\0'; + if(i == 0) { + stat = NC_EEMPTY; + } else { + stat = NCJparse(text,0,&json); + if(!stat) { + if(jsonoptions.trace) jsontrace(json,0); + NCJreclaim(json); + } + } + printf("%d",(stat?0:1)); /* parse success|failure */ + if(f != stdin) fclose(f); + return 0; +} diff --git a/v3_nczarr_test/zmapio.c b/v3_nczarr_test/zmapio.c new file mode 100644 index 0000000000..37e0d639be --- /dev/null +++ b/v3_nczarr_test/zmapio.c @@ -0,0 +1,557 @@ +/* + * Copyright 2018, University Corporation for Atmospheric Research + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + */ + +#include +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_GETOPT_H +#include +#endif + +#if defined(_WIN32) && !defined(__MINGW32__) +#include "XGetopt.h" +#endif + +#include + +#include "zincludes.h" +#include "ncpathmgr.h" +#include "nclog.h" +#include "ncuri.h" + +#undef DEBUG + +#define DATANAME "data" + +typedef enum Mapop { +MOP_NONE=0, +MOP_OBJDUMP=1, +MOP_CLEAR=2 +} Mapop; + +typedef enum OBJKIND { +OK_NONE=0, +OK_META=1, +OK_CHUNK=2, +OK_GROUP=3, +OK_IGNORE=4 +} OBJKIND; + +static struct Mops { + Mapop mapop; + const char* opname; +} mapops[] = { +{MOP_NONE,"none"}, +{MOP_OBJDUMP,"objdump"}, +{MOP_CLEAR,"clear"}, +{MOP_NONE,NULL} +}; + +static struct Type { + const char* typename; + nc_type nctype; + size_t typesize; + const char format[16]; +} types[] = { +{"ubyte",NC_UBYTE,1,"%u"}, +{"byte",NC_BYTE,1,"%d"}, +{"ushort",NC_USHORT,2,"%u"}, +{"short",NC_SHORT,2,"%d"}, +{"uint",NC_UINT,4,"%u"}, +{"int",NC_INT,4,"%d"}, +{"uint64",NC_UINT64,8,"%llu"}, +{"int64",NC_INT64,8,"%lld"}, +{"float",NC_FLOAT,4,"%f"}, +{"double",NC_DOUBLE,8,"%lf"}, +{"char",NC_CHAR,1,"'%c'"}, +{"string",NC_STRING,sizeof(char*),"%*s"}, +{NULL,NC_NAT,0,""} +}; + +/* Command line options */ +struct Dumpptions { + int debug; + int meta_only; + Mapop mop; + char infile[4096]; + NCZM_IMPL impl; + char* rootpath; + const struct Type* nctype; + char format[16]; + int xflags; +# define XNOZMETADATA 1 + int strlen; + int zarrformat; +} dumpoptions; + +/* Forward */ +static int objdump(void); +static NCZM_IMPL implfor(const char* path); +static void printcontent(size64_t len, const char* content, OBJKIND kind); +static int depthfirst(NCZMAP* map, const char*, NClist* stack); +static char* rootpathfor(const char* path); +static OBJKIND keykind(const char* key); +static void sortlist(NClist* l); +static const char* filenamefor(const char* f0); + +#define NCCHECK(expr) nccheck((expr),__LINE__) +static void nccheck(int stat, int line) +{ + if(stat) { + fprintf(stderr,"%d: %s\n",line,nc_strerror(stat)); + fflush(stderr); + exit(1); + } +} + +static void +zmapusage(void) +{ + fprintf(stderr,"usage: zmapio [-2|-3][-t ][-d][-v][-h][-x] \n"); + exit(1); +} + +static Mapop +decodeop(const char* name) +{ + struct Mops* p = mapops; + for(;p->opname != NULL;p++) { + if(strcasecmp(p->opname,name)==0) return p->mapop; + } + return MOP_NONE; +} + +static const struct Type* +decodetype(const char* name, int* strlenp) +{ + struct Type* p = types; + + if(strncmp(name,"string/",strlen("string/"))==0) { + *strlenp = atoi(name+strlen("string/")); + name = "string"; + } + + for(;p->typename != NULL;p++) { + if(strcasecmp(p->typename,name)==0) return p; + } + return NULL; +} + +int +main(int argc, char** argv) +{ + int stat = NC_NOERR; + int c; + char* p; + + nc_initialize(); + + /* Init options */ + memset((void*)&dumpoptions,0,sizeof(dumpoptions)); + + while ((c = getopt(argc, argv, "23dhvx:t:F:T:X:")) != EOF) { + switch(c) { + case '2': + dumpoptions.zarrformat = 2; + break; + case '3': + dumpoptions.zarrformat = 3; + break; + case 'd': + dumpoptions.debug = 1; + break; + case 'h': + dumpoptions.meta_only = 1; + break; + case 't': + dumpoptions.nctype = decodetype(optarg,&dumpoptions.strlen); + if(dumpoptions.nctype == NULL) zmapusage(); + break; + case 'x': + dumpoptions.mop = decodeop(optarg); + if(dumpoptions.mop == MOP_NONE) zmapusage(); + break; + case 'v': + zmapusage(); + goto done; + case 'F': + strcpy(dumpoptions.format,optarg); + break; + case 'T': + nctracelevel(atoi(optarg)); + break; + case 'X': + for(p=optarg;*p;p++) { + switch (*p) { + case 'm': dumpoptions.xflags |= XNOZMETADATA; break; + default: fprintf(stderr,"Unknown -X argument: %c",*p); break; + } + }; + break; + case '?': + fprintf(stderr,"unknown option\n"); + goto fail; + } + } + + /* Default the kind */ + if(dumpoptions.nctype == NULL) { + dumpoptions.nctype = &types[0]; + fprintf(stderr,"Default type: %s\n",dumpoptions.nctype->typename); + } + + /* get file argument */ + argc -= optind; + argv += optind; + + if (argc > 1) { + fprintf(stderr, "zmapio: only one input file argument permitted\n"); + goto fail; + } + if (argc == 0) { + fprintf(stderr, "zmapio: no input file specified\n"); + goto fail; + } + + { + char* p = NC_shellUnescape(argv[0]); + strcpy(dumpoptions.infile,filenamefor(p)); + if(p) free(p); + } + + if((dumpoptions.impl = implfor(dumpoptions.infile))== NCZM_UNDEF) + zmapusage(); + + if((dumpoptions.rootpath = rootpathfor(dumpoptions.infile))== NULL) + zmapusage(); + + switch (dumpoptions.mop) { + default: + fprintf(stderr,"Default action: objdump\n"); + /* fall thru */ + case MOP_OBJDUMP: + if((stat = objdump())) goto done; + break; + } + +done: + /* Reclaim dumpoptions */ + nullfree(dumpoptions.rootpath); + nc_finalize(); + if(stat) + fprintf(stderr,"fail: %s\n",nc_strerror(stat)); + return (stat ? 1 : 0); +fail: + stat = NC_EINVAL; + goto done; +} + +static NCZM_IMPL +implfor(const char* path) +{ + NCURI* uri = NULL; + const char* mode = NULL; + NClist* segments = nclistnew(); + + NCZM_IMPL impl = NCZM_UNDEF; + + ncuriparse(path,&uri); + if(uri == NULL) goto done; + mode = ncurifragmentlookup(uri,"mode"); + if(mode == NULL) goto done; + /* split on commas */ + NCCHECK(nczm_split_delim(mode,',',segments)); + for(size_t i=0;ipath,segments))) goto done; + /* remove the bucket name */ + p = (char*)nclistremove(segments,0); + nullfree(p); p = NULL; + /* Put it back together */ + if((stat = nczm_join(segments,&rootpath))) goto done; + } break; +#endif + default: + stat = NC_EINVAL; + goto done; + } + +done: + nclistfreeall(segments); segments = NULL; + ncurifree(uri); uri = NULL; + if(stat) + {nullfree(rootpath); rootpath = NULL;} + return rootpath; +} + +static int +objdump(void) +{ + int stat = NC_NOERR; + NCZMAP* map = NULL; + NClist* stack = nclistnew(); + char* obj = NULL; + char* content = NULL; + size_t depth = 0; + + if((stat=nczmap_open(dumpoptions.impl, dumpoptions.infile, NC_NOCLOBBER, 0, NULL, &map))) + goto done; + + /* Depth first walk all the groups to get all keys */ + if((stat = depthfirst(map,"/",stack))) goto done; + + if(dumpoptions.debug) { + + fprintf(stderr,"stack:\n"); + for(size_t i=0;itypesize); + if((content = calloc(1,padlen+1))==NULL) {stat = NC_ENOMEM; goto done;} + content[len] = '\0'; + if(len > 0) { + if((stat=nczmap_read(map,obj,0,len,content))) goto done; + assert(content != NULL); + if(kind == OK_CHUNK) + len = ceildiv(len,dumpoptions.nctype->typesize); + printf("[%zu] %s : (%llu)",depth,obj,len); + if(kind == OK_CHUNK && dumpoptions.nctype->nctype != NC_STRING) + printf(" (%s)",dumpoptions.nctype->typename); + printf(" |"); + switch(kind) { + case OK_GROUP: + case OK_META: + printcontent(len,content,kind); + break; + case OK_CHUNK: + if(dumpoptions.meta_only) + printf("..."); + else + printcontent(len,content,kind); + break; + default: break; + } + printf("|\n"); + } else { + printf("[%zu] %s : (%llu) ||\n",depth,obj,len); + } + } +done: + nullfree(content); + nczmap_close(map,0); + nclistfreeall(stack); + return stat; +} + +/* Depth first walk all the groups to get all keys */ +static int +depthfirst(NCZMAP* map, const char* key, NClist* stack) +{ + int stat = NC_NOERR; + NCbytes* prefix = ncbytesnew(); + + if(key == NULL || key[0] == '\0') + key = "/"; + ncbytescat(prefix,key); + if(strlen(key) > 1 && key[strlen(key)-1]=='/') { + ncbytessetlength(prefix,ncbyteslength(prefix)-1); /* remove trailing '/' */ + ncbytesnull(prefix); + } + if((stat = nczmap_listall(map,ncbytescontents(prefix),stack))) goto done; + sortlist(stack); + +done: + ncbytesfree(prefix); + return stat; +} + + +static void +printcontent(size64_t len, const char* content, OBJKIND kind) +{ + size64_t i, count; + + const char* format = NULL; + size64_t strlen = (size64_t)dumpoptions.strlen; + + format = dumpoptions.nctype->format; + if(dumpoptions.format[0] != '\0') + format = dumpoptions.format; + count = len; + +#ifdef DEBUG + printf("debug: len=%d strlen=%d count=%d\n",(int)len,(int)strlen,(int)count); fflush(stdout); +#endif + + for(i=0;i 0) printf(", "); + switch(dumpoptions.nctype->nctype) { + case NC_BYTE: printf(format,((char*)content)[i]); break; + case NC_SHORT: printf(format,((short*)content)[i]); break; + case NC_INT: printf(format,((int*)content)[i]); break; + case NC_INT64: printf(format,((long long*)content)[i]); break; + case NC_UBYTE: printf(format,((unsigned char*)content)[i]); break; + case NC_USHORT: printf(format,((unsigned short*)content)[i]); break; + case NC_UINT: printf(format,((unsigned int*)content)[i]); break; + case NC_UINT64: printf(format,((unsigned long long*)content)[i]); break; + case NC_FLOAT: printf(format,((float*)content)[i]); break; + case NC_DOUBLE: printf(format,((double*)content)[i]); break; + case NC_CHAR: printf(format,((char*)content)[i]); break; + case NC_STRING: printf(format,(int)strlen,((char*)(&content[i*strlen]))); break; + default: abort(); + } + break; + case OK_META: + printf("%c",content[i]); + break; + default: + printf("%.2hhx", content[i]); + } + } +} + +static char chunkchars[] = "./c0123456789"; + +static OBJKIND +keykind(const char* key) +{ + OBJKIND kind = OK_NONE; + char* suffix = NULL; + if(nczm_divide_at(key,-1,NULL,&suffix) == NC_NOERR) { + if(suffix) { + if(suffix[0] != '/') + kind = OK_NONE; + else if(suffix[1] == '.') { + if(strcmp(&suffix[1],".zmetadata")==0 && (dumpoptions.xflags & XNOZMETADATA)) + kind = OK_IGNORE; + else + kind = OK_META; + if(!dumpoptions.zarrformat) dumpoptions.zarrformat = 2; + } else if(strcasecmp(&suffix[1],"zarr.json")==0) { + kind = OK_META; + if(!dumpoptions.zarrformat) dumpoptions.zarrformat = 3; + } else if(strcasecmp(&suffix[1],"zarr.json")==0) { + kind = OK_META; + if(!dumpoptions.zarrformat) dumpoptions.zarrformat = 3; + } else if(suffix[strlen(suffix)-1] == '/') + kind = OK_GROUP; + else { + char* p = suffix+1; + for(;*p;p++) { + if(strchr(chunkchars,*p) == NULL) break; + } + kind = OK_CHUNK; + } + } + } + nullfree(suffix); + return kind; +} + + +/* Define a static qsort comparator for strings for use with qsort */ +static int +cmp_strings(const void* a1, const void* a2) +{ + const char** s1 = (const char**)a1; + const char** s2 = (const char**)a2; + return strcmp(*s1,*s2); +} + +/* quick sort a list of strings */ +static void +sortlist(NClist* l) +{ + if(nclistlength(l) <= 1) return; + qsort(nclistcontents(l),nclistlength(l),sizeof(char*),cmp_strings); +#if 0 +for(i=0;i +#include +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_GETOPT_H +#include +#endif + +#if defined(_WIN32) && !defined(__MINGW32__) +#include "XGetopt.h" +#endif + +#include "zincludes.h" +#include "ncpathmgr.h" + +#undef DEBUG + +#define AWSHOST ".amazonaws.com" + +typedef enum S3op { +S3_NONE=0, +S3_HOST=1, +S3_BUCKET=2, +S3_KEY=3, +} S3op; + +/* Command line options */ +struct S3options { + int debug; + S3op op; + char* url; +} s3options; + +/*Forward*/ +static int processurl(S3op op, const char* url, char** piece); + +static void +zs3usage(void) +{ + fprintf(stderr,"usage: zs3parse [-h|-b|-k] |\n"); + exit(1); +} + +int +main(int argc, char** argv) +{ + int stat = NC_NOERR; + int c; + char* piece = NULL; + + memset((void*)&s3options,0,sizeof(s3options)); + + while ((c = getopt(argc, argv, "vhbk")) != EOF) { + switch(c) { + case 'b': + s3options.op = S3_BUCKET; + break; + case 'h': + s3options.op = S3_HOST; + break; + case 'k': + s3options.op = S3_KEY; + break; + case 'v': + zs3usage(); + goto done; + case '?': + fprintf(stderr,"unknown option: %c\n",c); + goto fail; + } + } + + /* get url|file argument */ + argc -= optind; + argv += optind; + + if (argc > 1) { + fprintf(stderr, "zs3parse: only one url|file argument permitted\n"); + goto fail; + } + if (argc == 0) { + fprintf(stderr, "zs3parse: no url|file specified\n"); + goto fail; + } + s3options.url = strdup(argv[0]); + + stat = processurl(s3options.op, s3options.url, &piece); + if(stat == NC_NOERR) { + if(piece == NULL) goto fail; + printf("%s",piece); + } +done: + nullfree(piece); + /* Reclaim s3options */ + nullfree(s3options.url); + if(stat) + fprintf(stderr,"fail: %s\n",nc_strerror(stat)); + return (stat ? 1 : 0); +fail: + stat = NC_EINVAL; + goto done; +} + +static int +processurl(S3op op, const char* surl, char** piece) +{ + int stat = NC_NOERR; + NClist* segments = NULL; + NCbytes* buf = ncbytesnew(); + char* value = NULL; + char* host = NULL; + char* bucket = NULL; + char* prefix = NULL; + NCURI* url = NULL; + + ncuriparse(surl,&url); + if(url == NULL) + {stat = NC_EURL; goto done;} + /* do some verification */ + if(strcmp(url->protocol,"https") != 0 + && strcmp(url->protocol,"http") != 0) + {stat = NC_EURL; goto done;} + + if(url->host == NULL || strlen(url->host) == 0) + {stat = NC_EURL; goto done;} + if((host = strdup(url->host))==NULL) + {stat = NC_ENOMEM; goto done;} + /* We have to process the path to get the bucket, + and remove it from the path */ + if(url->path == NULL || strlen(url->path) == 0) + {stat = NC_EURL; goto done;} + /* split the path by "/" */ + nclistfreeall(segments); + segments = nclistnew(); + if((stat = nczm_split_delim(url->path,'/',segments))) goto done; + if(nclistlength(segments) == 0) + {stat = NC_EURL; goto done;} + bucket = ((char*)nclistremove(segments,0)); + if((stat = nczm_join(segments,&prefix))) goto done; + nclistfreeall(segments); segments = NULL; + + switch (op) { + case S3_HOST: value = host; host = NULL; break; + case S3_BUCKET: value = bucket; bucket = NULL; break; + case S3_KEY: value = prefix; prefix = NULL; break; + default: stat = NC_EURL; goto done; + } + + if(piece) {*piece = value; value = NULL;} + +done: + ncurifree(url); + nullfree(value); + nullfree(host); + nullfree(bucket); + nullfree(prefix); + ncbytesfree(buf); + nclistfreeall(segments); + return stat; +} +