From 05a20057165c6277f7ff3e0381cd360035300b67 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 27 Jan 2024 12:37:00 -0700 Subject: [PATCH] add Makefiles --- nczarr_test/Makefile.core | 281 +++++++++++++++++++++++++++++++++++++ nczarr_test/Makefile.files | 155 ++++++++++++++++++++ 2 files changed, 436 insertions(+) create mode 100644 nczarr_test/Makefile.core create mode 100644 nczarr_test/Makefile.files diff --git a/nczarr_test/Makefile.core b/nczarr_test/Makefile.core new file mode 100644 index 0000000000..6a18bf978a --- /dev/null +++ b/nczarr_test/Makefile.core @@ -0,0 +1,281 @@ +TESTS_ENVIRONMENT = +TEST_EXTENSIONS = .sh +#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#TESTS_ENVIRONMENT += export SETX=1; + +AM_CFLAGS += -I${top_srcdir} +AM_CPPFLAGS += -I${top_srcdir} +AM_CPPFLAGS += -I${top_srcdir}/libnczarr +AM_CPPFLAGS += -I${top_srcdir}/plugins +LDADD = ${top_builddir}/liblib/libnetcdf.la +AM_LDFLAGS += ${top_builddir}/liblib/libnetcdf.la +AM_CXXFLAGS = + +testcommonsrc = test_utils.c test_utils.h +commonsrc = ut_util.c ut_test.c ut_includes.h ut_test.h ut_util.h test_nczarr_utils.h + +check_PROGRAMS += ut_map ut_mapapi ut_json +ut_map_SOURCES = ut_map.c ${commonsrc} +ut_mapapi_SOURCES = ut_mapapi.c ${commonsrc} +ut_json_SOURCES = ut_json.c ${commonsrc} + +test_fillonlyz_SOURCES = test_fillonlyz.c ${testcommonsrc} + +check_PROGRAMS += test_fillonlyz test_quantize test_notzarr + +# Unlimited Dimension tests +if USE_HDF5 +test_put_vars_two_unlim_dim_SOURCES = test_put_vars_two_unlim_dim.c ${testcommonsrc} +check_PROGRAMS += test_zchunks test_zchunks2 test_zchunks3 test_unlim_vars test_put_vars_two_unlim_dim +check_PROGRAMS += test_unlim_io +test_unlim_io_SOURCES = test_unlim_io.c ${testcommonsrc} +TESTS += test_put_vars_two_unlim_dim +endif + +if BUILD_UTILITIES + +TESTS += run_ut_misc.sh +TESTS += run_ut_map.sh +TESTS += run_ut_mapapi.sh + +if AX_DISABLE # Obsolete tests +TESTS += run_ut_chunk.sh +endif + +TESTS += run_ncgen4.sh run_nan.sh + +TESTS += run_quantize.sh +TESTS += run_purezarr.sh +TESTS += run_misc.sh +TESTS += run_jsonconvention.sh +TESTS += run_strings.sh +TESTS += run_scalar.sh +TESTS += run_nulls.sh +TESTS += run_notzarr.sh + +# Unlimited Dimension tests (at least in part) +TESTS += run_mud.sh +TESTS += test_unlim_vars + +if USE_HDF5 +# Unlimited Dimension tests +check_PROGRAMS += test_chunking +TESTS += run_nccopy5.sh +endif + +if USE_HDF5 +TESTS += run_unlim_io.sh run_nccopyz.sh +endif + +if LARGE_FILE_TESTS + check_PROGRAMS += test_writecaching test_readcaching + TESTS += run_cachetest.sh +endif + +endif #BUILD_UTILITIES + +if BUILD_UTILITIES + +if ENABLE_NCZARR_ZIP + TESTS += run_newformat.sh +endif + +if LARGE_FILE_TESTS +check_PROGRAMS += test_chunkcases +test_chunkcases_SOURCES = test_chunkcases.c ${testcommonsrc} +TESTS += run_chunkcases.sh +endif + +check_PROGRAMS += test_endians +TESTS += test_endians + +if USE_HDF5 +TESTS += run_fillonlyz.sh +endif + +if BUILD_BENCHMARKS + +UTILSRC = bm_utils.c timer_utils.c test_utils.c bm_utils.h bm_timer.h + +bm_chunks3_SOURCES = bm_chunks3.c ${UTILSRC} + +check_PROGRAMS += bm_chunks3 + +# The perf tests need modernization +if AX_DISABLE +TESTS += run_perf_chunks1.sh +endif + +endif # BUILD_BENCHMARKS + +if ENABLE_NCZARR_FILTERS +if ENABLE_FILTER_TESTING + +check_PROGRAMS += test_filter_avail test_filter_vlen +TESTS += run_filter_vlen.sh run_filter_misc.sh + +# Do simple filter functionality tests for nczarr +check_PROGRAMS += test_nczfilter +TESTS += run_nczfilter.sh + +# Echo filter tests from nc_test4 +check_PROGRAMS += testfilter testfilter_misc testfilter_order testfilter_repeat testfilter_multi +TESTS += run_filter.sh +TESTS += run_specific_filters.sh + +if AX_DISABLE +# This test is too dangerous to run in a parallel make environment. +# It causes race conditions. So suppress and only test by hand. +TESTS += run_unknown.sh +endif + +if ISMINGW +XFAIL_TESTS = run_filter.sh run_specific_filters.sh run_nczfilter.sh +endif # ISMINGW + +endif #ENABLE_FILTER_TESTING +endif #ENABLE_NCZARR_FILTERS + +endif #BUILD_UTILITIES + +# These programs are used by the test cases +noinst_PROGRAMS = zhex +zhex_SOURCES = zhex.c +noinst_PROGRAMS += zisjson +zisjson_SOURCES = zisjson.c +noinst_PROGRAMS += zmapio +zmapio_SOURCES = zmapio.c +noinst_PROGRAMS += zs3parse +zs3parse_SOURCES = zs3parse.c + +if ENABLE_S3 +noinst_PROGRAMS += s3util +s3util_SOURCES = s3util.c +if ENABLE_S3_TESTALL +if ENABLE_S3_AWS +check_PROGRAMS += tst_pure_awssdk +tst_pure_awssdk_SOURCES = tst_pure_awssdk.cpp +AM_CXXFLAGS += -std=c++11 +endif +endif +endif + +if ENABLE_NCDUMPCHUNKS +# Given a netcdf4|NCZarr file, dump the actual chunk contents. +# Used to validate nczarr chunking code. +AM_CPPFLAGS += -I$(top_srcdir)/libnczarr +noinst_PROGRAMS += ncdumpchunks +ncdumpchunks_SOURCES = ncdumpchunks.c +endif + +EXTRA_DIST += ${ALLSCRIPTS} ${ALLBASELINE} + +CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc tmp*.dump tmp*.tmp tmp*.zmap tmp_ngc.c ref_zarr_test_data.cdl tst_*.nc.zip ref_quotes.zip ref_power_901_constants.zip + +BUILT_SOURCES += ${ALLBUILTSRC} ${ALLBUILTSCRIPTS} + +test_quantize.c: $(top_srcdir)/nc_test4/tst_quantize.c + rm -f $@ + echo "#define TESTNCZARR" > $@ + cat $(top_srcdir)/nc_test4/tst_quantize.c >> $@ + +test_filter_vlen.c: $(top_srcdir)/nc_test4/tst_filter_vlen.c + rm -f $@ + echo "#define TESTNCZARR" > $@ + cat $(top_srcdir)/nc_test4/tst_filter_vlen.c >> $@ + +test_unlim_vars.c: $(top_srcdir)/nc_test4/tst_unlim_vars.c + rm -f $@ + echo "#define TESTNCZARR" > $@ + cat $(top_srcdir)/nc_test4/tst_unlim_vars.c >> $@ + +test_endians.c: $(top_srcdir)/nc_test4/tst_h5_endians.c + rm -f $@ + echo "#define TESTNCZARR" > $@ + cat $(top_srcdir)/nc_test4/tst_h5_endians.c >> $@ + +test_put_vars_two_unlim_dim.c: $(top_srcdir)/nc_test4/tst_put_vars_two_unlim_dim.c + rm -f $@ + echo "#define TESTNCZARR" > $@ + cat $(top_srcdir)/nc_test4/tst_put_vars_two_unlim_dim.c >> $@ + +test_chunking.c: $(top_srcdir)/ncdump/tst_chunking.c + rm -f $@ + echo "#define TESTNCZARR" > $@ + cat $(top_srcdir)/ncdump/tst_chunking.c >> $@ + +run_unknown.sh: $(top_srcdir)/nc_test4/tst_unknown.sh + rm -f $@ run_unknown.tmp + echo "#!/bin/bash" > run_unknown.tmp + echo "TESTNCZARR=1" >> run_unknown.tmp + cat $(top_srcdir)/nc_test4/tst_unknown.sh >> run_unknown.tmp + tr -d '\r' < run_unknown.tmp > $@ + chmod a+x $@ + rm -f run_unknown.tmp + +run_specific_filters.sh: $(top_srcdir)/nc_test4/tst_specific_filters.sh + rm -f $@ run_specific_filters.tmp + echo "#!/bin/bash" > run_specific_filters.tmp + echo "TESTNCZARR=1" >> run_specific_filters.tmp + cat $(top_srcdir)/nc_test4/tst_specific_filters.sh >> run_specific_filters.tmp + tr -d '\r' < run_specific_filters.tmp > $@ + chmod a+x $@ + rm -f run_specific_filters.tmp + +run_filter_vlen.sh: $(top_srcdir)/nc_test4/tst_filter_vlen.sh + rm -f $@ run_filter_vlen.tmp + echo "#!/bin/bash" > run_filter_vlen.tmp + echo "TESTNCZARR=1" >> run_filter_vlen.tmp + cat $(top_srcdir)/nc_test4/tst_filter_vlen.sh >> run_filter_vlen.tmp + tr -d '\r' < run_filter_vlen.tmp > $@ + chmod a+x $@ + rm -f run_filter_vlen.tmp + +run_filter_misc.sh: $(top_srcdir)/nc_test4/tst_filter_misc.sh + rm -f $@ run_filter_misc.tmp + echo "#!/bin/bash" > run_filter_misc.tmp + echo "TESTNCZARR=1" >> run_filter_misc.tmp + cat $(top_srcdir)/nc_test4/tst_filter_misc.sh >> run_filter_misc.tmp + tr -d '\r' < run_filter_misc.tmp > $@ + chmod a+x $@ + rm -f run_filter_misc.tmp + +run_filterinstall.sh: $(top_srcdir)/nc_test4/tst_filterinstall.sh + rm -f $@ run_filterinstall.tmp + echo "#!/bin/bash" > run_filterinstall.tmp + echo "TESTNCZARR=1" >> run_filterinstall.tmp + cat $(top_srcdir)/nc_test4/tst_filterinstall.sh >> run_filterinstall.tmp + tr -d '\r' < run_filterinstall.tmp > $@ + chmod a+x $@ + rm -f run_filterinstall.tmp + +run_mud.sh: $(top_srcdir)/ncdump/tst_mud.sh + rm -f $@ run_mud.tmp + echo "#!/bin/bash" > run_mud.tmp + echo "TESTNCZARR=1" >> run_mud.tmp + cat $(top_srcdir)/ncdump/tst_mud.sh >> run_mud.tmp + tr -d '\r' < run_mud.tmp > $@ + chmod a+x $@ + rm -f run_mud.tmp + +run_nccopy5.sh: $(top_srcdir)/ncdump/tst_nccopy5.sh + rm -f $@ run_nccopy5.tmp + echo "#!/bin/bash" > run_nccopy5.tmp + echo "TESTNCZARR=1" >> run_nccopy5.tmp + cat $(top_srcdir)/ncdump/tst_nccopy5.sh >> run_nccopy5.tmp + tr -d '\r' < run_nccopy5.tmp > $@ + chmod a+x $@ + rm -f run_nccopy5.tmp + +# Remove directories +clean-local: + rm -fr testdir_* testset_* + rm -fr tmp_*.nc tmp_*.zarr tst_quantize*.zarr tmp*.file results.file results.s3 results.zip + rm -fr rcmiscdir ref_power_901_constants.file + rm -fr v3_*.log v3_*.trs +if ENABLE_S3_TESTALL + bash ${abs_top_builddir}/s3cleanup.sh +endif + diff --git a/nczarr_test/Makefile.files b/nczarr_test/Makefile.files new file mode 100644 index 0000000000..5eade050a3 --- /dev/null +++ b/nczarr_test/Makefile.files @@ -0,0 +1,155 @@ +ALLSRC = +ALLBUILTSRC = +ALLBUILTSCRIPTS = +ALLV3SCRIPTS = +ALLV2SCRIPTS = +ALLSCRIPTS = +ALLBASELINE = +ALLV3BASELINE = +ALLV2BASELINE = + +ALLSCRIPTS += test_nczarr.sh + +ALLSRC += test_utils.c test_utils.h +ALLSRC += ut_util.c ut_test.c ut_includes.h ut_test.h ut_util.h test_nczarr_utils.h + +ALLSRC += ut_map.c ut_mapapi.c ut_json.c +ALLSRC += test_fillonlyz.c test_notzarr.c +ALLBUILTSRC += test_quantize.c + +# Unlimited Dimension tests +if USE_HDF5 +ALLSRC += test_zchunks.c test_zchunks2.c test_zchunks3.c +ALLSRC += test_unlim_io.c +ALLBUILTSRC += test_unlim_vars.c test_put_vars_two_unlim_dim.c +endif + +if BUILD_UTILITIES + +ALLSCRIPTS += run_ut_misc.sh +ALLSCRIPTS += run_ut_map.sh +ALLSCRIPTS += run_ut_mapapi.sh +ALLSCRIPTS += run_ncgen4.sh run_nan.sh +ALLSCRIPTS += run_quantize.sh +ALLSCRIPTS += run_purezarr.sh +ALLSCRIPTS += run_misc.sh +ALLSCRIPTS += run_jsonconvention.sh +ALLSCRIPTS += run_strings.sh +ALLSCRIPTS += run_scalar.sh +ALLSCRIPTS += run_nulls.sh +ALLSCRIPTS += run_notzarr.sh + +# Unlimited Dimension tests (at least in part) +ALLBUILTSCRIPTS += run_mud.sh + +if USE_HDF5 +ALLBUILTSRC += test_chunking.c +ALLBUILTSCRIPTS += run_nccopy5.sh +endif + +if USE_HDF5 +ALLSCRIPTS += run_unlim_io.sh run_nccopyz.sh +endif + +if LARGE_FILE_TESTS +ALLSRC += test_writecaching.c test_readcaching.c +ALLSCRIPTS += run_cachetest.sh +endif + +if ENABLE_NCZARR_ZIP +ALLSCRIPTS += run_newformat.sh +endif + +if LARGE_FILE_TESTS +ALLSRC += test_chunkcases.c +ALLSCRIPTS += run_chunkcases.sh +endif + +ALLBUILTSRC += test_endians.c + +if USE_HDF5 +ALLSCRIPTS += run_fillonlyz.sh +endif + +if BUILD_BENCHMARKS +ALLSRC += bm_utils.c timer_utils.c test_utils.c bm_utils.h bm_timer.h +ALLSRC += bm_chunks3.c +endif # BUILD_BENCHMARKS + +if ENABLE_NCZARR_FILTERS +if ENABLE_FILTER_TESTING + +ALLSRC += test_filter_avail.c +ALLBUILTSRC += test_filter_vlen.c +ALLBUILTSCRIPTS += run_filter_vlen.sh run_filter_misc.sh + +ALLSRC += test_nczfilter.c +ALLSCRIPTS += run_nczfilter.sh + +# Echo filter tests from nc_test4 +ALLSRC += testfilter.c testfilter_misc.c testfilter_order.c testfilter_repeat.c testfilter_multi.c +ALLSCRIPTS += run_filter.sh +ALLBUILTSCRIPTS += run_specific_filters.sh + +endif #ENABLE_FILTER_TESTING +endif #ENABLE_NCZARR_FILTERS + +endif #BUILD_UTILITIES + +# These programs are used by the test cases +ALLSRC += zhex.c +ALLSRC += zisjson.c +ALLSRC += zmapio.c +ALLSRC += zs3parse.c + +if ENABLE_S3 +ALLSRC += s3util.c +if ENABLE_S3_TESTALL +if ENABLE_S3_AWS +ALLSRC += tst_pure_awssdk.cpp +endif +endif +endif + +if ENABLE_NCDUMPCHUNKS +ALLSRC += ncdumpchunks.c +endif + +# V3 specific versions of files +ALLV3BASELINE += ref_nczarr2zarr.cdl ref_purezarr.cdl ref_xarray.cdl ref_misc2.cdl \ + ref_jsonconvention.cdl ref_jsonconvention.zmap \ + ref_nulls_zarr.baseline ref_string_zarr.baseline ref_string_nczarr.baseline \ + ref_zarr_test_data_2d.cdl.gz \ + ref_groups_regular.cdl ref_tst_nans.dmp ref_bzip2.cdl ref_filtered.cdl \ + ref_any.cdl ref_multi.cdl + +# V2 only +ALLV2BASELINE += ref_noshape.file.zip ref_power_901_constants.cdl ref_power_901_constants_orig.zip + +# Excludes ALLV3BASELINE +ALLBASELINE += \ +ref_avail1.cdl ref_avail1.dmp ref_avail1.txt \ +ref_byte.cdl ref_byte_fill_value_null.cdl ref_byte_fill_value_null.zarr.zip ref_byte.zarr.zip \ +ref_fillonly.cdl \ +ref_groups.h5 \ +ref_misc1.cdl ref_misc1.dmp \ +ref_ndims.cdl ref_ndims.dmp \ +ref_newformatpure.cdl \ +ref_notzarr.tar.gz \ +ref_nulls.cdl ref_nulls_nczarr.baseline \ +ref_oldformat.cdl ref_oldformat.zip \ +ref_perdimspecs.cdl \ +ref_purezarr_base.cdl \ +ref_quotes.cdl ref_quotes_orig.zip \ +ref_rem.cdl ref_rem.dmp \ +ref_scalar.cdl \ +ref_skip.cdl ref_skip.txt ref_skipw.cdl \ +ref_string.cdl \ +ref_t_meta_dim1.cdl ref_t_meta_var1.cdl \ +ref_ut_json_build.txt ref_ut_json_parse.txt \ +ref_ut_mapapi_create.cdl ref_ut_mapapi_data.cdl ref_ut_mapapi_meta.cdl ref_ut_mapapi_search.txt \ +ref_ut_map_create.cdl ref_ut_map_readmeta2.txt ref_ut_map_readmeta.txt ref_ut_map_search.txt ref_ut_map_writedata.cdl ref_ut_map_writemeta2.cdl ref_ut_map_writemeta.cdl \ +ref_ut_proj.txt \ +ref_ut_testmap_create.cdl \ +ref_whole.cdl ref_whole.txt \ +ref_zarr_test_data.cdl.gz