MONGOCRYPT-731 re-enable ASan tasks (#926)

* relocate misplaced comment.

Remove the comment from the YAML folded line to fix applying environment variables.

* suppress leak in dlerror.

To ignore a "Indirect leak" reported on Ubuntu 22.04 arm64. Assumed not to be an issue in libmongocrypt.

* remove unused `build-and-test-asan-s390x`

* document purpose of `build-and-test-asan-mac`

* set `ENABLE_EXTRA_ALIGNMENT=OFF` for imported libbson.

Fixes observed errors on older ASan with older gcc.

* print around tests in `pkgconfig-tests.sh` and `linker-tests.sh`.

To help identify a failing test.

* fix leaks in `_test_mc_FLE2TagAndEncryptedMetadataBlock_validate`

* document fix of disabled extra alignment. This may impact downstream consumers. Example: build with imported libbson, then load a different libbson at runtime.

This was the case in the pkgconfig tests. Disabling alignment was the default in 1.11 and accidentally changed in 1.12.0.

* apply `ENABLE_EXTRA_ALIGNMENT=OFF` in common CMake args for tests.

The `macos-1100` distro appears to have a system install of libbson with extra alignment enabled. libmongocrypt detects the system install and defaults to enable extra alignment to try to agree. This commit sets  `ENABLE_EXTRA_ALIGNMENT=OFF` for both libbson and libmongocrypt to ensure agreement.

* disable extra alignment in `linker-tests.sh`

To match libbson recommendeded defaults.

---------

Co-authored-by: Ezra Chung <88335979+eramongodb@users.noreply.github.com>

.evergreen/config.yml

@@ -403,10 +403,10 @@ tasks:
   - func: "fetch source"
   - func: "build and test"
     vars:
+      # Add detect_odr_violation=0 to ASAN_OPTIONS to ignore odr-violation in IntelDFP symbol: __dpml_bid_globals_table
       compile_env: >-
         ${compile_env|}
         LIBMONGOCRYPT_EXTRA_CFLAGS="-fsanitize=address -pthread"
-        # Add detect_odr_violation=0 to ASAN_OPTIONS to ignore odr-violation in IntelDFP symbol: __dpml_bid_globals_table
         ASAN_OPTIONS="detect_leaks=1 detect_odr_violation=0"
 
 - name: build-and-test-ubsan
@@ -424,13 +424,7 @@ tasks:
   commands:
   - func: "fetch source"
   - func: "build and test"
-    vars:
-      compile_env: ${compile_env|} LIBMONGOCRYPT_EXTRA_CFLAGS="-fsanitize=address"
-
-- name: build-and-test-asan-s390x
-  commands:
-  - func: "fetch source"
-  - func: "build and test"
+    # Exclude leak detection. clang on macos-11-amd64 reports: "detect_leaks is not supported on this platform"
     vars:
       compile_env: ${compile_env|} LIBMONGOCRYPT_EXTRA_CFLAGS="-fsanitize=address"
 

.evergreen/linker-tests.sh

@@ -25,7 +25,7 @@ linker_tests_deps_root="$EVG_DIR/linker_tests_deps"
 rm -rf -- "$linker_tests_root"
 mkdir -p "$linker_tests_root"/{install,libmongocrypt-cmake-build,app-cmake-build}
 
-# Make libbson1
+echo "Make libbson1 ..."
 run_chdir "$linker_tests_root" bash "$EVG_DIR/prep_c_driver_source.sh"
 MONGOC_DIR="$linker_tests_root/mongo-c-driver"
 
@@ -40,8 +40,11 @@ if [ "${MACOS_UNIVERSAL-}" = "ON" ]; then
     ADDITIONAL_CMAKE_FLAGS="$ADDITIONAL_CMAKE_FLAGS -DCMAKE_OSX_ARCHITECTURES='arm64;x86_64'"
 fi
 
+# Disable extra alignment in libbson and libmongocrypt to ensure agreement.
+# libmongocrypt disables by default, but may enable if a system install of libbson is detected with extra alignment.
 common_cmake_args=(
   $ADDITIONAL_CMAKE_FLAGS
+  -DENABLE_EXTRA_ALIGNMENT=OFF
   -DCMAKE_BUILD_TYPE=RelWithDebInfo
 )
 
@@ -69,15 +72,17 @@ run_cmake \
   "-H$SRC_PATH" \
   "-B$BUILD_PATH"
 run_cmake --build "$BUILD_PATH" --target install --config RelWithDebInfo
+echo "Make libbson1 ... done"
 
-# Prepare libbson2
+echo "Prepare libbson2 ..."
 run_chdir "$MONGOC_DIR" git reset --hard
 run_chdir "$MONGOC_DIR" git apply --ignore-whitespace "$linker_tests_deps_root/bson_patches/libbson2.patch"
 # Apply patch to fix compile on RHEL 6.2. TODO: try to remove once RHEL 6.2 is dropped (MONGOCRYPT-688).
 run_chdir "$MONGOC_DIR" git apply "$LIBMONGOCRYPT_DIR/etc/libbson-remove-GCC-diagnostic-pragma.patch"
 LIBBSON2_SRC_DIR="$MONGOC_DIR"
+echo "Prepare libbson2 ... done"
 
-# Build libmongocrypt, static linking against libbson2
+echo "Build libmongocrypt, static linking against libbson2 ..."
 BUILD_DIR="$linker_tests_root/libmongocrypt-cmake-build"
 LMC_INSTALL_PATH="$linker_tests_root/install/libmongocrypt"
 SRC_PATH="$LIBMONGOCRYPT_DIR"
@@ -88,8 +93,9 @@ run_cmake \
   "-H$SRC_PATH" \
   "-B$BUILD_DIR"
 run_cmake --build "$BUILD_DIR" --target install --config RelWithDebInfo
+echo "Build libmongocrypt, static linking against libbson2 ... done"
 
-echo "Test case: Modelling libmongoc's use"
+echo "Test case: Model libmongoc's use ..."
 # app links against libbson1.so
 # app links against libmongocrypt.so
 BUILD_DIR="$linker_tests_root/app-cmake-build"
@@ -118,4 +124,5 @@ check_output () {
     echo "ok"
 }
 check_output ".calling bson_malloc0..from libbson1..calling mongocrypt_binary_new..from libbson2."
+echo "Test case: Model libmongoc's use ... done"
 exit 0

.evergreen/pkgconfig-tests.sh

@@ -25,8 +25,11 @@ if [ "$MACOS_UNIVERSAL" = "ON" ]; then
     ADDITIONAL_CMAKE_FLAGS="$ADDITIONAL_CMAKE_FLAGS -DCMAKE_OSX_ARCHITECTURES='arm64;x86_64'"
 fi
 
+# Disable extra alignment in libbson and libmongocrypt to ensure agreement.
+# libmongocrypt disables by default, but may enable if a system install of libbson is detected with extra alignment.
 common_cmake_args=(
     -DCMAKE_BUILD_TYPE=RelWithDebInfo
+    -DENABLE_EXTRA_ALIGNMENT=OFF
     $ADDITIONAL_CMAKE_FLAGS
 )
 
@@ -40,6 +43,7 @@ if is_true USE_NINJA; then
     bash "$EVG_DIR/ensure-ninja.sh"
 fi
 
+echo "Building libbson ..."
 libbson_install_dir="$pkgconfig_tests_root/install/libbson"
 build_dir="$mongoc_src_dir/_build"
 run_cmake -DENABLE_MONGOC=OFF \
@@ -49,8 +53,9 @@ run_cmake -DENABLE_MONGOC=OFF \
        -B"$build_dir"
 run_cmake --build "$build_dir" --target install --config RelWithDebInfo
 libbson_pkg_config_path="$(native_path "$(dirname "$(find "$libbson_install_dir" -name libbson-1.0.pc)")")"
+echo "Building libbson ... done"
 
-# Build libmongocrypt, static linking against libbson and configured for the PPA
+echo "Build libmongocrypt, static linking against libbson and configured for the PPA ..."
 mongocrypt_install_dir="$pkgconfig_tests_root/install/libmongocrypt"
 build_dir=$pkgconfig_tests_root/mongocrypt-build
 run_cmake -DUSE_SHARED_LIBBSON=OFF \
@@ -60,92 +65,103 @@ run_cmake -DUSE_SHARED_LIBBSON=OFF \
        -H"$LIBMONGOCRYPT_DIR" \
        -B"$build_dir"
 run_cmake --build "$build_dir" --target install --config RelWithDebInfo
+echo "Build libmongocrypt, static linking against libbson and configured for the PPA ... done"
 
 # To validate the pkg-config scripts, we don't want the libbson script to be visible
 mongocrypt_pkg_config_path="$(native_path "$(dirname "$(find "$mongocrypt_install_dir" -name libmongocrypt.pc)")")"
 
 export PKG_CONFIG_PATH
 PKG_CONFIG_PATH="$mongocrypt_pkg_config_path:$libbson_pkg_config_path"
 
-echo "Validating pkg-config scripts"
+echo "Validating pkg-config scripts ..."
 pkg-config --debug --print-errors --exists libmongocrypt-static
 pkg-config --debug --print-errors --exists libmongocrypt
+echo "Validating pkg-config scripts ... done"
 
-# Build example-state-machine, static linking against libmongocrypt
+echo "Build example-state-machine, static linking against libmongocrypt ..."
 gcc $(pkg-config --cflags libmongocrypt-static libbson-static-1.0) \
     -o "$pkgconfig_tests_root/example-state-machine" \
     "$LIBMONGOCRYPT_DIR/test/example-state-machine.c" \
     $(pkg-config --libs libmongocrypt-static)
 run_chdir "$LIBMONGOCRYPT_DIR" "$pkgconfig_tests_root/example-state-machine"
+echo "Build example-state-machine, static linking against libmongocrypt ... done"
 
-# Build example-no-bson, static linking against libmongocrypt
+echo "Build example-no-bson, static linking against libmongocrypt ..."
 gcc $(pkg-config --cflags libmongocrypt-static) \
     -o "$pkgconfig_tests_root/example-no-bson" \
     "$LIBMONGOCRYPT_DIR/test/example-no-bson.c" \
     $(pkg-config --libs libmongocrypt-static)
 command "$pkgconfig_tests_root/example-no-bson"
+echo "Build example-no-bson, static linking against libmongocrypt ... done"
 
-# Build example-state-machine, dynamic linking against libmongocrypt
+echo "Build example-state-machine, dynamic linking against libmongocrypt ..."
 gcc $(pkg-config --cflags libmongocrypt libbson-static-1.0) \
     -o "$pkgconfig_tests_root/example-state-machine" \
     "$LIBMONGOCRYPT_DIR/test/example-state-machine.c" \
     $(pkg-config --libs libmongocrypt)
 run_chdir "$LIBMONGOCRYPT_DIR" \
     env LD_LIBRARY_PATH="$mongocrypt_install_dir/lib:$mongocrypt_install_dir/lib64" \
     "$pkgconfig_tests_root/example-state-machine"
+echo "Build example-state-machine, dynamic linking against libmongocrypt ... done"
 
-# Build example-no-bson, dynamic linking against libmongocrypt
+echo "Build example-no-bson, dynamic linking against libmongocrypt ..."
 gcc $(pkg-config --cflags libmongocrypt) \
     -o "$pkgconfig_tests_root/example-no-bson" \
     "$LIBMONGOCRYPT_DIR/test/example-no-bson.c" \
     $(pkg-config --libs libmongocrypt)
 env LD_LIBRARY_PATH="$mongocrypt_install_dir/lib:$mongocrypt_install_dir/lib64" \
     "$pkgconfig_tests_root/example-no-bson"
+echo "Build example-no-bson, dynamic linking against libmongocrypt ... done"
 
 # Clean up prior to next execution
 rm -r "$mongocrypt_install_dir"
 
-# Build libmongocrypt, dynamic linking against libbson
+echo "Build libmongocrypt, dynamic linking against libbson ..."
 run_cmake -DUSE_SHARED_LIBBSON=ON \
        -DENABLE_BUILD_FOR_PPA=OFF \
        "${common_cmake_args[@]}" \
        -DCMAKE_INSTALL_PREFIX="$mongocrypt_install_dir" \
        -H"$LIBMONGOCRYPT_DIR" \
        -B"$build_dir"
 run_cmake --build "$build_dir" --target install --config RelWithDebInfo
+echo "Build libmongocrypt, dynamic linking against libbson ... done"
 
-# Build example-state-machine, static linking against libmongocrypt
+echo "Build example-state-machine, static linking against libmongocrypt ..."
 gcc $(pkg-config --cflags libmongocrypt-static libbson-static-1.0) \
     -o "$pkgconfig_tests_root/example-state-machine" \
     "$LIBMONGOCRYPT_DIR/test/example-state-machine.c" \
     $(pkg-config --libs libmongocrypt-static)
 run_chdir "$LIBMONGOCRYPT_DIR" \
     env LD_LIBRARY_PATH="$libbson_install_dir/lib:/$libbson_install_dir/lib64" \
     "$pkgconfig_tests_root/example-state-machine"
+echo "Build example-state-machine, static linking against libmongocrypt ..."
 
-# Build example-no-bson, static linking against libmongocrypt
+echo "Build example-no-bson, static linking against libmongocrypt ..."
 gcc $(pkg-config --cflags libmongocrypt-static) \
     -o "$pkgconfig_tests_root/example-no-bson" \
     "$LIBMONGOCRYPT_DIR/test/example-no-bson.c" \
     $(pkg-config --libs libmongocrypt-static)
 env LD_LIBRARY_PATH="$libbson_install_dir/lib:/$libbson_install_dir/lib64" \
     "$pkgconfig_tests_root/example-no-bson"
+echo "Build example-no-bson, static linking against libmongocrypt ... done"
 
-# Build example-state-machine, dynamic linking against libmongocrypt
+echo "Build example-state-machine, dynamic linking against libmongocrypt ..."
 gcc $(pkg-config --cflags libmongocrypt libbson-static-1.0) \
     -o "$pkgconfig_tests_root/example-state-machine" \
     "$LIBMONGOCRYPT_DIR/test/example-state-machine.c" \
     $(pkg-config --libs libmongocrypt)
 run_chdir "$LIBMONGOCRYPT_DIR" \
     env LD_LIBRARY_PATH="$mongocrypt_install_dir/lib:$mongocrypt_install_dir/lib64:$libbson_install_dir/lib:$libbson_install_dir/lib64" \
     "$pkgconfig_tests_root/example-state-machine"
+echo "Build example-state-machine, dynamic linking against libmongocrypt ... done"
 
-# Build example-no-bson, dynamic linking against libmongocrypt
+echo "Build example-no-bson, dynamic linking against libmongocrypt ..."
 gcc $(pkg-config --cflags libmongocrypt) \
     -o "$pkgconfig_tests_root/example-no-bson" \
     "$LIBMONGOCRYPT_DIR/test/example-no-bson.c" \
     $(pkg-config --libs libmongocrypt)
 env LD_LIBRARY_PATH="$mongocrypt_install_dir/lib:$mongocrypt_install_dir/lib64:$libbson_install_dir/lib:$libbson_install_dir/lib64" \
     "$pkgconfig_tests_root/example-no-bson"
+echo "Build example-no-bson, dynamic linking against libmongocrypt ... done"
 
 echo "pkg-config tests PASS"

.lsan-suppressions

@@ -3,3 +3,4 @@ leak:ccrng_cryptographic_generate
 leak:CRYPTO_zalloc
 # Ignore leak reported in dlopen error.
 leak:_dlerror_run
+leak:_dlerror

CHANGELOG.md

@@ -1,5 +1,7 @@
 # ChangeLog
 ## 1.13.0 (Not yet released)
+### Fixed
+- Restore default behavior to disable extra alignment when importing libbson. This was the default behavior in 1.11. This can be overridden by setting the CMake option `ENABLE_EXTRA_ALIGNMENT=ON`.
 ### Removed
 - Support for macOS versions older than 11. libmongocrypt is supported and tested with macOS 11+.
 
@@ -9,6 +11,8 @@
 - Add opt-in retry behavior for KMS operations (`mongocrypt_setopt_retry_kms`)
 ### Removed
 - libmongocrypt is no longer published in the MongoDB package repository for RHEL 6. libmongocrypt may instead be built from source on RHEL 6, but support for RHEL 6 will be dropped in a future release.
+### Notes
+- This release unintentionally changes the default behavior of extra alignment with importing libbson. See 1.13.0 release notes.
 
 ## 1.11.0
 ### New features

cmake/ImportBSON.cmake

@@ -56,9 +56,9 @@ cmake_push_check_state ()
    # extra-alignment enabled. We want to match that setting as our default, for convenience
    # purposes only.
    find_path (SYSTEM_BSON_INCLUDE_DIR bson/bson.h PATH_SUFFIXES libbson-1.0)
+   set (_extra_alignment_default OFF)
    if (SYSTEM_BSON_INCLUDE_DIR AND NOT DEFINED ENABLE_EXTRA_ALIGNMENT)
       set (CMAKE_REQUIRED_INCLUDES "${SYSTEM_BSON_INCLUDE_DIR}")
-      set (_extra_alignment_default OFF)
       check_c_source_compiles ([[
          #include <bson/bson.h>
 
@@ -140,10 +140,8 @@ function (_import_bson)
       set (ENABLE_SNAPPY OFF CACHE BOOL "Toggle snappy for the mongoc subproject (not required by libmongocrypt)")
       # Disable deprecated automatic init and cleanup. (May be overridden by the user)
       set (ENABLE_AUTOMATIC_INIT_AND_CLEANUP OFF CACHE BOOL "Enable automatic init and cleanup (GCC only)")
-      if (DEFINED _extra_alignment_default)
-         # Disable over-alignment of bson types. (May be overridden by the user)
-         set (ENABLE_EXTRA_ALIGNMENT ${_extra_alignment_default} CACHE BOOL "Toggle extra alignment of bson_t")
-      endif ()
+      # Disable over-alignment of bson types. (May be overridden by the user)
+      set (ENABLE_EXTRA_ALIGNMENT ${_extra_alignment_default} CACHE BOOL "Toggle extra alignment of bson_t")
       # We don't want the subproject to find libmongocrypt
       set (ENABLE_CLIENT_SIDE_ENCRYPTION OFF CACHE BOOL "Disable client-side encryption for the libmongoc subproject")
       # Clear `BUILD_VERSION` so C driver does not use a `BUILD_VERSION` meant for libmongocrypt.

test/test-mc-fle2-tag-and-encrypted-metadata-block.c

@@ -78,11 +78,14 @@ static void _test_mc_FLE2TagAndEncryptedMetadataBlock_validate(_mongocrypt_teste
 
     // Metadata block should be valid.
     ASSERT(mc_FLE2TagAndEncryptedMetadataBlock_validate(&metadata, status));
+    _mongocrypt_buffer_cleanup(&input);
+    mc_FLE2TagAndEncryptedMetadataBlock_cleanup(&metadata);
+    mongocrypt_status_destroy(status);
 }
 
 #undef TEST_TAG_AND_ENCRYPTED_METADATA_BLOCK
 
 void _mongocrypt_tester_install_fle2_tag_and_encrypted_metadata_block(_mongocrypt_tester_t *tester) {
     INSTALL_TEST(_test_mc_FLE2TagAndEncryptedMetadataBlock_roundtrip);
     INSTALL_TEST(_test_mc_FLE2TagAndEncryptedMetadataBlock_validate);
-}
+}