Release 0.8.0

### Features * add AWS SQS ReceiveMessage latency histogram * Add command line flags for parameters * Add configurable thread count for realtime updater * Add functional test * Adding e2e latency measurement for realtime updates * Allow specifying explicit histogram bucket boundaries * Allow the blob_storage_util cp command to work with local files. * Allow the DeltaFileRecordChangeNotifier to read local files as well as from S3 * Batch delete SQS messages * Build delta files from csv * clean up realtime queues * Configure AWS hosted Prometheus. * Disable the use of exceptions * Enhance/Simplify local export of telemetry with OTLP. * Functional testing of local server with delta files * make AwsSnsSqsManager thread safe * Make the blob_storage_change_watcher tool work for local files * Make the blob_storage_util cat and rm commands work for local files * Make the blob_storage_util ls command work for local files and refactor out common parts from the AWS binary * Make the delta_file_watcher tool work for local files * Move the platform specific server configuration logic to a separate file * multi-threaded realtime notifier * realtime tester in a container * Reuse SQS client * Speed up test updates publisher * Tools for generating and inserting realtime test data. * Upgrade build-system to release-0.18.0 * Upgrade build-system to release-0.20.0 * Upgrade debian runtime images to 15 Feb 2023 * Upgrade to build-system 0.17.0 * Use a PlatformInitializer so the data_cli will compile for --platform=local ### Bug Fixes * Add ability to interrupt a SleepFor Duration. * Add minimum shard size threshold for concurrent reader. * Launch Envoy first before all other processes. * Make MetricsRecorder a shared global instance. * Only run functional tests against local server * Remove functionaltest/run-server, update docs accordingly * Remove submodule section from docs * Run server in background, and reduce noise * Run server using delta files in run-server-docker * Update generate_load_test_data to use bazel-debian * Use symlink for identical test replies * Use VLOG for concurrent reader debugging logs. * Wait for envoy to respond before launching enclave service. ### Documentation * Add a playbook template for new alerts * Add description for backup_poll_frequency_secs * Add docs about how to run the server locally * fix /tmp/deltas discrepancy * remove obsolete service * Updating instructions on how to copy `eif` manually. ### Terraform * Convert tfvar files to json * Support Prometheus service running in a different region Change-Id: I67f31c6d05015b4674e1a9600ce7f190af46e390 Bug: 275543946 GitOrigin-RevId: 41c735e45b2e5f6553caef9584a670d8a83a095b
privacysandbox · Mar 29, 2023 · c1c6d16 · c1c6d16
1 parent f01c07a
commit c1c6d16
Show file tree

Hide file tree

Showing 323 changed files with 11,184 additions and 1,869 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -1,7 +1,16 @@
 build --verbose_failures
 build --client_env=CC=clang
 build --cxxopt=-std=c++17
+build --host_cxxopt=-std=c++17
 build --copt=-Werror=return-type
+
+# Broadly disable C++ Exceptions
+# https://google.github.io/styleguide/cppguide.html#Exceptions
+build --copt=-fno-exceptions
+# Grant exceptions to some dependencies so they can use exceptions
+build --per_file_copt=.*boost.*@-fexceptions
+build --per_file_copt=.*cc/aws/proxy.*@-fexceptions
+
 build -c opt
 build --output_filter='^//((?!(third_party):).)*$'`
 build --color=yes
@@ -12,3 +21,40 @@ test --test_output=errors
 # Disable ICU linking for googleurl.
 build --@com_google_googleurl//build_config:system_icu=0
 build --@io_opentelemetry_cpp//api:with_abseil=true
+
+# Address sanitizer
+# To use it: bazel build --config asan
+build:asan --strip=never
+build:asan --copt -fsanitize=address
+build:asan --copt -DADDRESS_SANITIZER
+build:asan --copt -O1
+build:asan --copt -g
+build:asan --copt -fno-omit-frame-pointer
+build:asan --linkopt -fsanitize=address
+
+# Thread sanitizer
+# bazel build --config tsan
+build:tsan --strip=never
+build:tsan --copt -fsanitize=thread
+build:tsan --copt -DTHREAD_SANITIZER
+build:tsan --copt -DDYNAMIC_ANNOTATIONS_ENABLED=1
+build:tsan --copt -DDYNAMIC_ANNOTATIONS_EXTERNAL_IMPL=1
+build:tsan --copt -O1
+build:tsan --copt -fno-omit-frame-pointer
+build:tsan --linkopt -fsanitize=thread
+
+# --config msan: Memory sanitizer
+build:msan --strip=never
+build:msan --copt -fsanitize=memory
+build:msan --copt -DADDRESS_SANITIZER
+build:msan --copt -O1
+build:msan --copt -fno-omit-frame-pointer
+build:msan --linkopt -fsanitize=memory
+
+# --config ubsan: Undefined Behavior Sanitizer
+build:ubsan --strip=never
+build:ubsan --copt -fsanitize=undefined
+build:ubsan --copt -O1
+build:ubsan --copt -fno-omit-frame-pointer
+build:ubsan --linkopt -fsanitize=undefined
+build:ubsan --linkopt -lubsan
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -19,6 +19,7 @@ exclude: (?x)^(
              google_internal/coordinator_environments/.*$|
              .bazel_output/.*|
              builders/.*|
+             testing/functionaltest-system/.*|
              version.txt
          )$
 
@@ -40,11 +41,13 @@ repos:
   - id: check-vcs-permalinks
   - id: check-executables-have-shebangs
   - id: detect-private-key
+  - id: pretty-format-json
+    args: ['--autofix']
+    exclude: '^testing/functionaltest/test_cases/.*$'
 
 - repo: https://github.com/jumanjihouse/pre-commit-hooks
   rev: 3.0.0
   hooks:
-  - id: git-check
   - id: script-must-not-have-extension
     exclude: '^google_internal/.*/kokoro_(presubmit|continuous).sh$'
   - id: script-must-have-extension

diff --git a/.versionrc.json b/.versionrc.json
diff --git a/.versionrc.json b/.versionrc.json
@@ -0,0 +1,51 @@
+{
+  "bumpFiles": [
+    {
+      "filename": "version.txt",
+      "type": "plain-text"
+    }
+  ],
+  "commitUrlFormat": " ",
+  "issueUrlFormat": " ",
+  "packageFiles": [
+    {
+      "filename": "version.txt",
+      "type": "plain-text"
+    }
+  ],
+  "tagPrefix": "release-",
+  "types": [
+    {
+      "section": "Features",
+      "type": "feat"
+    },
+    {
+      "section": "Bug Fixes",
+      "type": "fix"
+    },
+    {
+      "section": "Documentation",
+      "type": "docs"
+    },
+    {
+      "section": "Terraform",
+      "type": "terraform"
+    },
+    {
+      "hidden": true,
+      "type": "internal"
+    },
+    {
+      "hidden": true,
+      "type": "chore"
+    },
+    {
+      "hidden": true,
+      "type": "test"
+    },
+    {
+      "hidden": true,
+      "type": "refactor"
+    }
+  ]
+}
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,76 @@
 
 All notable changes to this project will be documented in this file. See [commit-and-tag-version](https://github.com/absolute-version/commit-and-tag-version) for commit guidelines.
 
+## [0.8.0](https://github.com/privacysandbox/fledge-key-value-service/compare/release-0.7.0...release-0.8.0) (2023-03-28)
+
+
+### Features
+
+* add AWS SQS ReceiveMessage latency histogram
+* Add command line flags for parameters
+* Add configurable thread count for realtime updater
+* Add functional test
+* Adding e2e latency measurement for realtime updates
+* Allow specifying explicit histogram bucket boundaries
+* Allow the blob_storage_util cp command to work with local files.
+* Allow the DeltaFileRecordChangeNotifier to read local files as well as from S3
+* Batch delete SQS messages
+* Build delta files from csv
+* clean up realtime queues
+* Configure AWS hosted Prometheus.
+* Disable the use of exceptions
+* Enhance/Simplify local export of telemetry with OTLP.
+* Functional testing of local server with delta files
+* make AwsSnsSqsManager thread safe
+* Make the blob_storage_change_watcher tool work for local files
+* Make the blob_storage_util cat and rm commands work for local files
+* Make the blob_storage_util ls command work for local files and refactor out common parts from the AWS binary
+* Make the delta_file_watcher tool work for local files
+* Move the platform specific server configuration logic to a separate file
+* multi-threaded realtime notifier
+* realtime tester in a container
+* Reuse SQS client
+* Speed up test updates publisher
+* Tools for generating and inserting realtime test data.
+* Upgrade build-system to release-0.18.0
+* Upgrade build-system to release-0.20.0
+* Upgrade debian runtime images to 15 Feb 2023
+* Upgrade to build-system 0.17.0
+* Use a PlatformInitializer so the data_cli will compile for --platform=local
+
+
+### Bug Fixes
+
+* Add ability to interrupt a SleepFor Duration.
+* Add minimum shard size threshold for concurrent reader.
+* Launch Envoy first before all other processes.
+* Make MetricsRecorder a shared global instance.
+* Only run functional tests against local server
+* Remove functionaltest/run-server, update docs accordingly
+* Remove submodule section from docs
+* Run server in background, and reduce noise
+* Run server using delta files in run-server-docker
+* Update generate_load_test_data to use bazel-debian
+* Use symlink for identical test replies
+* Use VLOG for concurrent reader debugging logs.
+* Wait for envoy to respond before launching enclave service.
+
+
+### Documentation
+
+* Add a playbook template for new alerts
+* Add description for backup_poll_frequency_secs
+* Add docs about how to run the server locally
+* fix /tmp/deltas discrepancy
+* remove obsolete service
+* Updating instructions on how to copy `eif` manually.
+
+
+### Terraform
+
+* Convert tfvar files to json
+* Support Prometheus service running in a different region
+
 ## [0.7.0](https://github.com/privacysandbox/fledge-key-value-service/compare/release-0.6.0...release-0.7.0) (2023-02-16)
 
 

diff --git a/WORKSPACE b/WORKSPACE
@@ -1,5 +1,10 @@
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
+local_repository(
+    name = "google_privacysandbox_functionaltest_system",
+    path = "testing/functionaltest-system",
+)
+
 http_archive(
     name = "boringssl",
     sha256 = "0cd64ecff9e5f757988b84b7685e968775de08ea9157656d0b9fee0fa62d67ec",

diff --git a/builders/.pre-commit-config.yaml b/builders/.pre-commit-config.yaml
@@ -42,7 +42,6 @@ repos:
   hooks:
   - id: git-check
   - id: script-must-not-have-extension
-    exclude: '^google_internal/.*/kokoro_(presubmit|continuous).sh$'
   - id: script-must-have-extension
   - id: require-ascii
   - id: shellcheck
@@ -141,7 +140,7 @@ repos:
     - --quiet
 
 - repo: https://github.com/psf/black
-  rev: 22.12.0
+  rev: 23.1.0
   hooks:
   - id: black
     name: black python formatter
diff --git a/builders/CHANGELOG.md b/builders/CHANGELOG.md
@@ -2,6 +2,64 @@
 
 All notable changes to this project will be documented in this file. See [commit-and-tag-version](https://github.com/absolute-version/commit-and-tag-version) for commit guidelines.
 
+### [0.21.1](https://team/kiwi-air-force-eng-team/build-system/compare/release-0.21.0...release-0.21.1) (2023-03-07)
+
+
+### Bug Fixes
+
+* Relax pinned version for apache2-utils
+
+## [0.21.0](https://team/kiwi-air-force-eng-team/build-system/compare/release-0.20.0...release-0.21.0) (2023-03-06)
+
+
+### Features
+
+* Add wrapper scripts for utils
+
+## [0.20.0](https://team/kiwi-air-force-eng-team/build-system/compare/release-0.19.0...release-0.20.0) (2023-03-01)
+
+
+### Features
+
+* Add docker buildkit plugin
+* Add tests for CLI wrapper scripts
+* Permit testing of a single image
+* Relax pinned versions in build-debian, presubmit and test-tools
+
+## [0.19.0](https://team/kiwi-air-force-eng-team/build-system/compare/release-0.18.0...release-0.19.0) (2023-03-01)
+
+
+### Features
+
+* Add jq wrapper
+
+
+### Bug Fixes
+
+* Relax pinned version of openjdk to 11.0.*
+
+## [0.18.0](https://team/kiwi-air-force-eng-team/build-system/compare/release-0.17.0...release-0.18.0) (2023-02-23)
+
+
+### Features
+
+* Relax pinned versions for apk and yum packages to semver
+
+## [0.17.0](https://team/kiwi-air-force-eng-team/build-system/compare/release-0.16.0...release-0.17.0) (2023-02-21)
+
+
+### Features
+
+* Upgrade black to 23.1.0
+* Upgrade tar to 1.34-r1
+* Upgrade to buildozer 6.0.1
+
+
+### Bug Fixes
+
+* Minor code cleanup in images/presubmit/install_apps
+* Upgrade ghz to 0.114.0
+
 ## [0.16.0](https://team/kiwi-air-force-eng-team/build-system/compare/release-0.15.1...release-0.16.0) (2023-02-05)
 
 

diff --git a/builders/etc/.pre-commit-config.yaml b/builders/etc/.pre-commit-config.yaml
@@ -150,7 +150,7 @@ repos:
     - --quiet
 
 - repo: https://github.com/psf/black
-  rev: 22.12.0
+  rev: 23.1.0
   hooks:
   - id: black
     name: black python formatter
diff --git a/builders/images/build-amazonlinux2/install_apps b/builders/images/build-amazonlinux2/install_apps
@@ -46,7 +46,7 @@ function install_compilers() {
   amazon-linux-extras install -y \
     python3.8 \
     java-openjdk11
-  yum install -y java-11-openjdk-devel-11.0.16.0.8-1.amzn2.0.1
+  yum install -y "java-11-openjdk-devel-11.0.*"
   mkdir -p /opt/bin
   update-alternatives --install /opt/bin/python3 python3 /usr/bin/python3.8 100
 }
@@ -58,29 +58,29 @@ function install_nitro() {
 
 function install_gcc() {
   # install gcc, which is required to build socat
-  yum install -y gcc-7.3.1-15.amzn2
+  yum install -y "gcc-7.3.1*"
 }
 
 function install_misc() {
   yum install -y \
-    gettext-0.19.8.1-3.amzn2 \
-    git-2.38.1-1.amzn2.0.1 \
-    tar-1.26-35.amzn2 \
-    unzip-6.0-43.amzn2 \
-    zip-3.0-11.amzn2.0.2
+    "gettext-0.19.8.*" \
+    "git-2.38*" \
+    "tar-1.26*" \
+    "unzip-6.0*" \
+    "zip-3.0*"
 }
 
 function install_packer() {
-  yum install -y yum-utils-1.1.31-46.amzn2.0.1.noarch
+  yum install -y "yum-utils-1.1.31*"
   yum-config-manager --add-repo https://rpm.releases.hashicorp.com/AmazonLinux/hashicorp.repo
-  yum -y install packer-1.8.5-1
+  yum -y install "packer-1.8.5*"
   update-alternatives --install /usr/local/bin/packer packer /usr/bin/packer 100
 
   /usr/local/bin/packer version
 }
 
 function install_clang() {
-  yum install -y clang-11.1.0-1.amzn2.0.2
+  yum install -y "clang-11.1.0*"
   clang --version
 }
 

diff --git a/builders/images/build-debian/Dockerfile b/builders/images/build-debian/Dockerfile
@@ -12,11 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+FROM docker/buildx-bin:v0.10 AS buildx-bin
 FROM ubuntu:20.04
 
 COPY install_apps install_golang_apps install_go.sh generate_system_bazelrc .bazelversion /scripts/
 COPY get_workspace_mount /usr/local/bin
 COPY gitconfig /etc
+COPY --from=buildx-bin /buildx /usr/libexec/docker/cli-plugins/docker-buildx
 
 ARG LOCALE=en_US.UTF-8
 ARG TARGETARCH