Add retry if needed in pre command hook

.buildkite/hooks/pre-command

@@ -2,6 +2,11 @@
 
 set -euo pipefail
 
+# needed in some systems where retry is not available
+if ! declare -f retry > /dev/null; then
+  source ../scripts/retry.sh
+fi
+
 # Secrets must be redacted
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
 PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account"

.buildkite/scripts/install_macos_tools.sh

@@ -8,24 +8,7 @@ PLATFORM_TYPE_LOWERCASE=$(uname | tr '[:upper:]' '[:lower:]')
 
 export BIN=${WORKSPACE:-$PWD}/bin
 
-retry() {
-  local retries=$1
-  shift
-  local count=0
-  until "$@"; do
-    exit=$?
-    wait=$((2 ** count))
-    count=$((count + 1))
-    if [ $count -lt "$retries" ]; then
-      >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
-      sleep $wait
-    else
-      >&2 echo "Retry $count/$retries exited $exit, no more retries left."
-      return $exit
-    fi
-  done
-  return 0
-}
+source ./retry.sh
 
 create_workspace() {
   if [[ ! -d "${BIN}" ]]; then

.buildkite/scripts/retry.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+retry() {
+  local retries=$1
+  shift
+  local count=0
+  until "$@"; do
+    exit=$?
+    wait=$((2 ** count))
+    count=$((count + 1))
+    if [ $count -lt "$retries" ]; then
+      >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+      sleep $wait
+    else
+      >&2 echo "Retry $count/$retries exited $exit, no more retries left."
+      return $exit
+    fi
+  done
+  return 0
+}