[CI] Avoid concurrent LLVM builds

scripts/buildkite/build_llvm.sh

@@ -18,6 +18,19 @@ LLVM_VERSION=$(llvm_version)
 
 echo "LLVM version: ${LLVM_VERSION}"
 
+LLVM_INSTALL_DIR=${LLVMROOT}/${LLVM_VERSION}
+LLVM_INSTALL_DIR=$(add_device_extensions ${LLVM_INSTALL_DIR} ${GPU})
+
+# If there's another process building it, wait.
+# Otherwise, make the dir quickly so others don't attempt at the same time
+if [ -d ${LLVM_INSTALL_DIR} ]; then
+  echo "LLVM being built by another process, results will be in ${LLVM_INSTALL_DIR}"
+  wait_for_file "${LLVM_INSTALL_DIR}" "bin/mlir-opt"
+  exit 0
+else
+  mkdir -p ${LLVM_INSTALL_DIR}
+fi
+
 # Destination for tar balls
 if [ ! "${LLVM_TAR_DIR}" ]; then
   LLVM_TAR_DIR="/tmp/tpp-llvm-tar"
@@ -47,8 +60,6 @@ fi
 rm ${LLVM_TAR_FILE}
 
 LLVM_PROJECT_DIR=${LLVM_TAR_DIR}/llvm-project-${LLVM_VERSION}
-LLVM_INSTALL_DIR=${LLVMROOT}/${LLVM_VERSION}
-LLVM_INSTALL_DIR=$(add_device_extensions ${LLVM_INSTALL_DIR} ${GPU})
 
 # Environment setup
 echo "--- ENVIRONMENT"

scripts/buildkite/check_llvm.sh

@@ -19,11 +19,11 @@ LLVM_VERSION=$(llvm_version)
 LLVM_INSTALL_DIR=${LLVMROOT}/${LLVM_VERSION}
 LLVM_INSTALL_DIR=$(add_device_extensions ${LLVM_INSTALL_DIR} ${GPU})
 
-if [ -d "${LLVM_INSTALL_DIR}" ]; then
+if [ -f "${LLVM_INSTALL_DIR}/bin/mlir-opt" ]; then
   echo "Found $LLVM_VERSION"
   exit 0
 else
-  echo "Not Found ${LLVM_INSTALL_DIR}"
+  echo "Not Found 'mlir-opt' in ${LLVM_INSTALL_DIR}"
 fi
 
 # LLVM not found.

scripts/ci/common.sh

@@ -25,28 +25,28 @@ git_commit() {
 
 # Check if a program is in the PATH
 check_program() {
-  PROG=$1
-  if ! which $PROG > /dev/null; then
-    echo "ERROR: '$PROG' not found!"
+  local PROG=${1}
+  if ! which ${PROG} > /dev/null; then
+    echo "ERROR: '${PROG}' not found!"
     exit 1
   fi
 }
 
 # Echoes and runs a program
 echo_run() {
-  PROGRAM=$*
+  local PROGRAM=$*
   echo "${PROGRAM}"
   ${PROGRAM}
 }
 
 # Get the LLVM version for this build
 llvm_version() {
-  LLVM_VERSION_FILE=$(git_root)/build_tools/llvm_version.txt
+  local LLVM_VERSION_FILE=$(git_root)/build_tools/llvm_version.txt
   if [ ! -f "${LLVM_VERSION_FILE}" ]; then
     echo "ERROR: cannot find ${LLVM_VERSION_FILE} for ${PWD}!"
     exit 1
   fi
-  LLVM_VERSION=$(cat "${LLVM_VERSION_FILE}")
+  local LLVM_VERSION=$(cat "${LLVM_VERSION_FILE}")
   if [ ! "${LLVM_VERSION}" ]; then
     echo "ERROR: cannot find LLVM version in ${LLVM_VERSION_FILE}!"
     exit 1
@@ -69,3 +69,19 @@ add_device_extensions() {
 
   echo ${BASE}
 }
+
+# Wait for a file to appear on an existing directory
+wait_for_file() {
+  local DIR="${1}"
+  local FILE="${1}/${2}"
+
+  if [ ! -d ${DIR} ]; then
+    echo "ERROR: Directory ${DIR} not found"
+  fi
+  echo "Waiting for ${FILE}..."
+  while [ ! -f ${FILE} ]; do
+    sleep 30
+    echo "."
+  done
+  echo "Found"
+}