From 80a8957f0475f22a55352e767826270baa3c087d Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Sat, 6 Apr 2024 21:50:30 +1300 Subject: [PATCH 01/24] delete unused variable remove unused var --- packages/data-api/scripts/installMvRefreshModule.sh | 1 - scripts/bash/getInternalDependencies.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/packages/data-api/scripts/installMvRefreshModule.sh b/packages/data-api/scripts/installMvRefreshModule.sh index f6800f1ece..ad38e7d89b 100755 --- a/packages/data-api/scripts/installMvRefreshModule.sh +++ b/packages/data-api/scripts/installMvRefreshModule.sh @@ -1,5 +1,4 @@ #!/bin/bash -DIR=$(dirname "$0") DIR=$(pwd "$0") diff --git a/scripts/bash/getInternalDependencies.sh b/scripts/bash/getInternalDependencies.sh index c38095854e..d44e4265b8 100755 --- a/scripts/bash/getInternalDependencies.sh +++ b/scripts/bash/getInternalDependencies.sh @@ -49,7 +49,6 @@ for dependency in ${internal_dependencies[@]}; do done # remove any duplicates -deduplicated_union=() for i in "${!internal_dependencies[@]}"; do for j in "${!internal_dependencies[@]}"; do if [[ i -ne j ]] && [[ ${internal_dependencies[i]} = ${internal_dependencies[j]} ]]; then From 0a80dbeda3b68184841edf16c730d41a5c7d8ffe Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Sat, 6 Apr 2024 21:51:07 +1300 Subject: [PATCH 02/24] portable shebang for Bash scripts portable shebang in `mergeEnvForDB.sh` --- packages/data-api/scripts/buildAnalyticsMaterializedView.sh | 3 ++- packages/data-api/scripts/dropAnalyticsMaterializedView.sh | 3 ++- packages/data-api/scripts/fastRefreshAnalyticsTable.sh | 6 ++++-- packages/data-api/scripts/fullRefreshAnalyticsTable.sh | 3 ++- packages/data-api/scripts/installMvRefreshModule.sh | 5 ++--- packages/data-api/scripts/patchMvRefreshModule.sh | 2 +- packages/data-api/scripts/refreshAnalyticsTable.sh | 3 ++- packages/data-api/scripts/uninstallMvRefreshModule.sh | 2 +- packages/data-lake-api/scripts/checkTestDataLakeExists.sh | 3 ++- packages/data-lake-api/scripts/setupTestDataLake.sh | 3 ++- packages/database/scripts/checkTestDatabaseExists.sh | 3 ++- packages/database/scripts/dumpDatabase.sh | 3 ++- packages/database/scripts/migrateCreate.sh | 2 +- packages/database/scripts/setupTestDatabase.sh | 3 ++- packages/devops/scripts/ci/triggerRedeploy.sh | 3 ++- packages/devops/scripts/ci/validateBranchName.sh | 3 ++- packages/devops/scripts/ci/validateNewMigrations.sh | 3 ++- packages/devops/scripts/ci/validateTests.sh | 3 ++- .../devops/scripts/ci/validateTypesAndDbSchemaInSync.sh | 3 ++- packages/devops/scripts/deployment/checkoutLatest.sh | 3 ++- packages/devops/scripts/deployment/configureNginx.sh | 3 ++- packages/devops/scripts/utility/getEC2TagValue.sh | 3 ++- scripts/bash/backendStartDev.sh | 3 ++- scripts/bash/buildInternalDependencies.sh | 3 ++- scripts/bash/buildNonInternalDependencies.sh | 3 ++- scripts/bash/downloadEnvironmentVariables.sh | 4 ++-- scripts/bash/getDeployablePackages.sh | 3 ++- scripts/bash/getInternalDependencies.sh | 3 ++- scripts/bash/getPackagesWithEnvFiles.sh | 3 ++- scripts/bash/mergeEnvForDB.sh | 3 ++- scripts/bash/pm2startInline.sh | 3 ++- scripts/bash/validate.sh | 3 ++- scripts/docker/downloadEnvironmentVariables.sh | 3 ++- 33 files changed, 65 insertions(+), 37 deletions(-) diff --git a/packages/data-api/scripts/buildAnalyticsMaterializedView.sh b/packages/data-api/scripts/buildAnalyticsMaterializedView.sh index 05a4638d62..eb37d715c3 100755 --- a/packages/data-api/scripts/buildAnalyticsMaterializedView.sh +++ b/packages/data-api/scripts/buildAnalyticsMaterializedView.sh @@ -1,4 +1,5 @@ -#!/bin/bash -e +#!/usr/bin/env bash +set -e DIR=$(pwd "$0") source "$DIR/../../scripts/bash/mergeEnvForDB.sh" diff --git a/packages/data-api/scripts/dropAnalyticsMaterializedView.sh b/packages/data-api/scripts/dropAnalyticsMaterializedView.sh index c961a67eaa..7d9e1025b6 100755 --- a/packages/data-api/scripts/dropAnalyticsMaterializedView.sh +++ b/packages/data-api/scripts/dropAnalyticsMaterializedView.sh @@ -1,4 +1,5 @@ -#!/bin/bash -e +#!/usr/bin/env bash +set -e DIR=$(pwd "$0") source "$DIR/../../scripts/bash/mergeEnvForDB.sh" diff --git a/packages/data-api/scripts/fastRefreshAnalyticsTable.sh b/packages/data-api/scripts/fastRefreshAnalyticsTable.sh index 5df9fd9863..7da9239afe 100755 --- a/packages/data-api/scripts/fastRefreshAnalyticsTable.sh +++ b/packages/data-api/scripts/fastRefreshAnalyticsTable.sh @@ -1,8 +1,10 @@ -#!/bin/bash -e +#!/usr/bin/env bash +set -e + echo "Fast refreshing analytics table" DIR=$(pwd "$0") -source "$DIR/../../scripts/bash/mergeEnvForDB.sh" +source "$DIR/../../scripts/bash/mergeEnvForDB.sh" # Set default port in case it wasn't in .env : "${DB_PORT:=5432}" diff --git a/packages/data-api/scripts/fullRefreshAnalyticsTable.sh b/packages/data-api/scripts/fullRefreshAnalyticsTable.sh index 65da86c341..6dc389b085 100755 --- a/packages/data-api/scripts/fullRefreshAnalyticsTable.sh +++ b/packages/data-api/scripts/fullRefreshAnalyticsTable.sh @@ -1,4 +1,5 @@ -#!/bin/bash -e +#!/usr/bin/env bash +set -e echo "Fully refreshing analytics table" DIR=$(pwd "$0") diff --git a/packages/data-api/scripts/installMvRefreshModule.sh b/packages/data-api/scripts/installMvRefreshModule.sh index ad38e7d89b..21b752898d 100755 --- a/packages/data-api/scripts/installMvRefreshModule.sh +++ b/packages/data-api/scripts/installMvRefreshModule.sh @@ -1,8 +1,7 @@ -#!/bin/bash - +#!/usr/bin/env bash DIR=$(pwd "$0") -source "$DIR/../../scripts/bash/mergeEnvForDB.sh" +source "$DIR/../../scripts/bash/mergeEnvForDB.sh" # Set default port in case it wasn't in .env : "${DB_PORT:=5432}" diff --git a/packages/data-api/scripts/patchMvRefreshModule.sh b/packages/data-api/scripts/patchMvRefreshModule.sh index d38e9d73c9..cfcce8b4ac 100755 --- a/packages/data-api/scripts/patchMvRefreshModule.sh +++ b/packages/data-api/scripts/patchMvRefreshModule.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash DIR=$(pwd "$0") source "$DIR/../../scripts/bash/mergeEnvForDB.sh" diff --git a/packages/data-api/scripts/refreshAnalyticsTable.sh b/packages/data-api/scripts/refreshAnalyticsTable.sh index 52e1a6aa2a..35fb801b2c 100755 --- a/packages/data-api/scripts/refreshAnalyticsTable.sh +++ b/packages/data-api/scripts/refreshAnalyticsTable.sh @@ -1,4 +1,5 @@ -#!/bin/bash -e +#!/usr/bin/env bash +set -e if [[ "$1" == "--full" || "$1" == "-f" ]]; then ./scripts/fullRefreshAnalyticsTable.sh diff --git a/packages/data-api/scripts/uninstallMvRefreshModule.sh b/packages/data-api/scripts/uninstallMvRefreshModule.sh index 9880e0ab60..d131c96244 100755 --- a/packages/data-api/scripts/uninstallMvRefreshModule.sh +++ b/packages/data-api/scripts/uninstallMvRefreshModule.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash DIR=$(pwd "$0") source "$DIR/../../scripts/bash/mergeEnvForDB.sh" diff --git a/packages/data-lake-api/scripts/checkTestDataLakeExists.sh b/packages/data-lake-api/scripts/checkTestDataLakeExists.sh index 927ee9aef7..0e1f20820c 100755 --- a/packages/data-lake-api/scripts/checkTestDataLakeExists.sh +++ b/packages/data-lake-api/scripts/checkTestDataLakeExists.sh @@ -1,4 +1,5 @@ -#!/bin/bash -e +#!/usr/bin/env bash +set -e DIR=$(pwd "$0") source "$DIR/../../scripts/bash/mergeEnvForDB.sh" diff --git a/packages/data-lake-api/scripts/setupTestDataLake.sh b/packages/data-lake-api/scripts/setupTestDataLake.sh index f9563a4065..8d7a87b47e 100755 --- a/packages/data-lake-api/scripts/setupTestDataLake.sh +++ b/packages/data-lake-api/scripts/setupTestDataLake.sh @@ -1,4 +1,5 @@ -#!/bin/bash -e +#!/usr/bin/env bash +set -e DIR=$(pwd "$0") source "$DIR/../../scripts/bash/mergeEnvForDB.sh" diff --git a/packages/database/scripts/checkTestDatabaseExists.sh b/packages/database/scripts/checkTestDatabaseExists.sh index 0d40d2da14..c3f57408bd 100755 --- a/packages/database/scripts/checkTestDatabaseExists.sh +++ b/packages/database/scripts/checkTestDatabaseExists.sh @@ -1,4 +1,5 @@ -#!/bin/bash -e +#!/usr/bin/env bash +set -e DIR=$(pwd "$0") source "$DIR/../../scripts/bash/mergeEnvForDB.sh" diff --git a/packages/database/scripts/dumpDatabase.sh b/packages/database/scripts/dumpDatabase.sh index 7b028cb779..978f9595ea 100755 --- a/packages/database/scripts/dumpDatabase.sh +++ b/packages/database/scripts/dumpDatabase.sh @@ -1,4 +1,5 @@ -#!/bin/bash -e +#!/usr/bin/env bash +set -e function print_help() { cat < Date: Sat, 6 Apr 2024 21:54:54 +1300 Subject: [PATCH 03/24] prefer `.` to `source` --- packages/data-api/scripts/buildAnalyticsMaterializedView.sh | 2 +- packages/data-api/scripts/dropAnalyticsMaterializedView.sh | 2 +- packages/data-api/scripts/fastRefreshAnalyticsTable.sh | 2 +- packages/data-api/scripts/fullRefreshAnalyticsTable.sh | 2 +- packages/data-api/scripts/installMvRefreshModule.sh | 2 +- packages/data-api/scripts/patchMvRefreshModule.sh | 2 +- packages/data-api/scripts/uninstallMvRefreshModule.sh | 2 +- packages/data-lake-api/scripts/checkTestDataLakeExists.sh | 2 +- packages/data-lake-api/scripts/setupTestDataLake.sh | 2 +- packages/database/scripts/checkTestDatabaseExists.sh | 2 +- packages/database/scripts/dumpDatabase.sh | 2 +- packages/database/scripts/setupTestDatabase.sh | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/data-api/scripts/buildAnalyticsMaterializedView.sh b/packages/data-api/scripts/buildAnalyticsMaterializedView.sh index eb37d715c3..e1f188fe33 100755 --- a/packages/data-api/scripts/buildAnalyticsMaterializedView.sh +++ b/packages/data-api/scripts/buildAnalyticsMaterializedView.sh @@ -2,7 +2,7 @@ set -e DIR=$(pwd "$0") -source "$DIR/../../scripts/bash/mergeEnvForDB.sh" +. "$DIR/../../scripts/bash/mergeEnvForDB.sh" # Set default port in case it wasn't in .env : "${DB_PORT:=5432}" diff --git a/packages/data-api/scripts/dropAnalyticsMaterializedView.sh b/packages/data-api/scripts/dropAnalyticsMaterializedView.sh index 7d9e1025b6..18847a153d 100755 --- a/packages/data-api/scripts/dropAnalyticsMaterializedView.sh +++ b/packages/data-api/scripts/dropAnalyticsMaterializedView.sh @@ -2,7 +2,7 @@ set -e DIR=$(pwd "$0") -source "$DIR/../../scripts/bash/mergeEnvForDB.sh" +. "$DIR/../../scripts/bash/mergeEnvForDB.sh" # Set default port in case it wasn't in .env : "${DB_PORT:=5432}" diff --git a/packages/data-api/scripts/fastRefreshAnalyticsTable.sh b/packages/data-api/scripts/fastRefreshAnalyticsTable.sh index 7da9239afe..08c475a5db 100755 --- a/packages/data-api/scripts/fastRefreshAnalyticsTable.sh +++ b/packages/data-api/scripts/fastRefreshAnalyticsTable.sh @@ -4,7 +4,7 @@ set -e echo "Fast refreshing analytics table" DIR=$(pwd "$0") -source "$DIR/../../scripts/bash/mergeEnvForDB.sh" +. "$DIR/../../scripts/bash/mergeEnvForDB.sh" # Set default port in case it wasn't in .env : "${DB_PORT:=5432}" diff --git a/packages/data-api/scripts/fullRefreshAnalyticsTable.sh b/packages/data-api/scripts/fullRefreshAnalyticsTable.sh index 6dc389b085..9991269a96 100755 --- a/packages/data-api/scripts/fullRefreshAnalyticsTable.sh +++ b/packages/data-api/scripts/fullRefreshAnalyticsTable.sh @@ -3,7 +3,7 @@ set -e echo "Fully refreshing analytics table" DIR=$(pwd "$0") -source "$DIR/../../scripts/bash/mergeEnvForDB.sh" +. "$DIR/../../scripts/bash/mergeEnvForDB.sh" # Set default port in case it wasn't in .env : "${DB_PORT:=5432}" diff --git a/packages/data-api/scripts/installMvRefreshModule.sh b/packages/data-api/scripts/installMvRefreshModule.sh index 21b752898d..7066460132 100755 --- a/packages/data-api/scripts/installMvRefreshModule.sh +++ b/packages/data-api/scripts/installMvRefreshModule.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash DIR=$(pwd "$0") -source "$DIR/../../scripts/bash/mergeEnvForDB.sh" +. "$DIR/../../scripts/bash/mergeEnvForDB.sh" # Set default port in case it wasn't in .env : "${DB_PORT:=5432}" diff --git a/packages/data-api/scripts/patchMvRefreshModule.sh b/packages/data-api/scripts/patchMvRefreshModule.sh index cfcce8b4ac..333eab5f7e 100755 --- a/packages/data-api/scripts/patchMvRefreshModule.sh +++ b/packages/data-api/scripts/patchMvRefreshModule.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash DIR=$(pwd "$0") -source "$DIR/../../scripts/bash/mergeEnvForDB.sh" +. "$DIR/../../scripts/bash/mergeEnvForDB.sh" COMMAND=$1 diff --git a/packages/data-api/scripts/uninstallMvRefreshModule.sh b/packages/data-api/scripts/uninstallMvRefreshModule.sh index d131c96244..291ec8fd4f 100755 --- a/packages/data-api/scripts/uninstallMvRefreshModule.sh +++ b/packages/data-api/scripts/uninstallMvRefreshModule.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash DIR=$(pwd "$0") -source "$DIR/../../scripts/bash/mergeEnvForDB.sh" +. "$DIR/../../scripts/bash/mergeEnvForDB.sh" # Set default port in case it wasn't in .env : "${DB_PORT:=5432}" diff --git a/packages/data-lake-api/scripts/checkTestDataLakeExists.sh b/packages/data-lake-api/scripts/checkTestDataLakeExists.sh index 0e1f20820c..e6a9629691 100755 --- a/packages/data-lake-api/scripts/checkTestDataLakeExists.sh +++ b/packages/data-lake-api/scripts/checkTestDataLakeExists.sh @@ -2,7 +2,7 @@ set -e DIR=$(pwd "$0") -source "$DIR/../../scripts/bash/mergeEnvForDB.sh" +. "$DIR/../../scripts/bash/mergeEnvForDB.sh" # Set default port in case it wasn't in .env : "${DATA_LAKE_DB_PORT:=5432}" diff --git a/packages/data-lake-api/scripts/setupTestDataLake.sh b/packages/data-lake-api/scripts/setupTestDataLake.sh index 8d7a87b47e..78c4946980 100755 --- a/packages/data-lake-api/scripts/setupTestDataLake.sh +++ b/packages/data-lake-api/scripts/setupTestDataLake.sh @@ -2,7 +2,7 @@ set -e DIR=$(pwd "$0") -source "$DIR/../../scripts/bash/mergeEnvForDB.sh" +. "$DIR/../../scripts/bash/mergeEnvForDB.sh" # Set default port in case it wasn't in .env : "${DATA_LAKE_DB_PORT:=5432}" diff --git a/packages/database/scripts/checkTestDatabaseExists.sh b/packages/database/scripts/checkTestDatabaseExists.sh index c3f57408bd..836f5f8e6b 100755 --- a/packages/database/scripts/checkTestDatabaseExists.sh +++ b/packages/database/scripts/checkTestDatabaseExists.sh @@ -2,7 +2,7 @@ set -e DIR=$(pwd "$0") -source "$DIR/../../scripts/bash/mergeEnvForDB.sh" +. "$DIR/../../scripts/bash/mergeEnvForDB.sh" # Set default port in case it wasn't in .env : "${DB_PORT:=5432}" diff --git a/packages/database/scripts/dumpDatabase.sh b/packages/database/scripts/dumpDatabase.sh index 978f9595ea..00669d4389 100755 --- a/packages/database/scripts/dumpDatabase.sh +++ b/packages/database/scripts/dumpDatabase.sh @@ -31,7 +31,7 @@ function show_loading_spinner() { } DIR=$(pwd "$0") -source "$DIR/../../scripts/bash/mergeEnvForDB.sh" +. "$DIR/../../scripts/bash/mergeEnvForDB.sh" DUMP_FILE_NAME="dump.sql" diff --git a/packages/database/scripts/setupTestDatabase.sh b/packages/database/scripts/setupTestDatabase.sh index 646c165415..e23a184bd4 100755 --- a/packages/database/scripts/setupTestDatabase.sh +++ b/packages/database/scripts/setupTestDatabase.sh @@ -2,7 +2,7 @@ set -e DIR=$(pwd "$0") -source "$DIR/../../scripts/bash/mergeEnvForDB.sh" +. "$DIR/../../scripts/bash/mergeEnvForDB.sh" # Set default port in case it wasn't in .env : "${DB_PORT:=5432}" From fa29c90eac6e3b2165860aec4329d665d2c986f5 Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Sat, 6 Apr 2024 21:59:30 +1300 Subject: [PATCH 04/24] prefer `echo` without `""` for blank lines --- packages/database/scripts/checkTestDatabaseExists.sh | 4 ++-- packages/database/scripts/dumpDatabase.sh | 2 +- packages/devops/scripts/lambda/resources/startup.sh | 2 +- scripts/bash/buildInternalDependencies.sh | 4 ++-- scripts/bash/pm2startInline.sh | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/database/scripts/checkTestDatabaseExists.sh b/packages/database/scripts/checkTestDatabaseExists.sh index 836f5f8e6b..302e6c9f17 100755 --- a/packages/database/scripts/checkTestDatabaseExists.sh +++ b/packages/database/scripts/checkTestDatabaseExists.sh @@ -14,8 +14,8 @@ fi echo -e "\033[31mError: $DB_NAME database does not exist!\033[m" echo "To create it, get the .env file from LastPass then run:" -echo "" +echo echo -e " \033[1myarn workspace @tupaia/database setup-test-database\033[m" -echo "" +echo exit 1 diff --git a/packages/database/scripts/dumpDatabase.sh b/packages/database/scripts/dumpDatabase.sh index 00669d4389..09a56517c5 100755 --- a/packages/database/scripts/dumpDatabase.sh +++ b/packages/database/scripts/dumpDatabase.sh @@ -27,7 +27,7 @@ function show_loading_spinner() { sleep .5 done printf "\r$1 " - echo "" # reset prompt + echo # reset prompt } DIR=$(pwd "$0") diff --git a/packages/devops/scripts/lambda/resources/startup.sh b/packages/devops/scripts/lambda/resources/startup.sh index 6a93fff68d..0fca1dee82 100755 --- a/packages/devops/scripts/lambda/resources/startup.sh +++ b/packages/devops/scripts/lambda/resources/startup.sh @@ -48,7 +48,7 @@ mkdir -m 777 -p $LOGS_DIR if [[ $DEPLOYMENT_NAME == "production" ]]; then \. "$HOME_DIR/.nvm/nvm.sh" # Load nvm so node is available on $PATH sudo -u ubuntu echo "10 13 * * * PATH=$PATH $HOME_DIR/tupaia/packages/web-config-server/run_preaggregation.sh | while IFS= read -r line; do printf '\%s \%s\\n' \"\$(date)\" \"\$line\"; done > $LOGS_DIR/preaggregation.txt" > tmp.cron - sudo -u ubuntu crontab -l >> tmp.cron || echo "" >> tmp.cron + sudo -u ubuntu crontab -l >> tmp.cron || echo >> tmp.cron sudo -u ubuntu crontab tmp.cron rm tmp.cron fi diff --git a/scripts/bash/buildInternalDependencies.sh b/scripts/bash/buildInternalDependencies.sh index e86ecff71a..900136122a 100755 --- a/scripts/bash/buildInternalDependencies.sh +++ b/scripts/bash/buildInternalDependencies.sh @@ -47,11 +47,11 @@ done if [[ $watch == "true" ]]; then echo -e "\033[1mConcurrently building and watching all internal dependencies\033[m" echo "> ${CONCURRENTLY_BIN} --names \"${build_prefixes[*]}\" ${build_commands[@]}" - echo "" + echo eval "${CONCURRENTLY_BIN} --names \"${build_prefixes[*]}\" ${build_commands[@]}" else echo -e "\033[1mConcurrently building internal dependencies in batches of ${CONCURRENT_BUILD_BATCH_SIZE}\033[m" echo "> ${CONCURRENTLY_BIN} -m $CONCURRENT_BUILD_BATCH_SIZE --names \"${build_prefixes[*]}\" -k ${build_commands[*]}" - echo "" + echo eval "${CONCURRENTLY_BIN} -m $CONCURRENT_BUILD_BATCH_SIZE --names \"${build_prefixes[*]}\" -k ${build_commands[*]}" fi diff --git a/scripts/bash/pm2startInline.sh b/scripts/bash/pm2startInline.sh index 36c7de308d..22ce065e68 100755 --- a/scripts/bash/pm2startInline.sh +++ b/scripts/bash/pm2startInline.sh @@ -5,10 +5,10 @@ cd "$(dirname "${BASH_SOURCE[0]}")" if [ -z "$1" ]; then echo -e "Usage: \033[1myarn start-stack\033[0m \033[4mstack\033[0m" - echo "" + echo echo -e "All \033[4mstack\033[0ms:" ls -1 ../../packages/devops/configs/pm2/ | sed 's|.config.js||g' | grep -v 'base' | awk '$0=" "$0' - echo "" + echo echo "Tips:" echo -e " - Normal PM2 commands work e.g. \033[1myarn pm2 status\033[0m" echo " - Start multiple stacks by calling this command multiple times" From 3807364eaa442daaf277ca1c30d1573ada4c1f27 Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Sat, 6 Apr 2024 22:07:10 +1300 Subject: [PATCH 05/24] `download-env-vars`: log in to Bitwarden first --- scripts/bash/downloadEnvironmentVariables.sh | 32 +++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/scripts/bash/downloadEnvironmentVariables.sh b/scripts/bash/downloadEnvironmentVariables.sh index 13c449f0e1..827df9d25b 100755 --- a/scripts/bash/downloadEnvironmentVariables.sh +++ b/scripts/bash/downloadEnvironmentVariables.sh @@ -5,8 +5,13 @@ DEPLOYMENT_NAME=$1 DIR=$(dirname "$0") COLLECTION_PATH="Engineering/Tupaia General/Environment Variables" # Collection in BitWarden where .env vars are kept +# Log in to bitwarden +bw login --check || bw login $BITWARDEN_EMAIL $BITWARDEN_PASSWORD +eval "$(bw unlock $BITWARDEN_PASSWORD | grep -o -m 1 'export BW_SESSION=.*$')" -# can provide one or more packages as command line arguments, or will default to all +COLLECTION_ID=$(bw get collection "$COLLECTION_PATH" | jq .id) + +# Can provide one or more packages as command line arguments, or will default to all if [ -z $2 ]; then echo "Fetching all .env files" PACKAGES=$(${DIR}/getPackagesWithEnvFiles.sh) @@ -15,11 +20,6 @@ else echo "Fetching environment variables for ${PACKAGES}" fi -# Login to bitwarden -bw login --check || bw login $BITWARDEN_EMAIL $BITWARDEN_PASSWORD -eval "$(bw unlock $BITWARDEN_PASSWORD | grep -o -m 1 'export BW_SESSION=.*$')" - -COLLECTION_ID=$(bw get collection "$COLLECTION_PATH" | jq .id) load_env_file_from_bw () { FILE_NAME=$1 @@ -29,12 +29,9 @@ load_env_file_from_bw () { echo "Fetching environment variables for $FILE_NAME: $ENV_FILE_PATH" - echo "Fetching environment variables for $FILE_NAME" - # checkout deployment specific env vars, or dev as fallback DEPLOYMENT_ENV_VARS=$(bw list items --search ${FILE_NAME}.${DEPLOYMENT_NAME}.env | jq --raw-output "map(select(.collectionIds[] | contains ($COLLECTION_ID))) | .[] .notes") - if [ -n "$DEPLOYMENT_ENV_VARS" ]; then echo "$DEPLOYMENT_ENV_VARS" > ${ENV_FILE_PATH} else @@ -60,11 +57,12 @@ load_env_file_from_bw () { # (after removing prefix, if there are duplicate keys, dotenv uses the last one in the file) sed -i -e 's/^###DEV_ONLY###//g' ${ENV_FILE_PATH} fi - - echo "downloaded .env vars for $FILE_NAME" + + echo "downloaded .env vars for $FILE_NAME" } - + + for PACKAGE in $PACKAGES; do # only download the env file if there is an example file in the package. If there isn't, this means it is a package that doesn't need env vars has_example_env_in_package=$(find $DIR/../../packages/$PACKAGE -type f -name '*.env.example' | wc -l) @@ -74,17 +72,15 @@ for PACKAGE in $PACKAGES; do done -# get all .env.*.example files in the env directory +# Get all *.env.example files in the env directory file_names=$(find $DIR/../../env -type f -name '*.env.example' -exec basename {} \;) - - -# for each file, get the extract the filename without the .example extension + +# For each file, get its basename without the .env.example extension for file_name in $file_names; do env_name=$(echo $file_name | sed 's/\.env.example//') load_env_file_from_bw $env_name $DIR/../../env $env_name done - - +# Log out of Bitwarden bw logout From 92433da1dc98e7b479e8b6c280c146ea5a148e14 Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Sat, 6 Apr 2024 22:08:30 +1300 Subject: [PATCH 06/24] simplify loop over `*.env.example` files --- scripts/bash/downloadEnvironmentVariables.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts/bash/downloadEnvironmentVariables.sh b/scripts/bash/downloadEnvironmentVariables.sh index 827df9d25b..b9468a005e 100755 --- a/scripts/bash/downloadEnvironmentVariables.sh +++ b/scripts/bash/downloadEnvironmentVariables.sh @@ -72,12 +72,8 @@ for PACKAGE in $PACKAGES; do done -# Get all *.env.example files in the env directory -file_names=$(find $DIR/../../env -type f -name '*.env.example' -exec basename {} \;) - -# For each file, get its basename without the .env.example extension -for file_name in $file_names; do - env_name=$(echo $file_name | sed 's/\.env.example//') +for file_name in *.env.example; do + env_name="${file_name%.env.example}" # Get its basename without the .env.example extension load_env_file_from_bw $env_name $DIR/../../env $env_name done From d34a6468bcc58ae24fee1e98e1199d4ba55e8c71 Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Sat, 6 Apr 2024 22:13:46 +1300 Subject: [PATCH 07/24] convenience variables for ANSI codes/sequences --- .../scripts/checkTestDatabaseExists.sh | 5 +- scripts/bash/ansiControlSequences.sh | 72 +++++++++++++++++++ scripts/bash/buildInternalDependencies.sh | 7 +- scripts/bash/pm2startInline.sh | 9 +-- 4 files changed, 84 insertions(+), 9 deletions(-) create mode 100644 scripts/bash/ansiControlSequences.sh diff --git a/packages/database/scripts/checkTestDatabaseExists.sh b/packages/database/scripts/checkTestDatabaseExists.sh index 302e6c9f17..08cf3ca4e2 100755 --- a/packages/database/scripts/checkTestDatabaseExists.sh +++ b/packages/database/scripts/checkTestDatabaseExists.sh @@ -2,6 +2,7 @@ set -e DIR=$(pwd "$0") +. "$DIR/../../scripts/bash/ansiControlSequences.sh" . "$DIR/../../scripts/bash/mergeEnvForDB.sh" # Set default port in case it wasn't in .env @@ -12,10 +13,10 @@ then exit 0 fi -echo -e "\033[31mError: $DB_NAME database does not exist!\033[m" +echo -e "${RED}Error: $DB_NAME database does not exist!${RESET}" echo "To create it, get the .env file from LastPass then run:" echo -echo -e " \033[1myarn workspace @tupaia/database setup-test-database\033[m" +echo -e " ${BOLD}yarn workspace @tupaia/database setup-test-database${RESET}" echo exit 1 diff --git a/scripts/bash/ansiControlSequences.sh b/scripts/bash/ansiControlSequences.sh new file mode 100644 index 0000000000..3a6cda434f --- /dev/null +++ b/scripts/bash/ansiControlSequences.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Defines (and exports) a handful of the more widely supported ANSI control sequences for +# manipulating font and colour when writing to stdout. Honours the NO_COLOR environment variable if +# it is set. +# +# EXAMPLE USAGE +# source "path/to/ansiControlSequence.sh" +# echo -e "${BOLD}${WHITE}${ON_RED}ERROR${RESET} File does not exist." +# printf 'See %bhttp://bes.au%b for more info.\n' "$MAGENTA" "$RESET" +# +# REMARKS +# - If using echo, remember to use -e. Otherwise, the ANSI codes will be printed literally. +# - Avoid using variables in the printf format string: https://github.com/koalaman/shellcheck/wiki/SC2059 +# +# REFERENCE +# https://en.wikipedia.org/wiki/ANSI_escape_code + +# ANSI control sequences + +export CURSOR_UP='\033[A' +export CURSOR_DOWN='\033[B' +export CURSOR_FORWARD='\033[C' +export CURSOR_BACK='\033[D' +export CURSOR_NEXT_LINE='\033[E' +export CURSOR_PREV_LINE='\033[F' +export CURSOR_START_OF_LINE='\033[G' +export CLEAR_LINE="\033[2K${CURSOR_START_OF_LINE}" + +# Select graphic rendition (SGR) parameters + +if [[ $NO_COLOR != '' ]]; then + # See https://no-color.org + export RESET='' + export BOLD='' + export UNDERLINE='' + export BLACK='' + export RED='' + export GREEN='' + export YELLOW='' + export BLUE='' + export MAGENTA='' + export CYAN='' + export WHITE='' + export ON_BLACK='' + export ON_RED='' + export ON_GREEN='' + export ON_YELLOW='' + export ON_BLUE='' + export ON_MAGENTA='' + export ON_CYAN='' + export ON_WHITE='' +else + export RESET='\033[m' + export BOLD='\033[1m' + export UNDERLINE='\033[4m' + export BLACK='\033[30m' + export RED='\033[31m' + export GREEN='\033[32m' + export YELLOW='\033[33m' + export BLUE='\033[34m' + export MAGENTA='\033[35m' + export CYAN='\033[36m' + export WHITE='\033[37m' + export ON_BLACK='\033[40m' + export ON_RED='\033[41m' + export ON_GREEN='\033[42m' + export ON_YELLOW='\033[43m' + export ON_BLUE='\033[44m' + export ON_MAGENTA='\033[45m' + export ON_CYAN='\033[46m' + export ON_WHITE='\033[47m' +fi diff --git a/scripts/bash/buildInternalDependencies.sh b/scripts/bash/buildInternalDependencies.sh index 900136122a..0834ba2189 100755 --- a/scripts/bash/buildInternalDependencies.sh +++ b/scripts/bash/buildInternalDependencies.sh @@ -2,11 +2,12 @@ set -e DIR=$(dirname "$0") +. "$DIR/ansiControlSequences.sh" CONCURRENT_BUILD_BATCH_SIZE=1 CONCURRENTLY_BIN="${DIR}/../../node_modules/.bin/concurrently" -USAGE="Usage: \033[1mbuildInternalDependencies.sh\033[m [\033[1m--watch\033[m] [\033[1m--packagePath\033[m|\033[1m-p\033[m]" +USAGE="Usage: ${BOLD}buildInternalDependencies.sh${RESET} [${BOLD}--watch${RESET}] [${BOLD}--packagePath${RESET}|${BOLD}-p${RESET}]" watch=false package_path="" @@ -45,12 +46,12 @@ for PACKAGE in $(${DIR}/getInternalDependencies.sh ${package_path}); do done if [[ $watch == "true" ]]; then - echo -e "\033[1mConcurrently building and watching all internal dependencies\033[m" + echo -e "${BOLD}Concurrently building and watching all internal dependencies${RESET}" echo "> ${CONCURRENTLY_BIN} --names \"${build_prefixes[*]}\" ${build_commands[@]}" echo eval "${CONCURRENTLY_BIN} --names \"${build_prefixes[*]}\" ${build_commands[@]}" else - echo -e "\033[1mConcurrently building internal dependencies in batches of ${CONCURRENT_BUILD_BATCH_SIZE}\033[m" + echo -e "${BOLD}Concurrently building internal dependencies in batches of ${CONCURRENT_BUILD_BATCH_SIZE}${RESET}" echo "> ${CONCURRENTLY_BIN} -m $CONCURRENT_BUILD_BATCH_SIZE --names \"${build_prefixes[*]}\" -k ${build_commands[*]}" echo eval "${CONCURRENTLY_BIN} -m $CONCURRENT_BUILD_BATCH_SIZE --names \"${build_prefixes[*]}\" -k ${build_commands[*]}" diff --git a/scripts/bash/pm2startInline.sh b/scripts/bash/pm2startInline.sh index 22ce065e68..44b89cc505 100755 --- a/scripts/bash/pm2startInline.sh +++ b/scripts/bash/pm2startInline.sh @@ -2,15 +2,16 @@ set -e cd "$(dirname "${BASH_SOURCE[0]}")" +. ./ansiControlSequences.sh if [ -z "$1" ]; then - echo -e "Usage: \033[1myarn start-stack\033[0m \033[4mstack\033[0m" + echo -e "Usage: ${BOLD}yarn start-stack${RESET} ${UNDERLINE}stack${RESET}" echo - echo -e "All \033[4mstack\033[0ms:" + echo -e "All ${UNDERLINE}stack${RESET}s:" ls -1 ../../packages/devops/configs/pm2/ | sed 's|.config.js||g' | grep -v 'base' | awk '$0=" "$0' echo echo "Tips:" - echo -e " - Normal PM2 commands work e.g. \033[1myarn pm2 status\033[0m" + echo -e " - Normal PM2 commands work e.g. ${BOLD}yarn pm2 status${RESET}" echo " - Start multiple stacks by calling this command multiple times" exit 1 fi @@ -18,6 +19,6 @@ fi yarn pm2 start "../../packages/devops/configs/pm2/$1.config.js" # When user quits logs, stop everything -trap "echo -e '\n\033[1;41;97m Stopping... \033[0m' && yarn pm2 delete all" EXIT +trap 'echo -e "\n${RED}[start-stack]${RESET} Stopping..." && yarn pm2 delete all' EXIT yarn pm2 logs --lines 0 From 64b9ac7ed0c01a0e506219c88903439d979bf818 Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Sat, 6 Apr 2024 22:15:23 +1300 Subject: [PATCH 08/24] inline `then` in `if` header --- packages/database/scripts/checkTestDatabaseExists.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/database/scripts/checkTestDatabaseExists.sh b/packages/database/scripts/checkTestDatabaseExists.sh index 08cf3ca4e2..01cf12d22d 100755 --- a/packages/database/scripts/checkTestDatabaseExists.sh +++ b/packages/database/scripts/checkTestDatabaseExists.sh @@ -8,8 +8,7 @@ DIR=$(pwd "$0") # Set default port in case it wasn't in .env : "${DB_PORT:=5432}" -if [ "$(PGPASSWORD=$DB_PG_PASSWORD psql -p $DB_PORT -X -A -h $DB_URL -U $DB_PG_USER -t -c "SELECT 1 FROM pg_database WHERE datname='$DB_NAME'" )" = '1' ] -then +if [ "$(PGPASSWORD=$DB_PG_PASSWORD psql -p $DB_PORT -X -A -h $DB_URL -U $DB_PG_USER -t -c "SELECT 1 FROM pg_database WHERE datname='$DB_NAME'" )" = '1' ]; then exit 0 fi From 2213a9af7b2be11266ca92798bcadd9d6d61e68e Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Sat, 6 Apr 2024 22:25:38 +1300 Subject: [PATCH 09/24] update env var message to refer to Bitwarden --- .../data-lake-api/scripts/checkTestDataLakeExists.sh | 9 ++++++++- packages/database/scripts/checkTestDatabaseExists.sh | 3 ++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/data-lake-api/scripts/checkTestDataLakeExists.sh b/packages/data-lake-api/scripts/checkTestDataLakeExists.sh index e6a9629691..0ff54e5112 100755 --- a/packages/data-lake-api/scripts/checkTestDataLakeExists.sh +++ b/packages/data-lake-api/scripts/checkTestDataLakeExists.sh @@ -3,6 +3,7 @@ set -e DIR=$(pwd "$0") . "$DIR/../../scripts/bash/mergeEnvForDB.sh" +. "$DIR/../../scripts/bash/ansiControlSequences.sh" # Set default port in case it wasn't in .env : "${DATA_LAKE_DB_PORT:=5432}" @@ -12,5 +13,11 @@ then exit 0 fi -echo -e "Error: $DATA_LAKE_DB_NAME database does not exist!\n\nTo create it, please get the .env file from lastpass then run:\nyarn workspace @tupaia/data-lake-api setup-test-data-lake\n" +echo -e "${RED}Error: $DATA_LAKE_DB_NAME database does not exist!${RESET}" +echo 'To create it, make sure you have the environment variables from Bitwarden and run:' +echo +echo -e " ${BOLD}yarn workspace @tupaia/data-lake-api setup-test-data-lake${RESET}" +echo +echo -e "If you’re missing environment variables, see ${MAGENTA}https://beyond-essential.slab.com/posts/tupaia-monorepo-setup-v5egpdpq#hvfnz-set-environment-variables${RESET}." + exit 1 diff --git a/packages/database/scripts/checkTestDatabaseExists.sh b/packages/database/scripts/checkTestDatabaseExists.sh index 01cf12d22d..14f7c027b5 100755 --- a/packages/database/scripts/checkTestDatabaseExists.sh +++ b/packages/database/scripts/checkTestDatabaseExists.sh @@ -13,9 +13,10 @@ if [ "$(PGPASSWORD=$DB_PG_PASSWORD psql -p $DB_PORT -X -A -h $DB_URL -U $DB_PG_U fi echo -e "${RED}Error: $DB_NAME database does not exist!${RESET}" -echo "To create it, get the .env file from LastPass then run:" +echo 'To create it, make sure you have the environment variables from Bitwarden and run:' echo echo -e " ${BOLD}yarn workspace @tupaia/database setup-test-database${RESET}" echo +echo -e "If you’re missing environment variables, see ${MAGENTA}https://beyond-essential.slab.com/posts/tupaia-monorepo-setup-v5egpdpq#hvfnz-set-environment-variables${RESET}." exit 1 From 8702ea2224f26bf67c289e63ff2c75956163a4d5 Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Sat, 6 Apr 2024 22:35:29 +1300 Subject: [PATCH 10/24] prettify `download-env-vars` output --- scripts/bash/downloadEnvironmentVariables.sh | 28 +++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/scripts/bash/downloadEnvironmentVariables.sh b/scripts/bash/downloadEnvironmentVariables.sh index b9468a005e..3f285b1660 100755 --- a/scripts/bash/downloadEnvironmentVariables.sh +++ b/scripts/bash/downloadEnvironmentVariables.sh @@ -3,21 +3,27 @@ set -e +x # Do not output commands in this script, as some would show credential DEPLOYMENT_NAME=$1 DIR=$(dirname "$0") -COLLECTION_PATH="Engineering/Tupaia General/Environment Variables" # Collection in BitWarden where .env vars are kept +. "$DIR/ansiControlSequences.sh" + +# Collection in BitWarden where .env vars are kept +COLLECTION_PATH='Engineering/Tupaia General/Environment Variables' # Log in to bitwarden +echo -e "${BLUE}==>️${RESET} ${BOLD}Logging into Bitwarden${RESET}" bw login --check || bw login $BITWARDEN_EMAIL $BITWARDEN_PASSWORD eval "$(bw unlock $BITWARDEN_PASSWORD | grep -o -m 1 'export BW_SESSION=.*$')" COLLECTION_ID=$(bw get collection "$COLLECTION_PATH" | jq .id) +echo + # Can provide one or more packages as command line arguments, or will default to all if [ -z $2 ]; then - echo "Fetching all .env files" PACKAGES=$(${DIR}/getPackagesWithEnvFiles.sh) + echo -e "${BLUE}==>️${RESET} ${BOLD}Fetching environment variables for all packages${RESET}" else PACKAGES=${@:2} - echo "Fetching environment variables for ${PACKAGES}" + echo -e "${BLUE}==>️${RESET} ${BOLD}Fetching environment variables for ${PACKAGES}${RESET}" fi @@ -27,7 +33,7 @@ load_env_file_from_bw () { NEW_FILE_NAME=$3 ENV_FILE_PATH=${BASE_FILE_PATH}/${NEW_FILE_NAME}.env - echo "Fetching environment variables for $FILE_NAME: $ENV_FILE_PATH" + echo -en "${YELLOW}🚚 Fetching variables for ${BOLD}${FILE_NAME}...${RESET}" # checkout deployment specific env vars, or dev as fallback DEPLOYMENT_ENV_VARS=$(bw list items --search ${FILE_NAME}.${DEPLOYMENT_NAME}.env | jq --raw-output "map(select(.collectionIds[] | contains ($COLLECTION_ID))) | .[] .notes") @@ -39,11 +45,10 @@ load_env_file_from_bw () { echo "$DEV_ENV_VARS" > ${ENV_FILE_PATH} fi - # Replace any instances of the placeholder [deployment-name] in the .env file with the actual deployment - # name (e.g. [deployment-name]-api.tupaia.org -> specific-deployment-api.tupaia.org) + # Replace any instances of the placeholder [deployment-name] in the .env file with the actual + # deployment name (e.g. [deployment-name]-api.tupaia.org -> specific-deployment-api.tupaia.org) sed -i -e "s/\[deployment-name\]/${DEPLOYMENT_NAME}/g" "${ENV_FILE_PATH}" - if [[ "${DEPLOYMENT_NAME}" == *-e2e || "${DEPLOYMENT_NAME}" == e2e ]]; then # Update e2e environment variables if [[ ${FILE_NAME} == "aggregation" ]]; then @@ -58,13 +63,14 @@ load_env_file_from_bw () { sed -i -e 's/^###DEV_ONLY###//g' ${ENV_FILE_PATH} fi - - echo "downloaded .env vars for $FILE_NAME" + echo -en "$CLEAR_LINE" + echo -e "${GREEN}✅ Downloaded variables for ${BOLD}${FILE_NAME}${RESET} → $ENV_FILE_PATH" } for PACKAGE in $PACKAGES; do - # only download the env file if there is an example file in the package. If there isn't, this means it is a package that doesn't need env vars + # Only download the env file if there is an example file in the package. If there isn’t, this + # means it is a package that doesn’t need env vars has_example_env_in_package=$(find $DIR/../../packages/$PACKAGE -type f -name '*.env.example' | wc -l) if [ $has_example_env_in_package -eq 1 ]; then load_env_file_from_bw $PACKAGE $DIR/../../packages/$PACKAGE "" @@ -79,4 +85,6 @@ done # Log out of Bitwarden +echo +echo -e "${BLUE}==>️${RESET} ${BOLD}Logging out of Bitwarden${RESET}" bw logout From 73e1797da1ff93d9a538490b891de73a0888ed6f Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Sat, 6 Apr 2024 22:58:09 +1300 Subject: [PATCH 11/24] prettify `backendStartDev.sh` output --- scripts/bash/backendStartDev.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/scripts/bash/backendStartDev.sh b/scripts/bash/backendStartDev.sh index 67c3c0fda8..2f79aeb511 100755 --- a/scripts/bash/backendStartDev.sh +++ b/scripts/bash/backendStartDev.sh @@ -1,15 +1,15 @@ #!/usr/bin/env bash -set -e - -## -# usage: +# Usage: # $1 - port to run babel inspector on # Optionally provide '-i' or '--include-internal' to include build and watching internal dependencies # Optionally provide '-ts' or '--typescript' to start typescript server -## -USAGE="Usage: backendStartDev babel_port_inspector [-i --include-internal] [-ts --typescript]" +set -e + DIR=$(dirname "$0") +. "$DIR/ansiControlSequences.sh" + +USAGE="Usage: ${BOLD}backendStartDev babel_port_inspector${RESET} [${BOLD}-i${RESET}|${BOLD}--include-internal${RESET}] [${BOLD}-ts${RESET}|${BOLD}--typescript${RESET}]" CONCURRENTLY_BIN="${DIR}/../../node_modules/.bin/concurrently" watch_flags="" include_internal=false @@ -30,11 +30,11 @@ while [ "$2" != "" ]; do shift ;; -s | --skip-internal) - echo "Skipping internal dependencies is now done by default. Remove the -s | --skip-internal flag, and if you want to include internal dependencies, add a -i (do try it - it's a lot faster than it used to be, because it only builds those relevant to the current package!)" + echo -e "Skipping internal dependencies is now done by default. Remove the ${BOLD}--skip-internal${RESET} (${BOLD}-s${RESET}) flag, and if you want to include internal dependencies, add a ${BOLD}-i${RESET}. (Do try it - it’s a lot faster than it used to be, because it only builds those relevant to the current package!)" exit 1 ;; *) - echo $USAGE + echo -e "$USAGE" exit 1 ;; esac @@ -45,7 +45,7 @@ if [[ ${type_script} == true ]]; then start_server="nodemon --watch src -e ts,json --exec node --inspect=${inspect_port} -r ts-node/register src/index.ts" fi -echo "Starting server" +echo -e "${BOLD}Starting server...${RESET}" if [[ ${include_internal} == true ]]; then echo "Internal dependencies are under watch for hot reload" @@ -57,6 +57,6 @@ if [[ ${include_internal} == true ]]; then start_server="${start_server} --delay 1 ${watch_flags}" ${CONCURRENTLY_BIN} "${DIR}/buildInternalDependencies.sh --watch --packagePath ." "eval ${start_server}" else - echo "Starting server without internal dependency build and watch. To include internal dependencies, add the -i flag - it's much faster than it used to be!" + echo -e "Starting server without internal dependency build and watch. To include internal dependencies, add the ${BOLD}-i${RESET} flag - it’s much faster than it used to be!" eval ${start_server} fi From a1db38f1a8ede431f4f595b988b46fe09e81cf08 Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Sat, 6 Apr 2024 23:17:32 +1300 Subject: [PATCH 12/24] prettify `dumpDatabase.sh` output --- packages/database/scripts/dumpDatabase.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/database/scripts/dumpDatabase.sh b/packages/database/scripts/dumpDatabase.sh index 09a56517c5..1f261427dc 100755 --- a/packages/database/scripts/dumpDatabase.sh +++ b/packages/database/scripts/dumpDatabase.sh @@ -32,6 +32,7 @@ function show_loading_spinner() { DIR=$(pwd "$0") . "$DIR/../../scripts/bash/mergeEnvForDB.sh" +. "$DIR/../../scripts/bash/ansiControlSequences.sh" DUMP_FILE_NAME="dump.sql" @@ -73,7 +74,7 @@ if [ "$identity_file" == "" ]; then fi if [ "$DB_PG_USER" == "" ] || [ "$DB_PG_PASSWORD" == "" ]; then - echo "Missing postgres user credential env vars in @tupaia/database .env file. Check LastPass for variables and add them to the .env file" + echo -e "${RED}Missing Postgres user credential env vars in @tupaia/database .env file.${RESET} Check Bitwarden for variables and add them to the .env file" exit 1 fi @@ -87,6 +88,5 @@ target_zip_path="$target_path.gz" show_loading_spinner "Dumping database to $target_zip_path" "PGPASSWORD=$DB_PG_PASSWORD pg_dump \"host=$host user=$DB_PG_USER dbname=tupaia sslmode=require sslkey=$identity_file\" -Z1 -f $target_zip_path" show_loading_spinner "Unzipping $target_zip_path" "gunzip -f $target_zip_path" -echo "Dump file available at $target_path" - -echo "Done!" +echo "Dump file available at $target_path" +echo -e "${GREEN}Done!${RESET}" From 77b76b239196e587bb2944dd2acac70d57f8eed5 Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Sat, 6 Apr 2024 23:37:16 +1300 Subject: [PATCH 13/24] prefer ``[[`` over `[`, and `=` over `==` in Bash --- .../scripts/buildAnalyticsMaterializedView.sh | 2 +- packages/data-api/scripts/installMvRefreshModule.sh | 2 +- packages/data-api/scripts/patchMvRefreshModule.sh | 10 +++++----- packages/data-api/scripts/refreshAnalyticsTable.sh | 2 +- .../data-api/scripts/uninstallMvRefreshModule.sh | 2 +- .../data-lake-api/scripts/checkTestDataLakeExists.sh | 3 +-- packages/data-lake-api/scripts/setupTestDataLake.sh | 2 +- packages/database/scripts/dumpDatabase.sh | 8 ++++---- packages/database/scripts/setupTestDatabase.sh | 6 +++--- packages/devops/scripts/ci/triggerRedeploy.sh | 2 +- packages/devops/scripts/ci/utils.sh | 2 +- packages/devops/scripts/ci/validateBranchName.sh | 6 +++--- packages/devops/scripts/ci/validateNewMigrations.sh | 8 ++++---- packages/devops/scripts/deployment/startBackEnds.sh | 6 +++--- packages/devops/scripts/lambda/resources/startup.sh | 8 ++++---- scripts/bash/backendStartDev.sh | 8 ++++---- scripts/bash/buildInternalDependencies.sh | 8 ++++---- scripts/bash/downloadEnvironmentVariables.sh | 12 ++++++------ scripts/bash/getInternalDependencies.sh | 10 +++++----- scripts/bash/mergeEnvForDB.sh | 6 +++--- scripts/bash/pm2startInline.sh | 2 +- scripts/docker/downloadEnvironmentVariables.sh | 8 ++++---- 22 files changed, 61 insertions(+), 62 deletions(-) diff --git a/packages/data-api/scripts/buildAnalyticsMaterializedView.sh b/packages/data-api/scripts/buildAnalyticsMaterializedView.sh index e1f188fe33..0d60df4ed7 100755 --- a/packages/data-api/scripts/buildAnalyticsMaterializedView.sh +++ b/packages/data-api/scripts/buildAnalyticsMaterializedView.sh @@ -9,7 +9,7 @@ DIR=$(pwd "$0") cd scripts export PGPASSWORD=$DB_PASSWORD -if [[ "$1" == "--force" || "$1" == "-f" ]]; then +if [[ $1 = '--force' || $1 = '-f' ]]; then psql -p $DB_PORT -h $DB_URL -d $DB_NAME -U $DB_USER -tc "SELECT build_analytics_table(true);" else psql -p $DB_PORT -h $DB_URL -d $DB_NAME -U $DB_USER -tc "SELECT build_analytics_table();" diff --git a/packages/data-api/scripts/installMvRefreshModule.sh b/packages/data-api/scripts/installMvRefreshModule.sh index 7066460132..a4600ef6f9 100755 --- a/packages/data-api/scripts/installMvRefreshModule.sh +++ b/packages/data-api/scripts/installMvRefreshModule.sh @@ -9,7 +9,7 @@ DIR=$(pwd "$0") export PGPASSWORD=$DB_PG_PASSWORD MV_REFRESH_EXISTS=`psql -p $DB_PORT -X -A -h $DB_URL -d $DB_NAME -U $DB_PG_USER -t -c "SELECT schema_name FROM information_schema.schemata WHERE schema_name = '$DB_MV_USER'"` -if [ "$MV_REFRESH_EXISTS" == "$DB_MV_USER" ]; then +if [[ $MV_REFRESH_EXISTS = "$DB_MV_USER" ]]; then echo "Fast Refresh module already exists, skipping installation" else git submodule update --init scripts/pg-mv-fast-refresh diff --git a/packages/data-api/scripts/patchMvRefreshModule.sh b/packages/data-api/scripts/patchMvRefreshModule.sh index 333eab5f7e..2770cab6d3 100755 --- a/packages/data-api/scripts/patchMvRefreshModule.sh +++ b/packages/data-api/scripts/patchMvRefreshModule.sh @@ -5,19 +5,19 @@ DIR=$(pwd "$0") COMMAND=$1 -if [[ "$COMMAND" == "" ]]; then +if [[ $COMMAND = '' ]]; then echo "Error: missing patch command! Must be one of: up, down, create" exit 1 fi -if [[ "$COMMAND" == "create" ]]; then +if [[ $COMMAND = create ]]; then echo "Enter patch name: " read PATCH_NAME fi VERSION=$2 -if [[ "$VERSION" == "" ]]; then +if [[ $VERSION = '' ]]; then echo "Version unspecified, defaulting to database mvrefresh version" # Set default port in case it wasn't in .env @@ -27,7 +27,7 @@ if [[ "$VERSION" == "" ]]; then VERSION_SQL_FUNC="SELECT mv\$version()" VERSION=`psql -p $DB_PORT -X -A -h $DB_URL -d $DB_NAME -U $DB_USER -t -c "$VERSION_SQL_FUNC"` - if [[ "$VERSION" == "" ]]; then + if [[ $VERSION = '' ]]; then echo "Error: failed to detect mvrefresh version from database" exit 1 fi @@ -35,7 +35,7 @@ if [[ "$VERSION" == "" ]]; then echo "Using version: $VERSION" fi -if [[ ! -d "./scripts/patches/$VERSION" && ! "$COMMAND" == "create" ]]; then +if [[ ! -d ./scripts/patches/$VERSION && $COMMAND != create ]]; then echo "No patches exist for version: $VERSION, skipping" else ts-node ./scripts/patchMvRefresh.ts $COMMAND:$VERSION $PATCH_NAME --migrations-dir "./scripts/patches" --table "patches" -v --config-file "../../babel.config.json" diff --git a/packages/data-api/scripts/refreshAnalyticsTable.sh b/packages/data-api/scripts/refreshAnalyticsTable.sh index 35fb801b2c..8810ca311f 100755 --- a/packages/data-api/scripts/refreshAnalyticsTable.sh +++ b/packages/data-api/scripts/refreshAnalyticsTable.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e -if [[ "$1" == "--full" || "$1" == "-f" ]]; then +if [[ $1 = '--full' || $1 = '-f' ]]; then ./scripts/fullRefreshAnalyticsTable.sh else ./scripts/fastRefreshAnalyticsTable.sh diff --git a/packages/data-api/scripts/uninstallMvRefreshModule.sh b/packages/data-api/scripts/uninstallMvRefreshModule.sh index 291ec8fd4f..5a7520025a 100755 --- a/packages/data-api/scripts/uninstallMvRefreshModule.sh +++ b/packages/data-api/scripts/uninstallMvRefreshModule.sh @@ -9,7 +9,7 @@ DIR=$(pwd "$0") export PGPASSWORD=$DB_PG_PASSWORD MV_REFRESH_EXISTS=`psql -X -A -p $DB_PORT -h $DB_URL -d $DB_NAME -U $DB_PG_USER -t -c "SELECT schema_name FROM information_schema.schemata WHERE schema_name = '$DB_MV_USER'"` -if [ "$MV_REFRESH_EXISTS" != "$DB_MV_USER" ]; then +if [[ $MV_REFRESH_EXISTS != "$DB_MV_USER" ]]; then echo "Fast Refresh module does not exist, skipping uninstallation" else git submodule update --init scripts/pg-mv-fast-refresh diff --git a/packages/data-lake-api/scripts/checkTestDataLakeExists.sh b/packages/data-lake-api/scripts/checkTestDataLakeExists.sh index 0ff54e5112..9e35e149b9 100755 --- a/packages/data-lake-api/scripts/checkTestDataLakeExists.sh +++ b/packages/data-lake-api/scripts/checkTestDataLakeExists.sh @@ -8,8 +8,7 @@ DIR=$(pwd "$0") # Set default port in case it wasn't in .env : "${DATA_LAKE_DB_PORT:=5432}" -if [ "$(PGPASSWORD=$DB_PG_PASSWORD psql -p $DATA_LAKE_DB_PORT -X -A -h $DATA_LAKE_DB_URL -U $DB_PG_USER -t -c "SELECT 1 FROM pg_database WHERE datname='$DATA_LAKE_DB_NAME'" )" = '1' ] -then +if [[ "$(PGPASSWORD=$DB_PG_PASSWORD psql -p $DATA_LAKE_DB_PORT -X -A -h $DATA_LAKE_DB_URL -U $DB_PG_USER -t -c "SELECT 1 FROM pg_database WHERE datname='$DATA_LAKE_DB_NAME'" )" = '1' ]]; then exit 0 fi diff --git a/packages/data-lake-api/scripts/setupTestDataLake.sh b/packages/data-lake-api/scripts/setupTestDataLake.sh index 78c4946980..aa6f5c4ead 100755 --- a/packages/data-lake-api/scripts/setupTestDataLake.sh +++ b/packages/data-lake-api/scripts/setupTestDataLake.sh @@ -9,7 +9,7 @@ DIR=$(pwd "$0") TUPAIA_USER_EXISTS=`PGPASSWORD=$DB_PG_PASSWORD psql -p $DATA_LAKE_DB_PORT -X -A -h $DATA_LAKE_DB_URL -U $DB_PG_USER -t -c "SELECT rolname FROM pg_catalog.pg_roles WHERE rolname = '$DATA_LAKE_DB_USER'"` -if [ -z "$TUPAIA_USER_EXISTS" ]; then +if [[ -z $TUPAIA_USER_EXISTS ]]; then PGPASSWORD=$DB_PG_PASSWORD psql -h $DATA_LAKE_DB_URL -p $DATA_LAKE_DB_PORT -U $DB_PG_USER -c "CREATE ROLE $DATA_LAKE_DB_USER LOGIN PASSWORD '$DATA_LAKE_DB_PASSWORD'" fi diff --git a/packages/database/scripts/dumpDatabase.sh b/packages/database/scripts/dumpDatabase.sh index 1f261427dc..886707aca1 100755 --- a/packages/database/scripts/dumpDatabase.sh +++ b/packages/database/scripts/dumpDatabase.sh @@ -40,7 +40,7 @@ identity_file="" server="dev" target_dir="." -while [ "$1" != "" ]; do +while [[ $1 != '' ]]; do case $1 in -s | --server) shift @@ -57,7 +57,7 @@ while [ "$1" != "" ]; do exit ;; *) - if [ "$identity_file" == "" ]; then + if [[ $identity_file = '' ]]; then identity_file=$1 shift else @@ -68,12 +68,12 @@ while [ "$1" != "" ]; do esac done -if [ "$identity_file" == "" ]; then +if [[ $identity_file = '' ]]; then print_help exit 1 fi -if [ "$DB_PG_USER" == "" ] || [ "$DB_PG_PASSWORD" == "" ]; then +if [[ $DB_PG_USER = '' || $DB_PG_PASSWORD = '' ]]; then echo -e "${RED}Missing Postgres user credential env vars in @tupaia/database .env file.${RESET} Check Bitwarden for variables and add them to the .env file" exit 1 fi diff --git a/packages/database/scripts/setupTestDatabase.sh b/packages/database/scripts/setupTestDatabase.sh index e23a184bd4..c9187e4e80 100755 --- a/packages/database/scripts/setupTestDatabase.sh +++ b/packages/database/scripts/setupTestDatabase.sh @@ -10,7 +10,7 @@ DIR=$(pwd "$0") TUPAIA_USER_EXISTS=`PGPASSWORD=$DB_PG_PASSWORD psql -p $DB_PORT -X -A -h $DB_URL -U $DB_PG_USER -t -c "SELECT rolname FROM pg_catalog.pg_roles WHERE rolname = '$DB_USER'"` IS_RDS=`PGPASSWORD=$DB_PG_PASSWORD psql -p $DB_PORT -X -A -h $DB_URL -U $DB_PG_USER -t -c "SELECT rolname FROM pg_catalog.pg_roles WHERE rolname = 'rds_superuser'"` -if [ -z "$TUPAIA_USER_EXISTS" ]; then +if [[ -z $TUPAIA_USER_EXISTS ]]; then PGPASSWORD=$DB_PG_PASSWORD psql -h $DB_URL -p $DB_PORT -U $DB_PG_USER -c "CREATE ROLE $DB_USER LOGIN SUPERUSER PASSWORD '$DB_PASSWORD'" fi @@ -18,7 +18,7 @@ PGPASSWORD=$DB_PG_PASSWORD psql -h $DB_URL -p $DB_PORT -U $DB_PG_USER -c "DROP D PGPASSWORD=$DB_PG_PASSWORD psql -h $DB_URL -p $DB_PORT -U $DB_PG_USER -c "CREATE DATABASE $DB_NAME WITH OWNER $DB_USER" PGPASSWORD=$DB_PG_PASSWORD psql -h $DB_URL -p $DB_PORT -U $DB_PG_USER -c "CREATE EXTENSION IF NOT EXISTS postgis" -if [ -z "$IS_RDS" ]; then +if [[ -z $IS_RDS ]]; then PGPASSWORD=$DB_PG_PASSWORD psql -h $DB_URL -p $DB_PORT -U $DB_PG_USER -c "ALTER USER $DB_USER WITH SUPERUSER" else PGPASSWORD=$DB_PG_PASSWORD psql -h $DB_URL -p $DB_PORT -U $DB_PG_USER -c "GRANT rds_superuser TO $DB_USER" @@ -26,7 +26,7 @@ fi PGPASSWORD=$DB_PASSWORD psql -h $DB_URL -p $DB_PORT -U $DB_USER -d $DB_NAME -f ./src/__tests__/testData/testDataDump.sql -if [ -z "$IS_RDS" ]; then +if [[ -z $IS_RDS ]]; then PGPASSWORD=$DB_PG_PASSWORD psql -h $DB_URL -p $DB_PORT -U $DB_PG_USER -c "ALTER USER $DB_USER WITH NOSUPERUSER" else PGPASSWORD=$DB_PG_PASSWORD psql -h $DB_URL -p $DB_PORT -U $DB_PG_USER -c "REVOKE rds_superuser FROM $DB_USER" diff --git a/packages/devops/scripts/ci/triggerRedeploy.sh b/packages/devops/scripts/ci/triggerRedeploy.sh index 6c7fd7f855..391cc61903 100755 --- a/packages/devops/scripts/ci/triggerRedeploy.sh +++ b/packages/devops/scripts/ci/triggerRedeploy.sh @@ -5,7 +5,7 @@ STOPPED_INSTANCES=$(aws ec2 describe-instances \ --filters Name=tag:Branch,Values=${CI_BRANCH} Name=tag:DeploymentType,Values=tupaia Name=instance-state-name,Values=stopped \ --no-cli-pager) -if [[ $STOPPED_INSTANCES == *"Instances"* ]]; then +if [[ $STOPPED_INSTANCES = *"Instances"* ]]; then echo "Can't redeploy while a deployment for ${CI_BRANCH} is stopped. Try again inside office hours, or start the app server and database then restart the build." exit 1 fi diff --git a/packages/devops/scripts/ci/utils.sh b/packages/devops/scripts/ci/utils.sh index f0aa34140c..1de488e125 100755 --- a/packages/devops/scripts/ci/utils.sh +++ b/packages/devops/scripts/ci/utils.sh @@ -39,7 +39,7 @@ function get_max_length() { function get_branch_name() { local branch_name="$CI_BRANCH" - if [[ $branch_name == "" ]]; then + if [[ $branch_name = '' ]]; then # Get currently checked out branch branch_name=$(git rev-parse --abbrev-ref HEAD) fi diff --git a/packages/devops/scripts/ci/validateBranchName.sh b/packages/devops/scripts/ci/validateBranchName.sh index 111e00d80e..a04240eeb0 100755 --- a/packages/devops/scripts/ci/validateBranchName.sh +++ b/packages/devops/scripts/ci/validateBranchName.sh @@ -18,12 +18,12 @@ function validate_name_ending() { local branch_name=$1 for suffix in ${SUBDOMAIN_SUFFIXES[@]}; do - if [[ "$branch_name" == *$suffix ]]; then + if [[ $branch_name = *$suffix ]]; then log_error "❌ Invalid branch name ending: '$suffix'" exit 1 fi # api is one of our suffixes so makes sure [branch]-api doesn't match any other api suffixes - if [[ "$suffix" == *-api && $branch_name-api == *$suffix ]]; then + if [[ $suffix = *-api && $branch_name-api = *$suffix ]]; then log_error "❌ Invalid branch name ending: '$suffix'" exit 1 fi @@ -49,7 +49,7 @@ function validate_name_chars() { fi for character in ${INVALID_CHARS[@]}; do - if [[ "$branch_name" == *"$character"* ]]; then + if [[ $branch_name = *"$character"* ]]; then log_error "❌ Invalid character in branch name: '$character'" exit 1 fi diff --git a/packages/devops/scripts/ci/validateNewMigrations.sh b/packages/devops/scripts/ci/validateNewMigrations.sh index 02bf5c77e4..a0c48ad834 100755 --- a/packages/devops/scripts/ci/validateNewMigrations.sh +++ b/packages/devops/scripts/ci/validateNewMigrations.sh @@ -7,7 +7,7 @@ ROOT="${DIR}/../../../../" . ${DIR}/utils.sh function get_date_command() { - if [[ $(uname) == "Darwin" ]]; then + if [[ $(uname) = Darwin ]]; then echo "gdate" # install gdate on MacOs: brew install coreutils else echo "date" @@ -48,13 +48,13 @@ function validate_migrations(){ local errors=""; while read -r migration_name; do - if [[ "$migration_name" == "" ]]; then + if [[ $migration_name = '' ]]; then break fi errors="$errors$(check_migration_outdated "$migration_name")" done <<< "$new_migration_names_in_string" - if [[ "$errors" != "" ]]; then + if [[ $errors != '' ]]; then echo $errors; exit 1; fi @@ -64,7 +64,7 @@ current_branch_name=$(get_branch_name) origin_branch_name="master" # Skip validation if current branch name is master -if [[ "$current_branch_name" == "$origin_branch_name" ]]; then +if [[ $current_branch_name = "$origin_branch_name" ]]; then echo "Skipping validation step while current branch is the same as origin" exit 0 fi diff --git a/packages/devops/scripts/deployment/startBackEnds.sh b/packages/devops/scripts/deployment/startBackEnds.sh index 988e02a433..8630e0dc9a 100755 --- a/packages/devops/scripts/deployment/startBackEnds.sh +++ b/packages/devops/scripts/deployment/startBackEnds.sh @@ -9,8 +9,8 @@ PACKAGES=$(${TUPAIA_DIR}/scripts/bash/getDeployablePackages.sh) # Start back end server packages for PACKAGE in ${PACKAGES[@]}; do - if [[ $PACKAGE == *server ]]; then - if [[ $PACKAGE == 'central-server' ]]; then + if [[ $PACKAGE = *server ]]; then + if [[ $PACKAGE = central-server ]]; then # reset cwd back to `/tupaia` cd ${TUPAIA_DIR} @@ -28,7 +28,7 @@ for PACKAGE in ${PACKAGES[@]}; do echo "Starting ${PACKAGE}" cd ${TUPAIA_DIR}/packages/$PACKAGE REPLICATION_PM2_CONFIG='' - if [ $PACKAGE == "web-config-server" ] || [ $PACKAGE == "report-server" ] ; then + if [[ $PACKAGE = web-config-server || $PACKAGE = report-server ]] ; then # as many replicas as cpu cores - 1 REPLICATION_PM2_CONFIG='-i -1' fi diff --git a/packages/devops/scripts/lambda/resources/startup.sh b/packages/devops/scripts/lambda/resources/startup.sh index 0fca1dee82..0ff4e5e0fa 100755 --- a/packages/devops/scripts/lambda/resources/startup.sh +++ b/packages/devops/scripts/lambda/resources/startup.sh @@ -25,7 +25,7 @@ BRANCH=$(${DEPLOYMENT_SCRIPTS}/../utility/getEC2TagValue.sh Branch) echo "Starting up ${DEPLOYMENT_NAME} (${BRANCH})" # Set bash prompt to have deployment name in it -if [[ $DEPLOYMENT_NAME == "production" ]]; then +if [[ $DEPLOYMENT_NAME = production ]]; then BASH_PROMPT_NAME="PROD" BASH_PROMPT_COLOR="31" else @@ -40,12 +40,12 @@ mkdir -m 777 -p $LOGS_DIR # Turn on cloudwatch agent for prod and dev (can be turned on manually if needed on feature instances) # TODO currently broken -# if [[ $DEPLOYMENT_NAME == "production" || $DEPLOYMENT_NAME == "dev" ]]; then +# if [[ $DEPLOYMENT_NAME = production || $DEPLOYMENT_NAME = dev ]]; then # $DEPLOYMENT_SCRIPTS/startCloudwatchAgent.sh |& while IFS= read -r line; do printf '\%s \%s\n' "$(date)" "$line"; done >> $LOGS_DIR/deployment_log.txt # fi # Add preaggregation cron job if production -if [[ $DEPLOYMENT_NAME == "production" ]]; then +if [[ $DEPLOYMENT_NAME = production ]]; then \. "$HOME_DIR/.nvm/nvm.sh" # Load nvm so node is available on $PATH sudo -u ubuntu echo "10 13 * * * PATH=$PATH $HOME_DIR/tupaia/packages/web-config-server/run_preaggregation.sh | while IFS= read -r line; do printf '\%s \%s\\n' \"\$(date)\" \"\$line\"; done > $LOGS_DIR/preaggregation.txt" > tmp.cron sudo -u ubuntu crontab -l >> tmp.cron || echo >> tmp.cron @@ -56,7 +56,7 @@ fi # Fetch the latest code cd $TUPAIA_DIR BRANCH_ON_REMOTE=$(sudo -Hu ubuntu git ls-remote --heads origin ${BRANCH}) -if [[ $BRANCH_ON_REMOTE == *${BRANCH} ]]; then +if [[ $BRANCH_ON_REMOTE = *${BRANCH} ]]; then echo "${BRANCH} exists" BRANCH_TO_USE=${BRANCH} else diff --git a/scripts/bash/backendStartDev.sh b/scripts/bash/backendStartDev.sh index 2f79aeb511..896a451749 100755 --- a/scripts/bash/backendStartDev.sh +++ b/scripts/bash/backendStartDev.sh @@ -19,7 +19,7 @@ inspect_port=${1} # Start server command for JS start_server="nodemon -w src --exec \"babel-node src --inspect=${inspect_port} --config-file '../../babel.config.json'\"" -while [ "$2" != "" ]; do +while [[ $2 != '' ]]; do case $2 in -ts | --typescript) type_script=true @@ -30,7 +30,7 @@ while [ "$2" != "" ]; do shift ;; -s | --skip-internal) - echo -e "Skipping internal dependencies is now done by default. Remove the ${BOLD}--skip-internal${RESET} (${BOLD}-s${RESET}) flag, and if you want to include internal dependencies, add a ${BOLD}-i${RESET}. (Do try it - it’s a lot faster than it used to be, because it only builds those relevant to the current package!)" + echo -e "Skipping internal dependencies is now done by default. Remove the ${BOLD}--skip-internal${RESET} (${BOLD}-s${RESET}) flag, and if you want to include internal dependencies, add a ${BOLD}-i${RESET}. (Do try it - it’s a lot faster than it used to be, because it only builds those relevant to the current package$EXCLAMATION_MARK)" exit 1 ;; *) @@ -41,13 +41,13 @@ while [ "$2" != "" ]; do done # Start server command for TS -if [[ ${type_script} == true ]]; then +if [[ $type_script = true ]]; then start_server="nodemon --watch src -e ts,json --exec node --inspect=${inspect_port} -r ts-node/register src/index.ts" fi echo -e "${BOLD}Starting server...${RESET}" -if [[ ${include_internal} == true ]]; then +if [[ $include_internal = true ]]; then echo "Internal dependencies are under watch for hot reload" for PACKAGE in $(${DIR}/getInternalDependencies.sh .); do watch_flags="${watch_flags} --watch ../${PACKAGE}/dist" diff --git a/scripts/bash/buildInternalDependencies.sh b/scripts/bash/buildInternalDependencies.sh index 0834ba2189..d48e60bf97 100755 --- a/scripts/bash/buildInternalDependencies.sh +++ b/scripts/bash/buildInternalDependencies.sh @@ -11,7 +11,7 @@ USAGE="Usage: ${BOLD}buildInternalDependencies.sh${RESET} [${BOLD}--watch${RESET watch=false package_path="" -while [ "$1" != "" ]; do +while [[ $1 != '' ]]; do case $1 in --watch) shift @@ -33,8 +33,8 @@ while [ "$1" != "" ]; do esac done -[[ $watch = "true" ]] && build_args="--watch" || build_args="" -[[ $watch = "true" ]] && build_ts_args="--watch --preserveWatchOutput" || build_ts_args="" +[[ $watch = true ]] && build_args="--watch" || build_args="" +[[ $watch = true ]] && build_ts_args="--watch --preserveWatchOutput" || build_ts_args="" build_commands=() build_prefixes=() @@ -45,7 +45,7 @@ for PACKAGE in $(${DIR}/getInternalDependencies.sh ${package_path}); do build_prefixes+=("${PACKAGE},") done -if [[ $watch == "true" ]]; then +if [[ $watch = true ]]; then echo -e "${BOLD}Concurrently building and watching all internal dependencies${RESET}" echo "> ${CONCURRENTLY_BIN} --names \"${build_prefixes[*]}\" ${build_commands[@]}" echo diff --git a/scripts/bash/downloadEnvironmentVariables.sh b/scripts/bash/downloadEnvironmentVariables.sh index 3f285b1660..aae3908c81 100755 --- a/scripts/bash/downloadEnvironmentVariables.sh +++ b/scripts/bash/downloadEnvironmentVariables.sh @@ -18,7 +18,7 @@ COLLECTION_ID=$(bw get collection "$COLLECTION_PATH" | jq .id) echo # Can provide one or more packages as command line arguments, or will default to all -if [ -z $2 ]; then +if [[ -z $2 ]]; then PACKAGES=$(${DIR}/getPackagesWithEnvFiles.sh) echo -e "${BLUE}==>️${RESET} ${BOLD}Fetching environment variables for all packages${RESET}" else @@ -38,7 +38,7 @@ load_env_file_from_bw () { # checkout deployment specific env vars, or dev as fallback DEPLOYMENT_ENV_VARS=$(bw list items --search ${FILE_NAME}.${DEPLOYMENT_NAME}.env | jq --raw-output "map(select(.collectionIds[] | contains ($COLLECTION_ID))) | .[] .notes") - if [ -n "$DEPLOYMENT_ENV_VARS" ]; then + if [[ -n $DEPLOYMENT_ENV_VARS ]]; then echo "$DEPLOYMENT_ENV_VARS" > ${ENV_FILE_PATH} else DEV_ENV_VARS=$(bw list items --search ${FILE_NAME}.dev.env | jq --raw-output "map(select(.collectionIds[] | contains ($COLLECTION_ID))) | .[] .notes") @@ -49,14 +49,14 @@ load_env_file_from_bw () { # deployment name (e.g. [deployment-name]-api.tupaia.org -> specific-deployment-api.tupaia.org) sed -i -e "s/\[deployment-name\]/${DEPLOYMENT_NAME}/g" "${ENV_FILE_PATH}" - if [[ "${DEPLOYMENT_NAME}" == *-e2e || "${DEPLOYMENT_NAME}" == e2e ]]; then + if [[ $DEPLOYMENT_NAME = *-e2e || $DEPLOYMENT_NAME = e2e ]]; then # Update e2e environment variables - if [[ ${FILE_NAME} == "aggregation" ]]; then sed -i -e 's/^AGGREGATION_URL_PREFIX="?dev-"?$/AGGREGATION_URL_PREFIX=e2e-/g' ${ENV_FILE_PATH} + if [[ $FILE_NAME = aggregation ]]; then fi fi - if [[ "${DEPLOYMENT_NAME}" == dev ]]; then + if [[ $DEPLOYMENT_NAME = dev ]]; then # Update dev specific environment variables # (removes ###DEV_ONLY### prefixes, leaving the key=value pair uncommented) # (after removing prefix, if there are duplicate keys, dotenv uses the last one in the file) @@ -72,7 +72,7 @@ for PACKAGE in $PACKAGES; do # Only download the env file if there is an example file in the package. If there isn’t, this # means it is a package that doesn’t need env vars has_example_env_in_package=$(find $DIR/../../packages/$PACKAGE -type f -name '*.env.example' | wc -l) - if [ $has_example_env_in_package -eq 1 ]; then + if [[ $has_example_env_in_package -eq 1 ]]; then load_env_file_from_bw $PACKAGE $DIR/../../packages/$PACKAGE "" fi done diff --git a/scripts/bash/getInternalDependencies.sh b/scripts/bash/getInternalDependencies.sh index 609abce037..6a91c8f420 100755 --- a/scripts/bash/getInternalDependencies.sh +++ b/scripts/bash/getInternalDependencies.sh @@ -2,7 +2,7 @@ set -e DIR=$(dirname "$0") -if [ "$1" != "" ]; then +if [[ $1 != '' ]]; then # pop the package_path off, and interpret the rest as dependencies that have been checked earlier # in the recursion package_path=$1 @@ -12,14 +12,14 @@ fi dependencies_already_visited=($@) # if no package.json entrypoint is specified, just return all internal dependencies -if [ -z ${package_path} ]; then +if [[ -z $package_path ]]; then echo "types" "utils" "tsutils" "ui-components" "ui-chart-components" "ui-map-components" "server-utils" "access-policy" "admin-panel" "aggregator" "api-client" "auth" "database" "data-api" "dhis-api" "data-lake-api" "expression-parser" "indicators" "weather-api" "kobo-api" "superset-api" "data-broker" "server-boilerplate" exit 0 fi # we are getting internal dependencies for a specific package.json internal_dependencies=($(sed -n '/"dependencies": {/,/}/p' ${PWD}/${package_path}/package.json | grep -o '@tupaia/[^"]*": "[0-9\.]*"' | cut -d / -f 2 | cut -d \" -f 1)) -if [ ${#internal_dependencies[@]} -eq 0 ]; then +if [[ ${#internal_dependencies[@]} -eq 0 ]]; then exit 0 # no internal dependencies of this package, early return fi @@ -41,7 +41,7 @@ unset array_without_gaps for dependency in ${internal_dependencies[@]}; do nested_dependencies=($(${DIR}/getInternalDependencies.sh "${package_path}/../${dependency}" ${dependencies_already_visited[@]} ${internal_dependencies[@]} )) - if [ ${#nested_dependencies[@]} -eq 0 ]; then + if [[ ${#nested_dependencies[@]} -eq 0 ]]; then continue fi @@ -52,7 +52,7 @@ done # remove any duplicates for i in "${!internal_dependencies[@]}"; do for j in "${!internal_dependencies[@]}"; do - if [[ i -ne j ]] && [[ ${internal_dependencies[i]} = ${internal_dependencies[j]} ]]; then + if [[ i -ne j && ${internal_dependencies[i]} = ${internal_dependencies[j]} ]]; then unset 'internal_dependencies[i]' fi done diff --git a/scripts/bash/mergeEnvForDB.sh b/scripts/bash/mergeEnvForDB.sh index d3ac22b0a5..38e5a3bcbe 100755 --- a/scripts/bash/mergeEnvForDB.sh +++ b/scripts/bash/mergeEnvForDB.sh @@ -5,7 +5,7 @@ set -e get_caller_package_directory() { local dir dir=$(dirname "$(readlink -f "$0")") - while [[ "$dir" != "/" ]]; do + while [[ $dir != '/' ]]; do if [[ -f "$dir/package.json" ]]; then echo "$dir" return @@ -30,7 +30,7 @@ common_files="$file1 $file2 $file3 $file4" # Remove files that don't exist for file in $common_files; do - if [ ! -f "$file" ]; then + if [[ ! -f $file ]]; then common_files=$(echo "$common_files" | sed "s|$file||g") fi done @@ -40,7 +40,7 @@ merged_content="$(cat $common_files)" # Process command line arguments, overwriting values if present for var in $(env); do - if [[ "$var" == *=* ]]; then + if [[ $var = *=* ]]; then key="${var%%=*}" value="${var#*=}" # Override values from command line diff --git a/scripts/bash/pm2startInline.sh b/scripts/bash/pm2startInline.sh index 44b89cc505..0fd7e8e10d 100755 --- a/scripts/bash/pm2startInline.sh +++ b/scripts/bash/pm2startInline.sh @@ -4,7 +4,7 @@ set -e cd "$(dirname "${BASH_SOURCE[0]}")" . ./ansiControlSequences.sh -if [ -z "$1" ]; then +if [[ -z $1 ]]; then echo -e "Usage: ${BOLD}yarn start-stack${RESET} ${UNDERLINE}stack${RESET}" echo echo -e "All ${UNDERLINE}stack${RESET}s:" diff --git a/scripts/docker/downloadEnvironmentVariables.sh b/scripts/docker/downloadEnvironmentVariables.sh index 1b10666a9f..915eac8f23 100755 --- a/scripts/docker/downloadEnvironmentVariables.sh +++ b/scripts/docker/downloadEnvironmentVariables.sh @@ -34,7 +34,7 @@ DIR=$(dirname "$0") # can provide one or more packages as command line arguments, or will default to all -if [ -z $3 ]; then +if [[ -z $3 ]]; then echo "Fetching all .env files" PACKAGES=$(${DIR}/../bash/getPackagesWithEnvFiles.sh) else @@ -55,14 +55,14 @@ for PACKAGE in $PACKAGES; do # name (e.g. [deployment-name]-api.tupaia.org -> specific-deployment-api.tupaia.org) sed -i -e "s/\[deployment-name\]/${DEPLOYMENT_NAME}/g" ${ENV_FILE_PATH} - if [[ "${DEPLOYMENT_NAME}" == *-e2e || "${DEPLOYMENT_NAME}" == e2e ]]; then + if [[ $DEPLOYMENT_NAME = *-e2e || $DEPLOYMENT_NAME = e2e ]]; then # Update e2e environment variables - if [[ ${PACKAGE} == "central-server" || ${PACKAGE} == "web-config-server" ]]; then + if [[ $PACKAGE = central-server || $PACKAGE = web-config-server ]]; then sed -i -E 's/^AGGREGATION_URL_PREFIX="?dev-"?$/AGGREGATION_URL_PREFIX=e2e-/g' ${ENV_FILE_PATH} fi fi - if [[ "${DEPLOYMENT_NAME}" == dev ]]; then + if [[ $DEPLOYMENT_NAME = 'dev' ]]; then # Update dev specific environment variables # (removes ###DEV_ONLY### prefixes, leaving the key=value pair uncommented) # (after removing prefix, if there are duplicate keys, dotenv uses the last one in the file) From 1d10de0058b32c6275b8605eaf460aeaa5030573 Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Sun, 7 Apr 2024 22:34:28 +1200 Subject: [PATCH 14/24] =?UTF-8?q?prefer=20`$(=E2=80=A6)`=20over=20backtick?= =?UTF-8?q?=20command=20substitution?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/data-api/scripts/installMvRefreshModule.sh | 2 +- packages/data-api/scripts/patchMvRefreshModule.sh | 2 +- packages/data-api/scripts/uninstallMvRefreshModule.sh | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/data-api/scripts/installMvRefreshModule.sh b/packages/data-api/scripts/installMvRefreshModule.sh index a4600ef6f9..91574daf43 100755 --- a/packages/data-api/scripts/installMvRefreshModule.sh +++ b/packages/data-api/scripts/installMvRefreshModule.sh @@ -7,7 +7,7 @@ DIR=$(pwd "$0") : "${DB_PORT:=5432}" export PGPASSWORD=$DB_PG_PASSWORD -MV_REFRESH_EXISTS=`psql -p $DB_PORT -X -A -h $DB_URL -d $DB_NAME -U $DB_PG_USER -t -c "SELECT schema_name FROM information_schema.schemata WHERE schema_name = '$DB_MV_USER'"` +MV_REFRESH_EXISTS=$(psql -p $DB_PORT -X -A -h $DB_URL -d $DB_NAME -U $DB_PG_USER -t -c "SELECT schema_name FROM information_schema.schemata WHERE schema_name = '$DB_MV_USER'") if [[ $MV_REFRESH_EXISTS = "$DB_MV_USER" ]]; then echo "Fast Refresh module already exists, skipping installation" diff --git a/packages/data-api/scripts/patchMvRefreshModule.sh b/packages/data-api/scripts/patchMvRefreshModule.sh index 2770cab6d3..e49e461dbd 100755 --- a/packages/data-api/scripts/patchMvRefreshModule.sh +++ b/packages/data-api/scripts/patchMvRefreshModule.sh @@ -25,7 +25,7 @@ if [[ $VERSION = '' ]]; then export PGPASSWORD=$DB_PASSWORD VERSION_SQL_FUNC="SELECT mv\$version()" - VERSION=`psql -p $DB_PORT -X -A -h $DB_URL -d $DB_NAME -U $DB_USER -t -c "$VERSION_SQL_FUNC"` + VERSION=$(psql -p $DB_PORT -X -A -h $DB_URL -d $DB_NAME -U $DB_USER -t -c "$VERSION_SQL_FUNC") if [[ $VERSION = '' ]]; then echo "Error: failed to detect mvrefresh version from database" diff --git a/packages/data-api/scripts/uninstallMvRefreshModule.sh b/packages/data-api/scripts/uninstallMvRefreshModule.sh index 5a7520025a..13a2c86179 100755 --- a/packages/data-api/scripts/uninstallMvRefreshModule.sh +++ b/packages/data-api/scripts/uninstallMvRefreshModule.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +set -e DIR=$(pwd "$0") . "$DIR/../../scripts/bash/mergeEnvForDB.sh" @@ -7,7 +8,7 @@ DIR=$(pwd "$0") : "${DB_PORT:=5432}" export PGPASSWORD=$DB_PG_PASSWORD -MV_REFRESH_EXISTS=`psql -X -A -p $DB_PORT -h $DB_URL -d $DB_NAME -U $DB_PG_USER -t -c "SELECT schema_name FROM information_schema.schemata WHERE schema_name = '$DB_MV_USER'"` +MV_REFRESH_EXISTS=$(psql -X -A -p $DB_PORT -h $DB_URL -d $DB_NAME -U $DB_PG_USER -t -c "SELECT schema_name FROM information_schema.schemata WHERE schema_name = '$DB_MV_USER'") if [[ $MV_REFRESH_EXISTS != "$DB_MV_USER" ]]; then echo "Fast Refresh module does not exist, skipping uninstallation" From 0be5e6f507fc655017c272f51810087d7eb6c828 Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Sun, 7 Apr 2024 22:48:53 +1200 Subject: [PATCH 15/24] prettify `patchMvRefereshModule.sh` output --- packages/data-api/scripts/patchMvRefreshModule.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/data-api/scripts/patchMvRefreshModule.sh b/packages/data-api/scripts/patchMvRefreshModule.sh index e49e461dbd..42104e7d75 100755 --- a/packages/data-api/scripts/patchMvRefreshModule.sh +++ b/packages/data-api/scripts/patchMvRefreshModule.sh @@ -2,11 +2,12 @@ DIR=$(pwd "$0") . "$DIR/../../scripts/bash/mergeEnvForDB.sh" +. "$DIR/../../scripts/bash/ansiControlSequences.sh" COMMAND=$1 if [[ $COMMAND = '' ]]; then - echo "Error: missing patch command! Must be one of: up, down, create" + echo "${RED}Error: missing patch command! Must be one of: up, down, create${RESET}" exit 1 fi @@ -28,11 +29,11 @@ if [[ $VERSION = '' ]]; then VERSION=$(psql -p $DB_PORT -X -A -h $DB_URL -d $DB_NAME -U $DB_USER -t -c "$VERSION_SQL_FUNC") if [[ $VERSION = '' ]]; then - echo "Error: failed to detect mvrefresh version from database" + echo "${RED}Error: failed to detect mvrefresh version from database${RESET}" exit 1 fi - echo "Using version: $VERSION" + echo "Using version: ${BOLD}${GREEN}$VERSION${RESET}" fi if [[ ! -d ./scripts/patches/$VERSION && $COMMAND != create ]]; then From 15b43de446eed6c87361830ff815f7719c0d857c Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Sun, 7 Apr 2024 23:00:51 +1200 Subject: [PATCH 16/24] prefer single quotes when no variables --- .../scripts/fastRefreshAnalyticsTable.sh | 2 +- .../scripts/fullRefreshAnalyticsTable.sh | 3 ++- .../scripts/installMvRefreshModule.sh | 2 +- .../data-api/scripts/patchMvRefreshModule.sh | 6 ++--- .../scripts/uninstallMvRefreshModule.sh | 2 +- packages/database/scripts/dumpDatabase.sh | 8 +++---- .../database/scripts/setupTestDatabase.sh | 8 +++---- packages/devops/scripts/ci/triggerRedeploy.sh | 10 ++++---- .../scripts/ci/validateNewMigrations.sh | 10 ++++---- .../deployment/buildDeployablePackages.sh | 2 +- .../scripts/deployment/checkoutLatest.sh | 2 +- .../scripts/deployment/startBackEnds.sh | 4 ++-- .../deployment/startCloudwatchAgent.sh | 2 +- scripts/bash/backendStartDev.sh | 3 ++- scripts/bash/buildInternalDependencies.sh | 4 ++-- scripts/bash/getDeployablePackages.sh | 18 +++++++++++++- scripts/bash/getInternalDependencies.sh | 24 ++++++++++++++++++- .../docker/downloadEnvironmentVariables.sh | 2 +- 18 files changed, 76 insertions(+), 36 deletions(-) diff --git a/packages/data-api/scripts/fastRefreshAnalyticsTable.sh b/packages/data-api/scripts/fastRefreshAnalyticsTable.sh index 08c475a5db..7e67d4a483 100755 --- a/packages/data-api/scripts/fastRefreshAnalyticsTable.sh +++ b/packages/data-api/scripts/fastRefreshAnalyticsTable.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e -echo "Fast refreshing analytics table" +echo 'Fast refreshing analytics table' DIR=$(pwd "$0") . "$DIR/../../scripts/bash/mergeEnvForDB.sh" diff --git a/packages/data-api/scripts/fullRefreshAnalyticsTable.sh b/packages/data-api/scripts/fullRefreshAnalyticsTable.sh index 9991269a96..297f1e2bb5 100755 --- a/packages/data-api/scripts/fullRefreshAnalyticsTable.sh +++ b/packages/data-api/scripts/fullRefreshAnalyticsTable.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash set -e -echo "Fully refreshing analytics table" + +echo 'Fully refreshing analytics table' DIR=$(pwd "$0") . "$DIR/../../scripts/bash/mergeEnvForDB.sh" diff --git a/packages/data-api/scripts/installMvRefreshModule.sh b/packages/data-api/scripts/installMvRefreshModule.sh index 91574daf43..74502e7621 100755 --- a/packages/data-api/scripts/installMvRefreshModule.sh +++ b/packages/data-api/scripts/installMvRefreshModule.sh @@ -10,7 +10,7 @@ export PGPASSWORD=$DB_PG_PASSWORD MV_REFRESH_EXISTS=$(psql -p $DB_PORT -X -A -h $DB_URL -d $DB_NAME -U $DB_PG_USER -t -c "SELECT schema_name FROM information_schema.schemata WHERE schema_name = '$DB_MV_USER'") if [[ $MV_REFRESH_EXISTS = "$DB_MV_USER" ]]; then - echo "Fast Refresh module already exists, skipping installation" + echo 'Fast Refresh module already exists, skipping installation' else git submodule update --init scripts/pg-mv-fast-refresh cd scripts/pg-mv-fast-refresh/ diff --git a/packages/data-api/scripts/patchMvRefreshModule.sh b/packages/data-api/scripts/patchMvRefreshModule.sh index 42104e7d75..5485454a2d 100755 --- a/packages/data-api/scripts/patchMvRefreshModule.sh +++ b/packages/data-api/scripts/patchMvRefreshModule.sh @@ -12,14 +12,14 @@ if [[ $COMMAND = '' ]]; then fi if [[ $COMMAND = create ]]; then - echo "Enter patch name: " + echo 'Enter patch name: ' read PATCH_NAME fi VERSION=$2 if [[ $VERSION = '' ]]; then - echo "Version unspecified, defaulting to database mvrefresh version" + echo 'Version unspecified, defaulting to database mvrefresh version' # Set default port in case it wasn't in .env : "${DB_PORT:=5432}" @@ -39,5 +39,5 @@ fi if [[ ! -d ./scripts/patches/$VERSION && $COMMAND != create ]]; then echo "No patches exist for version: $VERSION, skipping" else - ts-node ./scripts/patchMvRefresh.ts $COMMAND:$VERSION $PATCH_NAME --migrations-dir "./scripts/patches" --table "patches" -v --config-file "../../babel.config.json" + ts-node ./scripts/patchMvRefresh.ts $COMMAND:$VERSION $PATCH_NAME --migrations-dir './scripts/patches' --table 'patches' -v --config-file '../../babel.config.json' fi diff --git a/packages/data-api/scripts/uninstallMvRefreshModule.sh b/packages/data-api/scripts/uninstallMvRefreshModule.sh index 13a2c86179..ee99e5bed5 100755 --- a/packages/data-api/scripts/uninstallMvRefreshModule.sh +++ b/packages/data-api/scripts/uninstallMvRefreshModule.sh @@ -11,7 +11,7 @@ export PGPASSWORD=$DB_PG_PASSWORD MV_REFRESH_EXISTS=$(psql -X -A -p $DB_PORT -h $DB_URL -d $DB_NAME -U $DB_PG_USER -t -c "SELECT schema_name FROM information_schema.schemata WHERE schema_name = '$DB_MV_USER'") if [[ $MV_REFRESH_EXISTS != "$DB_MV_USER" ]]; then - echo "Fast Refresh module does not exist, skipping uninstallation" + echo 'Fast Refresh module does not exist, skipping uninstallation' else git submodule update --init scripts/pg-mv-fast-refresh cd scripts/pg-mv-fast-refresh/ diff --git a/packages/database/scripts/dumpDatabase.sh b/packages/database/scripts/dumpDatabase.sh index 886707aca1..52f8e5c7ac 100755 --- a/packages/database/scripts/dumpDatabase.sh +++ b/packages/database/scripts/dumpDatabase.sh @@ -34,11 +34,11 @@ DIR=$(pwd "$0") . "$DIR/../../scripts/bash/mergeEnvForDB.sh" . "$DIR/../../scripts/bash/ansiControlSequences.sh" -DUMP_FILE_NAME="dump.sql" +DUMP_FILE_NAME='dump.sql' -identity_file="" -server="dev" -target_dir="." +identity_file='' +server='dev' +target_dir='.' while [[ $1 != '' ]]; do case $1 in diff --git a/packages/database/scripts/setupTestDatabase.sh b/packages/database/scripts/setupTestDatabase.sh index c9187e4e80..7787c892ab 100755 --- a/packages/database/scripts/setupTestDatabase.sh +++ b/packages/database/scripts/setupTestDatabase.sh @@ -33,14 +33,14 @@ else fi -echo "Installing mvrefresh" +echo 'Installing mvrefresh' DB_NAME=$DB_NAME yarn workspace @tupaia/data-api install-mv-refresh -echo "Patching mvrefresh" +echo 'Patching mvrefresh' DB_NAME=$DB_NAME yarn workspace @tupaia/data-api patch-mv-refresh up -echo "Installing Analytics table" +echo 'Installing Analytics table' DB_NAME=$DB_NAME yarn workspace @tupaia/data-api build-analytics-table -echo "Deleting migrations that target data modifications, as there is no data to migrate on the test database" +echo 'Deleting migrations that target data modifications, as there is no data to migrate on the test database' rm -rf ./src/migrations-backup mkdir ./src/migrations-backup cp -r ./src/migrations/* ./src/migrations-backup/ diff --git a/packages/devops/scripts/ci/triggerRedeploy.sh b/packages/devops/scripts/ci/triggerRedeploy.sh index 391cc61903..e10472955e 100755 --- a/packages/devops/scripts/ci/triggerRedeploy.sh +++ b/packages/devops/scripts/ci/triggerRedeploy.sh @@ -16,7 +16,7 @@ RUNNING_INSTANCES=$(aws ec2 describe-instances \ --no-cli-pager) if [[ $RUNNING_INSTANCES != *"Instances"* ]]; then - echo "No deployment running, skipping redeploy" + echo 'No deployment running, skipping redeploy' exit 0 fi @@ -31,7 +31,7 @@ AWS_MAX_ATTEMPTS=1 aws lambda invoke \ $RESPONSE_FILE if grep -q errorMessage "$RESPONSE_FILE"; then - echo "Error while trying to redeploy" + echo 'Error while trying to redeploy' cat $RESPONSE_FILE exit 1 fi @@ -69,7 +69,7 @@ for DEPLOYMENT_BASE64 in $DEPLOYMENTS; do --cli-read-timeout 900 \ $SWAP_OUT_RESPONSE_FILE if grep -q errorMessage "$SWAP_OUT_RESPONSE_FILE"; then - echo "Error while trying to swap out instances" + echo 'Error while trying to swap out instances' cat $SWAP_OUT_RESPONSE_FILE exit 1 fi @@ -77,7 +77,7 @@ for DEPLOYMENT_BASE64 in $DEPLOYMENTS; do break else if [ "$WAIT_ATTEMPTS" -ge 75 ]; then # 75 * 200 seconds = 4.16 hours, sitting within codeship's 5 hour timeout - echo "Build failed! Waited 75 times, but new instance is still not reachable" + echo 'Build failed! Waited 75 times, but new instance is still not reachable' exit 1 else echo "Still waiting for ${DEPLOYMENT_NAME} startup build to complete. To watch detailed progress, connect to instance ${NEW_INSTANCE_ID} and run tail -f logs/deployment_log.txt" @@ -87,4 +87,4 @@ for DEPLOYMENT_BASE64 in $DEPLOYMENTS; do done done -echo "Redeploy complete" +echo 'Redeploy complete' diff --git a/packages/devops/scripts/ci/validateNewMigrations.sh b/packages/devops/scripts/ci/validateNewMigrations.sh index a0c48ad834..e75bcc93e5 100755 --- a/packages/devops/scripts/ci/validateNewMigrations.sh +++ b/packages/devops/scripts/ci/validateNewMigrations.sh @@ -8,9 +8,9 @@ ROOT="${DIR}/../../../../" function get_date_command() { if [[ $(uname) = Darwin ]]; then - echo "gdate" # install gdate on MacOs: brew install coreutils + echo 'gdate' # install gdate on MacOs: brew install coreutils else - echo "date" + echo 'date' fi } @@ -61,11 +61,11 @@ function validate_migrations(){ } current_branch_name=$(get_branch_name) -origin_branch_name="master" +origin_branch_name='master' # Skip validation if current branch name is master if [[ $current_branch_name = "$origin_branch_name" ]]; then - echo "Skipping validation step while current branch is the same as origin" + echo 'Skipping validation step while current branch is the same as origin' exit 0 fi @@ -81,4 +81,4 @@ git fetch origin $origin_branch_name:$origin_branch_name --quiet validate_migrations $current_branch_name $origin_branch_name log_success "✔ New migrations are valid!" -exit 0 \ No newline at end of file +exit 0 diff --git a/packages/devops/scripts/deployment/buildDeployablePackages.sh b/packages/devops/scripts/deployment/buildDeployablePackages.sh index 1500eceb2b..6b7ff986b2 100755 --- a/packages/devops/scripts/deployment/buildDeployablePackages.sh +++ b/packages/devops/scripts/deployment/buildDeployablePackages.sh @@ -4,7 +4,7 @@ DIR=$(dirname "$0") TUPAIA_DIR=$DIR/../../../.. DEPLOYMENT_NAME=$1 -echo "Building deployable packages" +echo 'Building deployable packages' PACKAGES=$(${TUPAIA_DIR}/scripts/bash/getDeployablePackages.sh) # Initialise NVM (which sets the path for access to npm, yarn etc. as well) diff --git a/packages/devops/scripts/deployment/checkoutLatest.sh b/packages/devops/scripts/deployment/checkoutLatest.sh index 0383db0332..579a651516 100755 --- a/packages/devops/scripts/deployment/checkoutLatest.sh +++ b/packages/devops/scripts/deployment/checkoutLatest.sh @@ -21,4 +21,4 @@ git reset --hard # clear out any manual changes that have been made, which would git checkout ${BRANCH_TO_USE} git reset --hard origin/${BRANCH_TO_USE} -echo "Checked out latest code" +echo 'Checked out latest code' diff --git a/packages/devops/scripts/deployment/startBackEnds.sh b/packages/devops/scripts/deployment/startBackEnds.sh index 8630e0dc9a..857d345c67 100755 --- a/packages/devops/scripts/deployment/startBackEnds.sh +++ b/packages/devops/scripts/deployment/startBackEnds.sh @@ -15,7 +15,7 @@ for PACKAGE in ${PACKAGES[@]}; do cd ${TUPAIA_DIR} # ensure that the analytics table is fully built - echo "Building analytics table" + echo 'Building analytics table' yarn workspace @tupaia/data-api install-mv-refresh yarn workspace @tupaia/data-api patch-mv-refresh up yarn workspace @tupaia/data-api build-analytics-table @@ -44,4 +44,4 @@ pm2 save # Log dump file grep status /home/ubuntu/.pm2/dump.pm2 -echo "Finished deploying latest" +echo 'Finished deploying latest' diff --git a/packages/devops/scripts/deployment/startCloudwatchAgent.sh b/packages/devops/scripts/deployment/startCloudwatchAgent.sh index c19d2692e9..d200d8bb83 100755 --- a/packages/devops/scripts/deployment/startCloudwatchAgent.sh +++ b/packages/devops/scripts/deployment/startCloudwatchAgent.sh @@ -1,4 +1,4 @@ #!/bin/bash -le DIR=$(dirname "$0") -echo "Turning on cloudwatch agent" +echo 'Turning on cloudwatch agent' /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -c file:$DIR/cloudwatchConfig.json -s diff --git a/scripts/bash/backendStartDev.sh b/scripts/bash/backendStartDev.sh index 896a451749..5b57426a29 100755 --- a/scripts/bash/backendStartDev.sh +++ b/scripts/bash/backendStartDev.sh @@ -12,6 +12,7 @@ DIR=$(dirname "$0") USAGE="Usage: ${BOLD}backendStartDev babel_port_inspector${RESET} [${BOLD}-i${RESET}|${BOLD}--include-internal${RESET}] [${BOLD}-ts${RESET}|${BOLD}--typescript${RESET}]" CONCURRENTLY_BIN="${DIR}/../../node_modules/.bin/concurrently" watch_flags="" +watch_flags='' include_internal=false type_script=false inspect_port=${1} @@ -48,7 +49,7 @@ fi echo -e "${BOLD}Starting server...${RESET}" if [[ $include_internal = true ]]; then - echo "Internal dependencies are under watch for hot reload" + echo 'Internal dependencies are under watch for hot reload' for PACKAGE in $(${DIR}/getInternalDependencies.sh .); do watch_flags="${watch_flags} --watch ../${PACKAGE}/dist" done diff --git a/scripts/bash/buildInternalDependencies.sh b/scripts/bash/buildInternalDependencies.sh index d48e60bf97..db9e54f3bb 100755 --- a/scripts/bash/buildInternalDependencies.sh +++ b/scripts/bash/buildInternalDependencies.sh @@ -33,8 +33,8 @@ while [[ $1 != '' ]]; do esac done -[[ $watch = true ]] && build_args="--watch" || build_args="" -[[ $watch = true ]] && build_ts_args="--watch --preserveWatchOutput" || build_ts_args="" +[[ $watch = true ]] && build_args='--watch' || build_args='' +[[ $watch = true ]] && build_ts_args='--watch --preserveWatchOutput' || build_ts_args='' build_commands=() build_prefixes=() diff --git a/scripts/bash/getDeployablePackages.sh b/scripts/bash/getDeployablePackages.sh index 872d9902d9..e6039a20b0 100755 --- a/scripts/bash/getDeployablePackages.sh +++ b/scripts/bash/getDeployablePackages.sh @@ -1,5 +1,21 @@ #!/usr/bin/env bash set -e -echo "admin-panel" "lesmis" "psss" "datatrak-web" "tupaia-web" "central-server" "data-table-server" "datatrak-web-server" "entity-server" "lesmis-server" "meditrak-app-server" "psss-server" "report-server" "tupaia-web-server" "web-config-server" "admin-panel-server" # admin-panel-server last as it depends on report-server +echo 'admin-panel' \ + 'lesmis' \ + 'psss' \ + 'datatrak-web' \ + 'tupaia-web' \ + 'central-server' \ + 'data-table-server' \ + 'datatrak-web-server' \ + 'entity-server' \ + 'lesmis-server' \ + 'meditrak-app-server' \ + 'psss-server' \ + 'report-server' \ + 'tupaia-web-server' \ + 'web-config-server' \ + 'admin-panel-server' # admin-panel-server last as it depends on report-server + exit 0 diff --git a/scripts/bash/getInternalDependencies.sh b/scripts/bash/getInternalDependencies.sh index 6a91c8f420..66faccc493 100755 --- a/scripts/bash/getInternalDependencies.sh +++ b/scripts/bash/getInternalDependencies.sh @@ -13,7 +13,29 @@ dependencies_already_visited=($@) # if no package.json entrypoint is specified, just return all internal dependencies if [[ -z $package_path ]]; then - echo "types" "utils" "tsutils" "ui-components" "ui-chart-components" "ui-map-components" "server-utils" "access-policy" "admin-panel" "aggregator" "api-client" "auth" "database" "data-api" "dhis-api" "data-lake-api" "expression-parser" "indicators" "weather-api" "kobo-api" "superset-api" "data-broker" "server-boilerplate" + echo 'types' \ + 'utils' \ + 'tsutils' \ + 'ui-components' \ + 'ui-chart-components' \ + 'ui-map-components' \ + 'server-utils' \ + 'access-policy' \ + 'admin-panel' \ + 'aggregator' \ + 'api-client' \ + 'auth' \ + 'database' \ + 'data-api' \ + 'dhis-api' \ + 'data-lake-api' \ + 'expression-parser' \ + 'indicators' \ + 'weather-api' \ + 'kobo-api' \ + 'superset-api' \ + 'data-broker' \ + 'server-boilerplate' exit 0 fi diff --git a/scripts/docker/downloadEnvironmentVariables.sh b/scripts/docker/downloadEnvironmentVariables.sh index 915eac8f23..99ce7c7564 100755 --- a/scripts/docker/downloadEnvironmentVariables.sh +++ b/scripts/docker/downloadEnvironmentVariables.sh @@ -35,7 +35,7 @@ DIR=$(dirname "$0") # can provide one or more packages as command line arguments, or will default to all if [[ -z $3 ]]; then - echo "Fetching all .env files" + echo 'Fetching all .env files' PACKAGES=$(${DIR}/../bash/getPackagesWithEnvFiles.sh) else PACKAGES=${@:3} From 507fa8352c23f9c078ec59d49bb62d1262f8a33a Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Sun, 7 Apr 2024 23:16:02 +1200 Subject: [PATCH 17/24] prefer ``(( ))`` over `[[ ]]` for arithmetic --- packages/devops/scripts/ci/triggerRedeploy.sh | 4 ++-- packages/devops/scripts/ci/utils.sh | 2 +- packages/devops/scripts/ci/validateNewMigrations.sh | 2 +- scripts/bash/downloadEnvironmentVariables.sh | 2 +- scripts/bash/getInternalDependencies.sh | 6 +++--- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/devops/scripts/ci/triggerRedeploy.sh b/packages/devops/scripts/ci/triggerRedeploy.sh index e10472955e..dc6bce542d 100755 --- a/packages/devops/scripts/ci/triggerRedeploy.sh +++ b/packages/devops/scripts/ci/triggerRedeploy.sh @@ -76,12 +76,12 @@ for DEPLOYMENT_BASE64 in $DEPLOYMENTS; do echo "ELB for ${DEPLOYMENT_NAME} now points to ${NEW_INSTANCE_ID}" break else - if [ "$WAIT_ATTEMPTS" -ge 75 ]; then # 75 * 200 seconds = 4.16 hours, sitting within codeship's 5 hour timeout + if (( WAIT_ATTEMPTS >= 75 )); then # 75 * 200 seconds = 4.16 hours, sitting within codeship's 5 hour timeout echo 'Build failed! Waited 75 times, but new instance is still not reachable' exit 1 else echo "Still waiting for ${DEPLOYMENT_NAME} startup build to complete. To watch detailed progress, connect to instance ${NEW_INSTANCE_ID} and run tail -f logs/deployment_log.txt" - WAIT_ATTEMPTS=$((WAIT_ATTEMPTS+1)) + (( WAIT_ATTEMPTS++ )) fi fi done diff --git a/packages/devops/scripts/ci/utils.sh b/packages/devops/scripts/ci/utils.sh index 1de488e125..8002798022 100755 --- a/packages/devops/scripts/ci/utils.sh +++ b/packages/devops/scripts/ci/utils.sh @@ -29,7 +29,7 @@ function get_max_length() { for item in "${array[@]}"; do length=${#item} - if [[ $length -gt $max ]]; then + if (( length > max )); then max=$length fi done diff --git a/packages/devops/scripts/ci/validateNewMigrations.sh b/packages/devops/scripts/ci/validateNewMigrations.sh index e75bcc93e5..bba31bfe25 100755 --- a/packages/devops/scripts/ci/validateNewMigrations.sh +++ b/packages/devops/scripts/ci/validateNewMigrations.sh @@ -35,7 +35,7 @@ function check_migration_outdated() { day=${migration_name:39:2} migration_timestamp=$($date_command -d "${year}-${month}-${day}" +%s) - if (( $migration_timestamp < $included_migrations_timestamp )); then + if (( migration_timestamp < included_migrations_timestamp )); then log_error "❌ New migration should be created after $valid_migration_date. Invalid migration name: '$migration_name'" fi } diff --git a/scripts/bash/downloadEnvironmentVariables.sh b/scripts/bash/downloadEnvironmentVariables.sh index aae3908c81..eb49c73911 100755 --- a/scripts/bash/downloadEnvironmentVariables.sh +++ b/scripts/bash/downloadEnvironmentVariables.sh @@ -72,8 +72,8 @@ for PACKAGE in $PACKAGES; do # Only download the env file if there is an example file in the package. If there isn’t, this # means it is a package that doesn’t need env vars has_example_env_in_package=$(find $DIR/../../packages/$PACKAGE -type f -name '*.env.example' | wc -l) - if [[ $has_example_env_in_package -eq 1 ]]; then load_env_file_from_bw $PACKAGE $DIR/../../packages/$PACKAGE "" + if (( has_example_env_in_package > 0 )); then fi done diff --git a/scripts/bash/getInternalDependencies.sh b/scripts/bash/getInternalDependencies.sh index 66faccc493..ced415f27a 100755 --- a/scripts/bash/getInternalDependencies.sh +++ b/scripts/bash/getInternalDependencies.sh @@ -41,7 +41,7 @@ fi # we are getting internal dependencies for a specific package.json internal_dependencies=($(sed -n '/"dependencies": {/,/}/p' ${PWD}/${package_path}/package.json | grep -o '@tupaia/[^"]*": "[0-9\.]*"' | cut -d / -f 2 | cut -d \" -f 1)) -if [[ ${#internal_dependencies[@]} -eq 0 ]]; then +if (( ${#internal_dependencies[@]} = 0 )); then exit 0 # no internal dependencies of this package, early return fi @@ -63,7 +63,7 @@ unset array_without_gaps for dependency in ${internal_dependencies[@]}; do nested_dependencies=($(${DIR}/getInternalDependencies.sh "${package_path}/../${dependency}" ${dependencies_already_visited[@]} ${internal_dependencies[@]} )) - if [[ ${#nested_dependencies[@]} -eq 0 ]]; then + if (( ${#nested_dependencies[@]} = 0 )); then continue fi @@ -74,7 +74,7 @@ done # remove any duplicates for i in "${!internal_dependencies[@]}"; do for j in "${!internal_dependencies[@]}"; do - if [[ i -ne j && ${internal_dependencies[i]} = ${internal_dependencies[j]} ]]; then + if (( i != j )) && [[ ${internal_dependencies[i]} = ${internal_dependencies[j]} ]]; then unset 'internal_dependencies[i]' fi done From 1160b28d04e21cb99a0c3f507e16c121c288b92f Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Sun, 7 Apr 2024 23:58:01 +1200 Subject: [PATCH 18/24] prefer array over string in `buildNonInternalDeps` --- scripts/bash/buildNonInternalDependencies.sh | 24 ++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/scripts/bash/buildNonInternalDependencies.sh b/scripts/bash/buildNonInternalDependencies.sh index 9dc7436dbb..1c9a07779b 100755 --- a/scripts/bash/buildNonInternalDependencies.sh +++ b/scripts/bash/buildNonInternalDependencies.sh @@ -1,7 +1,23 @@ #!/usr/bin/env bash set -ex -PACKAGES="report-server admin-panel-server central-server data-table-server datatrak-web datatrak-web-server entity-server lesmis lesmis-server meditrak-app-server psss psss-server web-config-server tupaia-web tupaia-web-server" +PACKAGES=( + 'report-server' + 'admin-panel-server' + 'central-server' + 'data-table-server' + 'datatrak-web' + 'datatrak-web-server' + 'entity-server' + 'lesmis' + 'lesmis-server' + 'meditrak-app-server' + 'psss' + 'psss-server' + 'web-config-server' + 'tupaia-web' + 'tupaia-web-server' +) CONCURRENT_BUILD_BATCH_SIZE=1 @@ -9,9 +25,9 @@ build_commands=() build_prefixes=() # Build dependencies -for PACKAGE in $PACKAGES; do - build_commands+=("\"yarn workspace @tupaia/${PACKAGE} build\"") - build_prefixes+=("${PACKAGE},") +for PACKAGE in "${PACKAGES[@]}"; do + build_commands+=("\"yarn workspace @tupaia/$PACKAGE build\"") + build_prefixes+=("$PACKAGE,") done eval "yarn concurrently -m $CONCURRENT_BUILD_BATCH_SIZE --names \"${build_prefixes[*]}\" -k ${build_commands[*]}" From 0ba73f16e11b16a5429315e7ddc6da44e7604e61 Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Mon, 8 Apr 2024 21:17:38 +1200 Subject: [PATCH 19/24] prefer array for `getDeployablePackages.sh` --- scripts/bash/getDeployablePackages.sh | 33 ++++++++++++++----------- scripts/bash/getPackagesWithEnvFiles.sh | 7 +++--- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/scripts/bash/getDeployablePackages.sh b/scripts/bash/getDeployablePackages.sh index e6039a20b0..c762d1c50d 100755 --- a/scripts/bash/getDeployablePackages.sh +++ b/scripts/bash/getDeployablePackages.sh @@ -1,21 +1,24 @@ #!/usr/bin/env bash set -e -echo 'admin-panel' \ - 'lesmis' \ - 'psss' \ - 'datatrak-web' \ - 'tupaia-web' \ - 'central-server' \ - 'data-table-server' \ - 'datatrak-web-server' \ - 'entity-server' \ - 'lesmis-server' \ - 'meditrak-app-server' \ - 'psss-server' \ - 'report-server' \ - 'tupaia-web-server' \ - 'web-config-server' \ +DEPLOYABLE_PACKAGES=( + 'admin-panel' + 'lesmis' + 'psss' + 'datatrak-web' + 'tupaia-web' + 'central-server' + 'data-table-server' + 'datatrak-web-server' + 'entity-server' + 'lesmis-server' + 'meditrak-app-server' + 'psss-server' + 'report-server' + 'tupaia-web-server' + 'web-config-server' 'admin-panel-server' # admin-panel-server last as it depends on report-server +) +echo "${DEPLOYABLE_PACKAGES[@]}" exit 0 diff --git a/scripts/bash/getPackagesWithEnvFiles.sh b/scripts/bash/getPackagesWithEnvFiles.sh index a96f62cad8..773409d813 100755 --- a/scripts/bash/getPackagesWithEnvFiles.sh +++ b/scripts/bash/getPackagesWithEnvFiles.sh @@ -4,7 +4,8 @@ set -e DIR=$(dirname "$0") # packages with .env files are (currently) all deployable, plus auth, data-api, and database -PACKAGES=$(${DIR}/getDeployablePackages.sh) -PACKAGES+=" data-api" -echo $PACKAGES +PACKAGES=$("$DIR/getDeployablePackages.sh") +PACKAGES+=('data-api') +echo "${PACKAGES[@]}" + exit 0 From ca00437cc1a96debf6a9f81fa615c78015a700cb Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Mon, 8 Apr 2024 23:02:04 +1200 Subject: [PATCH 20/24] refactor: clearer way to get current branch --- packages/devops/scripts/ci/utils.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/devops/scripts/ci/utils.sh b/packages/devops/scripts/ci/utils.sh index 8002798022..06bc588446 100755 --- a/packages/devops/scripts/ci/utils.sh +++ b/packages/devops/scripts/ci/utils.sh @@ -41,8 +41,8 @@ function get_branch_name() { local branch_name="$CI_BRANCH" if [[ $branch_name = '' ]]; then # Get currently checked out branch - branch_name=$(git rev-parse --abbrev-ref HEAD) + branch_name=$(git branch --show-current) fi echo $branch_name -} \ No newline at end of file +} From 97c51d6cb19905853089c6abbeaad1a11d323574 Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Tue, 9 Apr 2024 09:23:56 +1200 Subject: [PATCH 21/24] =?UTF-8?q?double-quote=20variables,=20avoiding=20`$?= =?UTF-8?q?{=E2=80=A6}`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../scripts/buildAnalyticsMaterializedView.sh | 6 ++-- .../scripts/dropAnalyticsMaterializedView.sh | 4 +-- .../scripts/fastRefreshAnalyticsTable.sh | 2 +- .../scripts/fullRefreshAnalyticsTable.sh | 6 ++-- .../scripts/installMvRefreshModule.sh | 4 +-- .../data-api/scripts/patchMvRefreshModule.sh | 2 +- .../scripts/uninstallMvRefreshModule.sh | 4 +-- packages/database/scripts/dumpDatabase.sh | 4 +-- packages/devops/scripts/ci/triggerRedeploy.sh | 30 ++++++++-------- packages/devops/scripts/ci/utils.sh | 12 +++---- .../devops/scripts/ci/validateBranchName.sh | 14 ++++---- .../scripts/ci/validateNewMigrations.sh | 14 ++++---- packages/devops/scripts/ci/validateTests.sh | 4 +-- .../ci/validateTypesAndDbSchemaInSync.sh | 2 +- scripts/bash/backendStartDev.sh | 10 +++--- scripts/bash/buildInternalDependencies.sh | 10 +++--- scripts/bash/downloadEnvironmentVariables.sh | 34 +++++++++---------- scripts/bash/getInternalDependencies.sh | 4 +-- scripts/bash/pm2startInline.sh | 2 +- scripts/bash/validate.sh | 2 +- 20 files changed, 85 insertions(+), 85 deletions(-) diff --git a/packages/data-api/scripts/buildAnalyticsMaterializedView.sh b/packages/data-api/scripts/buildAnalyticsMaterializedView.sh index 0d60df4ed7..16c1ad4d6a 100755 --- a/packages/data-api/scripts/buildAnalyticsMaterializedView.sh +++ b/packages/data-api/scripts/buildAnalyticsMaterializedView.sh @@ -10,8 +10,8 @@ DIR=$(pwd "$0") cd scripts export PGPASSWORD=$DB_PASSWORD if [[ $1 = '--force' || $1 = '-f' ]]; then - psql -p $DB_PORT -h $DB_URL -d $DB_NAME -U $DB_USER -tc "SELECT build_analytics_table(true);" + psql -p "$DB_PORT" -h "$DB_URL" -d "$DB_NAME" -U "$DB_USER" -tc "SELECT build_analytics_table(true);" else - psql -p $DB_PORT -h $DB_URL -d $DB_NAME -U $DB_USER -tc "SELECT build_analytics_table();" + psql -p "$DB_PORT" -h "$DB_URL" -d "$DB_NAME" -U "$DB_USER" -tc "SELECT build_analytics_table();" fi -psql -p $DB_PORT -h $DB_URL -d $DB_NAME -U $DB_USER -tc "SELECT create_analytics_table_indexes();" +psql -p "$DB_PORT" -h "$DB_URL" -d "$DB_NAME" -U "$DB_USER" -tc "SELECT create_analytics_table_indexes();" diff --git a/packages/data-api/scripts/dropAnalyticsMaterializedView.sh b/packages/data-api/scripts/dropAnalyticsMaterializedView.sh index 18847a153d..c47ac0ca89 100755 --- a/packages/data-api/scripts/dropAnalyticsMaterializedView.sh +++ b/packages/data-api/scripts/dropAnalyticsMaterializedView.sh @@ -9,5 +9,5 @@ DIR=$(pwd "$0") cd scripts export PGPASSWORD=$DB_PASSWORD -psql -p $DB_PORT -h $DB_URL -d $DB_NAME -U $DB_USER -tc "SELECT drop_analytics_table();" -psql -p $DB_PORT -h $DB_URL -d $DB_NAME -U $DB_USER -tc "SELECT drop_analytics_log_tables();" +psql -p "$DB_PORT" -h "$DB_URL" -d "$DB_NAME" -U "$DB_USER" -tc "SELECT drop_analytics_table();" +psql -p "$DB_PORT" -h "$DB_URL" -d "$DB_NAME" -U "$DB_USER" -tc "SELECT drop_analytics_log_tables();" diff --git a/packages/data-api/scripts/fastRefreshAnalyticsTable.sh b/packages/data-api/scripts/fastRefreshAnalyticsTable.sh index 7e67d4a483..6d74943671 100755 --- a/packages/data-api/scripts/fastRefreshAnalyticsTable.sh +++ b/packages/data-api/scripts/fastRefreshAnalyticsTable.sh @@ -11,4 +11,4 @@ DIR=$(pwd "$0") cd scripts export PGPASSWORD=$DB_PASSWORD -psql -p $DB_PORT -h $DB_URL -d $DB_NAME -U $DB_USER -tc "SELECT mv\$refreshMaterializedView('analytics', 'public', true);" +psql -p "$DB_PORT" -h "$DB_URL" -d "$DB_NAME" -U "$DB_USER" -tc "SELECT mv\$refreshMaterializedView('analytics', 'public', true);" diff --git a/packages/data-api/scripts/fullRefreshAnalyticsTable.sh b/packages/data-api/scripts/fullRefreshAnalyticsTable.sh index 297f1e2bb5..159101fd5d 100755 --- a/packages/data-api/scripts/fullRefreshAnalyticsTable.sh +++ b/packages/data-api/scripts/fullRefreshAnalyticsTable.sh @@ -11,6 +11,6 @@ DIR=$(pwd "$0") cd scripts export PGPASSWORD=$DB_PASSWORD -psql -p $DB_PORT -h $DB_URL -d $DB_NAME -U $DB_USER -tc "SELECT drop_analytics_table_indexes();" -psql -p $DB_PORT -h $DB_URL -d $DB_NAME -U $DB_USER -tc "SELECT mv\$refreshMaterializedView('analytics', 'public');" -psql -p $DB_PORT -h $DB_URL -d $DB_NAME -U $DB_USER -tc "SELECT create_analytics_table_indexes();" +psql -p "$DB_PORT" -h "$DB_URL" -d "$DB_NAME" -U "$DB_USER" -tc "SELECT drop_analytics_table_indexes();" +psql -p "$DB_PORT" -h "$DB_URL" -d "$DB_NAME" -U "$DB_USER" -tc "SELECT mv\$refreshMaterializedView('analytics', 'public');" +psql -p "$DB_PORT" -h "$DB_URL" -d "$DB_NAME" -U "$DB_USER" -tc "SELECT create_analytics_table_indexes();" diff --git a/packages/data-api/scripts/installMvRefreshModule.sh b/packages/data-api/scripts/installMvRefreshModule.sh index 74502e7621..15dcbcd9eb 100755 --- a/packages/data-api/scripts/installMvRefreshModule.sh +++ b/packages/data-api/scripts/installMvRefreshModule.sh @@ -7,7 +7,7 @@ DIR=$(pwd "$0") : "${DB_PORT:=5432}" export PGPASSWORD=$DB_PG_PASSWORD -MV_REFRESH_EXISTS=$(psql -p $DB_PORT -X -A -h $DB_URL -d $DB_NAME -U $DB_PG_USER -t -c "SELECT schema_name FROM information_schema.schemata WHERE schema_name = '$DB_MV_USER'") +MV_REFRESH_EXISTS=$(psql -p "$DB_PORT" -X -A -h "$DB_URL" -d "$DB_NAME" -U "$DB_PG_USER" -t -c "SELECT schema_name FROM information_schema.schemata WHERE schema_name = '$DB_MV_USER'") if [[ $MV_REFRESH_EXISTS = "$DB_MV_USER" ]]; then echo 'Fast Refresh module already exists, skipping installation' @@ -21,4 +21,4 @@ else fi export PGPASSWORD=$DB_PG_PASSWORD -psql -p $DB_PORT --set=db_user="$DB_USER" --set=mv_user="$DB_MV_USER" -h $DB_URL -d $DB_NAME -U $DB_PG_USER -f scripts/grantMvRefreshPermissions.sql +psql -p "$DB_PORT" --set=db_user="$DB_USER" --set=mv_user="$DB_MV_USER" -h "$DB_URL" -d "$DB_NAME" -U "$DB_PG_USER" -f scripts/grantMvRefreshPermissions.sql diff --git a/packages/data-api/scripts/patchMvRefreshModule.sh b/packages/data-api/scripts/patchMvRefreshModule.sh index 5485454a2d..86b936711e 100755 --- a/packages/data-api/scripts/patchMvRefreshModule.sh +++ b/packages/data-api/scripts/patchMvRefreshModule.sh @@ -26,7 +26,7 @@ if [[ $VERSION = '' ]]; then export PGPASSWORD=$DB_PASSWORD VERSION_SQL_FUNC="SELECT mv\$version()" - VERSION=$(psql -p $DB_PORT -X -A -h $DB_URL -d $DB_NAME -U $DB_USER -t -c "$VERSION_SQL_FUNC") + VERSION=$(psql -p "$DB_PORT" -X -A -h "$DB_URL" -d "$DB_NAME" -U "$DB_USER" -t -c "$VERSION_SQL_FUNC") if [[ $VERSION = '' ]]; then echo "${RED}Error: failed to detect mvrefresh version from database${RESET}" diff --git a/packages/data-api/scripts/uninstallMvRefreshModule.sh b/packages/data-api/scripts/uninstallMvRefreshModule.sh index ee99e5bed5..c21ffe8b55 100755 --- a/packages/data-api/scripts/uninstallMvRefreshModule.sh +++ b/packages/data-api/scripts/uninstallMvRefreshModule.sh @@ -8,7 +8,7 @@ DIR=$(pwd "$0") : "${DB_PORT:=5432}" export PGPASSWORD=$DB_PG_PASSWORD -MV_REFRESH_EXISTS=$(psql -X -A -p $DB_PORT -h $DB_URL -d $DB_NAME -U $DB_PG_USER -t -c "SELECT schema_name FROM information_schema.schemata WHERE schema_name = '$DB_MV_USER'") +MV_REFRESH_EXISTS=$(psql -X -A -p "$DB_PORT" -h "$DB_URL" -d "$DB_NAME" -U "$DB_PG_USER" -t -c "SELECT schema_name FROM information_schema.schemata WHERE schema_name = '$DB_MV_USER'") if [[ $MV_REFRESH_EXISTS != "$DB_MV_USER" ]]; then echo 'Fast Refresh module does not exist, skipping uninstallation' @@ -20,5 +20,5 @@ else cd ../.. git submodule deinit scripts/pg-mv-fast-refresh export PGPASSWORD=$DB_PG_PASSWORD - psql -p $DB_PORT -h $DB_URL -d $DB_NAME -U $DB_PG_USER -tc "REVOKE ALL PRIVILEGES on schema public FROM $DB_MV_USER; DROP ROLE IF EXISTS $DB_MV_USER;" + psql -p "$DB_PORT" -h "$DB_URL" -d "$DB_NAME" -U "$DB_PG_USER" -tc "REVOKE ALL PRIVILEGES on schema public FROM $DB_MV_USER; DROP ROLE IF EXISTS $DB_MV_USER;" fi diff --git a/packages/database/scripts/dumpDatabase.sh b/packages/database/scripts/dumpDatabase.sh index 52f8e5c7ac..673506efbf 100755 --- a/packages/database/scripts/dumpDatabase.sh +++ b/packages/database/scripts/dumpDatabase.sh @@ -14,13 +14,13 @@ EOF # https://stackoverflow.com/questions/12498304/using-bash-to-display-a-progress-indicator function show_loading_spinner() { - eval $2 & + eval "$2" & pid=$! # Process Id of the previous running command spin='-\|/' i=0 - while kill -0 $pid 2>/dev/null + while kill -0 "$pid" 2>/dev/null do i=$(( (i+1) %4 )) printf "\r$1 ${spin:$i:1}" diff --git a/packages/devops/scripts/ci/triggerRedeploy.sh b/packages/devops/scripts/ci/triggerRedeploy.sh index dc6bce542d..a92e893ae7 100755 --- a/packages/devops/scripts/ci/triggerRedeploy.sh +++ b/packages/devops/scripts/ci/triggerRedeploy.sh @@ -36,30 +36,30 @@ if grep -q errorMessage "$RESPONSE_FILE"; then exit 1 fi -DEPLOYMENTS=$(cat $RESPONSE_FILE | jq -r '.[] | @base64') +DEPLOYMENTS=$(cat "$RESPONSE_FILE" | jq -r '.[] | @base64') for DEPLOYMENT_BASE64 in $DEPLOYMENTS; do - DEPLOYMENT=$(echo $DEPLOYMENT_BASE64 | base64 --decode) - DEPLOYMENT_NAME=$(echo $DEPLOYMENT | jq -r '.DeploymentName') - NEW_INSTANCE_ID=$(echo $DEPLOYMENT | jq -r '.NewInstanceId') + DEPLOYMENT=$(echo "$DEPLOYMENT_BASE64" | base64 --decode) + DEPLOYMENT_NAME=$(echo "$DEPLOYMENT" | jq -r '.DeploymentName') + NEW_INSTANCE_ID=$(echo "$DEPLOYMENT" | jq -r '.NewInstanceId') - echo "Waiting for ${DEPLOYMENT_NAME} to run its startup build script. To watch detailed progress, connect to instance ${NEW_INSTANCE_ID} and run tail -f logs/deployment_log.txt" + echo "Waiting for $DEPLOYMENT_NAME to run its startup build script. To watch detailed progress, connect to instance $NEW_INSTANCE_ID and run tail -f logs/deployment_log.txt" WAIT_ATTEMPTS=0 while true; do STARTUP_COMPLETE=false aws ec2 wait instance-exists \ - --instance-ids ${NEW_INSTANCE_ID} \ + --instance-ids "$NEW_INSTANCE_ID" \ --filters Name=tag:StartupBuildProgress,Values=complete,errored \ --no-cli-pager && STARTUP_COMPLETE=true - if [ $STARTUP_COMPLETE == true ]; then + if [[ $STARTUP_COMPLETE = true ]]; then INSTANCE_ERRORED_RESPONSE=$(aws ec2 describe-instances \ - --instance-ids ${NEW_INSTANCE_ID} \ + --instance-ids "$NEW_INSTANCE_ID" \ --filters Name=tag:StartupBuildProgress,Values=errored \ --no-cli-pager) - if [[ $INSTANCE_ERRORED_RESPONSE == *"Instances"* ]]; then - echo "Build failed! Connect to instance ${NEW_INSTANCE_ID} and check the logs at ~/logs/deployment_log.txt and /var/log/cloud-init-output.log" + if [[ $INSTANCE_ERRORED_RESPONSE = *"Instances"* ]]; then + echo "Build failed! Connect to instance $NEW_INSTANCE_ID and check the logs at ~/logs/deployment_log.txt and /var/log/cloud-init-output.log" exit 1 fi - echo "New instance ${NEW_INSTANCE_ID} is ready, swapping over ELB" + echo "New instance $NEW_INSTANCE_ID is ready, swapping over ELB" SWAP_OUT_RESPONSE_FILE=lambda_swap_out_response.json AWS_MAX_ATTEMPTS=1 aws lambda invoke \ --function-name deployment \ @@ -67,20 +67,20 @@ for DEPLOYMENT_BASE64 in $DEPLOYMENTS; do --no-cli-pager \ --cli-binary-format raw-in-base64-out \ --cli-read-timeout 900 \ - $SWAP_OUT_RESPONSE_FILE + "$SWAP_OUT_RESPONSE_FILE" if grep -q errorMessage "$SWAP_OUT_RESPONSE_FILE"; then echo 'Error while trying to swap out instances' - cat $SWAP_OUT_RESPONSE_FILE + cat "$SWAP_OUT_RESPONSE_FILE" exit 1 fi - echo "ELB for ${DEPLOYMENT_NAME} now points to ${NEW_INSTANCE_ID}" + echo "ELB for $DEPLOYMENT_NAME now points to $NEW_INSTANCE_ID" break else if (( WAIT_ATTEMPTS >= 75 )); then # 75 * 200 seconds = 4.16 hours, sitting within codeship's 5 hour timeout echo 'Build failed! Waited 75 times, but new instance is still not reachable' exit 1 else - echo "Still waiting for ${DEPLOYMENT_NAME} startup build to complete. To watch detailed progress, connect to instance ${NEW_INSTANCE_ID} and run tail -f logs/deployment_log.txt" + echo "Still waiting for $DEPLOYMENT_NAME startup build to complete. To watch detailed progress, connect to instance $NEW_INSTANCE_ID and run tail -f logs/deployment_log.txt" (( WAIT_ATTEMPTS++ )) fi fi diff --git a/packages/devops/scripts/ci/utils.sh b/packages/devops/scripts/ci/utils.sh index 06bc588446..135ea69e5e 100755 --- a/packages/devops/scripts/ci/utils.sh +++ b/packages/devops/scripts/ci/utils.sh @@ -8,19 +8,19 @@ function ansi_color() { } function log_with_color() { - echo -e "$(ansi_color $2)$1$(ansi_color $COLOR_RESET)" + echo -e "$(ansi_color "$2")$1$(ansi_color "$COLOR_RESET")" } function log_error() { - log_with_color "$1" $COLOR_RED + log_with_color "$1" "$COLOR_RED" } function log_warn() { - log_with_color "$1" $COLOR_YELLOW + log_with_color "$1" "$COLOR_YELLOW" } function log_success() { - log_with_color "$1" $COLOR_GREEN + log_with_color "$1" "$COLOR_GREEN" } function get_max_length() { @@ -34,7 +34,7 @@ function get_max_length() { fi done - echo $max + echo "$max" } function get_branch_name() { @@ -44,5 +44,5 @@ function get_branch_name() { branch_name=$(git branch --show-current) fi - echo $branch_name + echo "$branch_name" } diff --git a/packages/devops/scripts/ci/validateBranchName.sh b/packages/devops/scripts/ci/validateBranchName.sh index a04240eeb0..4b5c680d82 100755 --- a/packages/devops/scripts/ci/validateBranchName.sh +++ b/packages/devops/scripts/ci/validateBranchName.sh @@ -2,7 +2,7 @@ set -e DIR=$(dirname "$0") -. ${DIR}/utils.sh +. "$DIR/utils.sh" INVALID_CHARS=('/' '\' '.' '&' '?' '_') SUBDOMAIN_SUFFIXES=(admin admin-api aggregation api config db export lesmis lesmis-api mobile psss psss-api report report-api ssh entity entity-api meditrak-api data-table-api tonga-aggregation www tupaia-web-api datatrak-web-api datatrak) @@ -11,13 +11,13 @@ SUBDOMAIN_SUFFIXES=(admin admin-api aggregation api config db export lesmis lesm # to create deployment urls, eg {{branchName}}-tonga-aggregation.tupaia.org MAX_SUBDOMAIN_LENGTH=64 MAX_SUBDOMAIN_SUFFIX_LENGTH=$(get_max_length "${SUBDOMAIN_SUFFIXES[@]}") -MAX_BRANCH_NAME_LENGTH=$((MAX_SUBDOMAIN_LENGTH - ${MAX_SUBDOMAIN_SUFFIX_LENGTH} - 1)) # Subtract 1 for the connecting `-` +MAX_BRANCH_NAME_LENGTH=$((MAX_SUBDOMAIN_LENGTH - MAX_SUBDOMAIN_SUFFIX_LENGTH - 1)) # Subtract 1 for the connecting `-` # As of 11/08/21, MAX_BRANCH_NAME_LENGTH = 64 - 17 - 1 = 46 (Longest subdomain "tonga-aggregation") function validate_name_ending() { local branch_name=$1 - for suffix in ${SUBDOMAIN_SUFFIXES[@]}; do + for suffix in "${SUBDOMAIN_SUFFIXES[@]}"; do if [[ $branch_name = *$suffix ]]; then log_error "❌ Invalid branch name ending: '$suffix'" exit 1 @@ -48,7 +48,7 @@ function validate_name_chars() { exit 1 fi - for character in ${INVALID_CHARS[@]}; do + for character in "${INVALID_CHARS[@]}"; do if [[ $branch_name = *"$character"* ]]; then log_error "❌ Invalid character in branch name: '$character'" exit 1 @@ -57,9 +57,9 @@ function validate_name_chars() { } branch_name=$(get_branch_name) -validate_name_ending $branch_name -validate_name_length $branch_name -validate_name_chars $branch_name +validate_name_ending "$branch_name" +validate_name_length "$branch_name" +validate_name_chars "$branch_name" log_success "✔ Branch name is valid!" exit 0 diff --git a/packages/devops/scripts/ci/validateNewMigrations.sh b/packages/devops/scripts/ci/validateNewMigrations.sh index bba31bfe25..8c81b96792 100755 --- a/packages/devops/scripts/ci/validateNewMigrations.sh +++ b/packages/devops/scripts/ci/validateNewMigrations.sh @@ -2,9 +2,9 @@ set -e DIR=$(dirname "$0") -ROOT="${DIR}/../../../../" +ROOT="$DIR/../../../../" -. ${DIR}/utils.sh +. "$DIR/utils.sh" function get_date_command() { if [[ $(uname) = Darwin ]]; then @@ -19,7 +19,7 @@ date_command=$(get_date_command) function convert_timestamp_to_date() { local timestamp=$1 local date=$($date_command -d @$timestamp '+%Y-%m-%d') - echo $date + echo "$date" } @@ -55,7 +55,7 @@ function validate_migrations(){ done <<< "$new_migration_names_in_string" if [[ $errors != '' ]]; then - echo $errors; + echo "$errors"; exit 1; fi } @@ -74,11 +74,11 @@ fi git remote remove origin git remote add origin https://github.com/beyondessential/tupaia.git # Remove this sub module because it uses ssh -git rm $ROOT/packages/data-api/scripts/pg-mv-fast-refresh +git rm "$ROOT/packages/data-api/scripts/pg-mv-fast-refresh" git fetch --quiet -git fetch origin $origin_branch_name:$origin_branch_name --quiet -validate_migrations $current_branch_name $origin_branch_name +git fetch origin "$origin_branch_name:$origin_branch_name" --quiet +validate_migrations "$current_branch_name" "$origin_branch_name" log_success "✔ New migrations are valid!" exit 0 diff --git a/packages/devops/scripts/ci/validateTests.sh b/packages/devops/scripts/ci/validateTests.sh index 481a50f826..566db82635 100755 --- a/packages/devops/scripts/ci/validateTests.sh +++ b/packages/devops/scripts/ci/validateTests.sh @@ -2,6 +2,6 @@ set -e DIR=$(dirname "$0") -ROOT="${DIR}/../../../../" +ROOT="$DIR/../../../../" -node ${ROOT}/scripts/node/validateTests +node "$ROOT/scripts/node/validateTests" diff --git a/packages/devops/scripts/ci/validateTypesAndDbSchemaInSync.sh b/packages/devops/scripts/ci/validateTypesAndDbSchemaInSync.sh index d29715cc85..3658746931 100755 --- a/packages/devops/scripts/ci/validateTypesAndDbSchemaInSync.sh +++ b/packages/devops/scripts/ci/validateTypesAndDbSchemaInSync.sh @@ -5,7 +5,7 @@ set -ex SCRIPT_DIR=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) # Load environment variables from .env files -source $SCRIPT_DIR/../../../../scripts/bash/mergeEnvForDB.sh +source "$SCRIPT_DIR/../../../../scripts/bash/mergeEnvForDB.sh" cd "$SCRIPT_DIR" diff --git a/scripts/bash/backendStartDev.sh b/scripts/bash/backendStartDev.sh index 5b57426a29..04aa66b6f3 100755 --- a/scripts/bash/backendStartDev.sh +++ b/scripts/bash/backendStartDev.sh @@ -50,14 +50,14 @@ echo -e "${BOLD}Starting server...${RESET}" if [[ $include_internal = true ]]; then echo 'Internal dependencies are under watch for hot reload' - for PACKAGE in $(${DIR}/getInternalDependencies.sh .); do - watch_flags="${watch_flags} --watch ../${PACKAGE}/dist" + for PACKAGE in $("$DIR/getInternalDependencies.sh" .); do + watch_flags+=" --watch ../$PACKAGE/dist" done # add the watch flags to the server start process, as well as a 1 second delay to debounce the # many restarts that otherwise happen during the initial build of internal dependencies - start_server="${start_server} --delay 1 ${watch_flags}" - ${CONCURRENTLY_BIN} "${DIR}/buildInternalDependencies.sh --watch --packagePath ." "eval ${start_server}" + start_server+=" --delay 1 $watch_flags" + "$CONCURRENTLY_BIN" "${DIR}/buildInternalDependencies.sh --watch --packagePath ." "eval ${start_server}" else echo -e "Starting server without internal dependency build and watch. To include internal dependencies, add the ${BOLD}-i${RESET} flag - it’s much faster than it used to be!" - eval ${start_server} + eval "$start_server" fi diff --git a/scripts/bash/buildInternalDependencies.sh b/scripts/bash/buildInternalDependencies.sh index db9e54f3bb..9fd3591739 100755 --- a/scripts/bash/buildInternalDependencies.sh +++ b/scripts/bash/buildInternalDependencies.sh @@ -40,9 +40,9 @@ build_commands=() build_prefixes=() # Build dependencies -for PACKAGE in $(${DIR}/getInternalDependencies.sh ${package_path}); do - build_commands+=("\"NODE_ENV=production yarn workspace @tupaia/${PACKAGE} build-dev $build_args\"") - build_prefixes+=("${PACKAGE},") +for PACKAGE in $("$DIR/getInternalDependencies.sh" "$package_path"); do + build_commands+=("\"NODE_ENV=production yarn workspace @tupaia/$PACKAGE build-dev $build_args\"") + build_prefixes+=("$PACKAGE,") done if [[ $watch = true ]]; then @@ -52,7 +52,7 @@ if [[ $watch = true ]]; then eval "${CONCURRENTLY_BIN} --names \"${build_prefixes[*]}\" ${build_commands[@]}" else echo -e "${BOLD}Concurrently building internal dependencies in batches of ${CONCURRENT_BUILD_BATCH_SIZE}${RESET}" - echo "> ${CONCURRENTLY_BIN} -m $CONCURRENT_BUILD_BATCH_SIZE --names \"${build_prefixes[*]}\" -k ${build_commands[*]}" + echo "> $CONCURRENTLY_BIN -m $CONCURRENT_BUILD_BATCH_SIZE --names \"${build_prefixes[*]}\" -k ${build_commands[*]}" echo - eval "${CONCURRENTLY_BIN} -m $CONCURRENT_BUILD_BATCH_SIZE --names \"${build_prefixes[*]}\" -k ${build_commands[*]}" + eval "$CONCURRENTLY_BIN -m $CONCURRENT_BUILD_BATCH_SIZE --names \"${build_prefixes[*]}\" -k ${build_commands[*]}" fi diff --git a/scripts/bash/downloadEnvironmentVariables.sh b/scripts/bash/downloadEnvironmentVariables.sh index eb49c73911..b46565e4ec 100755 --- a/scripts/bash/downloadEnvironmentVariables.sh +++ b/scripts/bash/downloadEnvironmentVariables.sh @@ -10,8 +10,8 @@ COLLECTION_PATH='Engineering/Tupaia General/Environment Variables' # Log in to bitwarden echo -e "${BLUE}==>️${RESET} ${BOLD}Logging into Bitwarden${RESET}" -bw login --check || bw login $BITWARDEN_EMAIL $BITWARDEN_PASSWORD -eval "$(bw unlock $BITWARDEN_PASSWORD | grep -o -m 1 'export BW_SESSION=.*$')" +bw login --check || bw login "$BITWARDEN_EMAIL" "$BITWARDEN_PASSWORD" +eval "$(bw unlock "$BITWARDEN_PASSWORD" | grep -o -m 1 'export BW_SESSION=.*$')" COLLECTION_ID=$(bw get collection "$COLLECTION_PATH" | jq .id) @@ -19,11 +19,11 @@ echo # Can provide one or more packages as command line arguments, or will default to all if [[ -z $2 ]]; then - PACKAGES=$(${DIR}/getPackagesWithEnvFiles.sh) + PACKAGES=$("$DIR/getPackagesWithEnvFiles.sh") echo -e "${BLUE}==>️${RESET} ${BOLD}Fetching environment variables for all packages${RESET}" else - PACKAGES=${@:2} - echo -e "${BLUE}==>️${RESET} ${BOLD}Fetching environment variables for ${PACKAGES}${RESET}" + PACKAGES=("${@:2}") + echo -e "${BLUE}==>️${RESET} ${BOLD}Fetching environment variables for ${PACKAGES[*]}${RESET}" fi @@ -31,28 +31,28 @@ load_env_file_from_bw () { FILE_NAME=$1 BASE_FILE_PATH=$2 NEW_FILE_NAME=$3 - ENV_FILE_PATH=${BASE_FILE_PATH}/${NEW_FILE_NAME}.env + ENV_FILE_PATH=$BASE_FILE_PATH/$NEW_FILE_NAME.env echo -en "${YELLOW}🚚 Fetching variables for ${BOLD}${FILE_NAME}...${RESET}" # checkout deployment specific env vars, or dev as fallback - DEPLOYMENT_ENV_VARS=$(bw list items --search ${FILE_NAME}.${DEPLOYMENT_NAME}.env | jq --raw-output "map(select(.collectionIds[] | contains ($COLLECTION_ID))) | .[] .notes") + DEPLOYMENT_ENV_VARS=$(bw list items --search "$FILE_NAME.$DEPLOYMENT_NAME.env" | jq --raw-output "map(select(.collectionIds[] | contains ($COLLECTION_ID))) | .[] .notes") if [[ -n $DEPLOYMENT_ENV_VARS ]]; then - echo "$DEPLOYMENT_ENV_VARS" > ${ENV_FILE_PATH} + echo "$DEPLOYMENT_ENV_VARS" > "$ENV_FILE_PATH" else - DEV_ENV_VARS=$(bw list items --search ${FILE_NAME}.dev.env | jq --raw-output "map(select(.collectionIds[] | contains ($COLLECTION_ID))) | .[] .notes") - echo "$DEV_ENV_VARS" > ${ENV_FILE_PATH} + DEV_ENV_VARS=$(bw list items --search "$FILE_NAME.dev.env" | jq --raw-output "map(select(.collectionIds[] | contains ($COLLECTION_ID))) | .[] .notes") + echo "$DEV_ENV_VARS" > "$ENV_FILE_PATH" fi # Replace any instances of the placeholder [deployment-name] in the .env file with the actual # deployment name (e.g. [deployment-name]-api.tupaia.org -> specific-deployment-api.tupaia.org) - sed -i -e "s/\[deployment-name\]/${DEPLOYMENT_NAME}/g" "${ENV_FILE_PATH}" + sed -i -e "s/\[deployment-name\]/$DEPLOYMENT_NAME/g" "$ENV_FILE_PATH" if [[ $DEPLOYMENT_NAME = *-e2e || $DEPLOYMENT_NAME = e2e ]]; then # Update e2e environment variables - sed -i -e 's/^AGGREGATION_URL_PREFIX="?dev-"?$/AGGREGATION_URL_PREFIX=e2e-/g' ${ENV_FILE_PATH} if [[ $FILE_NAME = aggregation ]]; then + sed -i -e 's/^AGGREGATION_URL_PREFIX="?dev-"?$/AGGREGATION_URL_PREFIX=e2e-/g' "$ENV_FILE_PATH" fi fi @@ -60,7 +60,7 @@ load_env_file_from_bw () { # Update dev specific environment variables # (removes ###DEV_ONLY### prefixes, leaving the key=value pair uncommented) # (after removing prefix, if there are duplicate keys, dotenv uses the last one in the file) - sed -i -e 's/^###DEV_ONLY###//g' ${ENV_FILE_PATH} + sed -i -e 's/^###DEV_ONLY###//g' "$ENV_FILE_PATH" fi echo -en "$CLEAR_LINE" @@ -68,19 +68,19 @@ load_env_file_from_bw () { } -for PACKAGE in $PACKAGES; do +for PACKAGE in "${PACKAGES[@]}"; do # Only download the env file if there is an example file in the package. If there isn’t, this # means it is a package that doesn’t need env vars - has_example_env_in_package=$(find $DIR/../../packages/$PACKAGE -type f -name '*.env.example' | wc -l) - load_env_file_from_bw $PACKAGE $DIR/../../packages/$PACKAGE "" + has_example_env_in_package=$(find "$DIR/../../packages/$PACKAGE" -type f -name '*.env.example' | wc -l) if (( has_example_env_in_package > 0 )); then + load_env_file_from_bw "$PACKAGE" "$DIR/../../packages/$PACKAGE" '' fi done for file_name in *.env.example; do env_name="${file_name%.env.example}" # Get its basename without the .env.example extension - load_env_file_from_bw $env_name $DIR/../../env $env_name + load_env_file_from_bw "$env_name" "$DIR/../../env" "$env_name" done diff --git a/scripts/bash/getInternalDependencies.sh b/scripts/bash/getInternalDependencies.sh index ced415f27a..d7941d02db 100755 --- a/scripts/bash/getInternalDependencies.sh +++ b/scripts/bash/getInternalDependencies.sh @@ -60,7 +60,7 @@ internal_dependencies=("${array_without_gaps[@]}") unset array_without_gaps # recursively build up an array of all internal dependencies this package depends on -for dependency in ${internal_dependencies[@]}; do +for dependency in "${internal_dependencies[@]}"; do nested_dependencies=($(${DIR}/getInternalDependencies.sh "${package_path}/../${dependency}" ${dependencies_already_visited[@]} ${internal_dependencies[@]} )) if (( ${#nested_dependencies[@]} = 0 )); then @@ -86,6 +86,6 @@ internal_dependencies=("${array_without_gaps[@]}") unset array_without_gaps # echo out result for calling script to pick up -echo ${internal_dependencies[@]} +echo "${internal_dependencies[@]}" exit 0 diff --git a/scripts/bash/pm2startInline.sh b/scripts/bash/pm2startInline.sh index 0fd7e8e10d..dffc6b2e43 100755 --- a/scripts/bash/pm2startInline.sh +++ b/scripts/bash/pm2startInline.sh @@ -8,7 +8,7 @@ if [[ -z $1 ]]; then echo -e "Usage: ${BOLD}yarn start-stack${RESET} ${UNDERLINE}stack${RESET}" echo echo -e "All ${UNDERLINE}stack${RESET}s:" - ls -1 ../../packages/devops/configs/pm2/ | sed 's|.config.js||g' | grep -v 'base' | awk '$0=" "$0' + ls -1 ../../packages/devops/configs/pm2/ | sed 's|.config.js||g' | grep -v 'base' | awk '$0=" "$0' echo echo "Tips:" echo -e " - Normal PM2 commands work e.g. ${BOLD}yarn pm2 status${RESET}" diff --git a/scripts/bash/validate.sh b/scripts/bash/validate.sh index 129087182a..137af302a0 100755 --- a/scripts/bash/validate.sh +++ b/scripts/bash/validate.sh @@ -2,7 +2,7 @@ set -e DIR=$(dirname "$0") -. ${DIR}/../../packages/devops/scripts/ci/utils.sh +. "$DIR/../../packages/devops/scripts/ci/utils.sh" yarn workspace @tupaia/devops validate-branch-name yarn workspace @tupaia/devops validate-tests From 090d23c9709566ad90df47cf56bac46e41a58ea8 Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Tue, 9 Apr 2024 10:36:58 +1200 Subject: [PATCH 22/24] prefer array in `mergeEnvForDB.sh` --- scripts/bash/mergeEnvForDB.sh | 36 +++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/scripts/bash/mergeEnvForDB.sh b/scripts/bash/mergeEnvForDB.sh index 38e5a3bcbe..138a9be173 100755 --- a/scripts/bash/mergeEnvForDB.sh +++ b/scripts/bash/mergeEnvForDB.sh @@ -21,30 +21,30 @@ CALLING_SCRIPT_DIR=$(get_caller_package_directory) CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # Fixed paths to the .env files for the test db -file1="$CURRENT_DIR/../../env/db.env" -file2="$CURRENT_DIR/../../env/pg.env" -file3="$CURRENT_DIR/../../env/data-lake.env" -file4="$CALLING_SCRIPT_DIR/.env" - -common_files="$file1 $file2 $file3 $file4" - - # Remove files that don't exist -for file in $common_files; do - if [[ ! -f $file ]]; then - common_files=$(echo "$common_files" | sed "s|$file||g") - fi -done - -# Load environment variables from .env files -merged_content="$(cat $common_files)" +common_files=( + './../../env/db.env' + './../../env/pg.env' + './../../env/data-lake.env' + './.env' +) + +# Load environment variables from .env files (if they exist) +merged_content=$( + for file in "${common_files[@]}"; do + if [[ -f $file ]]; then + cat "$file" + fi + done +) # Process command line arguments, overwriting values if present for var in $(env); do if [[ $var = *=* ]]; then key="${var%%=*}" value="${var#*=}" - # Override values from command line - merged_content+=" $key=\"$value\"" + # Override values from command line, with each variable on its own line + merged_content+=" +$key=\"$value\"" fi done From 8da68e4cdc8c4d436f4f050a772ed22c4c681a8b Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Tue, 9 Apr 2024 19:06:42 +1200 Subject: [PATCH 23/24] exit on error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit if `cd` fails in this script, then the script’s behaviour can become unpredictable --- packages/data-api/scripts/installMvRefreshModule.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/data-api/scripts/installMvRefreshModule.sh b/packages/data-api/scripts/installMvRefreshModule.sh index 15dcbcd9eb..ec40669458 100755 --- a/packages/data-api/scripts/installMvRefreshModule.sh +++ b/packages/data-api/scripts/installMvRefreshModule.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +set -e DIR=$(pwd "$0") . "$DIR/../../scripts/bash/mergeEnvForDB.sh" From bc26c0ef6626286df255cbc14b17166626747817 Mon Sep 17 00:00:00 2001 From: Jasper Lai <33956381+jaskfla@users.noreply.github.com> Date: Mon, 6 Jan 2025 14:36:15 +1300 Subject: [PATCH 24/24] remove unused variable --- scripts/bash/backendStartDev.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/bash/backendStartDev.sh b/scripts/bash/backendStartDev.sh index f99acc3999..68c0e9fe9c 100755 --- a/scripts/bash/backendStartDev.sh +++ b/scripts/bash/backendStartDev.sh @@ -10,7 +10,6 @@ DIR=$(dirname "$0") . "$DIR/ansiControlSequences.sh" USAGE="Usage: ${BOLD}backendStartDev babel_port_inspector${RESET} [${BOLD}-i${RESET}|${BOLD}--include-internal${RESET}] [${BOLD}-ts${RESET}|${BOLD}--typescript${RESET}]" -CONCURRENTLY_BIN="${DIR}/../../node_modules/.bin/concurrently" watch_flags='' include_internal=false type_script=false