diff --git a/.env.example b/.env.example index 3a86c41..8288728 100644 --- a/.env.example +++ b/.env.example @@ -1,11 +1,18 @@ +# YOU DO NOT NEED TO CREATE A .env FILE MANUALLY +# Terraform will automatically create a local .env file with the required environment variables when you run `terraform apply`. +# This file is just to illustrate the required environment variables for the function to work locally. + # Required for the function to be able to look up the Discord Webhook URL in GCP Secret Manager. -# Get it via `gcloud projects list --filter="name:governance-watchdog*" --format="value(projectId)")` +# You can check it manually via `gcloud projects list --filter="name:governance-watchdog*" --format="value(projectId)"` GCP_PROJECT_ID= -# Required for the function to be able to look up the Discord Webhook URL and Telegram Bot Token in GCP Secret Manager. -# Get it via `gcloud secrets list` +# Required for the function to be able to look up secrets in GCP Secret Manager. +# You can check it manually via `gcloud secrets list` DISCORD_WEBHOOK_URL_SECRET_ID= TELEGRAM_BOT_TOKEN_SECRET_ID= -# Get it via inviting @MissRose_bot to the telegram group and then using the `/id` command (please remove the bot after you're done) +# You can check it manually either via +# a) `terraform state show "google_cloudfunctions2_function.watchdog_notifications" | grep TELEGRAM_CHAT_ID | awk -F '= ' '{print $2}' | tr -d '"'` +# OR +# b) inviting @MissRose_bot to the telegram group and then using the `/id` command (please remove the bot after you're done) TELEGRAM_CHAT_ID= \ No newline at end of file diff --git a/.gitignore b/.gitignore index 6a71ca3..e684319 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,7 @@ errored.tfstate .env.yaml dist/ function-source.zip -node_modules/ \ No newline at end of file +node_modules/ + +# Local Stuff +.project_vars_cache \ No newline at end of file diff --git a/README.md b/README.md index 3758601..5b73adb 100644 --- a/README.md +++ b/README.md @@ -8,18 +8,20 @@ A monorepo for our governance watchdog, a system that monitors Mento Governance ![Architecture Diagram](arch-diagram.png) -- [Requirements](#requirements) -- [Local Development of Cloud Function Code](#local-development-of-cloud-function-code) +- [Requirements for local development](#requirements-for-local-development) +- [Local Infra Setup (when project is deployed already)](#local-infra-setup-when-project-is-deployed-already) +- [Running and testing the Cloud Function locally](#running-and-testing-the-cloud-function-locally) - [Testing the Deployed Cloud Function](#testing-the-deployed-cloud-function) -- [Infra Setup (when project is deployed already)](#infra-setup-when-project-is-deployed-already) -- [First Time Infra Deployment via Terraform](#first-time-infra-deployment-via-terraform) +- [Updating the Cloud Function](#updating-the-cloud-function) +- [Infra Deployment via Terraform](#infra-deployment-via-terraform) - [Google Cloud Permission Requirements](#google-cloud-permission-requirements) - [Deployment from scratch](#deployment-from-scratch) - [Migrate Terraform State to Google Cloud](#migrate-terraform-state-to-google-cloud) -- [Updating the Cloud Function](#updating-the-cloud-function) +- [Debugging Problems](#debugging-problems) + - [View Logs](#view-logs) - [Teardown](#teardown) -## Requirements +## Requirements for local development 1. Install the `gcloud` CLI @@ -51,6 +53,15 @@ A monorepo for our governance watchdog, a system that monitors Mento Governance # For other systems, see https://developer.hashicorp.com/terraform/install ``` +1. Install `jq` (used in a few shell scripts) + + ```sh + # On macOS + brew install jq + + # For other systems, see https://jqlang.github.io/jq/ + ``` + 1. Authenticate with Google Cloud default credentials in your local shell ```sh @@ -70,46 +81,8 @@ A monorepo for our governance watchdog, a system that monitors Mento Governance 1. A Telegram group to send notifications to 1. A Telegram bot must be in the group to receive the notifications. - If you're doing this from scratch, here's how to create a bot - - - Open a new chat with @BotFather - - Use the `/newbot` command to create a new bot - - Copy the API key printed out at the end of the prompt and store it in your `terraform.tfvars` - - ```hcl - telegram_bot_token = "" - ``` - - - Get the Chat ID by inviting @MissRose_bot to the group and then using the `/id` command - - Add the Chat ID to your `terraform.tfvars` - - ```hcl - telegram_chat_id = "" - ``` - - - Remove @MissRose_bot after you got the Chat ID - -## Local Development of Cloud Function Code - -- `npm install` (couldn't use `pnpm` because Google Cloud Build failed trying to install pnpm at the time of writing) -- `cp .env.example .env` and fill in the required values (there are comments in the `.env.example` explaining how to get them) -- `npm start` to start a local cloud function -- `npm test` to call the local cloud function with a mocked payload, this will send a real Discord message into channel belonging to the webhook in `.env`: - - ```sh - curl -H "Content-Type: application/json" -d @src/proposal-created.fixture.json localhost:8080 - ``` - -## Testing the Deployed Cloud Function - -You can test the deployed cloud function manually by using the `proposal-created.fixture.json` which contains a similar payload to what a QuickAlert would send to the cloud function: -```sh -./test-deployed-function.sh -# or `npm run test-in-prod` if you prefer npm to call this script -``` - -## Infra Setup (when project is deployed already) +## Local Infra Setup (when project is deployed already) 1. Set your local `gcloud` project to the watchdog project: @@ -124,11 +97,11 @@ You can test the deployed cloud function manually by using the `proposal-created terraform init ``` -1. Create a `terraform.tfvars` file in the `./infra` folder, this is like `.env` for Terraform: +1. While inside the `infra` folder, create `terraform.tfvars` file. This is like `.env` for Terraform: ```sh - touch ./infra/terraform.tfvars - # This file should be `.gitignore`d to avoid accidentally leaking sensitive data + touch terraform.tfvars + # This file is `.gitignore`d to avoid accidentally leaking sensitive data ``` 1. Add the following values to your `terraform.tfvars`, you can look up all values in the Google Cloud console (or ask another dev to share his local `terraform.tfvars` with you) @@ -147,24 +120,89 @@ You can test the deployed cloud function manually by using the `proposal-created group_billing_admins = "" ``` -1. Add the Discord Webhook URL from Google Cloud Secret Manager into your local `terraform.tfvars`: +1. Add the Discord Webhook URL from Google Cloud Secret Manager to your local `terraform.tfvars`: ```sh - # You will need the "Secret Manager Secret Accessor" IAM role for this command to succeed + # You need the "Secret Manager Secret Accessor" IAM role for this command to succeed echo "discord_webhook_url = \"$(gcloud secrets versions access latest --secret discord-webhook-url)\"" >> terraform.tfvars ``` +1. Add the Telegram Bot Token and Chat ID to your local `terraform.tfvars` + + ```sh + # Get the chat ID from cloud function's terraform state + echo "\ntelegram_chat_id = \"$(terraform state show "google_cloudfunctions2_function.watchdog_notifications" | grep TELEGRAM_CHAT_ID | awk -F '= ' '{print $2}' | tr -d '"')\"" >> terraform.tfvars + + # Get the bot token from secret manager (you need the "Secret Manager Secret Accessor" IAM role for this command to succeed) + echo "telegram_bot_token = \"$(gcloud secrets versions access latest --secret telegram-bot-token)\"" >> terraform.tfvars + ``` + 1. [Get our QuickNode API key from the QuickNode dashboard](https://dashboard.quicknode.com/api-keys) and add it to your local `terraform.tfvars`: ```sh # ./infra/terraform.tfvars - discord_webhook_url = "" quicknode_api_key = "" ``` This is necessary for Terraform to be able to create & destroy QuickAlerts as part of `terraform apply` -## First Time Infra Deployment via Terraform +1. Get the VictorOps Webhook URL to your local `terraform.tfvars`. You can get it by going to VictorOps and clicking `Integrations` > `Stackdriver` and copying the URL. The routing key can be founder under the `Settings` tab: + + ```sh + # ./infra/terraform.tfvars + victorops_webhook_url = "/" + ``` + +1. Auto-generate a local `.env` file by running `npm run generate:env` + +## Running and testing the Cloud Function locally + +- Make sure you generated a local `.env` file via `npm run generate:env` earlier +- `npm install` (couldn't use `pnpm` because Google Cloud Build failed trying to install pnpm at the time of writing) +- `npm start` to start a local cloud function +- `npm test` to call the local cloud function with a mocked payload, this will send a real Discord message into the channel belonging to the configured Discord Webhook: + + ```sh + curl -H "Content-Type: application/json" -d @src/proposal-created.fixture.json localhost:8080 + ``` + +## Testing the Deployed Cloud Function + +You can test the deployed cloud function manually by using the `proposal-created.fixture.json` which contains a similar payload to what a QuickAlert would send to the cloud function: + +```sh +./test-deployed-function.sh +# or `npm run test:prod` if you prefer npm to call this script +``` + +## Updating the Cloud Function + +You have two options, using `terraform` or the `gcloud` cli. Both are perfectly fine to use. + +1. Via `terraform` by running `npm run deploy:via:tf` + - How? The npm task will: + - Call `terraform apply` which re-deploys the function with the latest code from your local machine + - Pros + - Keeps the terraform state clean + - Same command for all changes, regardless of infra or cloud function code + - Cons + - Less familiar way of deploying cloud functions (if you're used to `gcloud functions deploy`) + - Less log output + - Slightly slower because `terraform apply` will always fetch the current state from the cloud storage bucket before deploying +2. Via `gcloud` by running `npm run deploy:via:gcloud` + - How? The npm task will: + - Look up the service account used by the cloud function + - Call `gcloud functions deploy` with the correct parameters + - Pros + - Familiar way of deploying cloud functions + - More log output making deployment failures slightly faster to debug + - Slightly faster because we're skipping the terraform state lookup + - Cons + - Will lead to inconsistent terraform state (because terraform is tracking the function source code and its version) + - Different commands to remember when updating infra components vs cloud function source code + - Will only work for updating a pre-existing cloud function's code, will fail for a first-time deploy + +## Infra Deployment via Terraform ### Google Cloud Permission Requirements @@ -176,28 +214,47 @@ In order to create this project from scratch using the [terraform-google-bootstr ### Deployment from scratch +1. Outcomment the `backend` section in `main.tf` (because this bucket doesn't exist yet, it will be created by the first `terraform apply` run) + + ```hcl + # backend "gcs" { + # bucket = "governance-watchdog-terraform-state-" + # } + ``` + +1. Run `terraform init` to install the required providers and init a temporary local backend in a `terraform.tfstate` file + 1. [Create a Discord Webhook URL](https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks) for the channel you want to receive notifications in -2. Add the Discord Webhook URL to your local `terraform.tfvars`: +1. Add the Discord Webhook URL to your local `terraform.tfvars`: ```sh # This will be stored in Google Secret Manager upon deployment via Terraform echo "discord_webhook_url = \""" >> terraform.tfvars ``` -3. Outcomment the `backend` section in `main.tf` (because this bucket doesn't exist yet, it will be created by the first `terraform apply` run) +1. Create a Telegram group and invite a new bot into it - ```hcl - # backend "gcs" { - # bucket = "governance-watchdog-terraform-state-" - # } - ``` + - Open a new telegram chat with @BotFather + - Use the `/newbot` command to create a new bot + - Copy the API key printed out at the end of the prompt and store it in your `terraform.tfvars` + + ```hcl + telegram_bot_token = "" + ``` + + - Get the Chat ID by inviting @MissRose_bot to the group and then using the `/id` command + - Add the Chat ID to your `terraform.tfvars` + + ```hcl + telegram_chat_id = "" + ``` -4. Run `terraform init` to install the required providers and init a temporary local backend in a `terraform.tfstate` file + - Remove @MissRose_bot after you got the Chat ID -5. **Deploy the entire project via `terraform apply`** +1. **Deploy the entire project via `terraform apply`** - You will see an overview of all resources to be created. Review them if you like and then type "Yes" to confirm. - This command can take up to 10 minutes because it does a lot of work creating and configuring all defined Google Cloud Resources @@ -206,21 +263,17 @@ In order to create this project from scratch using the [terraform-google-bootstr **Often a simple retry of `terraform apply` helps**. Sometimes a dependency of a resource has simply not finished creating when terraform already tried to deploy the next one, so waiting a few minutes for things to settle can help. -6. Set your local `gcloud` project to our freshly created one: +1. Set your local `gcloud` project ID to our freshly created one: ```sh - # If that `awk` magic fails, just look up the project ID manually via `gcloud projects list` - project_id=$(terraform state show "module.bootstrap.module.seed_project.module.project-factory.google_project.main" | grep 'project_id' | awk -F '"' '{print $2}') - - gcloud config set project $project_id - gcloud auth application-default set-quota-project $project_id + ./set-project-id.sh ``` -7. Check that everything worked as expected +1. Check that everything worked as expected ```sh # 1. Call the deployed function via: - npm run test-in-prod # or call the script directly via ./test-deployed-function.sh + npm run test:prod # or call the script directly via ./test-deployed-function.sh # 2. Monitor the configured Discord channel for a message to appear open https://discord.com/channels/966739027782955068/1262714272476037212 @@ -271,34 +324,14 @@ For all team members to be able to manage the Google Cloud infrastructure, you n rm terraform.tfstate.backup ``` -## Updating the Cloud Function +## Debugging Problems -You have two options, using `terraform` or the `gcloud` cli. Both are perfectly fine to use. +### View Logs -1. Via `terraform` by running `npm run deploy:via:tf` - - How? The npm task will: - - Compile TS to JS - - Zip the `./dist` folder into `function-source.zip` - - And then call `terraform apply` which re-deploys the function with the new source code from the zip file - - Pros - - Keeps the terraform state clean - - Same command for all changes, regardless of infra or cloud function code - - Cons - - Less familiar way of deploying cloud functions (if you're used to `gcloud functions deploy`) - - Less log output - - Slightly slower because `terraform apply` will always fetch the current state from the cloud storage bucket before deploying -2. Via `gcloud` by running `npm run deploy:via:gcloud` - - How? The npm task will: - - Generate a temporary `.env.yaml` (because for some reason gcloud does not support normal `.env` files) - - Look up the service account used by the cloud function - - Call `gcloud functions deploy` with the correct parameters - - Pros - - Familiar way of deploying cloud functions - - More log output making deployment failures slightly faster to debug - - Slightly faster because we're skipping the terraform state lookup - - Cons - - Will lead to inconsistent terraform state (because terraform is tracking the function source code and its version) - - Different commands to remember when updating infra components vs cloud function source code +For most problems, you'll likely want to check the cloud function logs first. + +- `npm run logs` will print the latest 50 log entries into your local terminal for quick and easy access +- `npm run logs:url` will print the URL to the function logs in the Google Cloud Console for full access ## Teardown diff --git a/deploy-via-gcloud.sh b/deploy-via-gcloud.sh index 0f2529e..8208f8e 100755 --- a/deploy-via-gcloud.sh +++ b/deploy-via-gcloud.sh @@ -1,19 +1,14 @@ -#! /bin/bash -set -e # fail on any error -set -o pipefail # ensure non-zero exit codes are propagated in piped commands +#!/bin/bash +set -e # Fail on any error +set -o pipefail # Ensure piped commands propagate exit codes properly +set -u # Treat unset variables as an error when substituting -entry_point="watchdogNotifier" -function_name="watchdog-notifications" -region="europe-west1" +# Load the project variables +source ./set-project-vars.sh -printf "Looking up function name..." -function_name=$(gcloud functions list --format="value(name)" | grep '^watchdog-notifications') -printf ' \033[1m%s\033[0m\n' "${function_name}" - -printf "Looking up project ID..." -project_name="governance-watchdog" -project_id=$(gcloud projects list --filter="name:${project_name}*" --format="value(projectId)") -printf ' \033[1m%s\033[0m\n' "${project_id}" +printf "Looking up entry point..." +entry_point=$(gcloud functions describe "${function_name}" --region="${region}" --format json | jq .buildConfig.entryPoint) +printf ' \033[1m%s\033[0m\n' "${entry_point}" printf "Looking up service account for function..." service_account_email=$(gcloud functions describe "${function_name}" --region="${region}" --format="value(serviceConfig.serviceAccountEmail)") diff --git a/get-logs-url.sh b/get-logs-url.sh index 64ff39d..f034a3f 100755 --- a/get-logs-url.sh +++ b/get-logs-url.sh @@ -1,8 +1,10 @@ #!/bin/bash -set -e # fail on any error -set -o pipefail # ensure non-zero exit codes are propagated in piped commands +set -e # Fail on any error +set -o pipefail # Ensure piped commands propagate exit codes properly +set -u # Treat unset variables as an error when substituting -project_id=$(gcloud config get-value project) -region=europe-west1 -function_name=watchdog-notifications -echo "https://console.cloud.google.com/functions/details/${region}/${function_name}?project=${project_id}&tab=logs " +# Load the project variables +source ./set-project-vars.sh + +logs_url="https://console.cloud.google.com/functions/details/${region}/${function_name}?project=${project_id}&tab=logs " +printf '\n\033[1m%s\033[0m\n' "${logs_url}" diff --git a/get-logs.sh b/get-logs.sh index 92948ba..435b2c1 100755 --- a/get-logs.sh +++ b/get-logs.sh @@ -1,17 +1,20 @@ #!/bin/bash -set -e # fail on any error -set -o pipefail # ensure non-zero exit codes are propagated in piped commands +set -e # Fail on any error +set -o pipefail # Ensure piped commands propagate exit codes properly +set -u # Treat unset variables as an error when substituting -# Get the name of the watchdog-notifications function -function_name=$(gcloud functions list --format="value(name)" | grep '^watchdog-notifications') +# Load the project variables +source ./set-project-vars.sh # Fetch raw logs raw_logs=$(gcloud functions logs read "${function_name}" \ - --region europe-west1 \ + --region "${region}" \ --format json \ + --limit 50 \ --sort-by TIME_UTC) # Format logs +printf "\n\n" echo "${raw_logs}" | jq -r '.[] | if .level == "E" then "\u001b[31m[\(.level)]\u001b[0m \u001b[33m\(.time_utc)\u001b[0m: \(.log)" else diff --git a/infra/cloud_function.tf b/infra/cloud_function.tf index de6a4e9..037e17a 100644 --- a/infra/cloud_function.tf +++ b/infra/cloud_function.tf @@ -1,14 +1,12 @@ resource "google_cloudfunctions2_function" "watchdog_notifications" { project = module.bootstrap.seed_project_id location = var.region - name = "watchdog-notifications" + name = var.function_name description = "A cloud function that receives blockchain event data from QuickAlerts and sends notifications to a Discord channel" build_config { - runtime = "nodejs20" - entry_point = "watchdogNotifier" - - # Use the service account created by terraform for the project (not the default Compute Engine service account) + runtime = "nodejs20" + entry_point = var.function_entry_point service_account = "projects/${module.bootstrap.seed_project_id}/serviceAccounts/${module.bootstrap.terraform_sa_email}" source { @@ -20,11 +18,9 @@ resource "google_cloudfunctions2_function" "watchdog_notifications" { } service_config { - available_memory = "256M" - timeout_seconds = 60 - - # Use the service account created by terraform for the project (not the default Compute Engine service account) + available_memory = "256M" service_account_email = module.bootstrap.terraform_sa_email + timeout_seconds = 60 # šŸ”’ Security Note: Checkov recommends to only allow this function to be called from a cloud load balancer. # We're making a conscious security tradeoff here for lower complexity and faster delivery. It seems unlikely diff --git a/infra/local-dotenv-file.tf b/infra/local-dotenv-file.tf new file mode 100644 index 0000000..edde379 --- /dev/null +++ b/infra/local-dotenv-file.tf @@ -0,0 +1,9 @@ +resource "local_file" "env_file" { + filename = "${path.module}/../.env" + content = <<-EOT + GCP_PROJECT_ID=${module.bootstrap.seed_project_id} + DISCORD_WEBHOOK_URL_SECRET_ID=${var.discord_webhook_url_secret_id} + TELEGRAM_BOT_TOKEN_SECRET_ID=${var.telegram_bot_token_secret_id} + TELEGRAM_CHAT_ID=${var.telegram_chat_id} + EOT +} diff --git a/infra/main.tf b/infra/main.tf index 68f0c9c..83cb35b 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -10,6 +10,10 @@ terraform { source = "hashicorp/archive" version = ">= 2.4.2" } + local = { + source = "hashicorp/local" + version = ">= 2.5.1" + } quicknode = { source = "jmtx1020/quicknode" version = "0.0.2" diff --git a/infra/secret-manager.tf b/infra/secret-manager.tf index ba0af8c..37971c5 100644 --- a/infra/secret-manager.tf +++ b/infra/secret-manager.tf @@ -3,7 +3,7 @@ # and if it can't find it locally it will prompt the user to enter it manually. resource "google_secret_manager_secret" "discord_webhook_url" { project = module.bootstrap.seed_project_id - secret_id = "discord-webhook-url" + secret_id = var.discord_webhook_url_secret_id replication { auto {} @@ -20,7 +20,7 @@ resource "google_secret_manager_secret_version" "discord_webhook_url" { # and if it can't find it locally it will prompt the user to enter it manually. resource "google_secret_manager_secret" "telegram_bot_token" { project = module.bootstrap.seed_project_id - secret_id = "telegram-bot-token" + secret_id = var.telegram_bot_token_secret_id replication { auto {} diff --git a/infra/variables.tf b/infra/variables.tf index 50fbf15..410067f 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -31,6 +31,13 @@ variable "group_billing_admins" { type = string } +# You can look this up via: +# `gcloud secrets list` +variable "discord_webhook_url_secret_id" { + type = string + default = "discord-webhook-url" +} + # You can look this up either on the Discord Channel settings, or fetch it from Secret Manager via: # `gcloud secrets versions access latest --secret discord-webhook-url` variable "discord_webhook_url" { @@ -43,6 +50,13 @@ variable "telegram_chat_id" { type = string } +# You can look this up via: +# `gcloud secrets list` +variable "telegram_bot_token_secret_id" { + type = string + default = "telegram-bot-token" +} + # You can look this up via: # `gcloud secrets versions access latest --secret telegram-bot-token` variable "telegram_bot_token" { @@ -56,6 +70,16 @@ variable "quicknode_api_key" { sensitive = true } +variable "function_name" { + type = string + default = "watchdog-notifications" +} + +variable "function_entry_point" { + type = string + default = "watchdogNotifier" +} + # Webhook URL to send monitoring alerts from within GCP Monitoring # You can find this URL in Victorops by going to "Integrations" -> "Stackdriver". # The routing key can be found under "Settings" -> "Routing Keys" diff --git a/package.json b/package.json index 35d120a..f498244 100644 --- a/package.json +++ b/package.json @@ -3,22 +3,26 @@ "version": "1.0.0", "description": "", "license": "ISC", - "author": "Philip Paetz ", + "author": "Mento Labs ", "main": "dist/index.js", "scripts": { "build": "rimraf dist && tsc", + "cache:clear": "./set-project-vars.sh --no-cache", "deploy": "npm run deploy:via:tf", "deploy:via:gcloud": "./deploy-via-gcloud.sh", "deploy:via:terraform": "npm run deploy:via:tf", "deploy:via:tf": "terraform -chdir=infra apply", "gcp-build": "npm run build", + "generate:env": "terraform -chdir=infra apply -target=local_file.env_file", "logs": "./get-logs.sh", "logs:url": "./get-logs-url.sh", "prestart": "npm run build", "start": "NODE_ENV=development functions-framework --target=watchdogNotifier", - "test": "curl -H \"Content-Type: application/json\" -d @src/proposal-created.fixture.json localhost:8080", - "test-in-prod": "./test-deployed-function.sh", - "test:healthcheck": "curl -H \"Content-Type: application/json\" -d @src/health-check.fixture.json localhost:8080" + "test": "npm run test:local", + "test:healthcheck": "curl -H \"Content-Type: application/json\" -d @src/health-check.fixture.json localhost:8080", + "test:local": "curl -H \"Content-Type: application/json\" -d @src/proposal-created.fixture.json localhost:8080", + "test:prod": "./test-deployed-function.sh", + "todo": "git ls-files -c --exclude-standard | grep -v \"package.json\" | xargs grep -n -i --color \"TODO:\\|FIXME:\"" }, "dependencies": { "@google-cloud/functions-framework": "^3.4.0", diff --git a/set-project-id.sh b/set-project-id.sh index b653e7b..a107559 100755 --- a/set-project-id.sh +++ b/set-project-id.sh @@ -1,23 +1,50 @@ -#! /bin/bash -set -e # fail on any error -set -o pipefail # ensure non-zero exit codes are propagated in piped commands +#!/bin/bash +set -e # Fail on any error +set -o pipefail # Ensure piped commands propagate exit codes properly +set -u # Treat unset variables as an error when substituting -# Get the project ID for the governance-watchdog project -printf "Fetching the project ID for the governance-watchdog project:" -project_id=$(gcloud projects list --filter="name:governance-watchdog" --format="value(projectId)") +printf "Looking up project name in variables.tf..." +project_name=$(awk '/variable "project_name"/{f=1} f==1&&/default/{print $3; exit}' ./infra/variables.tf | tr -d '",') +printf ' \033[1m%s\033[0m\n' "${project_name}" + +printf "Fetching the project ID..." +project_id=$(gcloud projects list --filter="name:${project_name}" --format="value(projectId)") printf ' \033[1m%s\033[0m\n' "${project_id}" -# Set your local default project to the governance-watchdog project -echo "Setting your default project to '${project_id}'..." -gcloud config set project "${project_id}" +# Set your local default project +printf "Setting your default project to %s..." "${project_id}" +{ + output=$(gcloud config set project "${project_id}" 2>&1 >/dev/null) + status=$? +} +if [[ ${status} -ne 0 ]]; then + echo "Error: ${output}" + exit "${status}" +fi +printf "āœ…\n" # Set the quota project to the governance-watchdog project, some gcloud commands require this to be set -echo "Setting the quota project to '${project_id}'..." -gcloud auth application-default set-quota-project "${project_id}" +printf "Setting the quota project to %s..." "${project_id}" +{ + output=$(gcloud auth application-default set-quota-project "${project_id}" 2>&1 >/dev/null) + status=$? +} +if [[ ${status} -ne 0 ]]; then + echo "Error: ${output}" + exit "${status}" +fi +printf "āœ…\n" # Update the project ID in your .env file so your cloud function points to the correct project when running locally -printf "\n\nUpdating the project ID in your .env file..." -sed -i '' "s/^GCP_PROJECT_ID=.*/GCP_PROJECT_ID=${project_id}/" .env +printf "Updating the project ID in your .env file..." +# Check if .env file exists +if [[ ! -f .env ]]; then + # If .env doesn't exist, create it with the initial value + echo "GCP_PROJECT_ID=${project_id}" >.env +else + # If .env exists, perform the sed replacement + sed -i '' "s/^GCP_PROJECT_ID=.*/GCP_PROJECT_ID=${project_id}/" .env +fi +printf "āœ…\n\n" -printf "\n\nāœ… All Done!" -exit 0 +echo "āœ… All Done!" diff --git a/set-project-vars.sh b/set-project-vars.sh new file mode 100755 index 0000000..1fc1709 --- /dev/null +++ b/set-project-vars.sh @@ -0,0 +1,79 @@ +#!/bin/bash +set -e # Fail on any error +set -o pipefail # Ensure piped commands propagate exit codes properly +set -u # Treat unset variables as an error when substituting + +cache_file=".project_vars_cache" +current_local_project_id=$(gcloud config get project) +current_tf_state_project_id=$(terraform -chdir=infra state show module.bootstrap.module.seed_project.module.project-factory.google_project.main | grep project_id | awk '{print $3}' | tr -d '"') + +if [[ ${current_local_project_id} != "${current_tf_state_project_id}" ]]; then + printf 'ļø\nšŸšØ Your local gcloud is set to the wrong project: \033[1m%s\033[0m šŸšØ\n' "${current_local_project_id}" + printf "\nRunning ./set-project-id.sh in an attempt to fix this...\n\n" + source ./set-project-id.sh + printf "\n\n" +fi + +cache_vars() { + if [[ $* != *"--no-cache"* ]]; then + printf "No cache file found at %s.\n\n" "${cache_file}" + fi + + printf "Loading and caching project values...\n\n" + + printf " - Project ID:" + project_id=${current_tf_state_project_id} + printf ' \033[1m%s\033[0m\n' "${project_id}" + + printf " - Project Name:" + project_name=$(awk '/variable "project_name"/{f=1} f==1&&/default/{print $3; exit}' ./infra/variables.tf | tr -d '",') + printf ' \033[1m%s\033[0m\n' "${project_name}" + + printf " - Region:" + region=$(awk '/variable "region"/{f=1} f==1&&/default/{print $3; exit}' ./infra/variables.tf | tr -d '",') + printf ' \033[1m%s\033[0m\n' "${region}" + + printf " - Service Account:" + service_account_email=$(terraform -chdir=infra state show "module.bootstrap.google_service_account.org_terraform[0]" | grep email | awk '{print $3}' | tr -d '"') + printf ' \033[1m%s\033[0m\n' "${service_account_email}" + + printf " - Function Name:" + function_name=$(awk '/variable "function_name"/{f=1} f==1&&/default/{print $3; exit}' ./infra/variables.tf | tr -d '",') + printf ' \033[1m%s\033[0m\n' "${function_name}" + + printf " - Function Entry Point:" + function_entry_point=$(awk '/variable "function_entry_point"/{f=1} f==1&&/default/{print $3; exit}' ./infra/variables.tf | tr -d '",') + printf ' \033[1m%s\033[0m\n' "${function_entry_point}" + + printf "\nCaching values in" + printf ' \033[1m%s\033[0m...' "${cache_file}" + + { + echo "project_id=${project_id}" + echo "project_name=${project_name}" + echo "region=${region}" + echo "service_account_email=${service_account_email}" + echo "function_name=${function_name}" + echo "function_entry_point=${function_entry_point}" + } >>"${cache_file}" + printf "āœ…\n\n" +} + +if [[ $* == *"--no-cache"* ]]; then + echo "Invalidating cache..." + rm -f "${cache_file}" + cache_vars --no-cache +elif [[ ! -f ${cache_file} ]]; then + cache_vars +else + # shellcheck disable=SC1090 + source "${cache_file}" + printf "Using cached values:\n" + printf " - Project ID: \033[1m%s\033[0m\n" "${project_id}" + printf " - Project Name: \033[1m%s\033[0m\n" "${project_name}" + printf " - Region: \033[1m%s\033[0m\n" "${region}" + printf " - Service Account: \033[1m%s\033[0m\n" "${service_account_email}" + printf " - Function Name: \033[1m%s\033[0m\n" "${function_name}" + printf " - Function Entry Point: \033[1m%s\033[0m\n" "${function_entry_point}" + printf "\n" +fi diff --git a/test-deployed-function.sh b/test-deployed-function.sh index 9403ea4..a159d5f 100755 --- a/test-deployed-function.sh +++ b/test-deployed-function.sh @@ -1,6 +1,9 @@ -#! /bin/bash -# This only works if the function has been deployed and your `terraform` can access the state backend +#!/bin/bash +set -e # Fail on any error +set -o pipefail # Ensure piped commands propagate exit codes properly +set -u # Treat unset variables as an error when substituting +# This only works if the function has been deployed and your `terraform` can access the state backend raw_function_url=$(terraform -chdir=infra output -json function_uri) function_url=$(echo "${raw_function_url}" | jq -r)