From ec974536cd15b25c3168fdd2d7fe3d5ed770e411 Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Thu, 3 Oct 2024 08:57:45 +0200 Subject: [PATCH 1/6] CI: add simple Azure Login workflow We currently depend on the azure/login GitHub Action in various places. It simply runs an "az login" and "az account set" behind the scenes, and has support for plenty of other scenarios. As we're moving away from azure/cli in more and more places, we could also replace the login script with our own. Let's create a Composite Workflow to achieve this goal. Signed-off-by: Dennis Ameling --- .github/workflows/azure-login/action.yml | 43 ++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/azure-login/action.yml diff --git a/.github/workflows/azure-login/action.yml b/.github/workflows/azure-login/action.yml new file mode 100644 index 00000000..01408fd9 --- /dev/null +++ b/.github/workflows/azure-login/action.yml @@ -0,0 +1,43 @@ +name: Azure Login +description: Logs into Azure using a service principal +inputs: + credentials: + description: Your credentials in JSON format + required: true + +runs: + using: "composite" + steps: + - name: Process Azure credentials + uses: actions/github-script@v7 + env: + AZURE_CREDENTIALS: ${{ inputs.credentials }} + with: + script: | + if (!process.env.AZURE_CREDENTIALS) { + core.setFailed('The AZURE_CREDENTIALS secret is required.') + process.exit(1) + } + + const azureCredentials = JSON.parse(process.env.AZURE_CREDENTIALS) + const {clientId, clientSecret, tenantId, subscriptionId} = azureCredentials + + core.setSecret(clientId) + core.exportVariable('AZURE_CLIENT_ID', clientId) + + core.setSecret(clientSecret) + core.exportVariable('AZURE_CLIENT_SECRET', clientSecret) + + core.setSecret(tenantId) + core.exportVariable('AZURE_TENANT_ID', tenantId) + + core.setSecret(subscriptionId) + core.exportVariable('AZURE_SUBSCRIPTION_ID', subscriptionId) + + - name: Azure Login + shell: bash + run: | + echo "Logging into Azure..." + az login --service-principal -u ${{ env.AZURE_CLIENT_ID }} -p ${{ env.AZURE_CLIENT_SECRET }} --tenant ${{ env.AZURE_TENANT_ID }} + echo "Setting subscription..." + az account set --subscription ${{ env.AZURE_SUBSCRIPTION_ID }} --output none From 65c3a8ecd31f5a3ca52b24987ad9c9aeeff71b9c Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Thu, 3 Oct 2024 09:00:29 +0200 Subject: [PATCH 2/6] Create self-hosted runners: switch Azure login logic to our own We now have our own simple workflow to log into Azure, using "az login". As we're moving way from things like the Azure CLI, let's also start using our own login script instead. Signed-off-by: Dennis Ameling --- .github/workflows/create-azure-self-hosted-runners.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/create-azure-self-hosted-runners.yml b/.github/workflows/create-azure-self-hosted-runners.yml index 08cf34af..a113fe77 100644 --- a/.github/workflows/create-azure-self-hosted-runners.yml +++ b/.github/workflows/create-azure-self-hosted-runners.yml @@ -144,11 +144,11 @@ jobs: ) echo "AZURE_ARM_PARAMETERS=$AZURE_ARM_PARAMETERS" >> $GITHUB_ENV - + - name: Azure Login - uses: azure/login@v2 + uses: ./.github/workflows/azure-login with: - creds: ${{ secrets.AZURE_CREDENTIALS }} + credentials: ${{ secrets.AZURE_CREDENTIALS }} - uses: azure/arm-deploy@v2 id: deploy-arm-template From 71aa60ef3dbd0ec2a18d55696e3f2a0e55a3ee1d Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Thu, 3 Oct 2024 09:02:08 +0200 Subject: [PATCH 3/6] Azure self-hosted runners: show logs on failure We're seeing intermittent `OSProvisioningTimedOut` errors on VM creation, which typically happens within 20-40 mins of the deployment hanging. Let's try and find out what's going wrong, so we can see if we need to fix things on our end. Ref: https://github.com/git-for-windows/git-for-windows-automation/actions/runs/11025772033/job/30621189230 Signed-off-by: Dennis Ameling --- .../create-azure-self-hosted-runners.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/create-azure-self-hosted-runners.yml b/.github/workflows/create-azure-self-hosted-runners.yml index a113fe77..9fbbd40e 100644 --- a/.github/workflows/create-azure-self-hosted-runners.yml +++ b/.github/workflows/create-azure-self-hosted-runners.yml @@ -158,6 +158,22 @@ jobs: template: ./azure-self-hosted-runners/azure-arm-template.json parameters: ./azure-self-hosted-runners/azure-arm-template-example-parameters.json ${{ env.AZURE_ARM_PARAMETERS }} scope: resourcegroup + + - name: Show some more information on failure + if: failure() + run: | + echo "::group::VM status" + az vm get-instance-view --resource-group ${{ secrets.AZURE_RESOURCE_GROUP }} --name ${{ steps.generate-vm-name.outputs.vm_name }} --query "instanceView.statuses" + az vm get-instance-view --resource-group ${{ secrets.AZURE_RESOURCE_GROUP }} --name ${{ steps.generate-vm-name.outputs.vm_name }} --query "statuses" + echo "::endgroup::" + + echo "::group::Deployment logs" + az group deployment show --resource-group ${{ secrets.AZURE_RESOURCE_GROUP }} --name deploy-${{ steps.generate-vm-name.outputs.vm_name }} + echo "::endgroup::" + + echo "::group::Extension logs" + az vm extension show --resource-group ${{ secrets.AZURE_RESOURCE_GROUP }} --vm-name ${{ steps.generate-vm-name.outputs.vm_name }} --name CustomScriptExtension + echo "::endgroup::" - name: Show post-deployment script output env: From 117d8bdae2d14ed346b50dbf9b2244acf6ae4245 Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Thu, 3 Oct 2024 09:03:53 +0200 Subject: [PATCH 4/6] Azure self-hosted runner: always show post-deployment script output Even if the deployment of a runner failed, we'd like to see the post-deployment script output. Maybe something failed there. This is under the assumption that the azure/arm-deploy action will output the information, even if the deployment failed. Let's see if it works. Signed-off-by: Dennis Ameling --- .github/workflows/create-azure-self-hosted-runners.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/create-azure-self-hosted-runners.yml b/.github/workflows/create-azure-self-hosted-runners.yml index 9fbbd40e..91104f8f 100644 --- a/.github/workflows/create-azure-self-hosted-runners.yml +++ b/.github/workflows/create-azure-self-hosted-runners.yml @@ -176,6 +176,7 @@ jobs: echo "::endgroup::" - name: Show post-deployment script output + if: always() env: CUSTOM_SCRIPT_OUTPUT: ${{ steps.deploy-arm-template.outputs.customScriptInstanceView }} run: echo "$CUSTOM_SCRIPT_OUTPUT" | jq -r '.substatuses[0].message' From 582bde81343d7d62a08409822b0f0e2cba2e3b37 Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Thu, 3 Oct 2024 09:05:01 +0200 Subject: [PATCH 5/6] Azure self-hosted runners: stop using Azure CLI The azure/cli action runs in a Docker container that only has a very limited subset of tools compared to the GitHub Actions runners. Luckily, the Azure CLI comes pre-installed with Hosted Runners. Let's use use it directly, so we can enjoy all the other tools that it offers. Signed-off-by: Dennis Ameling --- .github/workflows/create-azure-self-hosted-runners.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/create-azure-self-hosted-runners.yml b/.github/workflows/create-azure-self-hosted-runners.yml index 91104f8f..62bb5539 100644 --- a/.github/workflows/create-azure-self-hosted-runners.yml +++ b/.github/workflows/create-azure-self-hosted-runners.yml @@ -183,8 +183,4 @@ jobs: - name: Deallocate the VM for later use if: env.DEALLOCATE_IMMEDIATELY == 'true' - uses: azure/CLI@v2 - with: - azcliversion: 2.64.0 - inlineScript: | - az vm deallocate -n ${{ steps.generate-vm-name.outputs.vm_name }} -g ${{ secrets.AZURE_RESOURCE_GROUP }} --verbose + run: az vm deallocate -n ${{ steps.generate-vm-name.outputs.vm_name }} -g ${{ secrets.AZURE_RESOURCE_GROUP }} --verbose From ab3fad10e91c9ccac2e5611bd07bbb286aa9f62e Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Thu, 3 Oct 2024 09:06:13 +0200 Subject: [PATCH 6/6] Cleanup self-hosted runners: switch Azure login logic to our own We now have our own simple workflow to log into Azure, using "az login". As we're moving way from things like the Azure CLI, let's also start using our own login script instead. Signed-off-by: Dennis Ameling --- .../workflows/cleanup-self-hosted-runners.yml | 28 ++----------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/.github/workflows/cleanup-self-hosted-runners.yml b/.github/workflows/cleanup-self-hosted-runners.yml index 320e1940..aa8aff4b 100644 --- a/.github/workflows/cleanup-self-hosted-runners.yml +++ b/.github/workflows/cleanup-self-hosted-runners.yml @@ -21,38 +21,16 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Process Azure credentials - uses: actions/github-script@v7 - env: - AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} + - name: Azure Login + uses: ./.github/workflows/azure-login with: - script: | - if (!process.env.AZURE_CREDENTIALS) { - core.setFailed('The AZURE_CREDENTIALS secret is required.') - process.exit(1) - } - - const azureCredentials = JSON.parse(process.env.AZURE_CREDENTIALS) - const {clientId, clientSecret, tenantId, subscriptionId} = azureCredentials - - core.setSecret(clientId) - core.exportVariable('AZURE_CLIENT_ID', clientId) - - core.setSecret(clientSecret) - core.exportVariable('AZURE_CLIENT_SECRET', clientSecret) - - core.setSecret(tenantId) - core.exportVariable('AZURE_TENANT_ID', tenantId) + credentials: ${{ secrets.AZURE_CREDENTIALS }} - core.setSecret(subscriptionId) - core.exportVariable('AZURE_SUBSCRIPTION_ID', subscriptionId) - name: Discover VMs to delete env: GH_APP_ID: ${{ secrets.GH_APP_ID }} GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} run: | - az login --service-principal -u ${{ env.AZURE_CLIENT_ID }} -p ${{ env.AZURE_CLIENT_SECRET }} --tenant ${{ env.AZURE_TENANT_ID }} - az account set --subscription ${{ env.AZURE_SUBSCRIPTION_ID }} active_vms=$(az vm list -g ${{ secrets.AZURE_RESOURCE_GROUP }} | jq -c '.[] | {name,timeCreated}') current_time=$(date +%s) one_hour_ago=$(($current_time - 3600))