Update hatch_build.py

.github/workflows/additional-ci-image-checks.yml

@@ -20,10 +20,17 @@ name: Additional CI image checks
 on:  # yamllint disable-line rule:truthy
   workflow_call:
     inputs:
-      runs-on:
-        description: "The array of labels (in json form) determining type of the runner to use for the build."
-        required: false
-        default: '["ubuntu-22.04"]'
+      runs-on-as-json-default:
+        description: "The array of labels (in json form) determining default runner used for the build."
+        required: true
+        type: string
+      runs-on-as-json-public:
+        description: "The array of labels (in json form) determining public runners."
+        required: true
+        type: string
+      runs-on-as-json-self-hosted:
+        description: "The array of labels (in json form) determining self-hosted runners."
+        required: true
         type: string
       image-tag:
         description: "Tag to set for the image"
@@ -93,7 +100,8 @@ jobs:
       packages: write
     secrets: inherit
     with:
-      # Runs on Public runners
+      runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
+      runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }}
       cache-type: "Early"
       include-prod-images: "false"
       push-latest-images: "false"
@@ -110,7 +118,7 @@ jobs:
   check-that-image-builds-quickly:
     timeout-minutes: 11
     name: Check that image builds quickly
-    runs-on: ["ubuntu-22.04"]
+    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
     env:
       UPGRADE_TO_NEWER_DEPENDENCIES: false
       PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.default-python-version }}
@@ -150,7 +158,8 @@ jobs:
     secrets: inherit
     with:
       push-image: "false"
-      runs-on: ${{ inputs.runs-on }}
+      runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
+      runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }}
       image-tag: ${{ inputs.image-tag }}
       python-versions: ${{ inputs.python-versions }}
       platform: "linux/arm64"

.github/workflows/additional-prod-image-tests.yml

@@ -20,10 +20,9 @@ name: Additional PROD image tests
 on:  # yamllint disable-line rule:truthy
   workflow_call:
     inputs:
-      runs-on:
-        description: "The array of labels (in json form) determining type of the runner to use for the build."
-        required: false
-        default: '["ubuntu-22.04"]'
+      runs-on-as-json-public:
+        description: "The array of labels (in json form) determining public runners."
+        required: true
         type: string
       default-branch:
         description: "The default branch for the repository"
@@ -62,6 +61,7 @@ jobs:
     name: PROD image extra checks (main)
     uses: ./.github/workflows/prod-image-extra-checks.yml
     with:
+      runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
       python-versions: "[ '${{ inputs.default-python-version }}' ]"
       default-python-version: ${{ inputs.default-python-version }}
       branch: ${{ inputs.default-branch }}
@@ -77,6 +77,7 @@ jobs:
     name: PROD image extra checks (release)
     uses: ./.github/workflows/prod-image-extra-checks.yml
     with:
+      runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
       python-versions: "[ '${{ inputs.default-python-version }}' ]"
       default-python-version: ${{ inputs.default-python-version }}
       branch: ${{ inputs.default-branch }}
@@ -91,7 +92,7 @@ jobs:
   test-examples-of-prod-image-building:
     timeout-minutes: 60
     name: "Test examples of POD image building"
-    runs-on: ["ubuntu-22.04"]
+    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
     env:
       GITHUB_REPOSITORY: ${{ github.repository }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -135,7 +136,7 @@ jobs:
   test-docker-compose-quick-start:
     timeout-minutes: 60
     name: "Docker-compose quick start with PROD image verifying"
-    runs-on: ["ubuntu-22.04"]
+    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
     env:
       IMAGE_TAG: "${{ inputs.image-tag }}"
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"

.github/workflows/basic-tests.yml

@@ -20,6 +20,10 @@ name: Basic tests
 on:  # yamllint disable-line rule:truthy
   workflow_call:
     inputs:
+      runs-on-as-json-public:
+        description: "The array of labels (in json form) determining public runners."
+        required: true
+        type: string
       run-www-tests:
         description: "Whether to run WWW tests (true/false)"
         required: true
@@ -52,7 +56,7 @@ jobs:
   run-breeze-tests:
     timeout-minutes: 10
     name: Breeze unit tests
-    runs-on: ["ubuntu-22.04"]
+    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
     steps:
       - name: "Cleanup repo"
         shell: bash
@@ -76,7 +80,7 @@ jobs:
   tests-www:
     timeout-minutes: 10
     name: React WWW tests
-    runs-on: ["ubuntu-22.04"]
+    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
     if: inputs.run-www-tests == 'true'
     steps:
       - name: "Cleanup repo"
@@ -105,7 +109,7 @@ jobs:
   test-openapi-client:
     timeout-minutes: 10
     name: "Test OpenAPI client"
-    runs-on: ["ubuntu-22.04"]
+    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
     if: inputs.needs-api-codegen == 'true'
     steps:
       - name: "Cleanup repo"
@@ -195,7 +199,7 @@ jobs:
   static-checks-basic-checks-only:
     timeout-minutes: 30
     name: "Static checks: basic checks only"
-    runs-on: ["ubuntu-22.04"]
+    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
     if: inputs.basic-checks-only == 'true'
     steps:
       - name: "Cleanup repo"
@@ -262,7 +266,7 @@ jobs:
   upgrade-check:
     timeout-minutes: 45
     name: "Upgrade checks"
-    runs-on: ["ubuntu-22.04"]
+    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
     env:
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
     if: inputs.canary-run == 'true' && inputs.latest-versions-only != 'true'
@@ -330,7 +334,7 @@ jobs:
   test-airflow-release-commands:
     timeout-minutes: 80
     name: "Test Airflow release commands"
-    runs-on: ["ubuntu-22.04"]
+    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
     env:
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
       IMAGE_TAG: ${{ inputs.image-tag }}

.github/workflows/build-images.yml

@@ -51,6 +51,7 @@ jobs:
   build-info:
     timeout-minutes: 10
     name: "Build Info"
+    # At build-info stage we do not yet have outputs so we need to hard-code the runs-on to public runners
     runs-on: ["ubuntu-22.04"]
     env:
       TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
@@ -67,7 +68,9 @@ jobs:
       docker-cache: ${{ steps.selective-checks.outputs.docker-cache }}
       default-branch: ${{ steps.selective-checks.outputs.default-branch }}
       constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
-      runs-on: ${{steps.selective-checks.outputs.runs-on}}
+      runs-on-as-json-default: ${{ steps.selective-checks.outputs.runs-on-as-json-default }}
+      runs-on-as-json-public: ${{ steps.selective-checks.outputs.runs-on-as-json-public }}
+      runs-on-as-json-self-hosted: ${{ steps.selective-checks.outputs.runs-on-as-json-self-hosted }}
       is-self-hosted-runner: ${{ steps.selective-checks.outputs.is-self-hosted-runner }}
       is-committer-build: ${{ steps.selective-checks.outputs.is-committer-build }}
       is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner }}
@@ -80,7 +83,6 @@ jobs:
           github.event.pull_request.head.sha ||
           github.sha
         }}"
-    if: github.repository == 'apache/airflow'
     steps:
       - name: "Cleanup repo"
         shell: bash
@@ -171,7 +173,8 @@ jobs:
       needs.build-info.outputs.ci-image-build == 'true' &&
       github.event.pull_request.head.repo.full_name != 'apache/airflow'
     with:
-      runs-on: '["ubuntu-22.04"]'
+      runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }}
+      runs-on-as-json-self-hosted: ${{ needs.build-info.outputs.runs-on-as-json-self-hosted }}
       do-build: ${{ needs.build-info.outputs.ci-image-build }}
       target-commit-sha: ${{ needs.build-info.outputs.target-commit-sha }}
       pull-request-target: "true"
@@ -191,6 +194,7 @@ jobs:
     needs: [build-info, build-ci-images]
     uses: ./.github/workflows/generate-constraints.yml
     with:
+      runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }}
       python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
       # For regular PRs we do not need "no providers" constraints - they are only needed in canary builds
       generate-no-providers-constraints: "false"
@@ -206,12 +210,8 @@ jobs:
     secrets: inherit
     needs: [build-info, generate-constraints]
     uses: ./.github/workflows/prod-image-build.yml
-    # Only run this it if the PR comes from fork, otherwise build will be done "in-PR-workflow"
-    if: |
-      needs.build-info.outputs.prod-image-build == 'true' &&
-      github.event.pull_request.head.repo.full_name != 'apache/airflow'
     with:
-      runs-on: '["ubuntu-22.04"]'
+      runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }}
       build-type: "Regular"
       do-build: ${{ needs.build-info.outputs.ci-image-build }}
       upload-package-artifact: "true"

.github/workflows/check-providers.yml

@@ -20,10 +20,9 @@ name: Provider tests
 on:  # yamllint disable-line rule:truthy
   workflow_call:
     inputs:
-      runs-on:
-        description: "The array of labels (in json form) determining type of the runner to use for the build."
-        required: false
-        default: '["ubuntu-22.04"]'
+      runs-on-as-json-default:
+        description: "The array of labels (in json form) determining default runner used for the build."
+        required: true
         type: string
       image-tag:
         description: "Tag to set for the image"
@@ -59,7 +58,7 @@ jobs:
   prepare-install-verify-provider-packages-wheel:
     timeout-minutes: 80
     name: "Provider packages wheel build and verify"
-    runs-on: ${{fromJSON(inputs.runs-on)}}
+    runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
     env:
       GITHUB_REPOSITORY: ${{ github.repository }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -124,7 +123,7 @@ jobs:
   prepare-install-provider-packages-sdist:
     timeout-minutes: 80
     name: "Provider packages sdist build and install"
-    runs-on: ${{fromJSON(inputs.runs-on)}}
+    runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
     env:
       GITHUB_REPOSITORY: ${{ github.repository }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -192,11 +191,11 @@ jobs:
   providers-compatibility-checks:
     timeout-minutes: 80
     name: Compat ${{ matrix.airflow-version }}:P${{ matrix.python-version }} provider check
-    runs-on: ${{fromJSON(inputs.runs-on)}}
+    runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
     strategy:
       fail-fast: false
       matrix:
-        include: ${{fromJson(inputs.providers-compatibility-checks)}}
+        include: ${{fromJSON(inputs.providers-compatibility-checks)}}
     env:
       GITHUB_REPOSITORY: ${{ github.repository }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci-image-build.yml

@@ -20,10 +20,13 @@ name: Build CI images
 on:  # yamllint disable-line rule:truthy
   workflow_call:
     inputs:
-      runs-on:
-        description: "The array of labels (in json form) determining type of the runner to use for the build."
-        required: false
-        default: '["ubuntu-22.04"]'
+      runs-on-as-json-public:
+        description: "The array of labels (in json form) determining public runners."
+        required: true
+        type: string
+      runs-on-as-json-self-hosted:
+        description: "The array of labels (in json form) determining self-hosted runners."
+        required: true
         type: string
       do-build:
         description: >
@@ -98,14 +101,24 @@ jobs:
       fail-fast: true
       matrix:
         # yamllint disable-line rule:line-length
-        python-version: ${{ inputs.do-build == 'true' && fromJson(inputs.python-versions) || fromJson('[""]') }}
+        python-version: ${{ inputs.do-build == 'true' && fromJSON(inputs.python-versions) || fromJSON('[""]') }}
     timeout-minutes: 110
     name: "\
 ${{ inputs.do-build == 'true' && 'Build' || 'Skip building' }} \
 CI ${{ inputs.platform }} image\
 ${{ matrix.python-version }}${{ inputs.do-build == 'true' && ':' || '' }}\
 ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
-    runs-on: ${{ fromJSON(inputs.runs-on) }}
+    # The ARM images need to be built using self-hosted runners as ARM macos public runners
+    # do not yet allow us to run docker effectively and fast.
+    # https://github.com/actions/runner-images/issues/9254#issuecomment-1917916016
+    # https://github.com/abiosoft/colima/issues/970
+    # https://github.com/actions/runner/issues/1456
+    # See https://github.com/apache/airflow/pull/38640
+    # NOTE!!!!! This has to be put in one line for runs-on to recognize the "fromJSON" properly !!!!
+    # adding space before (with >) apparently turns the `runs-on` processed line into a string "Array"
+    # instead of an array of strings.
+    # yamllint disable-line rule:line-length
+    runs-on: ${{ (inputs.platform == 'linux/amd64') && fromJSON(inputs.runs-on-as-json-public) || fromJSON(inputs.runs-on-as-json-self-hosted) }}
     env:
       BACKEND: sqlite
       DEFAULT_BRANCH: ${{ inputs.branch }}