From d9433ba1aa2f2421d29790127dfda7ecd5c4521b Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 28 Mar 2024 12:58:46 +0100 Subject: [PATCH 1/3] Use explicit and easier to use runs-on approach for CI workflows Depending on selective checks, but also on the job executed, we choose whether to run job on public runners or self-hosted runners. So far the set of labels to select the runners were passed in a bit inconsistent way. Outputs of selective checks can only be strings and the `run-as` accepts array of strings (labels) - so we were using fromJSON to convert between the two. And we used runs-on inputs on a number of our workflows to pass the selection. However this meant that runs-on could be either string or array and that sometimes we passed public/self-hosted labels as strings directly and some of those were hard-coded. This PR changes it consistently across the board to introduce consistent approach: * build info have no selective checks results yet, so for them runs-on is hardcoded * similarly for "windows" and release jobs that are manually run without running selective checks * selective checks will produce three outuputs - JSON stringiified array of labels: * default (one that is selected depending on who runs the build) * public (for cases where we want to force the builds to use public runners * self-hosted (for cases where we want to force the builds to use self-hosted runners * all the outputs are named `-runs-on-as-string` to make it clear they are all strings * all inputs of workflows expectings strings are named the same (with as-string suffix and prefix) * whenever a job is run, we pass "runs-on" parameter to be `fromJSON` with appropriate type we want to use passed as input This will make it easier to reason on which job is using which type of runner and it will make it easier in the future to make it more flexible when we add ASF self-hosted runners and possibly our own K8S runners, or when we would want to change labels for public runners or self-hosted runners. --- .../workflows/additional-ci-image-checks.yml | 23 ++++++--- .../workflows/additional-prod-image-tests.yml | 13 ++--- .github/workflows/basic-tests.yml | 16 +++--- .github/workflows/build-images.yml | 11 ++-- .github/workflows/check-providers.yml | 15 +++--- .github/workflows/ci-image-build.yml | 25 +++++++--- .github/workflows/ci.yml | 50 +++++++++++-------- .github/workflows/finalize-tests.yml | 21 +++++--- .github/workflows/generate-constraints.yml | 6 ++- .github/workflows/helm-tests.yml | 17 ++++--- .github/workflows/integration-tests.yml | 9 ++-- .github/workflows/k8s-tests.yml | 9 ++-- .github/workflows/prod-image-build.yml | 13 +++-- .github/workflows/prod-image-extra-checks.yml | 7 +++ .github/workflows/push-image-cache.yml | 27 +++++++--- .github/workflows/release_dockerhub_image.yml | 6 +-- .github/workflows/run-unit-tests.yml | 15 +++--- .github/workflows/special-tests.yml | 23 ++++----- .github/workflows/static-checks-mypy-docs.yml | 15 +++--- dev/breeze/doc/ci/04_selective_checks.md | 6 ++- .../airflow_breeze/utils/selective_checks.py | 18 +++++-- dev/breeze/tests/test_selective_checks.py | 14 +++--- pyproject.toml | 2 +- 23 files changed, 213 insertions(+), 148 deletions(-) diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index 8ff00ba617194..25ecd5f9524ac 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -20,10 +20,17 @@ name: Additional CI image checks on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on: - description: "The array of labels (in json form) determining type of the runner to use for the build." - required: false - default: '["ubuntu-22.04"]' + runs-on-as-json-default: + description: "The array of labels (in json form) determining default runner used for the build." + required: true + type: string + runs-on-as-json-public: + description: "The array of labels (in json form) determining public runners." + required: true + type: string + runs-on-as-json-self-hosted: + description: "The array of labels (in json form) determining self-hosted runners." + required: true type: string image-tag: description: "Tag to set for the image" @@ -93,7 +100,8 @@ jobs: packages: write secrets: inherit with: - # Runs on Public runners + runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} + runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }} cache-type: "Early" include-prod-images: "false" push-latest-images: "false" @@ -110,7 +118,7 @@ jobs: check-that-image-builds-quickly: timeout-minutes: 11 name: Check that image builds quickly - runs-on: ["ubuntu-22.04"] + runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} env: UPGRADE_TO_NEWER_DEPENDENCIES: false PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.default-python-version }} @@ -150,7 +158,8 @@ jobs: secrets: inherit with: push-image: "false" - runs-on: ${{ inputs.runs-on }} + runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} + runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }} image-tag: ${{ inputs.image-tag }} python-versions: ${{ inputs.python-versions }} platform: "linux/arm64" diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml index c6a2146ecf9f3..500a4371a675e 100644 --- a/.github/workflows/additional-prod-image-tests.yml +++ b/.github/workflows/additional-prod-image-tests.yml @@ -20,10 +20,9 @@ name: Additional PROD image tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on: - description: "The array of labels (in json form) determining type of the runner to use for the build." - required: false - default: '["ubuntu-22.04"]' + runs-on-as-json-public: + description: "The array of labels (in json form) determining public runners." + required: true type: string default-branch: description: "The default branch for the repository" @@ -62,6 +61,7 @@ jobs: name: PROD image extra checks (main) uses: ./.github/workflows/prod-image-extra-checks.yml with: + runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} python-versions: "[ '${{ inputs.default-python-version }}' ]" default-python-version: ${{ inputs.default-python-version }} branch: ${{ inputs.default-branch }} @@ -77,6 +77,7 @@ jobs: name: PROD image extra checks (release) uses: ./.github/workflows/prod-image-extra-checks.yml with: + runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} python-versions: "[ '${{ inputs.default-python-version }}' ]" default-python-version: ${{ inputs.default-python-version }} branch: ${{ inputs.default-branch }} @@ -91,7 +92,7 @@ jobs: test-examples-of-prod-image-building: timeout-minutes: 60 name: "Test examples of POD image building" - runs-on: ["ubuntu-22.04"] + runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} env: GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -135,7 +136,7 @@ jobs: test-docker-compose-quick-start: timeout-minutes: 60 name: "Docker-compose quick start with PROD image verifying" - runs-on: ["ubuntu-22.04"] + runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} env: IMAGE_TAG: "${{ inputs.image-tag }}" PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index f96997c957aac..eea256b6a22dd 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -20,6 +20,10 @@ name: Basic tests on: # yamllint disable-line rule:truthy workflow_call: inputs: + runs-on-as-json-public: + description: "The array of labels (in json form) determining public runners." + required: true + type: string run-www-tests: description: "Whether to run WWW tests (true/false)" required: true @@ -52,7 +56,7 @@ jobs: run-breeze-tests: timeout-minutes: 10 name: Breeze unit tests - runs-on: ["ubuntu-22.04"] + runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} steps: - name: "Cleanup repo" shell: bash @@ -76,7 +80,7 @@ jobs: tests-www: timeout-minutes: 10 name: React WWW tests - runs-on: ["ubuntu-22.04"] + runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} if: inputs.run-www-tests == 'true' steps: - name: "Cleanup repo" @@ -105,7 +109,7 @@ jobs: test-openapi-client: timeout-minutes: 10 name: "Test OpenAPI client" - runs-on: ["ubuntu-22.04"] + runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} if: inputs.needs-api-codegen == 'true' steps: - name: "Cleanup repo" @@ -195,7 +199,7 @@ jobs: static-checks-basic-checks-only: timeout-minutes: 30 name: "Static checks: basic checks only" - runs-on: ["ubuntu-22.04"] + runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} if: inputs.basic-checks-only == 'true' steps: - name: "Cleanup repo" @@ -262,7 +266,7 @@ jobs: upgrade-check: timeout-minutes: 45 name: "Upgrade checks" - runs-on: ["ubuntu-22.04"] + runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" if: inputs.canary-run == 'true' && inputs.latest-versions-only != 'true' @@ -330,7 +334,7 @@ jobs: test-airflow-release-commands: timeout-minutes: 80 name: "Test Airflow release commands" - runs-on: ["ubuntu-22.04"] + runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" IMAGE_TAG: ${{ inputs.image-tag }} diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index b135da93cba06..e9104effee0b8 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -51,6 +51,7 @@ jobs: build-info: timeout-minutes: 10 name: "Build Info" + # At build-info stage we do not yet have outputs so we need to hard-code the runs-on to public runners runs-on: ["ubuntu-22.04"] env: TARGET_BRANCH: ${{ github.event.pull_request.base.ref }} @@ -67,7 +68,9 @@ jobs: docker-cache: ${{ steps.selective-checks.outputs.docker-cache }} default-branch: ${{ steps.selective-checks.outputs.default-branch }} constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} - runs-on: ${{steps.selective-checks.outputs.runs-on}} + runs-on-as-json-default: ${{ steps.selective-checks.outputs.runs-on-as-json-default }} + runs-on-as-json-public: ${{ steps.selective-checks.outputs.runs-on-as-json-public }} + runs-on-as-json-self-hosted: ${{ steps.selective-checks.outputs.runs-on-as-json-self-hosted }} is-self-hosted-runner: ${{ steps.selective-checks.outputs.is-self-hosted-runner }} is-committer-build: ${{ steps.selective-checks.outputs.is-committer-build }} is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner }} @@ -171,7 +174,8 @@ jobs: needs.build-info.outputs.ci-image-build == 'true' && github.event.pull_request.head.repo.full_name != 'apache/airflow' with: - runs-on: '["ubuntu-22.04"]' + runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} + runs-on-as-json-self-hosted: ${{ needs.build-info.outputs.runs-on-as-json-self-hosted }} do-build: ${{ needs.build-info.outputs.ci-image-build }} target-commit-sha: ${{ needs.build-info.outputs.target-commit-sha }} pull-request-target: "true" @@ -191,6 +195,7 @@ jobs: needs: [build-info, build-ci-images] uses: ./.github/workflows/generate-constraints.yml with: + runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} # For regular PRs we do not need "no providers" constraints - they are only needed in canary builds generate-no-providers-constraints: "false" @@ -211,7 +216,7 @@ jobs: needs.build-info.outputs.prod-image-build == 'true' && github.event.pull_request.head.repo.full_name != 'apache/airflow' with: - runs-on: '["ubuntu-22.04"]' + runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} build-type: "Regular" do-build: ${{ needs.build-info.outputs.ci-image-build }} upload-package-artifact: "true" diff --git a/.github/workflows/check-providers.yml b/.github/workflows/check-providers.yml index 7c62d5ab529ee..839b310046261 100644 --- a/.github/workflows/check-providers.yml +++ b/.github/workflows/check-providers.yml @@ -20,10 +20,9 @@ name: Provider tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on: - description: "The array of labels (in json form) determining type of the runner to use for the build." - required: false - default: '["ubuntu-22.04"]' + runs-on-as-json-default: + description: "The array of labels (in json form) determining default runner used for the build." + required: true type: string image-tag: description: "Tag to set for the image" @@ -59,7 +58,7 @@ jobs: prepare-install-verify-provider-packages-wheel: timeout-minutes: 80 name: "Provider packages wheel build and verify" - runs-on: ${{fromJSON(inputs.runs-on)}} + runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} env: GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -124,7 +123,7 @@ jobs: prepare-install-provider-packages-sdist: timeout-minutes: 80 name: "Provider packages sdist build and install" - runs-on: ${{fromJSON(inputs.runs-on)}} + runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} env: GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -192,11 +191,11 @@ jobs: providers-compatibility-checks: timeout-minutes: 80 name: Compat ${{ matrix.airflow-version }}:P${{ matrix.python-version }} provider check - runs-on: ${{fromJSON(inputs.runs-on)}} + runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} strategy: fail-fast: false matrix: - include: ${{fromJson(inputs.providers-compatibility-checks)}} + include: ${{fromJSON(inputs.providers-compatibility-checks)}} env: GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 75a611ba55abe..5864eea6a9baf 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -20,10 +20,13 @@ name: Build CI images on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on: - description: "The array of labels (in json form) determining type of the runner to use for the build." - required: false - default: '["ubuntu-22.04"]' + runs-on-as-json-public: + description: "The array of labels (in json form) determining public runners." + required: true + type: string + runs-on-as-json-self-hosted: + description: "The array of labels (in json form) determining self-hosted runners." + required: true type: string do-build: description: > @@ -98,14 +101,24 @@ jobs: fail-fast: true matrix: # yamllint disable-line rule:line-length - python-version: ${{ inputs.do-build == 'true' && fromJson(inputs.python-versions) || fromJson('[""]') }} + python-version: ${{ inputs.do-build == 'true' && fromJSON(inputs.python-versions) || fromJSON('[""]') }} timeout-minutes: 110 name: "\ ${{ inputs.do-build == 'true' && 'Build' || 'Skip building' }} \ CI ${{ inputs.platform }} image\ ${{ matrix.python-version }}${{ inputs.do-build == 'true' && ':' || '' }}\ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" - runs-on: ${{ fromJSON(inputs.runs-on) }} + # The ARM images need to be built using self-hosted runners as ARM macos public runners + # do not yet allow us to run docker effectively and fast. + # https://github.com/actions/runner-images/issues/9254#issuecomment-1917916016 + # https://github.com/abiosoft/colima/issues/970 + # https://github.com/actions/runner/issues/1456 + # See https://github.com/apache/airflow/pull/38640 + # NOTE!!!!! This has to be put in one line for runs-on to recognize the "fromJSON" properly !!!! + # adding space before (with >) apparently turns the `runs-on` processed line into a string "Array" + # instead of an array of strings. + # yamllint disable-line rule:line-length + runs-on: ${{ (inputs.platform == 'linux/amd64') && fromJSON(inputs.runs-on-as-json-public) || fromJSON(inputs.runs-on-as-json-self-hosted) }} env: BACKEND: sqlite DEFAULT_BRANCH: ${{ inputs.branch }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 438422aaf67b6..0fd4a3b5cdaf7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,10 +48,7 @@ jobs: build-info: name: "Build info" - # The runs-on cannot refer to env. or secrets. context, so we have no - # option but to specify a hard-coded list here. This is "safe", the - # runner checks if the user is an owner or collaborator of the repo - # before running the workflow. + # At build-info stage we do not yet have outputs so we need to hard-code the runs-on to public runners runs-on: ["ubuntu-22.04"] env: GITHUB_CONTEXT: ${{ toJson(github) }} @@ -105,7 +102,9 @@ jobs: providers-compatibility-checks: ${{ steps.selective-checks.outputs.providers-compatibility-checks }} helm-test-packages: ${{ steps.selective-checks.outputs.helm-test-packages }} debug-resources: ${{ steps.selective-checks.outputs.debug-resources }} - runs-on: ${{steps.selective-checks.outputs.runs-on}} + runs-on-as-json-default: ${{ steps.selective-checks.outputs.runs-on-as-json-default }} + runs-on-as-json-public: ${{ steps.selective-checks.outputs.runs-on-as-json-public }} + runs-on-as-json-self-hosted: ${{ steps.selective-checks.outputs.runs-on-as-json-self-hosted }} is-self-hosted-runner: ${{ steps.selective-checks.outputs.is-self-hosted-runner }} is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner }} is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }} @@ -160,6 +159,7 @@ jobs: needs: [build-info] uses: ./.github/workflows/basic-tests.yml with: + runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} run-www-tests: ${{needs.build-info.outputs.run-www-tests}} needs-api-codegen: ${{needs.build-info.outputs.needs-api-codegen}} default-python-version: ${{needs.build-info.outputs.default-python-version}} @@ -183,7 +183,8 @@ jobs: packages: write secrets: inherit with: - runs-on: '["ubuntu-22.04"]' + runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} + runs-on-as-json-self-hosted: ${{ needs.build-info.outputs.runs-on-as-json-self-hosted }} do-build: ${{ needs.build-info.outputs.in-workflow-build }} image-tag: ${{ needs.build-info.outputs.image-tag }} platform: "linux/amd64" @@ -197,7 +198,7 @@ jobs: wait-for-ci-images: timeout-minutes: 120 name: "Wait for CI images" - runs-on: ["ubuntu-22.04"] + runs-on: ${{ fromJSON(needs.build-info.outputs.runs-on-as-json-public) }} needs: [build-info, build-ci-images] if: needs.build-info.outputs.ci-image-build == 'true' env: @@ -238,7 +239,9 @@ jobs: uses: ./.github/workflows/additional-ci-image-checks.yml if: needs.build-info.outputs.canary-run == 'true' with: - runs-on: ${{ needs.build-info.outputs.runs-on }} + runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} + runs-on-as-json-self-hosted: ${{ needs.build-info.outputs.runs-on-as-json-self-hosted }} image-tag: ${{ needs.build-info.outputs.image-tag }} python-versions: ${{ needs.build-info.outputs.python-versions }} branch: ${{ needs.build-info.outputs.default-branch }} @@ -259,6 +262,7 @@ jobs: uses: ./.github/workflows/generate-constraints.yml if: needs.build-info.outputs.ci-image-build == 'true' with: + runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} # generate no providers constraints only in canary builds - they take quite some time to generate # they are not needed for regular builds, they are only needed to update constraints in canaries @@ -273,7 +277,7 @@ jobs: uses: ./.github/workflows/static-checks-mypy-docs.yml secrets: inherit with: - runs-on: ${{ needs.build-info.outputs.runs-on }} + runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} image-tag: ${{ needs.build-info.outputs.image-tag }} needs-mypy: ${{ needs.build-info.outputs.needs-mypy }} mypy-folders: ${{ needs.build-info.outputs.mypy-folders }} @@ -303,7 +307,7 @@ jobs: needs.build-info.outputs.skip-providers-tests != 'true' && needs.build-info.outputs.latest-versions-only != 'true' with: - runs-on: ${{ needs.build-info.outputs.runs-on }} + runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} image-tag: ${{ needs.build-info.outputs.image-tag }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} @@ -321,7 +325,8 @@ jobs: packages: read secrets: inherit with: - runs-on: ${{ needs.build-info.outputs.runs-on }} + runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }} image-tag: ${{ needs.build-info.outputs.image-tag }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} @@ -339,7 +344,7 @@ jobs: packages: read secrets: inherit with: - runs-on: ${{ needs.build-info.outputs.runs-on }} + runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} backend: "postgres" test-name: "Postgres" test-scope: "DB" @@ -363,7 +368,7 @@ jobs: packages: read secrets: inherit with: - runs-on: ${{ needs.build-info.outputs.runs-on }} + runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} backend: "mysql" test-name: "MySQL" test-scope: "DB" @@ -387,7 +392,7 @@ jobs: packages: read secrets: inherit with: - runs-on: ${{ needs.build-info.outputs.runs-on }} + runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} backend: "sqlite" test-name: "Sqlite" test-name-separator: "" @@ -413,7 +418,7 @@ jobs: packages: read secrets: inherit with: - runs-on: ${{ needs.build-info.outputs.runs-on }} + runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} backend: "sqlite" test-name: "" test-name-separator: "" @@ -443,7 +448,7 @@ jobs: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' || needs.build-info.outputs.full-tests-needed == 'true') with: - runs-on: ${{ needs.build-info.outputs.runs-on }} + runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} image-tag: ${{ needs.build-info.outputs.image-tag }} parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }} run-coverage: ${{ needs.build-info.outputs.run-coverage }} @@ -462,7 +467,7 @@ jobs: packages: read secrets: inherit with: - runs-on: ${{ needs.build-info.outputs.runs-on }} + runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} image-tag: ${{ needs.build-info.outputs.image-tag }} parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} @@ -489,7 +494,7 @@ jobs: packages: write secrets: inherit with: - runs-on: '["ubuntu-22.04"]' + runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} build-type: "Regular" do-build: ${{ needs.build-info.outputs.in-workflow-build }} upload-package-artifact: "true" @@ -509,7 +514,7 @@ jobs: wait-for-prod-images: timeout-minutes: 80 name: "Wait for PROD images" - runs-on: ["ubuntu-22.04"] + runs-on: ${{ fromJSON(needs.build-info.outputs.runs-on-as-json-public) }} needs: [build-info, wait-for-ci-images, build-prod-images] if: needs.build-info.outputs.prod-image-build == 'true' env: @@ -554,7 +559,7 @@ jobs: needs: [build-info, wait-for-prod-images, generate-constraints] uses: ./.github/workflows/additional-prod-image-tests.yml with: - runs-on: ${{ needs.build-info.outputs.runs-on }} + runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} default-branch: ${{ needs.build-info.outputs.default-branch }} constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} image-tag: ${{ needs.build-info.outputs.image-tag }} @@ -574,7 +579,7 @@ jobs: packages: read secrets: inherit with: - runs-on: ${{ needs.build-info.outputs.runs-on }} + runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} image-tag: ${{ needs.build-info.outputs.image-tag }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} kubernetes-versions-list-as-string: ${{ needs.build-info.outputs.kubernetes-versions-list-as-string }} @@ -605,7 +610,8 @@ jobs: - tests-special uses: ./.github/workflows/finalize-tests.yml with: - runs-on: ${{ needs.build-info.outputs.runs-on }} + runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} + runs-on-as-json-self-hosted: ${{ needs.build-info.outputs.runs-on-as-json-self-hosted }} image-tag: ${{ needs.build-info.outputs.image-tag }} python-versions: ${{ needs.build-info.outputs.python-versions }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml index 504f4a769850b..8b61225400ada 100644 --- a/.github/workflows/finalize-tests.yml +++ b/.github/workflows/finalize-tests.yml @@ -20,10 +20,13 @@ name: Finalize tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on: - description: "The array of labels (in json form) determining type of the runner to use for the build." - required: false - default: '["ubuntu-22.04"]' + runs-on-as-json-public: + description: "The array of labels (in json form) determining public runners." + required: true + type: string + runs-on-as-json-self-hosted: + description: "The array of labels (in json form) determining self-hosted runners." + required: true type: string image-tag: description: "Tag to set for the image" @@ -71,7 +74,7 @@ on: # yamllint disable-line rule:truthy type: string jobs: update-constraints: - runs-on: ["ubuntu-22.04"] + runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} timeout-minutes: 80 name: "Update constraints" permissions: @@ -133,7 +136,8 @@ jobs: packages: write secrets: inherit with: - runs-on: '["ubuntu-22.04"]' + runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} + runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }} cache-type: "Regular AMD" include-prod-images: "true" push-latest-images: "true" @@ -155,7 +159,8 @@ jobs: packages: write secrets: inherit with: - runs-on: ${{ inputs.runs-on }} + runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} + runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }} cache-type: "Regular ARM" include-prod-images: "true" push-latest-images: "true" @@ -171,7 +176,7 @@ jobs: summarize-warnings: timeout-minutes: 15 name: "Summarize warnings" - runs-on: ["ubuntu-22.04"] + runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} steps: - name: "Cleanup repo" shell: bash diff --git a/.github/workflows/generate-constraints.yml b/.github/workflows/generate-constraints.yml index 0adad4da3b327..57621102f857b 100644 --- a/.github/workflows/generate-constraints.yml +++ b/.github/workflows/generate-constraints.yml @@ -20,6 +20,10 @@ name: Generate constraints on: # yamllint disable-line rule:truthy workflow_call: inputs: + runs-on-as-json-public: + description: "The array of labels (in json form) determining public runners." + required: true + type: string python-versions-list-as-string: description: "Stringified array of all Python versions to test - separated by spaces." required: true @@ -46,7 +50,7 @@ jobs: contents: read timeout-minutes: 70 name: Generate constraints ${{ inputs.python-versions-list-as-string }} - runs-on: ['ubuntu-22.04'] + runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} env: DEBUG_RESOURCES: ${{ inputs.debug-resources }} GITHUB_REPOSITORY: ${{ github.repository }} diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml index f6a2df6041f22..96597840e717d 100644 --- a/.github/workflows/helm-tests.yml +++ b/.github/workflows/helm-tests.yml @@ -20,10 +20,13 @@ name: Helm tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on: - description: "The array of labels (in json form) determining type of the runner to use for the build." - required: false - default: '["ubuntu-22.04"]' + runs-on-as-json-default: + description: "The array of labels (in json form) determining default runner used for the build." + required: true + type: string + runs-on-as-json-public: + description: "The array of labels (in json form) determining public runners." + required: true type: string helm-test-packages: description: "Stringified JSON array of helm test packages to test" @@ -41,11 +44,11 @@ jobs: tests-helm: timeout-minutes: 80 name: "Unit tests Helm: ${{ matrix.helm-test-package }}" - runs-on: ${{ fromJSON(inputs.runs-on) }} + runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} strategy: fail-fast: false matrix: - helm-test-package: ${{ fromJson(inputs.helm-test-packages) }} + helm-test-package: ${{ fromJSON(inputs.helm-test-packages) }} env: # Always use default Python version of CI image for preparing packages PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" @@ -77,7 +80,7 @@ jobs: tests-helm-release: timeout-minutes: 80 name: "Release Helm" - runs-on: ["ubuntu-22.04"] + runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}" steps: diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index abc3c631f197c..054b745cb766d 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -20,10 +20,9 @@ name: Integration tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on: - description: "The array of labels (in json form) determining type of the runner to use for the build." - required: false - default: '["ubuntu-22.04"]' + runs-on-as-json-default: + description: "The array of labels (in json form) determining default runner used for the build." + required: true type: string image-tag: description: "Tag to set for the image" @@ -65,7 +64,7 @@ jobs: tests-integration: timeout-minutes: 130 name: Integration Tests ${{ matrix.backend }}:${{ matrix.backend-version}} - runs-on: ${{fromJSON(inputs.runs-on)}} + runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} strategy: fail-fast: false matrix: diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml index 7e09efa994038..74a40db287626 100644 --- a/.github/workflows/k8s-tests.yml +++ b/.github/workflows/k8s-tests.yml @@ -20,10 +20,9 @@ name: K8s tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on: - description: "The array of labels (in json form) determining type of the runner to use for the build." - required: false - default: '["ubuntu-22.04"]' + runs-on-as-json-default: + description: "The array of labels (in json form) determining default runner used for the build." + required: true type: string image-tag: description: "Tag to set for the image" @@ -55,7 +54,7 @@ jobs: name: "\ K8S System:${{ matrix.executor }} - ${{ matrix.use-standard-naming }} - \ ${{ inputs.kubernetes-versions-list-as-string }}" - runs-on: ${{ fromJSON(inputs.runs-on) }} + runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} strategy: matrix: executor: [KubernetesExecutor, CeleryExecutor, LocalExecutor] diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index ad9c35a445afa..37ef43259a497 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -20,10 +20,9 @@ name: Build PROD images on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on: - description: "The array of labels (in json form) determining type of the runner to use for the build." - required: false - default: '["ubuntu-22.04"]' + runs-on-as-json-public: + description: "The array of labels (in json form) determining default runner used for the build." + required: true type: string build-type: description: > @@ -120,7 +119,7 @@ jobs: build-prod-packages: name: "${{ inputs.do-build == 'true' && 'Build' || 'Skip building' }} Airflow and provider packages" timeout-minutes: 10 - runs-on: ["ubuntu-22.04"] + runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" steps: @@ -189,14 +188,14 @@ jobs: fail-fast: true matrix: # yamllint disable-line rule:line-length - python-version: ${{ inputs.do-build == 'true' && fromJson(inputs.python-versions) || fromJson('[""]') }} + python-version: ${{ inputs.do-build == 'true' && fromJSON(inputs.python-versions) || fromJSON('[""]') }} timeout-minutes: 80 name: "\ ${{ inputs.do-build == 'true' && 'Build' || 'Skip building' }} \ PROD ${{ inputs.build-type }} image\ ${{ matrix.python-version }}${{ inputs.do-build == 'true' && ':' || '' }}\ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" - runs-on: ${{ fromJSON(inputs.runs-on) }} + runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }} needs: - build-prod-packages env: diff --git a/.github/workflows/prod-image-extra-checks.yml b/.github/workflows/prod-image-extra-checks.yml index 48469af793d84..f17a81d9810c4 100644 --- a/.github/workflows/prod-image-extra-checks.yml +++ b/.github/workflows/prod-image-extra-checks.yml @@ -20,6 +20,10 @@ name: PROD images extra checks on: # yamllint disable-line rule:truthy workflow_call: inputs: + runs-on-as-json-public: + description: "The array of labels (in json form) determining public runners." + required: true + type: string python-versions: description: "JSON-formatted array of Python versions to build images from" required: true @@ -59,6 +63,7 @@ jobs: bullseye-image: uses: ./.github/workflows/prod-image-build.yml with: + runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} build-type: "Bullseye" upload-package-artifact: "false" image-tag: bullseye-${{ inputs.image-tag }} @@ -79,6 +84,7 @@ jobs: myssql-client-image: uses: ./.github/workflows/prod-image-build.yml with: + runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} build-type: "MySQL Client" upload-package-artifact: "false" image-tag: mysql-${{ inputs.image-tag }} @@ -99,6 +105,7 @@ jobs: pip-image: uses: ./.github/workflows/prod-image-build.yml with: + runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }} build-type: "pip" upload-package-artifact: "false" image-tag: mysql-${{ inputs.image-tag }} diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml index cb8c072e74179..83152426eabec 100644 --- a/.github/workflows/push-image-cache.yml +++ b/.github/workflows/push-image-cache.yml @@ -20,10 +20,13 @@ name: Push image cache on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on: - description: "The array of labels (in json form) determining type of the runner to use for the build." - required: false - default: "[\"ubuntu-22.04\"]" + runs-on-as-json-public: + description: "The array of labels (in json form) determining public runners." + required: true + type: string + runs-on-as-json-self-hosted: + description: "The array of labels (in json form) determining self-hosted runners." + required: true type: string cache-type: description: "Type of cache to push (Early / Regular)." @@ -76,11 +79,15 @@ on: # yamllint disable-line rule:truthy jobs: push-ci-image-cache: name: "Push CI ${{ inputs.cache-type }}:${{ matrix.python }} image cache " - runs-on: ${{ fromJSON(inputs.runs-on) }} + # NOTE!!!!! This has to be put in one line for runs-on to recognize the "fromJSON" properly !!!! + # adding space before (with >) apparently turns the `runs-on` processed line into a string "Array" + # instead of an array of strings. + # yamllint disable-line rule:line-length + runs-on: ${{ (inputs.platform == 'linux/amd64') && fromJSON(inputs.runs-on-as-json-public) || fromJSON(inputs.runs-on-as-json-self-hosted) }} strategy: fail-fast: false matrix: - python: ${{ fromJson(inputs.python-versions) }} + python: ${{ fromJSON(inputs.python-versions) }} env: COMMIT_SHA: ${{ github.sha }} # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the @@ -134,11 +141,15 @@ jobs: push-prod-image-cache: name: "Push PROD ${{ inputs.cache-type }}:${{ matrix.python }} image cache" - runs-on: ${{ fromJSON(inputs.runs-on) }} + # NOTE!!!!! This has to be put in one line for runs-on to recognize the "fromJSON" properly !!!! + # adding space before (with >) apparently turns the `runs-on` processed line into a string "Array" + # instead of an array of strings. + # yamllint disable-line rule:line-length + runs-on: ${{ (inputs.platform == 'linux/amd64') && fromJSON(inputs.runs-on-as-json-public) || fromJSON(inputs.runs-on-as-json-self-hosted) }} strategy: fail-fast: false matrix: - python: ${{ fromJson(inputs.python-versions) }} + python: ${{ fromJSON(inputs.python-versions) }} env: COMMIT_SHA: ${{ github.sha }} # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml index 2ce069cc0c3f2..d489feae94439 100644 --- a/.github/workflows/release_dockerhub_image.yml +++ b/.github/workflows/release_dockerhub_image.yml @@ -40,8 +40,6 @@ jobs: build-info: timeout-minutes: 10 name: "Build Info" - # TODO: when we have it properly set-up with labels we should check for - # "airflow-runner" presence in runs_on runs-on: ["ubuntu-22.04"] outputs: pythonVersions: ${{ steps.selective-checks.outputs.python-versions }} @@ -73,14 +71,12 @@ jobs: release-images: timeout-minutes: 120 name: "Release images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}" - # TODO: when we have it properly set-up with labels we should check for - # "airflow-runner" presence in runs_on runs-on: ["self-hosted", "Linux", "X64"] needs: [build-info] strategy: fail-fast: false matrix: - python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} + python-version: ${{ fromJSON(needs.build-info.outputs.pythonVersions) }} if: contains(fromJSON('[ "ashb", "eladkal", diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index a54019da211ac..65ff0694880d5 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -20,10 +20,9 @@ name: Unit tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on: - description: "The array of labels (in json form) determining type of the runner to use for the build." - required: false - default: '["ubuntu-22.04"]' + runs-on-as-json-default: + description: "The array of labels (in json form) determining default runner used for the build." + required: true type: string backend: description: "The backend to run the tests on" @@ -112,13 +111,13 @@ jobs: ${{ inputs.test-scope }}:\ ${{ inputs.test-name }}${{ inputs.test-name-separator }}${{ matrix.backend-version }}:\ ${{matrix.python-version}}: ${{ inputs.parallel-test-types-list-as-string }}" - runs-on: ${{fromJSON(inputs.runs-on)}} + runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} strategy: fail-fast: false matrix: - python-version: "${{fromJson(inputs.python-versions)}}" - backend-version: "${{fromJson(inputs.backend-versions)}}" - exclude: "${{fromJson(inputs.excludes)}}" + python-version: "${{fromJSON(inputs.python-versions)}}" + backend-version: "${{fromJSON(inputs.backend-versions)}}" + exclude: "${{fromJSON(inputs.excludes)}}" env: # yamllint disable rule:line-length AIRFLOW_ENABLE_AIP_44: "${{ inputs.enable-aip-44 }}" diff --git a/.github/workflows/special-tests.yml b/.github/workflows/special-tests.yml index a644c6d661d81..87012ade014c7 100644 --- a/.github/workflows/special-tests.yml +++ b/.github/workflows/special-tests.yml @@ -20,10 +20,9 @@ name: Special tests on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on: - description: "The array of labels (in json form) determining type of the runner to use for the build." - required: false - default: '["ubuntu-22.04"]' + runs-on-as-json-default: + description: "The array of labels (in json form) determining default runner used for the build." + required: true type: string image-tag: description: "Tag to set for the image" @@ -66,7 +65,7 @@ jobs: packages: read secrets: inherit with: - runs-on: ${{ inputs.runs-on }} + runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} downgrade-sqlalchemy: "true" test-name: "MinSQLAlchemy-Postgres" test-scope: "DB" @@ -88,7 +87,7 @@ jobs: packages: read secrets: inherit with: - runs-on: ${{ inputs.runs-on }} + runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} upgrade-boto: "true" test-name: "LatestBoto-Postgres" test-scope: "All" @@ -110,7 +109,7 @@ jobs: packages: read secrets: inherit with: - runs-on: ${{ inputs.runs-on }} + runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} pydantic: "v1" test-name: "Pydantic-V1-Postgres" test-scope: "All" @@ -132,7 +131,7 @@ jobs: packages: read secrets: inherit with: - runs-on: ${{ inputs.runs-on }} + runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} pydantic: "none" test-name: "Pydantic-Removed-Postgres" test-scope: "All" @@ -154,7 +153,7 @@ jobs: packages: read secrets: inherit with: - runs-on: ${{ inputs.runs-on }} + runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} downgrade-pendulum: "true" test-name: "Pendulum2-Postgres" test-scope: "All" @@ -176,7 +175,7 @@ jobs: packages: read secrets: inherit with: - runs-on: ${{ inputs.runs-on }} + runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} enable-aip-44: "false" test-name: "InProgressDisabled-Postgres" test-scope: "All" @@ -198,7 +197,7 @@ jobs: packages: read secrets: inherit with: - runs-on: ${{ inputs.runs-on }} + runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} test-name: "Postgres" test-scope: "Quarantined" backend: "postgres" @@ -219,7 +218,7 @@ jobs: packages: read secrets: inherit with: - runs-on: ${{ inputs.runs-on }} + runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }} test-name: "Postgres" test-scope: "ARM collection" backend: "postgres" diff --git a/.github/workflows/static-checks-mypy-docs.yml b/.github/workflows/static-checks-mypy-docs.yml index 2dcc64c9c37e4..bde5dd1b611e9 100644 --- a/.github/workflows/static-checks-mypy-docs.yml +++ b/.github/workflows/static-checks-mypy-docs.yml @@ -20,10 +20,9 @@ name: Static checks, mypy, docs on: # yamllint disable-line rule:truthy workflow_call: inputs: - runs-on: - description: "The array of labels (in json form) determining type of the runner to use for the build." - required: false - default: '["ubuntu-22.04"]' + runs-on-as-json-default: + description: "The array of labels (in json form) determining default runner used for the build." + required: true type: string image-tag: description: "Tag to set for the image" @@ -93,7 +92,7 @@ jobs: static-checks: timeout-minutes: 45 name: "Static checks" - runs-on: ${{ fromJSON(inputs.runs-on) }} + runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" UPGRADE_TO_NEWER_DEPENDENCIES: "${{ inputs.upgrade-to-newer-dependencies }}" @@ -140,12 +139,12 @@ jobs: mypy: timeout-minutes: 45 name: "MyPy checks" - runs-on: ${{ fromJSON(inputs.runs-on) }} + runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} if: inputs.needs-mypy == 'true' strategy: fail-fast: false matrix: - mypy-folder: ${{ fromJson(inputs.mypy-folders) }} + mypy-folder: ${{ fromJSON(inputs.mypy-folders) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}" IMAGE_TAG: "${{ inputs.image-tag }}" @@ -178,7 +177,7 @@ jobs: build-docs: timeout-minutes: 60 name: "Build documentation" - runs-on: ${{ fromJSON(inputs.runs-on) }} + runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }} strategy: fail-fast: false matrix: diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index b558061208a24..b93289627c32a 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -223,7 +223,9 @@ Github Actions to pass the list of parameters to a command to execute | run-kubernetes-tests | Whether Kubernetes tests should be run ("true"/"false") | true | | | run-tests | Whether unit tests should be run ("true"/"false") | true | | | run-www-tests | Whether WWW tests should be run ("true"/"false") | true | | -| runs-on | List of labels assigned for runners for that build (used to select runners) | ["ubuntu-22.04"] | | +| runs-on-as-json-default | List of labels assigned for runners for that build for default runs for that build (as string) | ["ubuntu-22.04"] | | +| runs-on-as-json-self-hosted | List of labels assigned for runners for that build for self hosted runners | ["self-hosted", "Linux", "X64"] | | +| runs-on-as-json-public | List of labels assigned for runners for that build for public runners | ["ubuntu-22.04"] | | | skip-pre-commits | Which pre-commits should be skipped during the static-checks run | check-provider-yaml-valid,flynt,identity | | | skip-provider-tests | When provider tests should be skipped (on non-main branch or when no provider changes detected) | true | | | sqlite-exclude | Which versions of Sqlite to exclude for tests as JSON array | [] | | @@ -308,7 +310,7 @@ am overview of possible labels and their meaning: | full tests needed | full-tests-needed | Run complete set of tests (might be with default or all python/k8s versions) | | non committer build | is-committer-build | If set then even for non-committer builds, the scripts used for images are used from target branch. | | upgrade to newer dependencies | upgrade-to-newer-dependencies | If set then dependencies in the CI image build are upgraded to the newer ones. | -| use public runners | runs-on | Force using public runners even for Committer runs. | +| use public runners | runs-on-as-json-default | Force using public runners even for Committer runs. | ----- diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 11189017481bf..9cbeacd3f0972 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -1111,7 +1111,7 @@ def affected_providers_list_as_string(self) -> str | None: return " ".join(sorted(affected_providers)) @cached_property - def runs_on(self) -> str: + def runs_on_as_json_default(self) -> str: if self._github_repository == APACHE_AIRFLOW_GITHUB_REPOSITORY: if self._github_event in [GithubEvents.SCHEDULE, GithubEvents.PUSH]: return RUNS_ON_SELF_HOSTED_RUNNER @@ -1133,6 +1133,14 @@ def runs_on(self) -> str: return RUNS_ON_SELF_HOSTED_RUNNER return RUNS_ON_PUBLIC_RUNNER + @cached_property + def runs_on_as_json_self_hosted(self) -> str: + return RUNS_ON_SELF_HOSTED_RUNNER + + @cached_property + def runs_on_as_json_public(self) -> str: + return RUNS_ON_PUBLIC_RUNNER + @cached_property def is_self_hosted_runner(self) -> bool: """ @@ -1140,7 +1148,7 @@ def is_self_hosted_runner(self) -> bool: All self-hosted runners have "self-hosted" label. """ - return "self-hosted" in json.loads(self.runs_on) + return "self-hosted" in json.loads(self.runs_on_as_json_default) @cached_property def is_airflow_runner(self) -> bool: @@ -1151,7 +1159,7 @@ def is_airflow_runner(self) -> bool: """ # TODO: when we have it properly set-up with labels we should just check for # "airflow-runner" presence in runs_on - runs_on_array = json.loads(self.runs_on) + runs_on_array = json.loads(self.runs_on_as_json_default) return "Linux" in runs_on_array and "X64" in runs_on_array and "self-hosted" in runs_on_array @cached_property @@ -1172,7 +1180,7 @@ def is_amd_runner(self) -> bool: or "x64" == label.lower() or "asf-runner" == label or ("ubuntu" in label and "arm" not in label.lower()) - for label in json.loads(self.runs_on) + for label in json.loads(self.runs_on_as_json_public) ] ) @@ -1188,7 +1196,7 @@ def is_arm_runner(self) -> bool: return any( [ "arm" == label.lower() or "arm64" == label.lower() or "asf-arm" == label - for label in json.loads(self.runs_on) + for label in json.loads(self.runs_on_as_json_public) ] ) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 441abdb1befea..a5af1ced6d3e0 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1624,9 +1624,7 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d @pytest.mark.parametrize( - "github_event, github_actor, github_repository, pr_labels, github_context_dict, " - "runs_on, " - "is_self_hosted_runner, is_airflow_runner, is_amd_runner, is_arm_runner, is_vm_runner, is_k8s_runner", + "github_event, github_actor, github_repository, pr_labels, github_context_dict, default_runs_on_as_string, is_self_hosted_runner, is_airflow_runner, is_amd_runner, is_arm_runner, is_vm_runner, is_k8s_runner", [ pytest.param( GithubEvents.PUSH, @@ -1816,7 +1814,7 @@ def test_runs_on( github_repository: str, pr_labels: list[str], github_context_dict: dict[str, Any], - runs_on: str, + default_runs_on_as_string, is_self_hosted_runner: str, is_airflow_runner: str, is_amd_runner: str, @@ -1834,7 +1832,7 @@ def test_runs_on( pr_labels=(), default_branch="main", ) - assert_outputs_are_printed({"runs-on": runs_on}, str(stderr)) + assert_outputs_are_printed({"runs-on-as-json-default": default_runs_on_as_string}, str(stderr)) assert_outputs_are_printed({"is-self-hosted-runner": is_self_hosted_runner}, str(stderr)) assert_outputs_are_printed({"is-airflow-runner": is_airflow_runner}, str(stderr)) assert_outputs_are_printed({"is-amd-runner": is_amd_runner}, str(stderr)) @@ -1999,7 +1997,7 @@ def test_mypy_matches( ("README.md",), { "is-committer-build": "false", - "runs-on": '["ubuntu-22.04"]', + "runs-on-as-json-default": '["ubuntu-22.04"]', }, "", (), @@ -2009,7 +2007,7 @@ def test_mypy_matches( ("README.md",), { "is-committer-build": "true", - "runs-on": '["self-hosted", "Linux", "X64"]', + "runs-on-as-json-default": '["self-hosted", "Linux", "X64"]', }, "potiuk", (), @@ -2019,7 +2017,7 @@ def test_mypy_matches( ("README.md",), { "is-committer-build": "false", - "runs-on": '["self-hosted", "Linux", "X64"]', + "runs-on-as-json-default": '["self-hosted", "Linux", "X64"]', }, "potiuk", ("non committer build",), diff --git a/pyproject.toml b/pyproject.toml index 79af6dba6f861..78721635ab483 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ # The dependencies can be automatically upgraded by running: # pre-commit run --hook-stage manual update-build-dependencies --all-files requires = [ - "GitPython==3.1.42", + "GitPython==3.1.43", "gitdb==4.0.11", "hatchling==1.22.4", "packaging==24.0", From cf01a30c6d3641c92555091029bbb82b2d2995df Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 1 Apr 2024 10:34:20 +0200 Subject: [PATCH 2/3] fixup! Use explicit and easier to use runs-on approach for CI workflows --- .github/workflows/build-images.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index e9104effee0b8..d228c4293a12b 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -83,7 +83,6 @@ jobs: github.event.pull_request.head.sha || github.sha }}" - if: github.repository == 'apache/airflow' steps: - name: "Cleanup repo" shell: bash @@ -211,10 +210,6 @@ jobs: secrets: inherit needs: [build-info, generate-constraints] uses: ./.github/workflows/prod-image-build.yml - # Only run this it if the PR comes from fork, otherwise build will be done "in-PR-workflow" - if: | - needs.build-info.outputs.prod-image-build == 'true' && - github.event.pull_request.head.repo.full_name != 'apache/airflow' with: runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} build-type: "Regular" From 2912856bb5ea25ebd3209e2968e3f8036a942ace Mon Sep 17 00:00:00 2001 From: higrys <43041407+higrys@users.noreply.github.com> Date: Mon, 1 Apr 2024 10:36:37 +0200 Subject: [PATCH 3/3] Update hatch_build.py --- hatch_build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hatch_build.py b/hatch_build.py index ac4b54b185ec3..0f4f5ec48c37c 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -16,6 +16,7 @@ # under the License. from __future__ import annotations + import itertools import json import logging