From 611afd3ef4fe55626455133e742c7c850ab627d5 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 28 Mar 2024 11:10:15 +0100 Subject: [PATCH] Better fix for constraint generation dependency for PROD build When we build PROD images in CI, we need constraints, to know which version of the depencies to use. Those constraints are build by earlier steps in the build and uploaded as artifacts so that they can be re-used by PROD image build. However depending on the type of PROD build (in main or in v2-*) different constraints are used: for main build we use source constraints because we are installing provider packages build from main sources, in v2* builds we are using PyPI constraints, because we are installing provider packages from PyPI. This made the dependency tree a bit complicated because PROD builds would have to wait for constraints generation rather than use quickly prepared source constraints and in #38533 we introduced an extra step in PROD build for v2* branches to additionally generate constraint in-the job that builds PROD images. This however has some consequences, such as potentially incosistencies between constraints used during other parts of the build, because PROD images can be potentially built much later than the generated constraints for the rest of the build. Also it introduced the need for the PROD builds to pull CI images to generate the constraints which adds an overhead in the process and makes PROD builds in v2 branches few minutes longer than they could be. This PR approaches it differently. Since now with `uv` constraints generation is way faster (few minutes, we could easily make PROD builds use always the same constraint artifacts generated in single job and pay a little delay penalty of starting the PROD builds but get it back at PROD build time (because constraints will be generated only once). Also that removes (back) the need of PROD builds to pull the CI images and removes the need to generate source constraints at CI image build time - which will decrease the build time a little. This PR makes necessary changes to make it happens: * removes source constraints generation in CI images * extracting generate-constraints job to be separate workflow and merging "docs" instead (we are hitting the limit of 20 workflows) * add dependency from all PROD build to wait for generate-constraints * remove CI image pulling and related steps from PROD image build * switches PROD to download and use the full set of constraints generated by `generate-constraints` job --- .github/workflows/build-images.yml | 19 +- .github/workflows/ci-image-build.yml | 18 -- .github/workflows/ci.yml | 51 +++--- .github/workflows/docs.yml | 130 ------------- .github/workflows/generate-constraints.yml | 121 ++++++++++++ .github/workflows/prod-image-build.yml | 33 +--- .github/workflows/prod-image-extra-checks.yml | 3 - ...ration.yml => static-checks-mypy-docs.yml} | 172 ++++++++++-------- 8 files changed, 252 insertions(+), 295 deletions(-) delete mode 100644 .github/workflows/docs.yml create mode 100644 .github/workflows/generate-constraints.yml rename .github/workflows/{static-checks-mypy-and-constraints-generation.yml => static-checks-mypy-docs.yml} (64%) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index e998ccbd87414..fc0eecf63c864 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -57,10 +57,9 @@ jobs: outputs: image-tag: ${{ github.event.pull_request.head.sha || github.sha }} python-versions: ${{ steps.selective-checks.outputs.python-versions }} - upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} - all-python-versions-list-as-string: >- - ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} + python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} default-python-version: ${{ steps.selective-checks.outputs.default-python-version }} + upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} run-tests: ${{ steps.selective-checks.outputs.run-tests }} run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }} @@ -178,7 +177,6 @@ jobs: pull-request-target: "true" is-committer-build: ${{ needs.build-info.outputs.is-committer-build }} push-image: "true" - upload-constraints: "true" use-uv: "true" image-tag: ${{ needs.build-info.outputs.image-tag }} platform: "linux/amd64" @@ -188,13 +186,23 @@ jobs: upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} docker-cache: ${{ needs.build-info.outputs.docker-cache }} + generate-constraints: + name: "Generate constraints" + needs: [build-info, build-ci-images] + uses: ./.github/workflows/generate-constraints.yml + with: + python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} + image-tag: ${{ needs.build-info.outputs.image-tag }} + chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + build-prod-images: name: Build PROD images permissions: contents: read packages: write secrets: inherit - needs: [build-info, build-ci-images] + needs: [build-info, generate-constraints] uses: ./.github/workflows/prod-image-build.yml # Only run this it if the PR comes from fork, otherwise build will be done "in-PR-workflow" if: | @@ -211,7 +219,6 @@ jobs: push-image: "true" use-uv: "true" image-tag: ${{ needs.build-info.outputs.image-tag }} - ci-image-tag: ${{ needs.build-info.outputs.image-tag }} platform: "linux/amd64" python-versions: ${{ needs.build-info.outputs.python-versions }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index d28a34daf5ac5..75a611ba55abe 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -56,10 +56,6 @@ on: # yamllint disable-line rule:truthy required: false default: "true" type: string - upload-constraints: - description: "Whether to upload constraints artifacts" - default: "false" - type: string debian-version: description: "Base Debian distribution to use for the build (bookworm/bullseye)" type: string @@ -216,17 +212,3 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" - name: "Stop ARM instance" run: ./scripts/ci/images/ci_stop_arm_instance.sh if: always() && inputs.do-build == 'true' && inputs.platform == 'linux/arm64' - - name: "Source constraints: ${{ matrix.python-version }}" - shell: bash - run: > - breeze release-management generate-constraints --python "${{ matrix.python-version }}" - --airflow-constraints-mode constraints-source-providers --answer yes - if: inputs.do-build == 'true' && inputs.upload-constraints == 'true' - - name: "Upload constraint artifacts" - uses: actions/upload-artifact@v4 - with: - name: source-constraints-${{ matrix.python-version }} - path: ./files/constraints-*/constraints-source-providers-*.txt - retention-days: 7 - if-no-files-found: error - if: inputs.do-build == 'true' && inputs.upload-constraints == 'true' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cea684bd2fb3b..8b86ce8a6eb6d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -188,7 +188,6 @@ jobs: image-tag: ${{ needs.build-info.outputs.image-tag }} platform: "linux/amd64" python-versions: ${{ needs.build-info.outputs.python-versions }} - upload-constraints: "true" branch: ${{ needs.build-info.outputs.default-branch }} use-uv: "true" upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} @@ -254,43 +253,38 @@ jobs: include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} debug-resources: ${{ needs.build-info.outputs.debug-resources }} - static-checks-mypy-and-constraints-generation: - name: "Static checks, mypy and constraints generation" + + generate-constraints: + name: "Generate constraints" needs: [build-info, wait-for-ci-images] - uses: ./.github/workflows/static-checks-mypy-and-constraints-generation.yml + uses: ./.github/workflows/generate-constraints.yml + if: needs.build-info.outputs.ci-image-build == 'true' + with: + python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} + image-tag: ${{ needs.build-info.outputs.image-tag }} + chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + + static-checks-mypy-docs: + name: "Static checks, mypy, docs" + needs: [build-info, wait-for-ci-images] + uses: ./.github/workflows/static-checks-mypy-docs.yml with: runs-on: ${{ needs.build-info.outputs.runs-on }} image-tag: ${{ needs.build-info.outputs.image-tag }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} branch: ${{ needs.build-info.outputs.default-branch }} + canary-run: ${{ needs.build-info.outputs.canary-run }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} + docs-list-as-string: ${{ needs.build-info.outputs.docs-list-as-string }} latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }} basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} skip-pre-commits: ${{ needs.build-info.outputs.skip-pre-commits }} chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} ci-image-build: ${{ needs.build-info.outputs.ci-image-build }} - debug-resources: ${{ needs.build-info.outputs.debug-resources }} - - docs: - name: "Docs" - uses: ./.github/workflows/docs.yml - needs: [build-info, wait-for-ci-images] - permissions: - contents: read - packages: read - secrets: inherit - if: > - needs.build-info.outputs.docs-build == 'true' && - needs.build-info.outputs.latest-versions-only != 'true' - with: - runs-on: ${{ needs.build-info.outputs.runs-on }} - image-tag: ${{ needs.build-info.outputs.image-tag }} - default-branch: ${{ needs.build-info.outputs.default-branch }} - docs-list-as-string: ${{ needs.build-info.outputs.docs-list-as-string }} - default-python-version: ${{ needs.build-info.outputs.default-python-version }} include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} - canary-run: ${{ needs.build-info.outputs.canary-run }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} providers: name: "Provider checks" @@ -475,7 +469,7 @@ jobs: name: > ${{ needs.build-info.outputs.in-workflow-build == 'true' && 'Build' || 'Skip building' }} PROD images in-workflow - needs: [build-info, build-ci-images] + needs: [build-info, build-ci-images, generate-constraints] uses: ./.github/workflows/prod-image-build.yml permissions: contents: read @@ -491,7 +485,6 @@ jobs: do-build: ${{ needs.build-info.outputs.in-workflow-build }} upload-package-artifact: "true" image-tag: ${{ needs.build-info.outputs.image-tag }} - ci-image-tag: ${{ needs.build-info.outputs.image-tag }} platform: "linux/amd64" python-versions: ${{ needs.build-info.outputs.python-versions }} default-python-version: ${{ needs.build-info.outputs.default-python-version }} @@ -549,7 +542,7 @@ jobs: additional-prod-image-tests: name: "Additional PROD image tests" - needs: [build-info, wait-for-prod-images, static-checks-mypy-and-constraints-generation] + needs: [build-info, wait-for-prod-images, generate-constraints] uses: ./.github/workflows/additional-prod-image-tests.yml with: runs-on: ${{ needs.build-info.outputs.runs-on }} @@ -591,16 +584,16 @@ jobs: secrets: inherit needs: - build-info - - docs + - generate-constraints - wait-for-ci-images - wait-for-prod-images + - static-checks-mypy-docs - tests-sqlite - tests-mysql - tests-postgres - tests-non-db - tests-special - tests-integration - - static-checks-mypy-and-constraints-generation uses: ./.github/workflows/finalize-tests.yml with: runs-on: ${{ needs.build-info.outputs.runs-on }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index f5fdcfa4c4af4..0000000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,130 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -name: Documentation -on: # yamllint disable-line rule:truthy - workflow_call: - inputs: - runs-on: - description: "The array of labels (in json form) determining type of the runner to use for the build." - required: false - default: '["ubuntu-22.04"]' - type: string - image-tag: - description: "Tag to set for the image" - required: true - type: string - default-branch: - description: "Default branch of the repository" - required: true - type: string - docs-list-as-string: - description: "Stringified list of docs to build (space separated)" - required: true - type: string - default-python-version: - description: "Which version of python should be used by default" - required: true - type: string - include-success-outputs: - description: "Whether to include success outputs" - required: true - type: string - canary-run: - description: "Whether this is a canary run (true/false)" - required: true - type: string -jobs: - build-docs: - timeout-minutes: 60 - name: "Build documentation" - runs-on: ${{ fromJSON(inputs.runs-on) }} - strategy: - fail-fast: false - matrix: - flag: ["--docs-only", "--spellcheck-only"] - env: - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_USERNAME: ${{ github.actor }} - IMAGE_TAG: "${{ inputs.image-tag }}" - INCLUDE_NOT_READY_PROVIDERS: "true" - INCLUDE_SUCCESS_OUTPUTS: "${{ inputs.include-success-outputs }}" - PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" - VERBOSE: "true" - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Cleanup docker - uses: ./.github/actions/cleanup-docker - - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{ inputs.image-tag }}" - uses: ./.github/actions/prepare_breeze_and_image - - uses: actions/cache@v4 - id: cache-doc-inventories - with: - path: ./docs/_inventory_cache/ - key: docs-inventory-${{ hashFiles('pyproject.toml;') }} - restore-keys: | - docs-inventory-${{ hashFiles('pyproject.toml;') }} - docs-inventory- - - name: "Building docs with ${{ matrix.flag }} flag" - run: > - breeze build-docs ${{ inputs.docs-list-as-string }} ${{ matrix.flag }} - - name: "Clone airflow-site" - run: > - git clone https://github.com/apache/airflow-site.git ${GITHUB_WORKSPACE}/airflow-site && - echo "AIRFLOW_SITE_DIRECTORY=${GITHUB_WORKSPACE}/airflow-site" >> "$GITHUB_ENV" - if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' - - name: "Publish docs" - run: > - breeze release-management publish-docs --override-versioned --run-in-parallel - ${{ inputs.docs-list-as-string }} - if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' - - name: "Generate back references for providers" - run: breeze release-management add-back-references all-providers - if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' - - name: "Generate back references for apache-airflow" - run: breeze release-management add-back-references apache-airflow - if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' - - name: "Generate back references for docker-stack" - run: breeze release-management add-back-references docker-stack - if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' - - name: "Generate back references for helm-chart" - run: breeze release-management add-back-references helm-chart - if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 - if: > - inputs.canary-run == 'true' && - inputs.default-branch == 'main' && - matrix.flag == '--docs-only' - with: - aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} - aws-region: eu-central-1 - - name: "Upload documentation to AWS S3" - if: > - inputs.canary-run == 'true' && - inputs.default-branch == 'main' && - matrix.flag == '--docs-only' - run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs diff --git a/.github/workflows/generate-constraints.yml b/.github/workflows/generate-constraints.yml new file mode 100644 index 0000000000000..f7930a1504408 --- /dev/null +++ b/.github/workflows/generate-constraints.yml @@ -0,0 +1,121 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: Generate constraints +on: # yamllint disable-line rule:truthy + workflow_call: + inputs: + python-versions-list-as-string: + description: "Stringified array of all Python versions to test - separated by spaces." + required: true + type: string + image-tag: + description: "Tag to set for the image" + required: true + type: string + chicken-egg-providers: + description: "Space-separated list of providers that should be installed from context files" + required: true + type: string + debug-resources: + description: "Whether to run in debug mode (true/false)" + required: true + type: string +jobs: + generate-constraints: + permissions: + contents: read + timeout-minutes: 70 + name: Generate constraints ${{ inputs.python-versions-list-as-string }} + runs-on: ['ubuntu-22.04'] + env: + DEBUG_RESOURCES: ${{ inputs.debug-resources }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + INCLUDE_SUCCESS_OUTPUTS: "true" + IMAGE_TAG: ${{ inputs.image-tag }} + PYTHON_VERSIONS: ${{ inputs.python-versions-list-as-string }} + VERBOSE: "true" + VERSION_SUFFIX_FOR_PYPI: "dev0" + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker + - name: "Install Breeze" + uses: ./.github/actions/breeze + - name: Login to ghcr.io + run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + - name: "\ + Pull CI images \ + ${{ inputs.python-versions-list-as-string }}:\ + ${{ inputs.image-tag }}" + run: breeze ci-image pull --run-in-parallel --tag-as-latest + - name: " + Verify CI images \ + ${{ inputs.python-versions-list-as-string }}:\ + ${{ inputs.image-tag }}" + run: breeze ci-image verify --run-in-parallel + - name: "Source constraints" + shell: bash + run: > + breeze release-management generate-constraints --run-in-parallel + --airflow-constraints-mode constraints-source-providers --answer yes + - name: "No providers constraints" + shell: bash + timeout-minutes: 25 + run: > + breeze release-management generate-constraints --run-in-parallel + --airflow-constraints-mode constraints-no-providers --answer yes + - name: "Prepare chicken-eggs provider packages" + # In case of provider packages which use latest dev0 version of providers, we should prepare them + # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency + # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0 + shell: bash + run: > + breeze release-management prepare-provider-packages --include-not-ready-providers + --package-format wheel --version-suffix-for-pypi dev0 + ${{ inputs.chicken-egg-providers }} + if: inputs.chicken-egg-providers != '' + - name: "PyPI constraints" + shell: bash + timeout-minutes: 25 + run: > + breeze release-management generate-constraints --run-in-parallel + --airflow-constraints-mode constraints --answer yes + --chicken-egg-providers "${{ inputs.chicken-egg-providers }}" + - name: "Dependency upgrade summary" + shell: bash + run: | + for PYTHON_VERSION in ${{ env.PYTHON_VERSIONS }}; do + echo "Summarizing Python $PYTHON_VERSION" + cat "files/constraints-${PYTHON_VERSION}"/*.md >> $GITHUB_STEP_SUMMARY || true + done + - name: "Upload constraint artifacts" + uses: actions/upload-artifact@v4 + with: + name: constraints + path: ./files/constraints-*/constraints-*.txt + retention-days: 7 + if-no-files-found: error diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index 4180516983ce4..ad9c35a445afa 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -79,10 +79,6 @@ on: # yamllint disable-line rule:truthy description: "Tag to set for the image" required: true type: string - ci-image-tag: - description: "Tag to use for the CI image used during the build" - required: true - type: string python-versions: description: "JSON-formatted array of Python versions to build images from" required: true @@ -295,39 +291,16 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" name: prod-packages path: ./docker-context-files if: inputs.do-build == 'true' - - name: "Download constraints from the CI build" + - name: "Download constraints" uses: actions/download-artifact@v4 with: - name: source-constraints-${{ matrix.python-version }} + name: constraints path: ./docker-context-files - if: inputs.do-build == 'true' && inputs.build-provider-packages == 'true' + if: inputs.do-build == 'true' - name: Login to ghcr.io shell: bash run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin if: inputs.do-build == 'true' - - name: Pull CI images ${{ matrix.python-version }}:${{ inputs.ci-image-tag }} - run: > - breeze ci-image pull --tag-as-latest --image-tag "${{ inputs.ci-image-tag }}" - --python "${{ matrix.python-version }}" - if: inputs.do-build == 'true' && inputs.build-provider-packages != 'true' - - name: "PyPI constraints" - timeout-minutes: 25 - run: > - breeze release-management generate-constraints --python "${{ matrix.python-version }}" - --airflow-constraints-mode constraints --answer yes - --chicken-egg-providers "${{ inputs.chicken-egg-providers }}" - env: - # always use UV for constraints generation even if we build `pip` image - USE_UV: "true" - if: inputs.do-build == 'true' && inputs.build-provider-packages != 'true' - # This cleanup is needed as we will run out of disk space if we keep both CI and PROD images - # on public runners. - - name: Cleanup docker - uses: ./.github/actions/cleanup-docker - if: inputs.do-build == 'true' && inputs.build-provider-packages != 'true' - - name: "Copy constraints to docker-context-files" - run: cp -vr --no-preserve=mode,ownership ./files/constraints-* ./docker-context-files - if: inputs.do-build == 'true' && inputs.build-provider-packages != 'true' - name: "Build PROD images w/ source providers ${{ matrix.python-version }}:${{ inputs.image-tag }}" shell: bash run: > diff --git a/.github/workflows/prod-image-extra-checks.yml b/.github/workflows/prod-image-extra-checks.yml index 7fc010f36f2e0..48469af793d84 100644 --- a/.github/workflows/prod-image-extra-checks.yml +++ b/.github/workflows/prod-image-extra-checks.yml @@ -62,7 +62,6 @@ jobs: build-type: "Bullseye" upload-package-artifact: "false" image-tag: bullseye-${{ inputs.image-tag }} - ci-image-tag: ${{ inputs.image-tag }} debian-version: "bullseye" python-versions: ${{ inputs.python-versions }} default-python-version: ${{ inputs.default-python-version }} @@ -83,7 +82,6 @@ jobs: build-type: "MySQL Client" upload-package-artifact: "false" image-tag: mysql-${{ inputs.image-tag }} - ci-image-tag: ${{ inputs.image-tag }} install-mysql-client-type: "mysql" python-versions: ${{ inputs.python-versions }} default-python-version: ${{ inputs.default-python-version }} @@ -104,7 +102,6 @@ jobs: build-type: "pip" upload-package-artifact: "false" image-tag: mysql-${{ inputs.image-tag }} - ci-image-tag: ${{ inputs.image-tag }} install-mysql-client-type: "mysql" python-versions: ${{ inputs.python-versions }} default-python-version: ${{ inputs.default-python-version }} diff --git a/.github/workflows/static-checks-mypy-and-constraints-generation.yml b/.github/workflows/static-checks-mypy-docs.yml similarity index 64% rename from .github/workflows/static-checks-mypy-and-constraints-generation.yml rename to .github/workflows/static-checks-mypy-docs.yml index b0e0c69bbccfa..60cc4b4b1cfb0 100644 --- a/.github/workflows/static-checks-mypy-and-constraints-generation.yml +++ b/.github/workflows/static-checks-mypy-docs.yml @@ -16,7 +16,7 @@ # under the License. # --- -name: Static checks, mypy and constraints generation +name: Static checks, mypy, docs on: # yamllint disable-line rule:truthy workflow_call: inputs: @@ -37,10 +37,18 @@ on: # yamllint disable-line rule:truthy description: "Branch used to run the CI jobs in (main/v2_*_test)." required: true type: string + canary-run: + description: "Whether this is a canary run (true/false)" + required: true + type: string default-python-version: description: "Which version of python should be used by default" required: true type: string + docs-list-as-string: + description: "Stringified list of docs to build (space separated)" + required: true + type: string upgrade-to-newer-dependencies: description: "Whether to upgrade to newer dependencies (true/false)" required: true @@ -65,88 +73,15 @@ on: # yamllint disable-line rule:truthy description: "Whether to skip pre-commits (true/false)" required: true type: string + include-success-outputs: + description: "Whether to include success outputs" + required: true + type: string debug-resources: description: "Whether to debug resources (true/false)" required: true type: string jobs: - generate-constraints: - permissions: - contents: read - timeout-minutes: 70 - name: Generate constraints ${{ inputs.python-versions-list-as-string }} - runs-on: ['ubuntu-22.04'] - env: - DEBUG_RESOURCES: ${{ inputs.debug-resources }} - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_USERNAME: ${{ github.actor }} - INCLUDE_SUCCESS_OUTPUTS: "true" - PYTHON_VERSIONS: ${{ inputs.python-versions-list-as-string }} - VERBOSE: "true" - VERSION_SUFFIX_FOR_PYPI: "dev0" - if: inputs.ci-image-build == 'true' - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Cleanup docker - uses: ./.github/actions/cleanup-docker - - name: "Install Breeze" - uses: ./.github/actions/breeze - - name: Login to ghcr.io - run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - name: Pull CI images ${{ inputs.python-versions-list-as-string }}:${{ inputs.image-tag }} - run: breeze ci-image pull --run-in-parallel --tag-as-latest - - name: Verify CI images ${{ inputs.python-versions-list-as-string }}:${{ inputs.image-tag }} - run: breeze ci-image verify --run-in-parallel - - name: "Source constraints" - shell: bash - run: > - breeze release-management generate-constraints --run-in-parallel - --airflow-constraints-mode constraints-source-providers --answer yes - - name: "No providers constraints" - shell: bash - timeout-minutes: 25 - run: > - breeze release-management generate-constraints --run-in-parallel - --airflow-constraints-mode constraints-no-providers --answer yes - - name: "Prepare chicken-eggs provider packages" - # In case of provider packages which use latest dev0 version of providers, we should prepare them - # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency - # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0 - shell: bash - run: > - breeze release-management prepare-provider-packages --include-not-ready-providers - --package-format wheel --version-suffix-for-pypi dev0 - ${{ inputs.chicken-egg-providers }} - if: inputs.chicken-egg-providers != '' - - name: "PyPI constraints" - shell: bash - timeout-minutes: 25 - run: > - breeze release-management generate-constraints --run-in-parallel - --airflow-constraints-mode constraints --answer yes - --chicken-egg-providers "${{ inputs.chicken-egg-providers }}" - - name: "Dependency upgrade summary" - shell: bash - run: | - for PYTHON_VERSION in ${{ env.PYTHON_VERSIONS }}; do - echo "Summarizing Python $PYTHON_VERSION" - cat "files/constraints-${PYTHON_VERSION}"/*.md >> $GITHUB_STEP_SUMMARY || true - done - - name: "Upload constraint artifacts" - uses: actions/upload-artifact@v4 - with: - name: constraints - path: ./files/constraints-*/constraints-*.txt - retention-days: 7 - if-no-files-found: error - static-checks: timeout-minutes: 45 name: "Static checks" @@ -229,6 +164,85 @@ jobs: VERBOSE: "false" COLUMNS: "250" SKIP_GROUP_OUTPUT: "true" - DEFAULT_BRANCH: ${{ inputs.default-branch }} + DEFAULT_BRANCH: ${{ inputs.branch }} RUFF_FORMAT: "github" INCLUDE_MYPY_VOLUME: "false" + + build-docs: + timeout-minutes: 60 + name: "Build documentation" + runs-on: ${{ fromJSON(inputs.runs-on) }} + strategy: + fail-fast: false + matrix: + flag: ["--docs-only", "--spellcheck-only"] + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + IMAGE_TAG: "${{ inputs.image-tag }}" + INCLUDE_NOT_READY_PROVIDERS: "true" + INCLUDE_SUCCESS_OUTPUTS: "${{ inputs.include-success-outputs }}" + PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" + VERBOSE: "true" + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker + - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{ inputs.image-tag }}" + uses: ./.github/actions/prepare_breeze_and_image + - uses: actions/cache@v4 + id: cache-doc-inventories + with: + path: ./docs/_inventory_cache/ + key: docs-inventory-${{ hashFiles('pyproject.toml;') }} + restore-keys: | + docs-inventory-${{ hashFiles('pyproject.toml;') }} + docs-inventory- + - name: "Building docs with ${{ matrix.flag }} flag" + run: > + breeze build-docs ${{ inputs.docs-list-as-string }} ${{ matrix.flag }} + - name: "Clone airflow-site" + run: > + git clone https://github.com/apache/airflow-site.git ${GITHUB_WORKSPACE}/airflow-site && + echo "AIRFLOW_SITE_DIRECTORY=${GITHUB_WORKSPACE}/airflow-site" >> "$GITHUB_ENV" + if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' + - name: "Publish docs" + run: > + breeze release-management publish-docs --override-versioned --run-in-parallel + ${{ inputs.docs-list-as-string }} + if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' + - name: "Generate back references for providers" + run: breeze release-management add-back-references all-providers + if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' + - name: "Generate back references for apache-airflow" + run: breeze release-management add-back-references apache-airflow + if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' + - name: "Generate back references for docker-stack" + run: breeze release-management add-back-references docker-stack + if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' + - name: "Generate back references for helm-chart" + run: breeze release-management add-back-references helm-chart + if: inputs.canary-run == 'true' && matrix.flag == '--docs-only' + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + if: > + inputs.canary-run == 'true' && + inputs.branch == 'main' && + matrix.flag == '--docs-only' + with: + aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} + aws-region: eu-central-1 + - name: "Upload documentation to AWS S3" + if: > + inputs.canary-run == 'true' && + inputs.branch == 'main' && + matrix.flag == '--docs-only' + run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs