From de6d83a00fd7af30586e58461428c8d3c74d55d3 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sat, 4 Jan 2025 22:36:45 +0100 Subject: [PATCH] Update uv, pip and pre-commit versions automatically in more places. (#45398) --- .github/actions/install-pre-commit/action.yml | 7 +- .github/workflows/basic-tests.yml | 3 +- .pre-commit-config.yaml | 8 +- contributing-docs/08_static_code_checks.rst | 2 +- dev/breeze/doc/ci/02_images.md | 56 ++--- .../doc/images/output_static-checks.svg | 2 +- .../doc/images/output_static-checks.txt | 2 +- .../src/airflow_breeze/pre_commit_ids.py | 2 +- scripts/ci/pre_commit/update_installers.py | 161 ------------- .../update_installers_and_pre_commit.py | 218 ++++++++++++++++++ 10 files changed, 260 insertions(+), 201 deletions(-) delete mode 100755 scripts/ci/pre_commit/update_installers.py create mode 100755 scripts/ci/pre_commit/update_installers_and_pre_commit.py diff --git a/.github/actions/install-pre-commit/action.yml b/.github/actions/install-pre-commit/action.yml index b4c6a6c9d546a..abdd3ea98ffc9 100644 --- a/.github/actions/install-pre-commit/action.yml +++ b/.github/actions/install-pre-commit/action.yml @@ -19,19 +19,18 @@ name: 'Install pre-commit' description: 'Installs pre-commit and related packages' inputs: - # TODO(potiuk): automate update of these versions python-version: description: 'Python version to use' default: "3.9" uv-version: description: 'uv version to use' - default: "0.5.14" + default: "0.5.14" # Keep this comment to allow automatic replacement of uv version pre-commit-version: description: 'pre-commit version to use' - default: "4.0.1" + default: "4.0.1" # Keep this comment to allow automatic replacement of pre-commit version pre-commit-uv-version: description: 'pre-commit-uv version to use' - default: "4.1.4" + default: "4.1.4" # Keep this comment to allow automatic replacement of pre-commit-uv version runs: using: "composite" steps: diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 297a912ea8122..353f65d9a6c9c 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -307,11 +307,12 @@ jobs: run: > pre-commit run --all-files --show-diff-on-failure --color always --verbose - --hook-stage manual update-installers || true + --hook-stage manual update-installers-and-pre-commit || true if: always() env: UPGRADE_UV: "true" UPGRADE_PIP: "false" + UPGRADE_PRE_COMMIT: "true" - name: "Run automated upgrade for pip" run: > pre-commit run diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 09690edc5db6c..c5d0d154b88a4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -197,12 +197,12 @@ repos: additional_dependencies: ['pyyaml'] pass_filenames: false require_serial: true - - id: update-installers - name: Update installers to latest (manual) - entry: ./scripts/ci/pre_commit/update_installers.py + - id: update-installers-and-pre-commit + name: Update installers and pre-commit to latest (manual) + entry: ./scripts/ci/pre_commit/update_installers_and_pre_commit.py stages: ['manual'] language: python - files: ^.pre-commit-config.yaml$|^scripts/ci/pre_commit/update_installers.py$ + files: ^.pre-commit-config.yaml$|^scripts/ci/pre_commit/update_installers_and_pre_commit.py$ pass_filenames: false require_serial: true additional_dependencies: ['pyyaml', 'rich>=12.4.4', 'requests'] diff --git a/contributing-docs/08_static_code_checks.rst b/contributing-docs/08_static_code_checks.rst index abfb737890460..78462afe3057f 100644 --- a/contributing-docs/08_static_code_checks.rst +++ b/contributing-docs/08_static_code_checks.rst @@ -374,7 +374,7 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------+---------+ | update-installed-providers-to-be-sorted | Sort and uniquify installed_providers.txt | | +-----------------------------------------------------------+--------------------------------------------------------+---------+ -| update-installers | Update installers to latest (manual) | | +| update-installers-and-pre-commit | Update installers and pre-commit to latest (manual) | | +-----------------------------------------------------------+--------------------------------------------------------+---------+ | update-local-yml-file | Update mounts in the local yml file | | +-----------------------------------------------------------+--------------------------------------------------------+---------+ diff --git a/dev/breeze/doc/ci/02_images.md b/dev/breeze/doc/ci/02_images.md index b613f67c72d04..3d1d7d8b53eb7 100644 --- a/dev/breeze/doc/ci/02_images.md +++ b/dev/breeze/doc/ci/02_images.md @@ -419,33 +419,35 @@ DOCKER_BUILDKIT=1 docker build . -f Dockerfile.ci \ The following build arguments (`--build-arg` in docker build command) can be used for CI images: -| Build argument | Default value | Description | -|-----------------------------------|----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `PYTHON_BASE_IMAGE` | `python:3.9-slim-bookworm` | Base Python image | -| `PYTHON_MAJOR_MINOR_VERSION` | `3.9` | major/minor version of Python (should match base image) | -| `DEPENDENCIES_EPOCH_NUMBER` | `2` | increasing this number will reinstall all apt dependencies | -| `ADDITIONAL_PIP_INSTALL_FLAGS` | | additional `pip` flags passed to the installation commands (except when reinstalling `pip` itself) | -| `HOME` | `/root` | Home directory of the root user (CI image has root user as default) | -| `AIRFLOW_HOME` | `/root/airflow` | Airflow's HOME (that's where logs and sqlite databases are stored) | -| `AIRFLOW_SOURCES` | `/opt/airflow` | Mounted sources of Airflow | -| `AIRFLOW_REPO` | `apache/airflow` | the repository from which PIP dependencies are pre-installed | -| `AIRFLOW_BRANCH` | `main` | the branch from which PIP dependencies are pre-installed | -| `AIRFLOW_CI_BUILD_EPOCH` | `1` | increasing this value will reinstall PIP dependencies from the repository from scratch | -| `AIRFLOW_CONSTRAINTS_LOCATION` | | If not empty, it will override the source of the constraints with the specified URL or file. | -| `AIRFLOW_CONSTRAINTS_REFERENCE` | | reference (branch or tag) from GitHub repository from which constraints are used. By default it is set to `constraints-main` but can be `constraints-2-X`. | -| `AIRFLOW_EXTRAS` | `all` | extras to install | -| `UPGRADE_INVALIDATION_STRING` | | If set to any random value the dependencies are upgraded to newer versions. In CI it is set to build id. | -| `ADDITIONAL_AIRFLOW_EXTRAS` | | additional extras to install | -| `ADDITIONAL_PYTHON_DEPS` | | additional Python dependencies to install | -| `DEV_APT_COMMAND` | | Dev apt command executed before dev deps are installed in the first part of image | -| `ADDITIONAL_DEV_APT_COMMAND` | | Additional Dev apt command executed before dev dep are installed in the first part of the image | -| `DEV_APT_DEPS` | | Dev APT dependencies installed in the first part of the image (default empty means default dependencies are used) | -| `ADDITIONAL_DEV_APT_DEPS` | | Additional apt dev dependencies installed in the first part of the image | -| `ADDITIONAL_DEV_APT_ENV` | | Additional env variables defined when installing dev deps | -| `AIRFLOW_PIP_VERSION` | `24.3.1` | PIP version used. | -| `AIRFLOW_UV_VERSION` | `0.5.14` | UV version used. | -| `AIRFLOW_USE_UV` | `true` | Whether to use UV for installation. | -| `PIP_PROGRESS_BAR` | `on` | Progress bar for PIP installation | +| Build argument | Default value | Description | +|---------------------------------|----------------------------|-------------------------------------------------------------------------------------------------------------------| +| `PYTHON_BASE_IMAGE` | `python:3.9-slim-bookworm` | Base Python image | +| `PYTHON_MAJOR_MINOR_VERSION` | `3.9` | major/minor version of Python (should match base image) | +| `DEPENDENCIES_EPOCH_NUMBER` | `2` | increasing this number will reinstall all apt dependencies | +| `ADDITIONAL_PIP_INSTALL_FLAGS` | | additional `pip` flags passed to the installation commands (except when reinstalling `pip` itself) | +| `HOME` | `/root` | Home directory of the root user (CI image has root user as default) | +| `AIRFLOW_HOME` | `/root/airflow` | Airflow's HOME (that's where logs and sqlite databases are stored) | +| `AIRFLOW_SOURCES` | `/opt/airflow` | Mounted sources of Airflow | +| `AIRFLOW_REPO` | `apache/airflow` | the repository from which PIP dependencies are pre-installed | +| `AIRFLOW_BRANCH` | `main` | the branch from which PIP dependencies are pre-installed | +| `AIRFLOW_CI_BUILD_EPOCH` | `1` | increasing this value will reinstall PIP dependencies from the repository from scratch | +| `AIRFLOW_CONSTRAINTS_LOCATION` | | If not empty, it will override the source of the constraints with the specified URL or file. | +| `AIRFLOW_CONSTRAINTS_REFERENCE` | `constraints-main` | reference (branch or tag) from GitHub repository from which constraints are used. | +| `AIRFLOW_EXTRAS` | `all` | extras to install | +| `UPGRADE_INVALIDATION_STRING` | | If set to any random value the dependencies are upgraded to newer versions. In CI it is set to build id. | +| `ADDITIONAL_AIRFLOW_EXTRAS` | | additional extras to install | +| `ADDITIONAL_PYTHON_DEPS` | | additional Python dependencies to install | +| `DEV_APT_COMMAND` | | Dev apt command executed before dev deps are installed in the first part of image | +| `ADDITIONAL_DEV_APT_COMMAND` | | Additional Dev apt command executed before dev dep are installed in the first part of the image | +| `DEV_APT_DEPS` | | Dev APT dependencies installed in the first part of the image (default empty means default dependencies are used) | +| `ADDITIONAL_DEV_APT_DEPS` | | Additional apt dev dependencies installed in the first part of the image | +| `ADDITIONAL_DEV_APT_ENV` | | Additional env variables defined when installing dev deps | +| `AIRFLOW_PIP_VERSION` | `24.3.1` | `pip` version used. | +| `AIRFLOW_UV_VERSION` | `0.5.14` | `uv` version used. | +| `AIRFLOW_PRE_COMMIT_VERSION` | `4.0.1` | `pre-commit` version used. | +| `AIRFLOW_PRE_COMMIT_UV_VERSION` | `4.1.4` | `pre-commit-uv` version used. | +| `AIRFLOW_USE_UV` | `true` | Whether to use UV for installation. | +| `PIP_PROGRESS_BAR` | `on` | Progress bar for PIP installation | Here are some examples of how CI images can built manually. CI is always diff --git a/dev/breeze/doc/images/output_static-checks.svg b/dev/breeze/doc/images/output_static-checks.svg index a2422824dd981..bfea55c495fb5 100644 --- a/dev/breeze/doc/images/output_static-checks.svg +++ b/dev/breeze/doc/images/output_static-checks.svg @@ -371,7 +371,7 @@ │update-breeze-readme-config-hash | update-chart-dependencies |                   │ │update-common-sql-api-stubs | update-er-diagram | update-extras |                │ │update-in-the-wild-to-be-sorted | update-inlined-dockerfile-scripts |            │ -│update-installed-providers-to-be-sorted | update-installers |                    │ +│update-installed-providers-to-be-sorted | update-installers-and-pre-commit |     │ │update-local-yml-file | update-migration-references |                            │ │update-openapi-spec-tags-to-be-sorted | update-providers-dependencies |          │ │update-providers-init-py | update-reproducible-source-date-epoch |               │ diff --git a/dev/breeze/doc/images/output_static-checks.txt b/dev/breeze/doc/images/output_static-checks.txt index a685635825ff4..38529eb9753cf 100644 --- a/dev/breeze/doc/images/output_static-checks.txt +++ b/dev/breeze/doc/images/output_static-checks.txt @@ -1 +1 @@ -1e545fdd89efbdd78a883519b6d93ce2 +6239e6a528459f731b6908ce668a8950 diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py b/dev/breeze/src/airflow_breeze/pre_commit_ids.py index a4495df607ecd..8667a2cc4b785 100644 --- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py +++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py @@ -141,7 +141,7 @@ "update-in-the-wild-to-be-sorted", "update-inlined-dockerfile-scripts", "update-installed-providers-to-be-sorted", - "update-installers", + "update-installers-and-pre-commit", "update-local-yml-file", "update-migration-references", "update-openapi-spec-tags-to-be-sorted", diff --git a/scripts/ci/pre_commit/update_installers.py b/scripts/ci/pre_commit/update_installers.py deleted file mode 100755 index 28de198824064..0000000000000 --- a/scripts/ci/pre_commit/update_installers.py +++ /dev/null @@ -1,161 +0,0 @@ -#!/usr/bin/env python -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from __future__ import annotations - -import os -import re -import sys -from pathlib import Path - -import requests - -sys.path.insert(0, str(Path(__file__).parent.resolve())) # make sure common_precommit_utils is imported -from common_precommit_utils import AIRFLOW_SOURCES_ROOT_PATH, console - -FILES_TO_UPDATE = [ - AIRFLOW_SOURCES_ROOT_PATH / "Dockerfile", - AIRFLOW_SOURCES_ROOT_PATH / "Dockerfile.ci", - AIRFLOW_SOURCES_ROOT_PATH / "scripts" / "ci" / "install_breeze.sh", - AIRFLOW_SOURCES_ROOT_PATH / "scripts" / "docker" / "common.sh", - AIRFLOW_SOURCES_ROOT_PATH / "scripts" / "tools" / "setup_breeze", - AIRFLOW_SOURCES_ROOT_PATH / "pyproject.toml", - AIRFLOW_SOURCES_ROOT_PATH / "dev" / "breeze" / "src" / "airflow_breeze" / "global_constants.py", - AIRFLOW_SOURCES_ROOT_PATH - / "dev" - / "breeze" - / "src" - / "airflow_breeze" - / "commands" - / "release_management_commands.py", -] - - -DOC_FILES_TO_UPDATE: list[Path] = [ - AIRFLOW_SOURCES_ROOT_PATH / "dev/" / "breeze" / "doc" / "ci" / "02_images.md" -] - - -def get_latest_pypi_version(package_name: str) -> str: - response = requests.get(f"https://pypi.org/pypi/{package_name}/json") - response.raise_for_status() # Ensure we got a successful response - data = response.json() - latest_version = data["info"]["version"] # The version info is under the 'info' key - return latest_version - - -AIRFLOW_PIP_PATTERN = re.compile(r"(AIRFLOW_PIP_VERSION=)([0-9.]+)") -AIRFLOW_PIP_QUOTED_PATTERN = re.compile(r"(AIRFLOW_PIP_VERSION = )(\"[0-9.]+\")") -PIP_QUOTED_PATTERN = re.compile(r"(PIP_VERSION = )(\"[0-9.]+\")") -PIP_QUOTED_PATTERN_NO_SPACES = re.compile(r"(PIP_VERSION=)(\"[0-9.]+\")") -AIRFLOW_PIP_DOC_PATTERN = re.compile(r"(\| *`AIRFLOW_PIP_VERSION` *\| *)(`[0-9.]+`)( *\|)") -AIRFLOW_PIP_UPGRADE_PATTERN = re.compile(r"(python -m pip install --upgrade pip==)([0-9.]+)") - -AIRFLOW_UV_PATTERN = re.compile(r"(AIRFLOW_UV_VERSION=)([0-9.]+)") -AIRFLOW_UV_QUOTED_PATTERN = re.compile(r"(AIRFLOW_UV_VERSION = )(\"[0-9.]+\")") -UV_QUOTED_PATTERN = re.compile(r"(UV_VERSION = )(\"[0-9.]+\")") -UV_QUOTED_PATTERN_NO_SPACES = re.compile(r"(UV_VERSION=)(\"[0-9.]+\")") -AIRFLOW_UV_DOC_PATTERN = re.compile(r"(\| *`AIRFLOW_UV_VERSION` *\| *)(`[0-9.]+`)( *\|)") -UV_GREATER_PATTERN = re.compile(r'"(uv>=)([0-9]+)"') - -UPGRADE_UV: bool = os.environ.get("UPGRADE_UV", "true").lower() == "true" -UPGRADE_PIP: bool = os.environ.get("UPGRADE_PIP", "true").lower() == "true" - - -def replace_group_2_while_keeping_total_length(pattern: re.Pattern[str], replacement: str, text: str) -> str: - def replacer(match): - original_length = len(match.group(2)) - padding = "" - if len(match.groups()) > 2: - padding = match.group(3) - new_length = len(replacement) - diff = new_length - original_length - if diff <= 0: - padding = " " * -diff + padding - else: - padding = padding[diff:] - padded_replacement = match.group(1) + replacement + padding - return padded_replacement.strip() - - return re.sub(pattern, replacer, text) - - -if __name__ == "__main__": - pip_version = get_latest_pypi_version("pip") - console.print(f"[bright_blue]Latest pip version: {pip_version}") - uv_version = get_latest_pypi_version("uv") - console.print(f"[bright_blue]Latest uv version: {uv_version}") - - changed = False - for file in FILES_TO_UPDATE: - console.print(f"[bright_blue]Updating {file}") - file_content = file.read_text() - new_content = file_content - if UPGRADE_PIP: - new_content = replace_group_2_while_keeping_total_length( - AIRFLOW_PIP_PATTERN, pip_version, new_content - ) - new_content = replace_group_2_while_keeping_total_length( - AIRFLOW_PIP_UPGRADE_PATTERN, pip_version, new_content - ) - new_content = replace_group_2_while_keeping_total_length( - AIRFLOW_PIP_QUOTED_PATTERN, f'"{pip_version}"', new_content - ) - new_content = replace_group_2_while_keeping_total_length( - PIP_QUOTED_PATTERN, f'"{pip_version}"', new_content - ) - new_content = replace_group_2_while_keeping_total_length( - PIP_QUOTED_PATTERN_NO_SPACES, f'"{pip_version}"', new_content - ) - if UPGRADE_UV: - new_content = replace_group_2_while_keeping_total_length( - AIRFLOW_UV_PATTERN, uv_version, new_content - ) - new_content = replace_group_2_while_keeping_total_length( - UV_GREATER_PATTERN, uv_version, new_content - ) - new_content = replace_group_2_while_keeping_total_length( - AIRFLOW_UV_QUOTED_PATTERN, f'"{uv_version}"', new_content - ) - new_content = replace_group_2_while_keeping_total_length( - UV_QUOTED_PATTERN, f'"{uv_version}"', new_content - ) - new_content = replace_group_2_while_keeping_total_length( - UV_QUOTED_PATTERN_NO_SPACES, f'"{uv_version}"', new_content - ) - if new_content != file_content: - file.write_text(new_content) - console.print(f"[bright_blue]Updated {file}") - changed = True - for file in DOC_FILES_TO_UPDATE: - console.print(f"[bright_blue]Updating {file}") - file_content = file.read_text() - new_content = file_content - if UPGRADE_PIP: - new_content = replace_group_2_while_keeping_total_length( - AIRFLOW_PIP_DOC_PATTERN, f"`{pip_version}`", new_content - ) - if UPGRADE_UV: - new_content = replace_group_2_while_keeping_total_length( - AIRFLOW_UV_DOC_PATTERN, f"`{uv_version}`", new_content - ) - if new_content != file_content: - file.write_text(new_content) - console.print(f"[bright_blue]Updated {file}") - changed = True - if changed: - sys.exit(1) diff --git a/scripts/ci/pre_commit/update_installers_and_pre_commit.py b/scripts/ci/pre_commit/update_installers_and_pre_commit.py new file mode 100755 index 0000000000000..9e5b2c68cf57e --- /dev/null +++ b/scripts/ci/pre_commit/update_installers_and_pre_commit.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import os +import re +import sys +from enum import Enum +from pathlib import Path + +import requests + +sys.path.insert(0, str(Path(__file__).parent.resolve())) # make sure common_precommit_utils is imported +from common_precommit_utils import AIRFLOW_SOURCES_ROOT_PATH, console + +# List of files to update and whether to keep total length of the original value when replacing. +FILES_TO_UPDATE: list[tuple[Path, bool]] = [ + (AIRFLOW_SOURCES_ROOT_PATH / "Dockerfile", False), + (AIRFLOW_SOURCES_ROOT_PATH / "Dockerfile.ci", False), + (AIRFLOW_SOURCES_ROOT_PATH / "scripts" / "ci" / "install_breeze.sh", False), + (AIRFLOW_SOURCES_ROOT_PATH / "scripts" / "docker" / "common.sh", False), + (AIRFLOW_SOURCES_ROOT_PATH / "scripts" / "tools" / "setup_breeze", False), + (AIRFLOW_SOURCES_ROOT_PATH / "pyproject.toml", False), + (AIRFLOW_SOURCES_ROOT_PATH / "dev" / "breeze" / "src" / "airflow_breeze" / "global_constants.py", False), + ( + AIRFLOW_SOURCES_ROOT_PATH + / "dev" + / "breeze" + / "src" + / "airflow_breeze" + / "commands" + / "release_management_commands.py", + False, + ), + (AIRFLOW_SOURCES_ROOT_PATH / ".github" / "actions" / "install-pre-commit" / "action.yml", False), + (AIRFLOW_SOURCES_ROOT_PATH / "dev/" / "breeze" / "doc" / "ci" / "02_images.md", True), +] + + +def get_latest_pypi_version(package_name: str) -> str: + response = requests.get(f"https://pypi.org/pypi/{package_name}/json") + response.raise_for_status() # Ensure we got a successful response + data = response.json() + latest_version = data["info"]["version"] # The version info is under the 'info' key + return latest_version + + +class Quoting(Enum): + UNQUOTED = 0 + SINGLE_QUOTED = 1 + DOUBLE_QUOTED = 2 + REVERSE_SINGLE_QUOTED = 3 + + +PIP_PATTERNS: list[tuple[re.Pattern, Quoting]] = [ + (re.compile(r"(AIRFLOW_PIP_VERSION=)([0-9.]+)"), Quoting.UNQUOTED), + (re.compile(r"(python -m pip install --upgrade pip==)([0-9.]+)"), Quoting.UNQUOTED), + (re.compile(r"(AIRFLOW_PIP_VERSION = )(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(PIP_VERSION = )(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(PIP_VERSION=)(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(\| *`AIRFLOW_PIP_VERSION` *\| *)(`[0-9.]+`)( *\|)"), Quoting.REVERSE_SINGLE_QUOTED), +] + +UV_PATTERNS: list[tuple[re.Pattern, Quoting]] = [ + (re.compile(r"(AIRFLOW_UV_VERSION=)([0-9.]+)"), Quoting.UNQUOTED), + (re.compile(r"(uv>=)([0-9]+)"), Quoting.UNQUOTED), + (re.compile(r"(AIRFLOW_UV_VERSION = )(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(UV_VERSION = )(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(UV_VERSION=)(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(\| *`AIRFLOW_UV_VERSION` *\| *)(`[0-9.]+`)( *\|)"), Quoting.REVERSE_SINGLE_QUOTED), + ( + re.compile( + r"(default: \")([0-9.]+)(\" # Keep this comment to " + r"allow automatic replacement of uv version)" + ), + Quoting.UNQUOTED, + ), +] + +PRE_COMMIT_PATTERNS: list[tuple[re.Pattern, Quoting]] = [ + (re.compile(r"(AIRFLOW_PRE_COMMIT_VERSION=)([0-9.]+)"), Quoting.UNQUOTED), + (re.compile(r"(AIRFLOW_PRE_COMMIT_VERSION = )(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(pre-commit>=)([0-9]+)"), Quoting.UNQUOTED), + (re.compile(r"(PRE_COMMIT_VERSION = )(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(PRE_COMMIT_VERSION=)(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + ( + re.compile(r"(\| *`AIRFLOW_PRE_COMMIT_VERSION` *\| *)(`[0-9.]+`)( *\|)"), + Quoting.REVERSE_SINGLE_QUOTED, + ), + ( + re.compile( + r"(default: \")([0-9.]+)(\" # Keep this comment to allow automatic " + r"replacement of pre-commit version)" + ), + Quoting.UNQUOTED, + ), +] + +PRE_COMMIT_UV_PATTERNS: list[tuple[re.Pattern, Quoting]] = [ + (re.compile(r"(AIRFLOW_PRE_COMMIT_UV_VERSION=)([0-9.]+)"), Quoting.UNQUOTED), + (re.compile(r"(AIRFLOW_PRE_COMMIT_UV_VERSION = )(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(pre-commit-uv>=)([0-9]+)"), Quoting.UNQUOTED), + (re.compile(r"(PRE_COMMIT_UV_VERSION = )(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + (re.compile(r"(PRE_COMMIT_UV_VERSION=)(\"[0-9.]+\")"), Quoting.DOUBLE_QUOTED), + ( + re.compile(r"(\| *`AIRFLOW_PRE_COMMIT_UV_VERSION` *\| *)(`[0-9.]+`)( *\|)"), + Quoting.REVERSE_SINGLE_QUOTED, + ), + ( + re.compile( + r"(default: \")([0-9.]+)(\" # Keep this comment to allow automatic " + r"replacement of pre-commit-uv version)" + ), + Quoting.UNQUOTED, + ), +] + + +def get_replacement(value: str, quoting: Quoting) -> str: + if quoting == Quoting.DOUBLE_QUOTED: + return f'"{value}"' + elif quoting == Quoting.SINGLE_QUOTED: + return f"'{value}'" + elif quoting == Quoting.REVERSE_SINGLE_QUOTED: + return f"`{value}`" + return value + + +UPGRADE_UV: bool = os.environ.get("UPGRADE_UV", "true").lower() == "true" +UPGRADE_PIP: bool = os.environ.get("UPGRADE_PIP", "true").lower() == "true" +UPGRADE_PRE_COMMIT: bool = os.environ.get("UPGRADE_PRE_COMMIT", "true").lower() == "true" + + +def replace_version(pattern: re.Pattern[str], version: str, text: str, keep_total_length: bool = True) -> str: + # Assume that the pattern has up to 3 replacement groups: + # 1. Prefix + # 2. Original version + # 3. Suffix + # + # (prefix)(version)(suffix) + # In case "keep_total_length" is set to True, the replacement will be padded with spaces to match + # the original length + def replacer(match): + prefix = match.group(1) + postfix = match.group(3) if len(match.groups()) > 2 else "" + if not keep_total_length: + return prefix + version + postfix + original_length = len(match.group(2)) + new_length = len(version) + diff = new_length - original_length + if diff <= 0: + postfix = " " * -diff + postfix + else: + postfix = postfix[diff:] + padded_replacement = prefix + version + postfix + return padded_replacement.strip() + + return re.sub(pattern, replacer, text) + + +if __name__ == "__main__": + changed = False + for file, keep_length in FILES_TO_UPDATE: + console.print(f"[bright_blue]Updating {file}") + file_content = file.read_text() + new_content = file_content + if UPGRADE_PIP: + pip_version = get_latest_pypi_version("pip") + console.print(f"[bright_blue]Latest pip version: {pip_version}") + for line_pattern, quoting in PIP_PATTERNS: + new_content = replace_version( + line_pattern, get_replacement(pip_version, quoting), new_content, keep_length + ) + if UPGRADE_UV: + uv_version = get_latest_pypi_version("uv") + console.print(f"[bright_blue]Latest uv version: {uv_version}") + for line_pattern, quoting in UV_PATTERNS: + new_content = replace_version( + line_pattern, get_replacement(uv_version, quoting), new_content, keep_length + ) + if UPGRADE_PRE_COMMIT: + pre_commit_version = get_latest_pypi_version("pre-commit") + console.print(f"[bright_blue]Latest pre-commit version: {pre_commit_version}") + for line_pattern, quoting in PRE_COMMIT_PATTERNS: + new_content = replace_version( + line_pattern, get_replacement(pre_commit_version, quoting), new_content, keep_length + ) + if UPGRADE_UV: + pre_commit_uv_version = get_latest_pypi_version("pre-commit-uv") + console.print(f"[bright_blue]Latest pre-commit-uv version: {pre_commit_uv_version}") + for line_pattern, quoting in PRE_COMMIT_UV_PATTERNS: + new_content = replace_version( + line_pattern, + get_replacement(pre_commit_uv_version, quoting), + new_content, + keep_length, + ) + if new_content != file_content: + file.write_text(new_content) + console.print(f"[bright_blue]Updated {file}") + changed = True + if changed: + sys.exit(1)