Separate out aiobotocore support to airlfow extra (#30161)

This is follow-up after #30127 and #30144 about handling async (deferrable) operators for amazon provider. Rather than making aiobotocre directly a devel extra dependency, we create a separate "aiobotocore" extra that allows for greater flexibility on how we handle the aiobotocore support. It allows for two approach: 1) (current) if we decide that by default we keep boto/botocore compatible with aiobotocore in our constraints/image then aiobotocore should be added to devel and it should be included as preselected extra in Dockerfile. This will lead to having aibotocore and compatible boto/botocore in both constraints and the PROD image. 2) (possible) if we decide that we prefer to keep to the latest version of boto/botocore in constraints/image, then we could remove aiobotocore from both constraints and PROD image. We should also in this case swap the "LatestBoto" CI job introduced in #30144 to be "WithAiobotocore" job - by installing aiobotocore and downgrading boto/botocore in the job.
apache · Mar 17, 2023 · 78b827c · 78b827c
1 parent c99201a
commit 78b827c
Show file tree

Hide file tree

Showing 9 changed files with 29 additions and 20 deletions.
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
@@ -610,14 +610,14 @@ all dependencies needed in the CI environment.
 This is the full list of those extras:
 
   .. START EXTRAS HERE
-airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.drill,
-apache.druid, apache.flink, apache.hdfs, apache.hive, apache.impala, apache.kylin, apache.livy,
-apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, arangodb, asana, async, atlas,
-atlassian.jira, aws, azure, cassandra, celery, cgroups, cloudant, cncf.kubernetes, common.sql,
-crypto, dask, databricks, datadog, dbt.cloud, deprecated_api, devel, devel_all, devel_ci,
-devel_hadoop, dingding, discord, doc, doc_gen, docker, druid, elasticsearch, exasol, facebook, ftp,
-gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http,
-imap, influxdb, jdbc, jenkins, kerberos, kubernetes, ldap, leveldb, microsoft.azure,
+aiobotocore, airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra,
+apache.drill, apache.druid, apache.flink, apache.hdfs, apache.hive, apache.impala, apache.kylin,
+apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, arangodb, asana,
+async, atlas, atlassian.jira, aws, azure, cassandra, celery, cgroups, cloudant, cncf.kubernetes,
+common.sql, crypto, dask, databricks, datadog, dbt.cloud, deprecated_api, devel, devel_all,
+devel_ci, devel_hadoop, dingding, discord, doc, doc_gen, docker, druid, elasticsearch, exasol,
+facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs,
+hive, http, imap, influxdb, jdbc, jenkins, kerberos, kubernetes, ldap, leveldb, microsoft.azure,
 microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openfaas,
 opsgenie, oracle, pagerduty, pandas, papermill, password, pinot, plexus, postgres, presto, qds,
 qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid, sentry, sftp, singularity, slack,

diff --git a/Dockerfile b/Dockerfile
@@ -35,7 +35,7 @@
 #                        much smaller.
 #
 # Use the same builder frontend version for everyone
-ARG AIRFLOW_EXTRAS="amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google_auth,grpc,hashicorp,http,ldap,microsoft.azure,mysql,odbc,pandas,postgres,redis,sendgrid,sftp,slack,snowflake,ssh,statsd,virtualenv"
+ARG AIRFLOW_EXTRAS="aiobotocore,amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google_auth,grpc,hashicorp,http,ldap,microsoft.azure,mysql,odbc,pandas,postgres,redis,sendgrid,sftp,slack,snowflake,ssh,statsd,virtualenv"
 ARG ADDITIONAL_AIRFLOW_EXTRAS=""
 ARG ADDITIONAL_PYTHON_DEPS=""
 

diff --git a/INSTALL b/INSTALL
@@ -94,14 +94,14 @@ pip install .[async,google,amazon] \
 The list of available extras:
 
 # START EXTRAS HERE
-airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra, apache.drill,
-apache.druid, apache.flink, apache.hdfs, apache.hive, apache.impala, apache.kylin, apache.livy,
-apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, arangodb, asana, async, atlas,
-atlassian.jira, aws, azure, cassandra, celery, cgroups, cloudant, cncf.kubernetes, common.sql,
-crypto, dask, databricks, datadog, dbt.cloud, deprecated_api, devel, devel_all, devel_ci,
-devel_hadoop, dingding, discord, doc, doc_gen, docker, druid, elasticsearch, exasol, facebook, ftp,
-gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http,
-imap, influxdb, jdbc, jenkins, kerberos, kubernetes, ldap, leveldb, microsoft.azure,
+aiobotocore, airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.cassandra,
+apache.drill, apache.druid, apache.flink, apache.hdfs, apache.hive, apache.impala, apache.kylin,
+apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, arangodb, asana,
+async, atlas, atlassian.jira, aws, azure, cassandra, celery, cgroups, cloudant, cncf.kubernetes,
+common.sql, crypto, dask, databricks, datadog, dbt.cloud, deprecated_api, devel, devel_all,
+devel_ci, devel_hadoop, dingding, discord, doc, doc_gen, docker, druid, elasticsearch, exasol,
+facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs,
+hive, http, imap, influxdb, jdbc, jenkins, kerberos, kubernetes, ldap, leveldb, microsoft.azure,
 microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openfaas,
 opsgenie, oracle, pagerduty, pandas, papermill, password, pinot, plexus, postgres, presto, qds,
 qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid, sentry, sftp, singularity, slack,

diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -257,6 +257,7 @@ def get_airflow_extras():
 
 DEFAULT_EXTRAS = [
     # BEGINNING OF EXTRAS LIST UPDATED BY PRE COMMIT
+    "aiobotocore",
     "amazon",
     "async",
     "celery",

diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst
@@ -40,6 +40,8 @@ python dependencies for the provided package.
 +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+
 | extra               | install command                                     | enables                                                                    |
 +=====================+=====================================================+============================================================================+
+| aiobotocore         | ``pip install 'apache-airflow[aiobotocore]'``       | Support for asynchronous (deferrable) operators for Amazon integration     |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+
 | async               | ``pip install 'apache-airflow[async]'``             | Async worker classes for Gunicorn                                          |
 +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+
 | celery              | ``pip install 'apache-airflow[celery]'``            | CeleryExecutor (also installs the celery provider package!)                |

diff --git a/docs/docker-stack/build-arg-ref.rst b/docs/docker-stack/build-arg-ref.rst
@@ -80,6 +80,7 @@ List of default extras in the production Dockerfile:
 
 .. BEGINNING OF EXTRAS LIST UPDATED BY PRE COMMIT
 
+* aiobotocore
 * amazon
 * async
 * celery

diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt
@@ -31,10 +31,10 @@ k8s:status:2b8aa0b9597a1790c184c7b776ebd9c3
 k8s:tests:dc44e907de908d1c4f11cf5d99915da1
 k8s:upload-k8s-image:b012457bb6a98524859ecaf5746b4439
 k8s:a04e7db8fb49c16a0633ac32f14463ec
-prod-image:build:5d0fcf0f0b1f4a8d13378253214162d6
+prod-image:build:a1608071b5a80b3cc442da9d00e1525b
 prod-image:pull:e3c89dd908fc44adf6e159c2950ebdd0
 prod-image:verify:31bc5efada1d70a0a31990025db1a093
-prod-image:2ac8e1e6b4afb259f00390ad0e90a06f
+prod-image:79bd4cc9de03ab7e1d75f025d75eee46
 release-management:create-minor-branch:6a01066dce15e09fb269a8385626657c
 release-management:generate-constraints:ae30d6ad49a1b2c15b61cb29080fd957
 release-management:generate-issue-content-providers:f3c00ba74e3afc054fe29b65156740ac

diff --git a/images/breeze/output_prod-image_build.svg b/images/breeze/output_prod-image_build.svg
diff --git a/setup.py b/setup.py
@@ -399,6 +399,9 @@ def write_version(filename: str = str(AIRFLOW_SOURCES_ROOT / "airflow" / "git_ve
     "wheel",
     "yamllint",
     "aioresponses",
+]
+
+aiobotocore = [
     # This required for AWS deferrable operators.
     # There is conflict between boto3 and aiobotocore dependency botocore.
     # TODO: We can remove it once boto3 and aiobotocore both have compatible botocore version or
@@ -421,6 +424,7 @@ def get_unique_dependency_list(req_list_iterable: Iterable[list[str]]):
 
 devel = get_unique_dependency_list(
     [
+        aiobotocore,
         cgroups,
         devel_only,
         doc,
@@ -458,6 +462,7 @@ def get_unique_dependency_list(req_list_iterable: Iterable[list[str]]):
 # Those are extras that are extensions of the 'core' Airflow. They provide additional features
 # To airflow core. They do not have separate providers because they do not have any operators/hooks etc.
 CORE_EXTRAS_DEPENDENCIES: dict[str, list[str]] = {
+    "aiobotocore": aiobotocore,
     "async": async_packages,
     "celery": celery,
     "cgroups": cgroups,