ENH: Add pre-commit check for setup.cfg options.extras_require

ci/deps/actions-38-minimum_versions.yaml

@@ -13,7 +13,7 @@ dependencies:
   - pytest-xdist>=1.31
   - psutil
   - pytest-asyncio>=0.17
-  - boto3
+
 
   # required dependencies
   - python-dateutil=2.8.2
@@ -23,6 +23,7 @@ dependencies:
   # optional dependencies
   - beautifulsoup4=4.9.3
   - blosc=1.21.0
+  - boto3=1.22.7
   - bottleneck=1.3.2
   - brotlipy=0.7.0
   - fastparquet=0.4.0
@@ -36,6 +37,7 @@ dependencies:
   - numba=0.53.1
   - numexpr=2.7.3
   - odfpy=1.4.1
+  - qtpy=2.2.0
   - openpyxl=3.0.7
   - pandas-gbq=0.15.0
   - psycopg2=2.8.6
@@ -54,3 +56,4 @@ dependencies:
   - xlrd=2.0.1
   - xlsxwriter=1.4.3
   - zstandard=0.15.2
+  - PyQt5=5.15.1

pandas/compat/_optional.py

@@ -15,6 +15,7 @@
     "bs4": "4.9.3",
     "blosc": "1.21.0",
     "bottleneck": "1.3.2",
+    "boto3": "1.22.7",
     "brotli": "0.7.0",
     "fastparquet": "0.4.0",
     "fsspec": "2021.07.0",
@@ -46,6 +47,8 @@
     "xlsxwriter": "1.4.3",
     "zstandard": "0.15.2",
     "tzdata": "2022.1",
+    "qtpy": "2.2.0",
+    "PyQt5": "5.15.1",
 }
 
 # A mapping from import name to package name (on PyPI) for packages where

scripts/validate_min_versions_in_sync.py

@@ -4,6 +4,7 @@
 
 ci/deps/actions-.*-minimum_versions.yaml
 pandas/compat/_optional.py
+setup.cfg
 
 TODO: doc/source/getting_started/install.rst
 
@@ -13,6 +14,7 @@
 """
 from __future__ import annotations
 
+import configparser
 import pathlib
 import sys
 
@@ -21,6 +23,7 @@
     pathlib.Path("ci/deps").absolute().glob("actions-*-minimum_versions.yaml")
 )
 CODE_PATH = pathlib.Path("pandas/compat/_optional.py").resolve()
+SETUP_PATH = pathlib.Path("setup.cfg").resolve()
 EXCLUDE_DEPS = {"tzdata"}
 # pandas package is not available
 # in pre-commit environment
@@ -63,23 +66,64 @@ def get_versions_from_ci(content: list[str]) -> tuple[dict[str, str], dict[str,
             if package in EXCLUDE_DEPS:
                 continue
             if not seen_optional:
-                required_deps[package] = version
+                required_deps[package.casefold()] = version
             else:
-                optional_deps[package] = version
+                optional_deps[package.casefold()] = version
     return required_deps, optional_deps
 
 
+def get_versions_from_setup() -> dict[str, str]:
+    install_map = _optional.INSTALL_MAPPING
+    optional_dependencies = {}
+
+    parser = configparser.ConfigParser()
+    parser.read(SETUP_PATH)
+    setup_optional = parser["options.extras_require"]["all"]
+    dependencies = setup_optional[1:].split("\n")
+
+    # remove test dependencies
+    test = parser["options.extras_require"]["test"]
+    test_dependencies = set(test[1:].split("\n"))
+    dependencies = [
+        package for package in dependencies if package not in test_dependencies
+    ]
+
+    for dependency in dependencies:
+        package, version = dependency.strip().split(">=")
+        optional_dependencies[install_map.get(package, package).casefold()] = version
+
+    for item in EXCLUDE_DEPS:
+        optional_dependencies.pop(item)
+
+    return optional_dependencies
+
+
 def main():
     with open(CI_PATH, encoding="utf-8") as f:
         _, ci_optional = get_versions_from_ci(f.readlines())
     code_optional = get_versions_from_code()
-    diff = set(ci_optional.items()).symmetric_difference(code_optional.items())
+    setup_optional = get_versions_from_setup()
+
+    diff = (ci_optional.items() | code_optional.items() | setup_optional.items()) - (
+        ci_optional.items() & code_optional.items() & setup_optional.items()
+    )
+
     if diff:
-        sys.stdout.write(
+        packages = {package for package, _ in diff}
+        out = sys.stdout
+        out.write(
             f"The follow minimum version differences were found between  "
-            f"{CI_PATH} and {CODE_PATH}. Please ensure these are aligned: "
-            f"{diff}\n"
+            f"{CI_PATH}, {CODE_PATH} AND {SETUP_PATH}. "
+            f"Please ensure these are aligned: \n\n"
         )
+
+        for package in packages:
+            out.write(
+                f"{package}\n"
+                f"{CI_PATH}: {ci_optional.get(package, 'Not specified')}\n"
+                f"{CODE_PATH}: {code_optional.get(package, 'Not specified')}\n"
+                f"{SETUP_PATH}: {setup_optional.get(package, 'Not specified')}\n\n"
+            )
         sys.exit(1)
     sys.exit(0)
 

setup.cfg

@@ -58,20 +58,20 @@ test =
 # see: doc/source/getting_started/install.rst
 performance =
     bottleneck>=1.3.2
-    numba>=0.53.0
+    numba>=0.53.1
     numexpr>=2.7.1
 timezone =
     tzdata>=2022.1
 computation =
     scipy>=1.7.1
     xarray>=0.19.0
 fss =
-    fsspec>=2021.7.0
+    fsspec>=2021.07.0
 aws =
     boto3>=1.22.7
     s3fs>=0.4.0
 gcp =
-    gcsfs>=2021.05.0
+    gcsfs>=2021.07.0
     pandas-gbq>=0.15.0
 excel =
     odfpy>=1.4.1
@@ -121,19 +121,19 @@ compression =
 all =
     beautifulsoup4>=4.9.3
     blosc>=1.21.0
-    bottleneck>=1.3.1
+    bottleneck>=1.3.2
     boto3>=1.22.7
     brotlipy>=0.7.0
     fastparquet>=0.4.0
-    fsspec>=2021.7.0
-    gcsfs>=2021.05.0
+    fsspec>=2021.07.0
+    gcsfs>=2021.07.0
     html5lib>=1.1
-    hypothesis>=5.5.3
+    hypothesis>=6.13.0
     jinja2>=3.0.0
     lxml>=4.6.3
     matplotlib>=3.3.2
-    numba>=0.53.0
-    numexpr>=2.7.1
+    numba>=0.53.1
+    numexpr>=2.7.3
     odfpy>=1.4.1
     openpyxl>=3.0.7
     pandas-gbq>=0.15.0
@@ -149,7 +149,7 @@ all =
     pyxlsb>=1.0.8
     qtpy>=2.2.0
     scipy>=1.7.1
-    s3fs>=0.4.0
+    s3fs>=2021.08.0
     SQLAlchemy>=1.4.16
     tables>=3.6.1
     tabulate>=0.8.9