From e9f1b8b27ac9ae2d11c04a2cb59786fc52f7fdde Mon Sep 17 00:00:00 2001 From: amanda Date: Thu, 8 Dec 2022 15:30:43 +0000 Subject: [PATCH 1/5] Investigate if ST2 runs with python 3.10 --- .github/workflows/ci.yaml | 40 ++++++++++++++++++- .github/workflows/microbenchmarks.yaml | 6 +++ .../workflows/orquesta-integration-tests.yaml | 6 +++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fb052f7928..c0620c4387 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -70,11 +70,19 @@ jobs: task: 'ci-compile' python-version-short: '3.8' python-version: '3.8.10' + - name: 'Lint Checks (black, flake8, etc.)' + task: 'ci-checks' + python-version-short: '3.10' + python-version: '3.10.9' + - name: 'Compile (pip deps, pylint, etc.)' + task: 'ci-compile' + python-version-short: '3.10' + python-version: '3.10.9' env: TASK: '${{ matrix.task }}' COLUMNS: '120' - PYLINT_CONCURRENCY: '4' + PYLINT_CONCURRENCY: '6' steps: - name: Checkout repository uses: actions/checkout@v2 @@ -174,6 +182,18 @@ jobs: nosetests_node_index: 1 python-version-short: '3.8' python-version: '3.8.10' + - name: 'Unit Tests (chunk 1)' + task: 'ci-unit' + nosetests_node_total: 2 + nosetests_node_index: 0 + python-version-short: '3.10' + python-version: '3.10.9' + - name: 'Unit Tests (chunk 2)' + task: 'ci-unit' + nosetests_node_total: 2 + nosetests_node_index: 1 + python-version-short: '3.10' + python-version: '3.10.9' # This job is slow so we only run in on a daily basis # - name: 'Micro Benchmarks' # task: 'micro-benchmarks' @@ -359,6 +379,24 @@ jobs: nosetests_node_index: 1 python-version-short: '3.8' python-version: '3.8.10' + - name: 'Pack Tests' + task: 'ci-packs-tests' + nosetests_node_total: 1 + nosetests_node_index: 0 + python-version-short: '3.10' + python-version: '3.10.9' + - name: 'Integration Tests (chunk 1)' + task: 'ci-integration' + nosetests_node_total: 2 + nosetests_node_index: 0 + python-version-short: '3.10' + python-version: '3.10.9' + - name: 'Integration Tests (chunk 2)' + task: 'ci-integration' + nosetests_node_total: 2 + nosetests_node_index: 1 + python-version-short: '3.10' + python-version: '3.10.9' services: mongo: image: mongo:4.4 diff --git a/.github/workflows/microbenchmarks.yaml b/.github/workflows/microbenchmarks.yaml index fcc7b5cca0..97663424ad 100644 --- a/.github/workflows/microbenchmarks.yaml +++ b/.github/workflows/microbenchmarks.yaml @@ -46,6 +46,12 @@ jobs: nosetests_node_index: 0 python-version-short: '3.8' python-version: '3.8.10' + - name: 'Microbenchmarks' + task: 'micro-benchmarks' + nosetests_node_total: 1 + nosetests_node_index: 0 + python-version-short: '3.10' + python-version: '3.10.9' services: mongo: image: mongo:4.4 diff --git a/.github/workflows/orquesta-integration-tests.yaml b/.github/workflows/orquesta-integration-tests.yaml index 5e0a4ebeef..a51680a1dc 100644 --- a/.github/workflows/orquesta-integration-tests.yaml +++ b/.github/workflows/orquesta-integration-tests.yaml @@ -67,6 +67,12 @@ jobs: nosetests_node_index: 0 python-version-short: '3.8' python-version: '3.8.10' + - name: 'Integration Tests (Orquesta)' + task: 'ci-orquesta' + nosetests_node_total: 1 + nosetests_node_index: 0 + python-version-short: '3.10' + python-version: '3.10.9' services: mongo: image: mongo:4.4 From 19c86e856341da999374ee686a630842a349275d Mon Sep 17 00:00:00 2001 From: amanda Date: Thu, 8 Dec 2022 16:05:21 +0000 Subject: [PATCH 2/5] Upgrade greenlet and eventlet --- fixed-requirements.txt | 4 ++-- requirements.txt | 4 ++-- st2actions/requirements.txt | 2 +- st2api/requirements.txt | 2 +- st2auth/requirements.txt | 2 +- st2common/requirements.txt | 4 ++-- st2reactor/requirements.txt | 2 +- st2stream/requirements.txt | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/fixed-requirements.txt b/fixed-requirements.txt index 306570a89c..1d57891cd9 100644 --- a/fixed-requirements.txt +++ b/fixed-requirements.txt @@ -10,13 +10,13 @@ dnspython>=1.16.0,<2.0.0 cryptography==3.4.7 # Note: 0.20.0 removed select.poll() on which some of our code and libraries we # depend on rely -eventlet==0.30.2 +eventlet==0.33.2 flex==6.14.1 gitpython==3.1.15 # Needed by gitpython, old versions used to bundle it gitdb==4.0.2 # Note: greenlet is used by eventlet -greenlet==1.0.0 +greenlet==1.1.3 gunicorn==20.1.0 jsonpath-rw==1.4.0 jsonschema==2.6.0 diff --git a/requirements.txt b/requirements.txt index 3d5395bb0f..088630d651 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,11 +17,11 @@ ciso8601 cryptography==3.4.7 decorator==4.4.2 dnspython>=1.16.0,<2.0.0 -eventlet==0.30.2 +eventlet==0.33.2 flex==6.14.1 gitdb==4.0.2 gitpython==3.1.15 -greenlet==1.0.0 +greenlet==1.1.3 gunicorn==20.1.0 importlib-metadata==3.10.1 jinja2==2.11.3 diff --git a/st2actions/requirements.txt b/st2actions/requirements.txt index acd17a961e..12e929d250 100644 --- a/st2actions/requirements.txt +++ b/st2actions/requirements.txt @@ -8,7 +8,7 @@ MarkupSafe<2.1.0,>=0.23 apscheduler==3.7.0 chardet<3.1.0 -eventlet==0.30.2 +eventlet==0.33.2 gitpython==3.1.15 jinja2==2.11.3 kombu==5.0.2 diff --git a/st2api/requirements.txt b/st2api/requirements.txt index f503ef97bc..9301de8eff 100644 --- a/st2api/requirements.txt +++ b/st2api/requirements.txt @@ -5,7 +5,7 @@ # If you want to update depdencies for a single component, modify the # in-requirements.txt for that component and then run 'make requirements' to # update the component requirements.txt -eventlet==0.30.2 +eventlet==0.33.2 gunicorn==20.1.0 jsonschema==2.6.0 kombu==5.0.2 diff --git a/st2auth/requirements.txt b/st2auth/requirements.txt index 1d6a06de81..f2b0cb1409 100644 --- a/st2auth/requirements.txt +++ b/st2auth/requirements.txt @@ -6,7 +6,7 @@ # in-requirements.txt for that component and then run 'make requirements' to # update the component requirements.txt bcrypt==3.2.0 -eventlet==0.30.2 +eventlet==0.33.2 gunicorn==20.1.0 oslo.config>=1.12.1,<1.13 passlib==1.7.4 diff --git a/st2common/requirements.txt b/st2common/requirements.txt index 4757263181..ba7dae77c4 100644 --- a/st2common/requirements.txt +++ b/st2common/requirements.txt @@ -14,11 +14,11 @@ ciso8601 cryptography==3.4.7 decorator==4.4.2 dnspython>=1.16.0,<2.0.0 -eventlet==0.30.2 +eventlet==0.33.2 flex==6.14.1 gitdb==4.0.2 gitpython==3.1.15 -greenlet==1.0.0 +greenlet==1.1.3 jinja2==2.11.3 jsonpath-rw==1.4.0 jsonschema==2.6.0 diff --git a/st2reactor/requirements.txt b/st2reactor/requirements.txt index 3388a8214b..18ef8a40b9 100644 --- a/st2reactor/requirements.txt +++ b/st2reactor/requirements.txt @@ -6,7 +6,7 @@ # in-requirements.txt for that component and then run 'make requirements' to # update the component requirements.txt apscheduler==3.7.0 -eventlet==0.30.2 +eventlet==0.33.2 jsonpath-rw==1.4.0 jsonschema==2.6.0 kombu==5.0.2 diff --git a/st2stream/requirements.txt b/st2stream/requirements.txt index faa39eafd8..b1cb255820 100644 --- a/st2stream/requirements.txt +++ b/st2stream/requirements.txt @@ -5,7 +5,7 @@ # If you want to update depdencies for a single component, modify the # in-requirements.txt for that component and then run 'make requirements' to # update the component requirements.txt -eventlet==0.30.2 +eventlet==0.33.2 gunicorn==20.1.0 jsonschema==2.6.0 kombu==5.0.2 From ab82669680ca5954cbcfaae46a97dd3eb0c42857 Mon Sep 17 00:00:00 2001 From: Amanda McGuinness Date: Thu, 8 Dec 2022 17:40:54 +0000 Subject: [PATCH 3/5] Update pytest for python3.10 and also try newer oslo.config --- fixed-requirements.txt | 2 +- requirements.txt | 2 +- st2actions/requirements.txt | 2 +- st2api/requirements.txt | 2 +- st2auth/requirements.txt | 2 +- st2common/requirements.txt | 2 +- st2reactor/requirements.txt | 2 +- st2stream/requirements.txt | 2 +- test-requirements.txt | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/fixed-requirements.txt b/fixed-requirements.txt index 1d57891cd9..b9177eda2e 100644 --- a/fixed-requirements.txt +++ b/fixed-requirements.txt @@ -34,7 +34,7 @@ networkx>=2.5.1,<2.6 decorator==4.4.2 # NOTE: Recent version substantially affect the performance and add big import time overhead # See https://github.com/StackStorm/st2/issues/4160#issuecomment-394386433 for details -oslo.config>=1.12.1,<1.13 +oslo.config==9.0.0 oslo.utils<5.0,>=4.0.0 paramiko==2.10.1 passlib==1.7.4 diff --git a/requirements.txt b/requirements.txt index 088630d651..8222b216c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,7 +38,7 @@ nose-parallel==0.4.0 nose-timer==1.0.1 orjson==3.5.2 orquesta@ git+https://github.com/StackStorm/orquesta.git@v1.5.0 -oslo.config>=1.12.1,<1.13 +oslo.config==9.0.0 oslo.utils<5.0,>=4.0.0 paramiko==2.10.1 passlib==1.7.4 diff --git a/st2actions/requirements.txt b/st2actions/requirements.txt index 12e929d250..22b7e1027f 100644 --- a/st2actions/requirements.txt +++ b/st2actions/requirements.txt @@ -14,7 +14,7 @@ jinja2==2.11.3 kombu==5.0.2 lockfile==0.12.2 logshipper@ git+https://github.com/StackStorm/logshipper.git@stackstorm_patched ; platform_system=="Linux" -oslo.config>=1.12.1,<1.13 +oslo.config==9.0.0 oslo.utils<5.0,>=4.0.0 pyinotify==0.9.6 ; platform_system=="Linux" pyparsing<3 diff --git a/st2api/requirements.txt b/st2api/requirements.txt index 9301de8eff..4f72144060 100644 --- a/st2api/requirements.txt +++ b/st2api/requirements.txt @@ -10,7 +10,7 @@ gunicorn==20.1.0 jsonschema==2.6.0 kombu==5.0.2 mongoengine==0.23.0 -oslo.config>=1.12.1,<1.13 +oslo.config==9.0.0 oslo.utils<5.0,>=4.0.0 pymongo==3.11.3 pyparsing<3 diff --git a/st2auth/requirements.txt b/st2auth/requirements.txt index f2b0cb1409..3037dcf583 100644 --- a/st2auth/requirements.txt +++ b/st2auth/requirements.txt @@ -8,7 +8,7 @@ bcrypt==3.2.0 eventlet==0.33.2 gunicorn==20.1.0 -oslo.config>=1.12.1,<1.13 +oslo.config==9.0.0 passlib==1.7.4 pymongo==3.11.3 six==1.13.0 diff --git a/st2common/requirements.txt b/st2common/requirements.txt index ba7dae77c4..e9cb56c015 100644 --- a/st2common/requirements.txt +++ b/st2common/requirements.txt @@ -28,7 +28,7 @@ mongoengine==0.23.0 networkx>=2.5.1,<2.6 orjson==3.5.2 orquesta@ git+https://github.com/StackStorm/orquesta.git@v1.5.0 -oslo.config>=1.12.1,<1.13 +oslo.config==9.0.0 paramiko==2.10.1 pyOpenSSL<=21.0.0 pymongo==3.11.3 diff --git a/st2reactor/requirements.txt b/st2reactor/requirements.txt index 18ef8a40b9..0e183b243a 100644 --- a/st2reactor/requirements.txt +++ b/st2reactor/requirements.txt @@ -10,6 +10,6 @@ eventlet==0.33.2 jsonpath-rw==1.4.0 jsonschema==2.6.0 kombu==5.0.2 -oslo.config>=1.12.1,<1.13 +oslo.config==9.0.0 python-dateutil==2.8.1 six==1.13.0 diff --git a/st2stream/requirements.txt b/st2stream/requirements.txt index b1cb255820..d2d6c92134 100644 --- a/st2stream/requirements.txt +++ b/st2stream/requirements.txt @@ -10,7 +10,7 @@ gunicorn==20.1.0 jsonschema==2.6.0 kombu==5.0.2 mongoengine==0.23.0 -oslo.config>=1.12.1,<1.13 +oslo.config==9.0.0 oslo.utils<5.0,>=4.0.0 pymongo==3.11.3 pyparsing<3 diff --git a/test-requirements.txt b/test-requirements.txt index 56b8b7ac2a..082f9afda9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -36,7 +36,7 @@ prance==0.15.0 # pip-tools 5.4 needs pip>=20.1 # pip-tools 6.0 needs pip>=20.3 pip-tools>=5.4,<6.1 -pytest==6.2.3 +pytest==6.2.5 pytest-benchmark==3.4.1 pytest-benchmark[histogram]==3.4.1 # zstandard is used for micro benchmarks From 8097a23e55755c2da4f5f8096aba69f6fcf56a63 Mon Sep 17 00:00:00 2001 From: Amanda McGuinness Date: Thu, 8 Dec 2022 17:49:43 +0000 Subject: [PATCH 4/5] Update pylint/astroid --- test-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 082f9afda9..140c873ad1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,8 +2,8 @@ coverage==4.5.2 pep8==1.7.1 st2flake8==0.1.0 -astroid==2.5.6 -pylint==2.8.2 +astroid==2.12.13 +pylint==2.15.8 pylint-plugin-utils>=0.4 black==22.3.0 pre-commit==2.1.0 From b9a373abf06b9f6a9bda81eb034a7e3d3c1a8936 Mon Sep 17 00:00:00 2001 From: Amanda McGuinness Date: Thu, 8 Dec 2022 18:04:37 +0000 Subject: [PATCH 5/5] Increment prompt-toolkit for python 3.10 for orquesta run --- fixed-requirements.txt | 2 +- requirements.txt | 2 +- st2client/requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fixed-requirements.txt b/fixed-requirements.txt index b9177eda2e..e5bafa636c 100644 --- a/fixed-requirements.txt +++ b/fixed-requirements.txt @@ -38,7 +38,7 @@ oslo.config==9.0.0 oslo.utils<5.0,>=4.0.0 paramiko==2.10.1 passlib==1.7.4 -prompt-toolkit==1.0.15 +prompt-toolkit==3.0.36 pyinotify==0.9.6 ; platform_system=="Linux" pymongo==3.11.3 pyparsing<3 diff --git a/requirements.txt b/requirements.txt index 8222b216c5..43044a59d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,7 +43,7 @@ oslo.utils<5.0,>=4.0.0 paramiko==2.10.1 passlib==1.7.4 prettytable==2.1.0 -prompt-toolkit==1.0.15 +prompt-toolkit==3.0.36 psutil==5.8.0 pyOpenSSL<=21.0.0 pyinotify==0.9.6 ; platform_system=="Linux" diff --git a/st2client/requirements.txt b/st2client/requirements.txt index dd430a635e..ef42f457e2 100644 --- a/st2client/requirements.txt +++ b/st2client/requirements.txt @@ -14,7 +14,7 @@ jsonpath-rw==1.4.0 jsonschema==2.6.0 orjson==3.5.2 prettytable==2.1.0 -prompt-toolkit==1.0.15 +prompt-toolkit==3.0.36 pyOpenSSL<=21.0.0 pysocks python-dateutil==2.8.1