Full CI support for public builds + switch to use cibuildwheel (#267)

* switch to cibuildwheel + some cleanups

* try setting up Python manually

* comment out un-needed code + propagate python-version

* fix: need to check out first

* only build natively; add -v; ensure targeting manylinux

* single quotes

* restore env setup to get artifact dir

* fix artifact dir

* fix artifact name

* restore & fix artifact name again

* build on all platforms!

* fix CIBW_BUILD for windows

* fix typo

* no quotes for wildcard matching

* move CIBW_BUILD logic to script

* fix win runner name

* try to find where pwsh is

* try to escape

* continue hunting..

* try to overwrite shell

* try to install ps

* be explicit about shell (why?)

* only build for win 64 bits

* try to install msvc

* install msvc ourselves

* fix typo

* skip custom cl ver check

* install to standard location

* try to locate Python include path

* switch to public windows runner for now

* windows image does not have sudo

* pwd on Windows Bash does not use Windows path format

* cover all Python versions!

* add quotes

* align the python version installed via GHA vs used at build time

* fix constraint syntax

* check if setup-python is causing interference

* fix typo

* apply a WAR on Linux

* fix unbound var

* detect Python path after it's installed (by CIBW)

* try CIBW_BEFORE_ALL_LINUX

* try to restore the pre-py-span setup...

* reduce build matrix to experiment with cuda.bindings builds

* fix parenthesis

* use abs path

* defer CUDA_PATH setting

* use CIBW_ENVIRONMENT to pass env var

* fetch cuda-profiler-api headers

* only rely on redist

* allow wheel repair to fix the triplet tags

* restore full build matrix!

* fix wget on Windows; pass PARALLEL_LEVEL to CIBW

* switch from wget to curl

* windows needs unzip not tar

* mv -> rsync

* git bash has no wget or rsync...

* ensure win-style path on win

* convert abs path

* debug

* another level down

* check if it is a race condition on win

* clean up unused (but still relevant) code

* clean up unused (but still relevant) code - cont'd

* consolidate with PYTHON_VERSION_FORMATTED

.github/actions/build/action.yml

@@ -13,73 +13,79 @@ inputs:
   host-platform:
     required: true
     type: string
-  use-container:
-    required: true
-    type: boolean
-  docker-image:
-    type: string
-    required: true
   upload-enabled:
     required: true
     type: boolean
-  python-version:
-    required: true
-    type: string
 
 runs:
   using: composite
   steps:
+    - name: Build cuda.core wheel
+      uses: pypa/cibuildwheel@v2.22.0
+      env:
+        CIBW_BUILD: ${{ env.CIBW_BUILD }}
+        CIBW_ARCHS_LINUX: "native"
+        CIBW_BUILD_VERBOSITY: 1
+        # # ensure Python.h & co can be found
+        # CIBW_BEFORE_BUILD_WINDOWS: >
+        #   python -c "import sysconfig; print(sysconfig.get_path('include'))" >> $env:INCLUDE
+      with:
+        package-dir: ./cuda_core/
+        output-dir: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
 
-    - if: ${{ inputs.use-container }}
-      name: Build (in container)
-      shell: bash --noprofile --norc -xeuo pipefail {0}
-      run: |
-
-        docker run \
-          -e AWS_REGION \
-          -e AWS_SESSION_TOKEN \
-          -e AWS_ACCESS_KEY_ID \
-          -e AWS_SECRET_ACCESS_KEY \
-          -e GITHUB_TOKEN \
-          -e BINDINGS_ARTIFACTS_DIR="$BINDINGS_ARTIFACTS_DIR" \
-          -e CORE_ARTIFACTS_DIR="$CORE_ARTIFACTS_DIR" \
-          -e UPLOAD_ENABLED="$UPLOAD_ENABLED" \
-          -e USE_CUDA="$USE_CUDA" \
-          -e REPO_DIR="$REPO_DIR" \
-          -e LEGATE_CORE_BUILD_MODE="$LEGATE_CORE_BUILD_MODE" \
-          -e PYTHON_VERSION="$PYTHON_VERSION" \
-          -v "${{ env.REPO_DIR }}:${{ env.REPO_DIR }}" \
-          -v "${{ env.BINDINGS_ARTIFACTS_DIR }}:${{ env.BINDINGS_ARTIFACTS_DIR }}" \
-          -v "${{ env.CORE_ARTIFACTS_DIR }}:${{ env.CORE_ARTIFACTS_DIR }}" \
-          --rm "${{ inputs.docker-image }}" \
-          /bin/bash -c "${{ env.REPO_DIR }}/continuous_integration/scripts/entrypoint ${{ env.REPO_DIR }}/continuous_integration/scripts/build ${{ inputs.build-type}} ${{ inputs.target-device }}"
-
-    - if: ${{ !inputs.use-container }}
-      name: Build (without container)
-      shell: bash --noprofile --norc -xeuo pipefail {0}
-      run: |
-        "${{ env.REPO_DIR }}/continuous_integration/scripts/entrypoint" "${{ env.REPO_DIR }}/continuous_integration/scripts/build" "${{ inputs.build-type}}" "${{ inputs.target-device }}"
-
-    - name: Display structure of the bindings artifacts folder (post build)
+    - name: List the cuda.core artifacts directory
       shell: bash --noprofile --norc -xeuo pipefail {0}
       run: |
-        sudo chown -R $(whoami) ${{ env.BINDINGS_ARTIFACTS_DIR }}
-        ls -lahR ${{ env.BINDINGS_ARTIFACTS_DIR }}
+        if [[ "${{ inputs.host-platform }}" == win* ]]; then
+          export CHOWN=chown
+        else
+          export CHOWN="sudo chown"
+        fi
+        $CHOWN -R $(whoami) ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
+        ls -lahR ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
 
-    - name: Upload bindings build artifacts
+    - name: Upload cuda.core build artifacts
       uses: actions/upload-artifact@v4
       with:
-        name: ${{ env.BINDINGS_ARTIFACT_NAME }}
-        path: ${{ env.BINDINGS_ARTIFACTS_DIR }}
+        name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}
+        path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl
+        if-no-files-found: error
+        overwrite: 'true'
+
+    - name: Build cuda.bindings wheel
+      uses: pypa/cibuildwheel@v2.22.0
+      env:
+        CIBW_BUILD: ${{ env.CIBW_BUILD }}
+        CIBW_ARCHS_LINUX: "native"
+        CIBW_BUILD_VERBOSITY: 1
+        CIBW_ENVIRONMENT_LINUX: >
+          CUDA_PATH="$(realpath ./cuda_toolkit)"
+          PARALLEL_LEVEL=${{ env.PARALLEL_LEVEL }}
+        CIBW_ENVIRONMENT_WINDOWS: >
+          CUDA_HOME="$(cygpath -w $(realpath ./cuda_toolkit))"
+        #  PARALLEL_LEVEL=${{ env.PARALLEL_LEVEL }}
+        # # ensure Python.h & co can be found
+        # CIBW_BEFORE_BUILD_WINDOWS: >
+        #   python -c "import sysconfig; print(sysconfig.get_path('include'))" >> $env:INCLUDE
+      with:
+        package-dir: ./cuda_bindings/
+        output-dir: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
 
-    - name: Display structure of the core artifacts folder (post build)
+    - name: List the cuda.bindings artifacts directory
       shell: bash --noprofile --norc -xeuo pipefail {0}
       run: |
-        sudo chown -R $(whoami) ${{ env.CORE_ARTIFACTS_DIR }}
-        ls -lahR ${{ env.CORE_ARTIFACTS_DIR }}
+        if [[ "${{ inputs.host-platform }}" == win* ]]; then
+          export CHOWN=chown
+        else
+          export CHOWN="sudo chown"
+        fi
+        $CHOWN -R $(whoami) ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
+        ls -lahR ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
 
-    - name: Upload core build artifacts
+    - name: Upload cuda.bindings build artifacts
       uses: actions/upload-artifact@v4
       with:
-        name: ${{ env.CORE_ARTIFACT_NAME }}
-        path: ${{ env.CORE_ARTIFACTS_DIR }}
+        name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}
+        path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl
+        if-no-files-found: error
+        overwrite: 'true'

.github/actions/setup/action.yml

@@ -22,51 +22,121 @@ inputs:
   python-version:
     required: true
     type: string
+  cuda-version:
+    required: true
+    type: string
 
 runs:
   using: composite
   steps:
-    - name: Set REPO_DIR and Dump environment
+    # WAR: setup-python is not relocatable...
+    # see https://github.com/actions/setup-python/issues/871
+    - name: Set up Python ${{ inputs.python-version }}
+      if: ${{ startsWith(inputs.host-platform, 'linux') }}
+      id: setup-python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.12"
+
+    - name: Set up MSVC
+      if: ${{ startsWith(inputs.host-platform, 'win') }}
+      uses: ilammy/msvc-dev-cmd@v1
+
+    - name: Dump environment
       shell: bash --noprofile --norc -xeuo pipefail {0}
       run: |
-        echo "REPO_DIR=$(pwd)" >> $GITHUB_ENV
         env
 
-    - name: Set environment variables
+    - name: Get CUDA components
       shell: bash --noprofile --norc -xeuo pipefail {0}
       run: |
+        CUDA_PATH="./cuda_toolkit"
+        mkdir $CUDA_PATH
 
-        WITH_TESTS_STR=''
-        if [[ ("${{ inputs.upload-enabled }}" == "false") && ("${{ inputs.build-type }}" != "ci") ]]; then
-          WITH_TESTS_STR='-with_tests'
+        # The binary archives (redist) are guaranteed to be updated as part of the release posting.
+        CTK_BASE_URL="https://developer.download.nvidia.com/compute/cuda/redist/"
+        CTK_JSON_URL="$CTK_BASE_URL/redistrib_${{ inputs.cuda-version }}.json"
+        if [[ "${{ inputs.host-platform }}" == linux* ]]; then
+          if [[ "${{ inputs.host-platform }}" == "linux-x64" ]]; then
+            CTK_SUBDIR="linux-x86_64"
+          elif [[ "${{ inputs.host-platform }}" == "linux-aarch64" ]]; then
+            CTK_SUBDIR="linux-sbsa"
+          fi
+          function extract() {
+            tar -xvf $1 -C $CUDA_PATH --strip-components=1
+          }
+        elif [[ "${{ inputs.host-platform }}" == "win-x64" ]]; then
+          CTK_SUBDIR="windows-x86_64"
+          function extract() {
+            _TEMP_DIR_=$(mktemp -d)
+            unzip $1 -d $_TEMP_DIR_
+            cp -r $_TEMP_DIR_/*/* $CUDA_PATH
+            rm -rf $_TEMP_DIR_
+          }
         fi
+        function populate_cuda_path() {
+          # take the component name as a argument
+          function download() {
+            curl -kLSs $1 -o $2
+          }
+          CTK_COMPONENT=$1
+          CTK_COMPONENT_REL_PATH="$(curl -s $CTK_JSON_URL |
+              python -c "import sys, json; print(json.load(sys.stdin)['${CTK_COMPONENT}']['${CTK_SUBDIR}']['relative_path'])")"
+          CTK_COMPONENT_URL="${CTK_BASE_URL}/${CTK_COMPONENT_REL_PATH}"
+          CTK_COMPONENT_COMPONENT_FILENAME="$(basename $CTK_COMPONENT_REL_PATH)"
+          download $CTK_COMPONENT_URL $CTK_COMPONENT_COMPONENT_FILENAME
+          extract $CTK_COMPONENT_COMPONENT_FILENAME
+          rm $CTK_COMPONENT_COMPONENT_FILENAME
+        }
 
-        TARGET_PLATFORM='linux-64'
-        if [[ "${{ inputs.host-platform }}" == "linux-aarch64" ]]; then
+        # Get headers and shared libraries in place
+        populate_cuda_path cuda_nvcc
+        populate_cuda_path cuda_cudart
+        populate_cuda_path cuda_nvrtc
+        populate_cuda_path cuda_profiler_api
+        ls -l $CUDA_PATH
+
+        # Note: the headers will be copied into the cibuildwheel manylinux container,
+        # so setting the CUDA_PATH env var here is meaningless.
+
+    - name: Set environment variables
+      shell: bash --noprofile --norc -xeuo pipefail {0}
+      run: |
+        # TODO: just align host-platform names with TARGET_PLATFORM...
+        if [[ "${{ inputs.host-platform }}" == "linux-x64" ]]; then
+          TARGET_PLATFORM='linux-64'
+        elif [[ "${{ inputs.host-platform }}" == "linux-aarch64" ]]; then
           TARGET_PLATFORM='linux-aarch64'
+        elif [[ "${{ inputs.host-platform }}" == "win-x64" ]]; then
+          TARGET_PLATFORM='win-64'
         fi
 
-        BUILD_MODE="${{ inputs.build-mode }}"
-        BUILD_MODE_STR=""
-        [ -n "${BUILD_MODE}" ] && BUILD_MODE_STR="-${BUILD_MODE}"
+        PYTHON_VERSION_FORMATTED=$(echo '${{ inputs.python-version }}' | tr -d '.')
+        if [[ "${{ inputs.host-platform }}" == linux* ]]; then
+          CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-manylinux*"
+          REPO_DIR=$(pwd)
+        elif [[ "${{ inputs.host-platform }}" == win* ]]; then
+          CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-win_amd64"
+          PWD=$(pwd)
+          REPO_DIR=$(cygpath -w $PWD)
+        fi
 
+        BUILD_MODE="${{ inputs.build-mode }}"
         if [[ ("${BUILD_MODE}" == "") || ("${BUILD_MODE}" == "release") ]]; then
           # We upload release versions in the default folder.
           PKG_DIR="${TARGET_PLATFORM}"
         else
           PKG_DIR="${BUILD_MODE}/${TARGET_PLATFORM}"
         fi
 
-        PYTHON_VERSION_FORMATTED=$(echo '${{ inputs.python-version }}' | tr -d '.')
-
-        echo "BINDINGS_ARTIFACT_NAME=${{ inputs.host-platform }}-${{ inputs.build-type }}-cuda_bindings-python${PYTHON_VERSION_FORMATTED}-${{ inputs.target-device }}${BUILD_MODE_STR}${WITH_TESTS_STR}-${{ github.sha }}" >> $GITHUB_ENV
-        echo "BINDINGS_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_bindings/dist")" >> $GITHUB_ENV
-        echo "CORE_ARTIFACT_NAME=${{ inputs.host-platform }}-${{ inputs.build-type }}-cuda_core-python${PYTHON_VERSION_FORMATTED}-${{ inputs.target-device }}${BUILD_MODE_STR}${WITH_TESTS_STR}-${{ github.sha }}" >> $GITHUB_ENV
-        echo "CORE_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_core/dist")" >> $GITHUB_ENV
-        echo "USE_CUDA=${{ (inputs.target-device == 'cpu' && 'OFF') || 'ON' }}" >> $GITHUB_ENV
+        echo "PARALLEL_LEVEL=$(nproc)" >> $GITHUB_ENV
+        echo "REPO_DIR=$REPO_DIR" >> $GITHUB_ENV
+        echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV
+        echo "CUDA_CORE_ARTIFACT_NAME=cuda-core-python${PYTHON_VERSION_FORMATTED}-${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ github.sha }}" >> $GITHUB_ENV
+        echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_core/dist")" >> $GITHUB_ENV
+        echo "CUDA_BINDINGS_ARTIFACT_NAME=cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${{ inputs.cuda-version }}-${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ github.sha }}" >> $GITHUB_ENV
+        echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/dist")" >> $GITHUB_ENV
         echo "UPLOAD_ENABLED=${{ (inputs.upload-enabled  == 'true' && 'ON') || 'OFF' }}" >> $GITHUB_ENV
-        echo "LEGATE_CORE_BUILD_MODE=${BUILD_MODE}" >> $GITHUB_ENV
         echo "BUILD_DATE=$(date +%Y%m%d)" >> $GITHUB_ENV
         echo "TARGET_PLATFORM=${TARGET_PLATFORM}" >> $GITHUB_ENV
-        echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV
-        echo "PYTHON_VERSION=${{ inputs.python-version }}" >> $GITHUB_ENV
+        echo "CIBW_BUILD=${CIBW_BUILD}" >> $GITHUB_ENV

.github/workflows/ci-gh.yml

@@ -18,15 +18,23 @@ jobs:
       matrix:
         host-platform:
           - linux-x64
+          - linux-aarch64
+          - win-x64
         target-device:
           - gpu
         build-mode:
           - release
         upload-enabled:
           - false
         python-version:
-          #TODO cover the whole python and cuda matrix
-          - 3.12
+          - "3.12"
+          - "3.11"
+          - "3.10"
+          - "3.9"
+        cuda-version:
+          # Note: this is for build-time only; the test-time matrix needs to be
+          # defined separately.
+          - "12.6.2"
     uses:
       ./.github/workflows/gh-build-and-test.yml
     with:
@@ -36,4 +44,5 @@ jobs:
       build-type: ci
       upload-enabled: ${{ matrix.upload-enabled }}
       python-version: ${{ matrix.python-version }}
+      cuda-version: ${{ matrix.cuda-version }}
     secrets: inherit

.github/workflows/gh-build-and-test.yml

@@ -1,16 +1,16 @@
 on:
   workflow_call:
     inputs:
-      host-platform:
+      target-device:
         type: string
         required: true
-      target-device:
+      build-type:
         type: string
         required: true
-      build-mode:
+      host-platform:
         type: string
         required: true
-      build-type:
+      build-mode:
         type: string
         required: true
       upload-enabled:
@@ -19,6 +19,10 @@ on:
       python-version:
         type: string
         required: true
+      cuda-version:
+        type: string
+        required: true
+
 jobs:
   build:
     if: ${{ github.repository_owner == 'nvidia' }}
@@ -28,13 +32,14 @@ jobs:
       client-repo: ${{ github.event.repository.name }}
       target-device: ${{ inputs.target-device }}
       runs-on: ${{ (inputs.host-platform == 'linux-x64' && 'linux-amd64-cpu8') ||
-                   (inputs.host-platform == 'linux-aarch64' && 'linux-arm64-cpu8') }}
+                   (inputs.host-platform == 'linux-aarch64' && 'linux-arm64-cpu8') ||
+                   (inputs.host-platform == 'win-x64' && 'windows-2019') }}
+                 #  (inputs.host-platform == 'win-x64' && 'windows-amd64-cpu8') }}
       build-type: ${{ inputs.build-type }}
-      use-container: ${{ inputs.host-platform == 'linux-x64' ||
-                         inputs.host-platform == 'linux-aarch64'}}
       host-platform: ${{ inputs.host-platform }}
-      dependencies-file: ""
       build-mode: ${{ inputs.build-mode }}
       upload-enabled: ${{ inputs.upload-enabled }}
       python-version: ${{ inputs.python-version }}
+      cuda-version: ${{ inputs.cuda-version }}
+      dependencies-file: ""
     secrets: inherit