From 70eb1123e1dbba9129379f1b213707189adcfb71 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Tue, 10 Oct 2023 18:43:14 +0100 Subject: [PATCH 01/90] GHA workflow to create new release branch on update to Project version --- .github/workflows/Release.yml | 57 ++++++++++++++++++++++++++++++ .github/workflows/artifacts_CI.yml | 1 + 2 files changed, 58 insertions(+) create mode 100644 .github/workflows/Release.yml diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml new file mode 100644 index 00000000..e7783324 --- /dev/null +++ b/.github/workflows/Release.yml @@ -0,0 +1,57 @@ +name: Release CI +on: + workflow_dispatch: + push: + branches: + - main + paths: + - "Project.toml" +jobs: + branch: + name: "Create Release Branch" + runs-on: ubuntu-20.04-16core + strategy: + fail-fast: false + steps: + - uses: actions/checkout@v3 + - uses: julia-actions/setup-julia@v1 + with: + version: "1.8" + arch: "x64" + - uses: julia-actions/cache@v1 + - name: Check Project version + id: version_check + shell: julia --color=yes --project {0} + run: | + using Pkg.Types + project = read_project("Project.toml") + latest_tag = readchomp(`git describe --tags --abbrev=0`) + @assert project.version == latest_tag "$(project.version) != $latest_tag" + open(ENV["GITHUB_OUTPUT"], "a") do io + println(io, "version=$(project.version)") + end + continue-on-error: true + - name: Checkout new branch + if: steps.version_check.outcome = 'success' + run: | + release_branch="release-v${{ steps.version_check.outputs.version }}" + # Error if branch already exists + if [[ ! -z $(git ls-remote --heads origin $release_branch) ]]; then + echo "Release branch already exists on remote: $release_branch" >&2 + exit 1 + fi + git checkout -b $release_branch + - name: Update Artifacts.toml + shell: julia --color=yes --project {0} + run: | + using Pkg + Pkg.instantiate() + include("download_tarballs.jl") # pulls artifacts from the latest CI job + include("upload_tarballs.jl") # uploads artifacts as assets to GH prerelease + include("bind_artifacts.jl") # updates Artifacts.toml with asset metadata + - name: Commit changes + run: | + git commit -m "Update Artifacts.toml" -- Artifacts.toml + git tag v${{ steps.version_check.outputs.version }} + git push --all origin + diff --git a/.github/workflows/artifacts_CI.yml b/.github/workflows/artifacts_CI.yml index 961b5afd..189b9a83 100644 --- a/.github/workflows/artifacts_CI.yml +++ b/.github/workflows/artifacts_CI.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - release-* tags: ["*"] # Triggers from tags ignore the `paths` filter: https://github.com/orgs/community/discussions/26273 paths: - "Artifacts.toml" From c5f11369ab80a3ca36fbf40e7a66ed105da5dd14 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Tue, 10 Oct 2023 18:56:05 +0100 Subject: [PATCH 02/90] compare VersionNumbers directly --- .github/workflows/Release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml index e7783324..0ba060e2 100644 --- a/.github/workflows/Release.yml +++ b/.github/workflows/Release.yml @@ -25,8 +25,8 @@ jobs: run: | using Pkg.Types project = read_project("Project.toml") - latest_tag = readchomp(`git describe --tags --abbrev=0`) - @assert project.version == latest_tag "$(project.version) != $latest_tag" + latest_tag = parse(VersionNumber, readchomp(`git describe --tags --abbrev=0`)) + @assert project.version > latest_tag "$(project.version) !> $latest_tag" open(ENV["GITHUB_OUTPUT"], "a") do io println(io, "version=$(project.version)") end From 0ecbc9e1b6fb9d0c982cd40f23c628579dc8ec1b Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 11 Oct 2023 16:25:13 +0100 Subject: [PATCH 03/90] move Release.yml into CI.yml --- .github/workflows/CI.yml | 49 ++++++++++++++++++++++++++++++ .github/workflows/Release.yml | 57 ----------------------------------- 2 files changed, 49 insertions(+), 57 deletions(-) delete mode 100644 .github/workflows/Release.yml diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index b69b7b2e..6192b5f5 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -182,3 +182,52 @@ jobs: with: key: ${{ steps.build-cache.outputs.cache-primary-key }} path: ~/.cache + release: + name: "Publish Pre-Release" + runs-on: ubuntu-20.04-16core + needs: [test] + strategy: + fail-fast: false + steps: + - uses: actions/checkout@v3 + - uses: julia-actions/setup-julia@v1 + with: + version: "1.8" + arch: "x64" + - uses: julia-actions/cache@v1 + - name: Check Project version + id: version_check + shell: julia --color=yes --project {0} + run: | + using Pkg.Types + project = read_project("Project.toml") + latest_tag = parse(VersionNumber, readchomp(`git describe --tags --abbrev=0`)) + @assert project.version > latest_tag "$(project.version) !> $latest_tag" + open(ENV["GITHUB_OUTPUT"], "a") do io + println(io, "version=$(project.version)") + end + continue-on-error: true + - name: Checkout new branch + if: steps.version_check.outcome = 'success' + run: | + release_branch="release-v${{ steps.version_check.outputs.version }}" + # Error if branch already exists + if [[ ! -z $(git ls-remote --heads origin $release_branch) ]]; then + echo "Release branch already exists on remote: $release_branch" >&2 + exit 1 + fi + git checkout -b $release_branch + - name: Update Artifacts.toml + shell: julia --color=yes --project {0} + run: | + using Pkg + Pkg.instantiate() + include("download_tarballs.jl") # pulls artifacts from the latest CI job + include("upload_tarballs.jl") # uploads artifacts as assets to GH prerelease + include("bind_artifacts.jl") # updates Artifacts.toml with asset metadata + - name: Commit changes + run: | + git commit -m "Update Artifacts.toml" -- Artifacts.toml + git tag v${{ steps.version_check.outputs.version }} + git push --all origin + diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml deleted file mode 100644 index 0ba060e2..00000000 --- a/.github/workflows/Release.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Release CI -on: - workflow_dispatch: - push: - branches: - - main - paths: - - "Project.toml" -jobs: - branch: - name: "Create Release Branch" - runs-on: ubuntu-20.04-16core - strategy: - fail-fast: false - steps: - - uses: actions/checkout@v3 - - uses: julia-actions/setup-julia@v1 - with: - version: "1.8" - arch: "x64" - - uses: julia-actions/cache@v1 - - name: Check Project version - id: version_check - shell: julia --color=yes --project {0} - run: | - using Pkg.Types - project = read_project("Project.toml") - latest_tag = parse(VersionNumber, readchomp(`git describe --tags --abbrev=0`)) - @assert project.version > latest_tag "$(project.version) !> $latest_tag" - open(ENV["GITHUB_OUTPUT"], "a") do io - println(io, "version=$(project.version)") - end - continue-on-error: true - - name: Checkout new branch - if: steps.version_check.outcome = 'success' - run: | - release_branch="release-v${{ steps.version_check.outputs.version }}" - # Error if branch already exists - if [[ ! -z $(git ls-remote --heads origin $release_branch) ]]; then - echo "Release branch already exists on remote: $release_branch" >&2 - exit 1 - fi - git checkout -b $release_branch - - name: Update Artifacts.toml - shell: julia --color=yes --project {0} - run: | - using Pkg - Pkg.instantiate() - include("download_tarballs.jl") # pulls artifacts from the latest CI job - include("upload_tarballs.jl") # uploads artifacts as assets to GH prerelease - include("bind_artifacts.jl") # updates Artifacts.toml with asset metadata - - name: Commit changes - run: | - git commit -m "Update Artifacts.toml" -- Artifacts.toml - git tag v${{ steps.version_check.outputs.version }} - git push --all origin - From 4bf7763e94c4b8c3fe834f7f71ecc524b520164b Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 11 Oct 2023 16:27:19 +0100 Subject: [PATCH 04/90] use actions/download-artifact instead of download_tarballs.jl --- .github/workflows/CI.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 6192b5f5..4cac0586 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -217,12 +217,16 @@ jobs: exit 1 fi git checkout -b $release_branch + - name: Download Artifacts + uses: actions/download-artifact@v3 + with: + name: tarballs + path: tarballs - name: Update Artifacts.toml shell: julia --color=yes --project {0} run: | using Pkg Pkg.instantiate() - include("download_tarballs.jl") # pulls artifacts from the latest CI job include("upload_tarballs.jl") # uploads artifacts as assets to GH prerelease include("bind_artifacts.jl") # updates Artifacts.toml with asset metadata - name: Commit changes From b6f8c828cf422c2a2dd7c7e3c7ce6293d3c4aea5 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 11 Oct 2023 16:28:41 +0100 Subject: [PATCH 05/90] git push tag and branch separately --- .github/workflows/CI.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 4cac0586..4035aee5 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -233,5 +233,6 @@ jobs: run: | git commit -m "Update Artifacts.toml" -- Artifacts.toml git tag v${{ steps.version_check.outputs.version }} - git push --all origin + git push origin release-v${{ steps.version_check.outputs.version }} + git push origin v${{ steps.version_check.outputs.version }} From f5bae6565bae17e1bf0e2f6cb743b2cb134f6a6d Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 11 Oct 2023 16:31:12 +0100 Subject: [PATCH 06/90] remove Artifacts.toml from main --- .github/workflows/artifacts_CI.yml | 1 - Artifacts.toml | 38 ------------------------------ 2 files changed, 39 deletions(-) delete mode 100644 Artifacts.toml diff --git a/.github/workflows/artifacts_CI.yml b/.github/workflows/artifacts_CI.yml index 189b9a83..ffa80389 100644 --- a/.github/workflows/artifacts_CI.yml +++ b/.github/workflows/artifacts_CI.yml @@ -3,7 +3,6 @@ on: workflow_dispatch: push: branches: - - main - release-* tags: ["*"] # Triggers from tags ignore the `paths` filter: https://github.com/orgs/community/discussions/26273 paths: diff --git a/Artifacts.toml b/Artifacts.toml deleted file mode 100644 index 39f8f81b..00000000 --- a/Artifacts.toml +++ /dev/null @@ -1,38 +0,0 @@ -[[ray_julia]] -arch = "x86_64" -git-tree-sha1 = "88eb04f4e4a87d0a369288ceb6fb605f4ca4ff03" -julia_version = "1.8.0" -libc = "glibc" -os = "linux" - - [[ray_julia.download]] - sha256 = "03bc890efc63375f901325d8f67a2f3403c7d1067658ed693390276ab37f6958" - url = "https://github.com/beacon-biosignals/Ray.jl/releases/download/v0.0.1/ray_julia.v0.0.1.x86_64-linux-gnu-julia_version+1.8.0.tar.gz" -[[ray_julia]] -arch = "aarch64" -git-tree-sha1 = "a24e4bb69ffa0baf615e27f3ae6d00084556881a" -julia_version = "1.8.0" -os = "macos" - - [[ray_julia.download]] - sha256 = "2d5ee4e6290e1207250a02e434d588af3321d7f978fe7d57e13d4827ca7aa51b" - url = "https://github.com/beacon-biosignals/Ray.jl/releases/download/v0.0.1/ray_julia.v0.0.1.aarch64-apple-darwin-julia_version+1.8.0.tar.gz" -[[ray_julia]] -arch = "x86_64" -git-tree-sha1 = "32c3aaecc5a1e2aed7e9ce6d9a95bef741d1ef9d" -julia_version = "1.9.0" -libc = "glibc" -os = "linux" - - [[ray_julia.download]] - sha256 = "1f96eaf5f35796f60b04172728caa29fc442ffcc30e8ffabfe147791dd711fc8" - url = "https://github.com/beacon-biosignals/Ray.jl/releases/download/v0.0.1/ray_julia.v0.0.1.x86_64-linux-gnu-julia_version+1.9.0.tar.gz" -[[ray_julia]] -arch = "aarch64" -git-tree-sha1 = "81cc7a633a617a01f1fc87674258d880baa4cafb" -julia_version = "1.9.0" -os = "macos" - - [[ray_julia.download]] - sha256 = "5969006375abafacf2fff0a1d499b492d1a883d1529e2a20fe69d3a4908f2729" - url = "https://github.com/beacon-biosignals/Ray.jl/releases/download/v0.0.1/ray_julia.v0.0.1.aarch64-apple-darwin-julia_version+1.9.0.tar.gz" From 8d93df0b7cd434743b091fc28d475885140d2bde Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 11 Oct 2023 16:44:18 +0100 Subject: [PATCH 07/90] refactor workflow --- .github/workflows/CI.yml | 36 ++++++++++++++---------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 4035aee5..21eeb0a9 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -188,6 +188,8 @@ jobs: needs: [test] strategy: fail-fast: false + env: + build_dir: ./build steps: - uses: actions/checkout@v3 - uses: julia-actions/setup-julia@v1 @@ -195,33 +197,19 @@ jobs: version: "1.8" arch: "x64" - uses: julia-actions/cache@v1 + - uses: actions/download-artifact@v3 + with: + name: tarballs + path: tarballs - name: Check Project version id: version_check shell: julia --color=yes --project {0} run: | using Pkg.Types project = read_project("Project.toml") - latest_tag = parse(VersionNumber, readchomp(`git describe --tags --abbrev=0`)) - @assert project.version > latest_tag "$(project.version) !> $latest_tag" open(ENV["GITHUB_OUTPUT"], "a") do io println(io, "version=$(project.version)") end - continue-on-error: true - - name: Checkout new branch - if: steps.version_check.outcome = 'success' - run: | - release_branch="release-v${{ steps.version_check.outputs.version }}" - # Error if branch already exists - if [[ ! -z $(git ls-remote --heads origin $release_branch) ]]; then - echo "Release branch already exists on remote: $release_branch" >&2 - exit 1 - fi - git checkout -b $release_branch - - name: Download Artifacts - uses: actions/download-artifact@v3 - with: - name: tarballs - path: tarballs - name: Update Artifacts.toml shell: julia --color=yes --project {0} run: | @@ -229,10 +217,14 @@ jobs: Pkg.instantiate() include("upload_tarballs.jl") # uploads artifacts as assets to GH prerelease include("bind_artifacts.jl") # updates Artifacts.toml with asset metadata - - name: Commit changes + working-directory: ${{ env.build_dir }} + - name: Push changes to branch + if: github.event_name != 'pull_request' run: | + version=v${{ steps.version_check.outputs.version }} + release_branch="release-$version + git checkout -b $release_branch git commit -m "Update Artifacts.toml" -- Artifacts.toml - git tag v${{ steps.version_check.outputs.version }} - git push origin release-v${{ steps.version_check.outputs.version }} - git push origin v${{ steps.version_check.outputs.version }} + git tag $version + git push origin $release_branch $version From 5aaa27e10faa2c55166629ce234984cf5d893857 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 11 Oct 2023 16:45:17 +0100 Subject: [PATCH 08/90] pull out common env variables --- .github/workflows/CI.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 21eeb0a9..3911f2e4 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -21,6 +21,9 @@ concurrency: # Cancel intermediate builds: only if it is a pull request build. group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} +env: + build_dir: ./build + ray_dir: ./build/ray jobs: test: name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} @@ -42,9 +45,6 @@ jobs: arch: aarch64 - os: macos-latest-xlarge arch: x64 - env: - build_dir: ./build - ray_dir: ./build/ray steps: - uses: actions/checkout@v3 - name: Determine Ray commit @@ -188,8 +188,6 @@ jobs: needs: [test] strategy: fail-fast: false - env: - build_dir: ./build steps: - uses: actions/checkout@v3 - uses: julia-actions/setup-julia@v1 From 4b33ab5dc85179283a181d0dd857864981567db8 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 11 Oct 2023 16:45:39 +0100 Subject: [PATCH 09/90] indenting --- .github/workflows/CI.yml | 74 ++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 3911f2e4..39f9fed4 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -189,40 +189,40 @@ jobs: strategy: fail-fast: false steps: - - uses: actions/checkout@v3 - - uses: julia-actions/setup-julia@v1 - with: - version: "1.8" - arch: "x64" - - uses: julia-actions/cache@v1 - - uses: actions/download-artifact@v3 - with: - name: tarballs - path: tarballs - - name: Check Project version - id: version_check - shell: julia --color=yes --project {0} - run: | - using Pkg.Types - project = read_project("Project.toml") - open(ENV["GITHUB_OUTPUT"], "a") do io - println(io, "version=$(project.version)") - end - - name: Update Artifacts.toml - shell: julia --color=yes --project {0} - run: | - using Pkg - Pkg.instantiate() - include("upload_tarballs.jl") # uploads artifacts as assets to GH prerelease - include("bind_artifacts.jl") # updates Artifacts.toml with asset metadata - working-directory: ${{ env.build_dir }} - - name: Push changes to branch - if: github.event_name != 'pull_request' - run: | - version=v${{ steps.version_check.outputs.version }} - release_branch="release-$version - git checkout -b $release_branch - git commit -m "Update Artifacts.toml" -- Artifacts.toml - git tag $version - git push origin $release_branch $version - + - uses: actions/checkout@v3 + - uses: julia-actions/setup-julia@v1 + with: + version: "1.8" + arch: "x64" + - uses: julia-actions/cache@v1 + - uses: actions/download-artifact@v3 + with: + name: tarballs + path: tarballs + - name: Check Project version + id: version_check + shell: julia --color=yes --project {0} + run: | + using Pkg.Types + project = read_project("Project.toml") + open(ENV["GITHUB_OUTPUT"], "a") do io + println(io, "version=$(project.version)") + end + - name: Update Artifacts.toml + shell: julia --color=yes --project {0} + run: | + using Pkg + Pkg.instantiate() + include("upload_tarballs.jl") # uploads artifacts as assets to GH prerelease + include("bind_artifacts.jl") # updates Artifacts.toml with asset metadata + working-directory: ${{ env.build_dir }} + - name: Push changes to branch + if: github.event_name != 'pull_request' + run: | + version=v${{ steps.version_check.outputs.version }} + release_branch="release-$version + git checkout -b $release_branch + git commit -m "Update Artifacts.toml" -- Artifacts.toml + git tag $version + git push origin $release_branch $version + From 7f7c28eb75ccebbc9f08c8147a06451276570624 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 11 Oct 2023 16:53:59 +0100 Subject: [PATCH 10/90] dont run Artifacts CI on pull requests --- .github/workflows/artifacts_CI.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/artifacts_CI.yml b/.github/workflows/artifacts_CI.yml index ffa80389..c9bf946f 100644 --- a/.github/workflows/artifacts_CI.yml +++ b/.github/workflows/artifacts_CI.yml @@ -8,10 +8,6 @@ on: paths: - "Artifacts.toml" - ".github/workflows/artifacts_CI.yml" - pull_request: - paths: - - "Artifacts.toml" - - ".github/workflows/artifacts_CI.yml" concurrency: # Skip intermediate builds: always. # Cancel intermediate builds: only if it is a pull request build. From 8e93fac1171994e45257bcfeb7baaa6e35d887e1 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 11 Oct 2023 17:02:31 +0100 Subject: [PATCH 11/90] Revert "remove Artifacts.toml from main" This reverts commit 463b534700611ed99a6aa9b9b3328b1b7a3887e4. --- .github/workflows/artifacts_CI.yml | 1 + Artifacts.toml | 38 ++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 Artifacts.toml diff --git a/.github/workflows/artifacts_CI.yml b/.github/workflows/artifacts_CI.yml index c9bf946f..088a8d83 100644 --- a/.github/workflows/artifacts_CI.yml +++ b/.github/workflows/artifacts_CI.yml @@ -3,6 +3,7 @@ on: workflow_dispatch: push: branches: + - main - release-* tags: ["*"] # Triggers from tags ignore the `paths` filter: https://github.com/orgs/community/discussions/26273 paths: diff --git a/Artifacts.toml b/Artifacts.toml new file mode 100644 index 00000000..39f8f81b --- /dev/null +++ b/Artifacts.toml @@ -0,0 +1,38 @@ +[[ray_julia]] +arch = "x86_64" +git-tree-sha1 = "88eb04f4e4a87d0a369288ceb6fb605f4ca4ff03" +julia_version = "1.8.0" +libc = "glibc" +os = "linux" + + [[ray_julia.download]] + sha256 = "03bc890efc63375f901325d8f67a2f3403c7d1067658ed693390276ab37f6958" + url = "https://github.com/beacon-biosignals/Ray.jl/releases/download/v0.0.1/ray_julia.v0.0.1.x86_64-linux-gnu-julia_version+1.8.0.tar.gz" +[[ray_julia]] +arch = "aarch64" +git-tree-sha1 = "a24e4bb69ffa0baf615e27f3ae6d00084556881a" +julia_version = "1.8.0" +os = "macos" + + [[ray_julia.download]] + sha256 = "2d5ee4e6290e1207250a02e434d588af3321d7f978fe7d57e13d4827ca7aa51b" + url = "https://github.com/beacon-biosignals/Ray.jl/releases/download/v0.0.1/ray_julia.v0.0.1.aarch64-apple-darwin-julia_version+1.8.0.tar.gz" +[[ray_julia]] +arch = "x86_64" +git-tree-sha1 = "32c3aaecc5a1e2aed7e9ce6d9a95bef741d1ef9d" +julia_version = "1.9.0" +libc = "glibc" +os = "linux" + + [[ray_julia.download]] + sha256 = "1f96eaf5f35796f60b04172728caa29fc442ffcc30e8ffabfe147791dd711fc8" + url = "https://github.com/beacon-biosignals/Ray.jl/releases/download/v0.0.1/ray_julia.v0.0.1.x86_64-linux-gnu-julia_version+1.9.0.tar.gz" +[[ray_julia]] +arch = "aarch64" +git-tree-sha1 = "81cc7a633a617a01f1fc87674258d880baa4cafb" +julia_version = "1.9.0" +os = "macos" + + [[ray_julia.download]] + sha256 = "5969006375abafacf2fff0a1d499b492d1a883d1529e2a20fe69d3a4908f2729" + url = "https://github.com/beacon-biosignals/Ray.jl/releases/download/v0.0.1/ray_julia.v0.0.1.aarch64-apple-darwin-julia_version+1.9.0.tar.gz" From 9adb74bc82e4702e3da013116a35993eae7bf13b Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 11 Oct 2023 17:13:33 +0100 Subject: [PATCH 12/90] fix download-artifact action --- .github/workflows/CI.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 39f9fed4..22a64fae 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -197,8 +197,8 @@ jobs: - uses: julia-actions/cache@v1 - uses: actions/download-artifact@v3 with: - name: tarballs - path: tarballs + name: ray_julia.v* + path: ${{ env.build_dir }}/tarballs - name: Check Project version id: version_check shell: julia --color=yes --project {0} From 044b73f38f58ede28ca62d1d24ab19974d5b27dc Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 11 Oct 2023 17:32:26 +0100 Subject: [PATCH 13/90] use download_tarballs again --- .github/workflows/CI.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 22a64fae..837039d3 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -195,10 +195,6 @@ jobs: version: "1.8" arch: "x64" - uses: julia-actions/cache@v1 - - uses: actions/download-artifact@v3 - with: - name: ray_julia.v* - path: ${{ env.build_dir }}/tarballs - name: Check Project version id: version_check shell: julia --color=yes --project {0} @@ -213,6 +209,7 @@ jobs: run: | using Pkg Pkg.instantiate() + include("download_tarballs.jl") # pulls artifacts from the latest CI job include("upload_tarballs.jl") # uploads artifacts as assets to GH prerelease include("bind_artifacts.jl") # updates Artifacts.toml with asset metadata working-directory: ${{ env.build_dir }} From a05fe9fc5445c038519e5798e979a5e1919491d6 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 11 Oct 2023 17:43:25 +0100 Subject: [PATCH 14/90] typo --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 837039d3..0c3c1d0d 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -217,7 +217,7 @@ jobs: if: github.event_name != 'pull_request' run: | version=v${{ steps.version_check.outputs.version }} - release_branch="release-$version + release_branch="release-$version" git checkout -b $release_branch git commit -m "Update Artifacts.toml" -- Artifacts.toml git tag $version From 1499bea2d59c3986afb87382324876712d386f0e Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 11 Oct 2023 17:45:00 +0100 Subject: [PATCH 15/90] fix include path --- .github/workflows/CI.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 0c3c1d0d..d10bb07f 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -209,9 +209,9 @@ jobs: run: | using Pkg Pkg.instantiate() - include("download_tarballs.jl") # pulls artifacts from the latest CI job - include("upload_tarballs.jl") # uploads artifacts as assets to GH prerelease - include("bind_artifacts.jl") # updates Artifacts.toml with asset metadata + include(joinpath(pwd(), "download_tarballs.jl")) # pulls artifacts from the latest CI job + include(joinpath(pwd(), "upload_tarballs.jl")) # uploads artifacts as assets to GH prerelease + include(joinpath(pwd(), "bind_artifacts.jl")) # updates Artifacts.toml with asset metadata working-directory: ${{ env.build_dir }} - name: Push changes to branch if: github.event_name != 'pull_request' From ec601079d37cb88306a56ed525dd5b25651aaba2 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 11 Oct 2023 18:55:10 +0100 Subject: [PATCH 16/90] Use action-gh-release GHA --- .github/workflows/CI.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d10bb07f..66794960 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -210,9 +210,16 @@ jobs: using Pkg Pkg.instantiate() include(joinpath(pwd(), "download_tarballs.jl")) # pulls artifacts from the latest CI job - include(joinpath(pwd(), "upload_tarballs.jl")) # uploads artifacts as assets to GH prerelease include(joinpath(pwd(), "bind_artifacts.jl")) # updates Artifacts.toml with asset metadata working-directory: ${{ env.build_dir }} + - name: Publish Pre-Release + uses: softprops/action-gh-release@v1 + if: github.event_name != 'pull_request' + with: + prerelease: true + generate_release_notes: true + tag_name: v${{ steps.version_check.outputs.version }} + files: ${{ env.build_dir }}/tarballs - name: Push changes to branch if: github.event_name != 'pull_request' run: | From 390405ee65190c640ddc07e801cd43cc47182f47 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 11 Oct 2023 19:02:54 +0100 Subject: [PATCH 17/90] don't need download_tarballs.jl --- .github/workflows/CI.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 66794960..f539af3a 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -209,8 +209,7 @@ jobs: run: | using Pkg Pkg.instantiate() - include(joinpath(pwd(), "download_tarballs.jl")) # pulls artifacts from the latest CI job - include(joinpath(pwd(), "bind_artifacts.jl")) # updates Artifacts.toml with asset metadata + include(joinpath(pwd(), "bind_artifacts.jl")) # updates Artifacts.toml with asset metadata working-directory: ${{ env.build_dir }} - name: Publish Pre-Release uses: softprops/action-gh-release@v1 From d6d06874eb5178adf311eec9e4e13347cd79a4d1 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 11 Oct 2023 19:19:49 +0100 Subject: [PATCH 18/90] reorganise actions --- .github/workflows/CI.yml | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f539af3a..9bef2112 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -202,7 +202,7 @@ jobs: using Pkg.Types project = read_project("Project.toml") open(ENV["GITHUB_OUTPUT"], "a") do io - println(io, "version=$(project.version)") + println(io, "version=v$(project.version)") end - name: Update Artifacts.toml shell: julia --color=yes --project {0} @@ -211,21 +211,18 @@ jobs: Pkg.instantiate() include(joinpath(pwd(), "bind_artifacts.jl")) # updates Artifacts.toml with asset metadata working-directory: ${{ env.build_dir }} - - name: Publish Pre-Release - uses: softprops/action-gh-release@v1 - if: github.event_name != 'pull_request' - with: - prerelease: true - generate_release_notes: true - tag_name: v${{ steps.version_check.outputs.version }} - files: ${{ env.build_dir }}/tarballs - - name: Push changes to branch - if: github.event_name != 'pull_request' + - name: Push Release Branch + if: ${{ github.ref == 'refs/heads/main' }} run: | - version=v${{ steps.version_check.outputs.version }} + version=${{ steps.version_check.outputs.version }} release_branch="release-$version" git checkout -b $release_branch git commit -m "Update Artifacts.toml" -- Artifacts.toml - git tag $version - git push origin $release_branch $version + git push origin $release_branch + - name: Publish Release + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') + with: + generate_release_notes: true + files: ${{ env.build_dir }}/tarballs From 5948599a29c6a5ca405c997d817713d211a0aaf2 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 11 Oct 2023 19:58:03 +0100 Subject: [PATCH 19/90] Dont run artifacts_CI on release branches It could lead to a race condition since we can't guarantee that the pre-release assets will have been uploaded before it runs. --- .github/workflows/artifacts_CI.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/artifacts_CI.yml b/.github/workflows/artifacts_CI.yml index 088a8d83..0d2f8a50 100644 --- a/.github/workflows/artifacts_CI.yml +++ b/.github/workflows/artifacts_CI.yml @@ -2,13 +2,7 @@ name: Artifacts CI on: workflow_dispatch: push: - branches: - - main - - release-* - tags: ["*"] # Triggers from tags ignore the `paths` filter: https://github.com/orgs/community/discussions/26273 - paths: - - "Artifacts.toml" - - ".github/workflows/artifacts_CI.yml" + tags: ["*"] concurrency: # Skip intermediate builds: always. # Cancel intermediate builds: only if it is a pull request build. From b9ef5d79b0623b337c19e075e6c952fbf00cac6a Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 11 Oct 2023 19:58:52 +0100 Subject: [PATCH 20/90] Push branch then publish pre-release --- .github/workflows/CI.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 9bef2112..99540bfa 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -203,6 +203,7 @@ jobs: project = read_project("Project.toml") open(ENV["GITHUB_OUTPUT"], "a") do io println(io, "version=v$(project.version)") + println(io, "release_branch=release-v$(project.version)") end - name: Update Artifacts.toml shell: julia --color=yes --project {0} @@ -214,15 +215,16 @@ jobs: - name: Push Release Branch if: ${{ github.ref == 'refs/heads/main' }} run: | - version=${{ steps.version_check.outputs.version }} - release_branch="release-$version" - git checkout -b $release_branch + git checkout -b ${{ steps.version_check.outputs.release_branch }} git commit -m "Update Artifacts.toml" -- Artifacts.toml - git push origin $release_branch - - name: Publish Release + git push origin ${{ steps.version_check.outputs.release_branch }} + git checkout main + - name: Publish Pre-Release uses: softprops/action-gh-release@v1 - if: startsWith(github.ref, 'refs/tags/') + if: ${{ github.ref == 'refs/heads/main' }} with: + prerelease: true generate_release_notes: true + target_commitish: ${{ steps.version_check.outputs.release_branch }} files: ${{ env.build_dir }}/tarballs From 2fca8435d2f86fdc2d81147d7402e10ba3b7bdaf Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 11 Oct 2023 20:07:13 +0100 Subject: [PATCH 21/90] actually we can run artifacts_CI on release branches --- .github/workflows/CI.yml | 12 ++++++++---- .github/workflows/artifacts_CI.yml | 7 ++++++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 99540bfa..9429c77e 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -212,19 +212,23 @@ jobs: Pkg.instantiate() include(joinpath(pwd(), "bind_artifacts.jl")) # updates Artifacts.toml with asset metadata working-directory: ${{ env.build_dir }} - - name: Push Release Branch - if: ${{ github.ref == 'refs/heads/main' }} + - name: Commit Changes + id: commit run: | git checkout -b ${{ steps.version_check.outputs.release_branch }} git commit -m "Update Artifacts.toml" -- Artifacts.toml - git push origin ${{ steps.version_check.outputs.release_branch }} + echo "commitish=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" git checkout main - name: Publish Pre-Release uses: softprops/action-gh-release@v1 if: ${{ github.ref == 'refs/heads/main' }} with: prerelease: true + target_commitish: ${{ steps.commit.outputs.commitish}} generate_release_notes: true - target_commitish: ${{ steps.version_check.outputs.release_branch }} files: ${{ env.build_dir }}/tarballs + - name: Push Release Branch + if: ${{ github.ref == 'refs/heads/main' }} + run: git push origin ${{ steps.version_check.outputs.release_branch }} + diff --git a/.github/workflows/artifacts_CI.yml b/.github/workflows/artifacts_CI.yml index 0d2f8a50..c9bf946f 100644 --- a/.github/workflows/artifacts_CI.yml +++ b/.github/workflows/artifacts_CI.yml @@ -2,7 +2,12 @@ name: Artifacts CI on: workflow_dispatch: push: - tags: ["*"] + branches: + - release-* + tags: ["*"] # Triggers from tags ignore the `paths` filter: https://github.com/orgs/community/discussions/26273 + paths: + - "Artifacts.toml" + - ".github/workflows/artifacts_CI.yml" concurrency: # Skip intermediate builds: always. # Cancel intermediate builds: only if it is a pull request build. From 1cb818a1035e9d36b01a90be2b64ee357cf61d72 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 11 Oct 2023 20:46:00 +0100 Subject: [PATCH 22/90] download then bind artifacts --- .github/workflows/CI.yml | 5 ++++- build/bind_artifacts.jl | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 9429c77e..98a4e8c9 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -195,6 +195,9 @@ jobs: version: "1.8" arch: "x64" - uses: julia-actions/cache@v1 + - uses: actions/download-artifact@v3 + with: + path: ${{ env.build_dir }}/tarballs - name: Check Project version id: version_check shell: julia --color=yes --project {0} @@ -210,7 +213,7 @@ jobs: run: | using Pkg Pkg.instantiate() - include(joinpath(pwd(), "bind_artifacts.jl")) # updates Artifacts.toml with asset metadata + include(joinpath(pwd(), "bind_artifacts.jl")) working-directory: ${{ env.build_dir }} - name: Commit Changes id: commit diff --git a/build/bind_artifacts.jl b/build/bind_artifacts.jl index 3ba9b8b6..cc0759eb 100644 --- a/build/bind_artifacts.jl +++ b/build/bind_artifacts.jl @@ -34,7 +34,7 @@ if abspath(PROGRAM_FILE) == @__FILE__ @info "Dowloading artifact $artifact_url" artifact_path = joinpath(TARBALL_DIR, artifact_name) - Downloads.download(artifact_url, artifact_path) + isfile(artifact_path) || error("No such file $artifact_path") @info "Adding artifact for $(triplet(platform))" bind_artifact!(ARTIFACTS_TOML, From a2ad5fbbd73f96c2d1c57c2727096506365e19b7 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 11 Oct 2023 21:06:23 +0100 Subject: [PATCH 23/90] call bind_artifacts from julia session --- .github/workflows/CI.yml | 3 ++- build/bind_artifacts.jl | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 98a4e8c9..24cf019b 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -213,7 +213,8 @@ jobs: run: | using Pkg Pkg.instantiate() - include(joinpath(pwd(), "bind_artifacts.jl")) + include("bind_artifacts.jl") + main() working-directory: ${{ env.build_dir }} - name: Commit Changes id: commit diff --git a/build/bind_artifacts.jl b/build/bind_artifacts.jl index cc0759eb..ea0199d7 100644 --- a/build/bind_artifacts.jl +++ b/build/bind_artifacts.jl @@ -23,7 +23,7 @@ function sha256sum(tarball_path) end end -if abspath(PROGRAM_FILE) == @__FILE__ +function main() # Start with a clean Artifacts.toml so that unsupported platforms are removed isfile(ARTIFACTS_TOML) && rm(ARTIFACTS_TOML) @@ -44,3 +44,7 @@ if abspath(PROGRAM_FILE) == @__FILE__ download_info=[(artifact_url, sha256sum(artifact_path))]) end end + +if abspath(PROGRAM_FILE) == @__FILE__ + main() +end From 9de13416ebae653b808ec4acf819f87945fe37a2 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 11 Oct 2023 21:45:22 +0100 Subject: [PATCH 24/90] use pwd again --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 24cf019b..084b0f3d 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -213,7 +213,7 @@ jobs: run: | using Pkg Pkg.instantiate() - include("bind_artifacts.jl") + include(joinpath(pwd(), "bind_artifacts.jl")) main() working-directory: ${{ env.build_dir }} - name: Commit Changes From 86cc97086c78d66cdff937220c324a419eac49dd Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 11 Oct 2023 22:08:56 +0100 Subject: [PATCH 25/90] download-artifacts is dumb --- .github/workflows/CI.yml | 7 +++---- build/bind_artifacts.jl | 5 ++--- build/download_tarballs.jl | 4 ++-- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 084b0f3d..5cd2ec4c 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -195,9 +195,6 @@ jobs: version: "1.8" arch: "x64" - uses: julia-actions/cache@v1 - - uses: actions/download-artifact@v3 - with: - path: ${{ env.build_dir }}/tarballs - name: Check Project version id: version_check shell: julia --color=yes --project {0} @@ -213,8 +210,10 @@ jobs: run: | using Pkg Pkg.instantiate() + include(joinpath(pwd(), "download_tarballs.jl")) include(joinpath(pwd(), "bind_artifacts.jl")) - main() + download_tarballs() + bind_artifacts() working-directory: ${{ env.build_dir }} - name: Commit Changes id: commit diff --git a/build/bind_artifacts.jl b/build/bind_artifacts.jl index ea0199d7..3855f71f 100644 --- a/build/bind_artifacts.jl +++ b/build/bind_artifacts.jl @@ -23,7 +23,7 @@ function sha256sum(tarball_path) end end -function main() +function bind_artifacts() # Start with a clean Artifacts.toml so that unsupported platforms are removed isfile(ARTIFACTS_TOML) && rm(ARTIFACTS_TOML) @@ -32,7 +32,6 @@ function main() artifact_url = gen_artifact_url(; repo_url=REPO_HTTPS_URL, tag=TAG, filename=artifact_name) - @info "Dowloading artifact $artifact_url" artifact_path = joinpath(TARBALL_DIR, artifact_name) isfile(artifact_path) || error("No such file $artifact_path") @@ -46,5 +45,5 @@ function main() end if abspath(PROGRAM_FILE) == @__FILE__ - main() + bind_artifacts() end diff --git a/build/download_tarballs.jl b/build/download_tarballs.jl index 76241865..db25ae3c 100644 --- a/build/download_tarballs.jl +++ b/build/download_tarballs.jl @@ -47,7 +47,7 @@ function download_ray_julia_artifacts(; commit_sha, token, tarball_dir=TARBALL_D return nothing end -function main() +function download_tarballs() token = github_token() @info "Retrieving CI built tarballs..." @@ -58,5 +58,5 @@ function main() end if abspath(PROGRAM_FILE) == @__FILE__ - main() + download_tarballs() end From c03c80b8a0fe2e3dc20a3d5a898d09c12a14bc35 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 12 Oct 2023 12:23:54 +0100 Subject: [PATCH 26/90] split download/update artifacts steps --- .github/workflows/CI.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 5cd2ec4c..67afffff 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -205,14 +205,20 @@ jobs: println(io, "version=v$(project.version)") println(io, "release_branch=release-v$(project.version)") end - - name: Update Artifacts.toml + - name: Download Artifacts shell: julia --color=yes --project {0} run: | using Pkg Pkg.instantiate() include(joinpath(pwd(), "download_tarballs.jl")) - include(joinpath(pwd(), "bind_artifacts.jl")) download_tarballs() + working-directory: ${{ env.build_dir }} + - name: Update Artifacts.toml + shell: julia --color=yes --project {0} + run: | + using Pkg + Pkg.instantiate() + include(joinpath(pwd(), "bind_artifacts.jl")) bind_artifacts() working-directory: ${{ env.build_dir }} - name: Commit Changes From 9930f695a377a104742415a935ac2d865a17d502 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 12 Oct 2023 12:31:12 +0100 Subject: [PATCH 27/90] space --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 67afffff..411186e1 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -233,7 +233,7 @@ jobs: if: ${{ github.ref == 'refs/heads/main' }} with: prerelease: true - target_commitish: ${{ steps.commit.outputs.commitish}} + target_commitish: ${{ steps.commit.outputs.commitish }} generate_release_notes: true files: ${{ env.build_dir }}/tarballs - name: Push Release Branch From 5ea305c7fc441ef0653cb1dc54eb010706e22798 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 12 Oct 2023 12:42:53 +0100 Subject: [PATCH 28/90] pass GITHUB_TOKEN as env variable --- .github/workflows/CI.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 411186e1..cab96f89 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -213,6 +213,8 @@ jobs: include(joinpath(pwd(), "download_tarballs.jl")) download_tarballs() working-directory: ${{ env.build_dir }} + env: + GITHUB_TOKEN: ${{ github.token }} - name: Update Artifacts.toml shell: julia --color=yes --project {0} run: | @@ -221,6 +223,8 @@ jobs: include(joinpath(pwd(), "bind_artifacts.jl")) bind_artifacts() working-directory: ${{ env.build_dir }} + env: + GITHUB_TOKEN: ${{ github.token }} - name: Commit Changes id: commit run: | From 53604707a7d0798f2d0167b3bc6f219859450223 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 12 Oct 2023 13:21:59 +0100 Subject: [PATCH 29/90] use different GHA to download artifacts --- .github/workflows/CI.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index cab96f89..18e0f537 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -206,15 +206,14 @@ jobs: println(io, "release_branch=release-v$(project.version)") end - name: Download Artifacts - shell: julia --color=yes --project {0} - run: | - using Pkg - Pkg.instantiate() - include(joinpath(pwd(), "download_tarballs.jl")) - download_tarballs() - working-directory: ${{ env.build_dir }} - env: - GITHUB_TOKEN: ${{ github.token }} + id: download + uses: dawidd6/action-download-artifact@v2 + with: + name: ray_julia* + name_is_regexp: true + skip_unpack: true + path: ${{ build_dir }}/tarballs + run_id: ${{ github.run_id }} - name: Update Artifacts.toml shell: julia --color=yes --project {0} run: | From b1ceca6b92839f920ca0678d59af638f0aaaba79 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 12 Oct 2023 13:26:45 +0100 Subject: [PATCH 30/90] indenting --- .github/workflows/CI.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 18e0f537..d297aa9e 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -208,12 +208,12 @@ jobs: - name: Download Artifacts id: download uses: dawidd6/action-download-artifact@v2 - with: - name: ray_julia* - name_is_regexp: true - skip_unpack: true - path: ${{ build_dir }}/tarballs - run_id: ${{ github.run_id }} + with: + name: ray_julia* + name_is_regexp: true + skip_unpack: true + path: ${{ build_dir }}/tarballs + run_id: ${{ github.run_id }} - name: Update Artifacts.toml shell: julia --color=yes --project {0} run: | From c384387a3e9acf4b0cb833ad088073f21ff22114 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 12 Oct 2023 13:30:58 +0100 Subject: [PATCH 31/90] fix --- .github/workflows/CI.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d297aa9e..19e1a55a 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -206,13 +206,12 @@ jobs: println(io, "release_branch=release-v$(project.version)") end - name: Download Artifacts - id: download uses: dawidd6/action-download-artifact@v2 with: name: ray_julia* name_is_regexp: true skip_unpack: true - path: ${{ build_dir }}/tarballs + path: ${{ env.build_dir }}/tarballs run_id: ${{ github.run_id }} - name: Update Artifacts.toml shell: julia --color=yes --project {0} From 91a3db7dca6180f4bdcb95dcb7b3b3435cc6d2d7 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 12 Oct 2023 13:44:21 +0100 Subject: [PATCH 32/90] more specific regex --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 19e1a55a..6b212b39 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -208,7 +208,7 @@ jobs: - name: Download Artifacts uses: dawidd6/action-download-artifact@v2 with: - name: ray_julia* + name: ^ray_julia\.v[0-9]+(\.[0-9]+){2}\.[a-z0-9_-]+-julia_version\+[0-9]+(\.[0-9]+){2}\.tar\.gz$ name_is_regexp: true skip_unpack: true path: ${{ env.build_dir }}/tarballs From bdadbfd65138a2e3deec467a3d4613dd48eaf09b Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 12 Oct 2023 14:21:44 +0100 Subject: [PATCH 33/90] download any artifacts at all whatsoever --- .github/workflows/CI.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 6b212b39..79dc2b79 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -208,11 +208,8 @@ jobs: - name: Download Artifacts uses: dawidd6/action-download-artifact@v2 with: - name: ^ray_julia\.v[0-9]+(\.[0-9]+){2}\.[a-z0-9_-]+-julia_version\+[0-9]+(\.[0-9]+){2}\.tar\.gz$ - name_is_regexp: true skip_unpack: true path: ${{ env.build_dir }}/tarballs - run_id: ${{ github.run_id }} - name: Update Artifacts.toml shell: julia --color=yes --project {0} run: | From c3c62055538d38c4168d017bd06b5116dc65d084 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 12 Oct 2023 14:55:07 +0100 Subject: [PATCH 34/90] download from this run_id --- .github/workflows/CI.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 79dc2b79..99437ba9 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -210,6 +210,7 @@ jobs: with: skip_unpack: true path: ${{ env.build_dir }}/tarballs + run_id: ${{ github.run_id }} - name: Update Artifacts.toml shell: julia --color=yes --project {0} run: | From 8fcd5928c22bf33b7fb8ac6b1b53a86279863308 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 12 Oct 2023 15:30:05 +0100 Subject: [PATCH 35/90] Revert using new GHA This reverts commit c3c62055538d38c4168d017bd06b5116dc65d084. --- .github/workflows/CI.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 99437ba9..cab96f89 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -206,11 +206,15 @@ jobs: println(io, "release_branch=release-v$(project.version)") end - name: Download Artifacts - uses: dawidd6/action-download-artifact@v2 - with: - skip_unpack: true - path: ${{ env.build_dir }}/tarballs - run_id: ${{ github.run_id }} + shell: julia --color=yes --project {0} + run: | + using Pkg + Pkg.instantiate() + include(joinpath(pwd(), "download_tarballs.jl")) + download_tarballs() + working-directory: ${{ env.build_dir }} + env: + GITHUB_TOKEN: ${{ github.token }} - name: Update Artifacts.toml shell: julia --color=yes --project {0} run: | From 3bbf8cfd805a9382794620ddcae417eb029089b0 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 12 Oct 2023 15:37:51 +0100 Subject: [PATCH 36/90] dont need GITHUB_TOKEN to bind_artifacts --- .github/workflows/CI.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index cab96f89..67a05b07 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -223,8 +223,6 @@ jobs: include(joinpath(pwd(), "bind_artifacts.jl")) bind_artifacts() working-directory: ${{ env.build_dir }} - env: - GITHUB_TOKEN: ${{ github.token }} - name: Commit Changes id: commit run: | From fb30ee9a567d69b03648d6bb52f84d0b606a03f0 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 12 Oct 2023 15:41:59 +0100 Subject: [PATCH 37/90] print commit sha --- build/download_tarballs.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/download_tarballs.jl b/build/download_tarballs.jl index db25ae3c..9d992898 100644 --- a/build/download_tarballs.jl +++ b/build/download_tarballs.jl @@ -50,8 +50,8 @@ end function download_tarballs() token = github_token() - @info "Retrieving CI built tarballs..." commit_sha = git_head_sha() + @info "Retrieving CI built tarballs for $commit_sha..." download_ray_julia_artifacts(; commit_sha, token) return nothing From eaceed2de3fda9f605683ca81471e1864bd8c3d1 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 12 Oct 2023 16:18:39 +0100 Subject: [PATCH 38/90] checkout pull request HEAD commit instead of merge commit --- .github/workflows/CI.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 67a05b07..9e7921fa 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -189,7 +189,9 @@ jobs: strategy: fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} - uses: julia-actions/setup-julia@v1 with: version: "1.8" From d5f1cd364a01a1e1fe572466abaaa05f5dc4a131 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 12 Oct 2023 17:12:55 +0100 Subject: [PATCH 39/90] PRINT ALL THE INFO --- build/download_tarballs.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build/download_tarballs.jl b/build/download_tarballs.jl index 9d992898..bb7fef35 100644 --- a/build/download_tarballs.jl +++ b/build/download_tarballs.jl @@ -23,11 +23,17 @@ function download_ray_julia_artifacts(; commit_sha, token, tarball_dir=TARBALL_D isdir(tarball_dir) || mkpath(tarball_dir) response = list_workflow_runs(; org=REPO_ORG, repo=REPO_NAME, head_sha=commit_sha) + @show response run_id = only(filter(r -> r.name == ARTIFACTS_WORKFLOW_NAME, response.workflow_runs)).id + @info "Polling run: $run_id" + response = list_workflow_run_artifacts(; org=REPO_ORG, repo=REPO_NAME, run_id) + @show response + artifacts = map(j -> j.name => j.archive_download_url, response.artifacts) + @show artifacts headers = ["Authorization" => "Bearer $token"] for (name, url) in artifacts startswith(name, "ray_julia") || continue From 39dd08c11e5ff9acc553ac9a971ff266253a9e7d Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 12 Oct 2023 17:51:55 +0100 Subject: [PATCH 40/90] pass headers to list_workflow_run_artifacts --- build/download_tarballs.jl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/build/download_tarballs.jl b/build/download_tarballs.jl index bb7fef35..b69c81ba 100644 --- a/build/download_tarballs.jl +++ b/build/download_tarballs.jl @@ -13,28 +13,29 @@ function list_workflow_runs(; org, repo, head_sha) end # https://docs.github.com/en/rest/actions/artifacts?apiVersion=2022-11-28#list-workflow-run-artifacts -function list_workflow_run_artifacts(; org, repo, run_id) +function list_workflow_run_artifacts(; org, repo, run_id, headers) url = "https://api.github.com/repos/$org/$repo/actions/runs/$run_id/artifacts" - b = Downloads.download(url, IOBuffer()) + b = Downloads.download(url, IOBuffer(); headers) return JSON3.read(seekstart(b)) end function download_ray_julia_artifacts(; commit_sha, token, tarball_dir=TARBALL_DIR) isdir(tarball_dir) || mkpath(tarball_dir) + headers = ["Authorization" => "Bearer $token"] + response = list_workflow_runs(; org=REPO_ORG, repo=REPO_NAME, head_sha=commit_sha) @show response run_id = only(filter(r -> r.name == ARTIFACTS_WORKFLOW_NAME, response.workflow_runs)).id @info "Polling run: $run_id" - response = list_workflow_run_artifacts(; org=REPO_ORG, repo=REPO_NAME, run_id) + response = list_workflow_run_artifacts(; org=REPO_ORG, repo=REPO_NAME, run_id, headers) @show response artifacts = map(j -> j.name => j.archive_download_url, response.artifacts) @show artifacts - headers = ["Authorization" => "Bearer $token"] for (name, url) in artifacts startswith(name, "ray_julia") || continue From 54d01fd67aa2ecfb38142f005eec2741927be24d Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 12 Oct 2023 12:54:08 -0500 Subject: [PATCH 41/90] Add `actions: read` permission to publish job --- .github/workflows/CI.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 9e7921fa..07aa4cf6 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -186,6 +186,8 @@ jobs: name: "Publish Pre-Release" runs-on: ubuntu-20.04-16core needs: [test] + permissions: + actions: read strategy: fail-fast: false steps: From 70bbd50d450e41f964576cb947b015e4bd8da5c2 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Fri, 13 Oct 2023 07:14:05 -0500 Subject: [PATCH 42/90] Attempt using `actions/download-artifact` (#200) * Try actions/download-artifact * fixup! Try actions/download-artifact * Use standard sized instance * Enable and update the rest of the job * Set Git author * fixup! Set Git author * Update git commit message * Iterating * Validate entire release job * Fix publish release step * Skip CI workflow for tags * Re-add conditionals --- .github/workflows/CI.yml | 62 ++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 07aa4cf6..d49c43c4 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -4,7 +4,6 @@ on: push: branches: - main - tags: ["*"] # Triggers from tags ignore the `paths` filter: https://github.com/orgs/community/discussions/26273 paths: - "src/**" - "test/**" @@ -24,6 +23,7 @@ concurrency: env: build_dir: ./build ray_dir: ./build/ray + tarballs_dir: ./build/tarballs jobs: test: name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} @@ -148,7 +148,6 @@ jobs: tarball_path = build_host_tarball() open(ENV["GITHUB_OUTPUT"], "a") do io println(io, "tarball_path=$tarball_path") - println(io, "tarball_filename=$(basename(tarball_path))") end working-directory: ${{ env.build_dir }} - uses: julia-actions/julia-buildpkg@v1 @@ -162,10 +161,12 @@ jobs: file: lcov.info flags: Ray.jl + # Add the ray_julia library from each job to the same artifact + # https://github.com/actions/upload-artifact#uploading-to-the-same-artifact - name: Save ray_julia library uses: actions/upload-artifact@v3 with: - name: ${{ steps.ray_julia.outputs.tarball_filename }} + name: ray_julia_libraries path: ${{ steps.ray_julia.outputs.tarball_path }} - name: Save Ray logs @@ -182,18 +183,20 @@ jobs: with: key: ${{ steps.build-cache.outputs.cache-primary-key }} path: ~/.cache + release: name: "Publish Pre-Release" - runs-on: ubuntu-20.04-16core + runs-on: ubuntu-latest needs: [test] - permissions: - actions: read - strategy: - fail-fast: false steps: - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} + - name: Retrieve ray_julia libraries + uses: actions/download-artifact@v3 + with: + name: ray_julia_libraries + path: ${{ env.tarballs_dir }} - uses: julia-actions/setup-julia@v1 with: version: "1.8" @@ -206,19 +209,9 @@ jobs: using Pkg.Types project = read_project("Project.toml") open(ENV["GITHUB_OUTPUT"], "a") do io - println(io, "version=v$(project.version)") - println(io, "release_branch=release-v$(project.version)") + println(io, "version=$(project.version)") + println(io, "tag=v$(project.version)") end - - name: Download Artifacts - shell: julia --color=yes --project {0} - run: | - using Pkg - Pkg.instantiate() - include(joinpath(pwd(), "download_tarballs.jl")) - download_tarballs() - working-directory: ${{ env.build_dir }} - env: - GITHUB_TOKEN: ${{ github.token }} - name: Update Artifacts.toml shell: julia --color=yes --project {0} run: | @@ -230,20 +223,27 @@ jobs: - name: Commit Changes id: commit run: | - git checkout -b ${{ steps.version_check.outputs.release_branch }} - git commit -m "Update Artifacts.toml" -- Artifacts.toml - echo "commitish=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - git checkout main + # Alternatively environment variables can be used but both author/committer need to be set + # https://git-scm.com/book/en/v2/Git-Internals-Environment-Variables#_committing + git config user.name beacon-buddy + git config user.email beacon-buddy@beacon.bio + + msg="Update Artifacts.toml to use ${{ steps.version_check.outputs.version }} assets" + git checkout --detach + git commit -m "$msg" -- Artifacts.toml + echo "commit=$(git rev-parse HEAD)" | tee -a "$GITHUB_OUTPUT" + - name: Tag and push + if: ${{ github.ref == 'refs/heads/main' }} + run: | + tag=${{ steps.version_check.outputs.tag }} + git tag $tag ${{ steps.commit.outputs.commit }} + git push origin $tag - name: Publish Pre-Release uses: softprops/action-gh-release@v1 if: ${{ github.ref == 'refs/heads/main' }} with: prerelease: true - target_commitish: ${{ steps.commit.outputs.commitish }} + tag_name: ${{ steps.version_check.outputs.tag }} + target_commitish: ${{ steps.commit.outputs.commit }} generate_release_notes: true - files: ${{ env.build_dir }}/tarballs - - name: Push Release Branch - if: ${{ github.ref == 'refs/heads/main' }} - run: git push origin ${{ steps.version_check.outputs.release_branch }} - - + files: ${{ env.tarballs_dir }}/*.tar.gz From 9d46971185dadd0f33163e9e9538d92d8a9e9957 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 13:20:13 +0100 Subject: [PATCH 43/90] artifacts_CI only runs on tags --- .github/workflows/CI.yml | 2 -- .github/workflows/artifacts_CI.yml | 5 ----- 2 files changed, 7 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d49c43c4..96edb27e 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -190,8 +190,6 @@ jobs: needs: [test] steps: - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - name: Retrieve ray_julia libraries uses: actions/download-artifact@v3 with: diff --git a/.github/workflows/artifacts_CI.yml b/.github/workflows/artifacts_CI.yml index c9bf946f..89aaa2f8 100644 --- a/.github/workflows/artifacts_CI.yml +++ b/.github/workflows/artifacts_CI.yml @@ -2,12 +2,7 @@ name: Artifacts CI on: workflow_dispatch: push: - branches: - - release-* tags: ["*"] # Triggers from tags ignore the `paths` filter: https://github.com/orgs/community/discussions/26273 - paths: - - "Artifacts.toml" - - ".github/workflows/artifacts_CI.yml" concurrency: # Skip intermediate builds: always. # Cancel intermediate builds: only if it is a pull request build. From 48ff20d38893bd87aaf010798a84488b4a7536b6 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 13:30:08 +0100 Subject: [PATCH 44/90] delete download/upload_tarballs.jl --- build/Project.toml | 2 - build/common.jl | 35 ------------------ build/download_tarballs.jl | 69 ----------------------------------- build/upload_tarballs.jl | 75 -------------------------------------- 4 files changed, 181 deletions(-) delete mode 100644 build/download_tarballs.jl delete mode 100644 build/upload_tarballs.jl diff --git a/build/Project.toml b/build/Project.toml index 14066777..3a89bb04 100644 --- a/build/Project.toml +++ b/build/Project.toml @@ -9,8 +9,6 @@ Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" -ZipFile = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea" -ghr_jll = "07c12ed4-43bc-5495-8a2a-d5838ef8d533" [compat] CxxWrap = "0.13.4" diff --git a/build/common.jl b/build/common.jl index fa191a7d..6fda4c11 100644 --- a/build/common.jl +++ b/build/common.jl @@ -6,19 +6,6 @@ using URIs: URI const TARBALL_DIR = joinpath(@__DIR__, "tarballs") const SO_FILE = "julia_core_worker_lib.so" -const TARBALL_REGEX = r""" - ^ray_julia\.v(?[0-9]+(\.[0-9]+){2})\. - (?[a-z0-9_-]+)- - julia_version\+(?[0-9]+(\.[0-9]+){2})\. - tar\.gz$ - """x - -const GH_RELEASE_ASSET_PATH_REGEX = r""" - ^/(?[^/]+)/(?[^/]+)/ - releases/download/ - (?[^/]+)/?$ - """x - const REQUIRED_BASE_TRIPLETS = ("x86_64-linux-gnu", "aarch64-apple-darwin") const REQUIRED_JULIA_VERSIONS = (v"1.8", v"1.9") const REQUIRED_PLATFORMS = let @@ -36,24 +23,6 @@ function remote_url(repo_root::AbstractString, name::AbstractString="origin") end end -function git_head_sha(repo_root::AbstractString=REPO_PATH) - return LibGit2.with(LibGit2.GitRepo(repo_root)) do repo - ref = LibGit2.head(repo) - commit = LibGit2.peel(LibGit2.GitCommit, ref) - return string(LibGit2.GitHash(commit)) - end -end - -function github_token() - return get(ENV, "GITHUB_TOKEN") do - s = Base.getpass("GitHub PAT") - println() - token = read(s, String) - Base.shred!(s) - return token - end -end - function convert_to_https_url(url) m = match(LibGit2.URL_REGEX, url) if m === nothing @@ -83,12 +52,8 @@ end const REPO_PATH = abspath(joinpath(@__DIR__, "..")) const REPO_HTTPS_URL = convert_to_https_url(remote_url(REPO_PATH)) -const REPO_ORG = split(parse(URI, REPO_HTTPS_URL).path, '/')[2] -const REPO_NAME = split(parse(URI, REPO_HTTPS_URL).path, '/')[3] const COMPILED_DIR = joinpath(REPO_PATH, "build", "bazel-bin") - const ARTIFACTS_TOML = joinpath(REPO_PATH, "Artifacts.toml") -const ARTIFACTS_WORKFLOW_NAME = "CI" const TAG = let project_toml = joinpath(REPO_PATH, "Project.toml") diff --git a/build/download_tarballs.jl b/build/download_tarballs.jl deleted file mode 100644 index b69c81ba..00000000 --- a/build/download_tarballs.jl +++ /dev/null @@ -1,69 +0,0 @@ -using Downloads: Downloads -using JSON3: JSON3 -using ZipFile: ZipFile - -include("common.jl") - -# https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#list-workflow-runs-for-a-repository -function list_workflow_runs(; org, repo, head_sha) - # Needs to be a full SHA - url = "https://api.github.com/repos/$org/$repo/actions/runs?head_sha=$head_sha" - b = Downloads.download(url, IOBuffer()) - return JSON3.read(seekstart(b)) -end - -# https://docs.github.com/en/rest/actions/artifacts?apiVersion=2022-11-28#list-workflow-run-artifacts -function list_workflow_run_artifacts(; org, repo, run_id, headers) - url = "https://api.github.com/repos/$org/$repo/actions/runs/$run_id/artifacts" - b = Downloads.download(url, IOBuffer(); headers) - return JSON3.read(seekstart(b)) -end - -function download_ray_julia_artifacts(; commit_sha, token, tarball_dir=TARBALL_DIR) - isdir(tarball_dir) || mkpath(tarball_dir) - - headers = ["Authorization" => "Bearer $token"] - - response = list_workflow_runs(; org=REPO_ORG, repo=REPO_NAME, head_sha=commit_sha) - @show response - run_id = only(filter(r -> r.name == ARTIFACTS_WORKFLOW_NAME, response.workflow_runs)).id - - @info "Polling run: $run_id" - - response = list_workflow_run_artifacts(; org=REPO_ORG, repo=REPO_NAME, run_id, headers) - @show response - - artifacts = map(j -> j.name => j.archive_download_url, response.artifacts) - - @show artifacts - for (name, url) in artifacts - startswith(name, "ray_julia") || continue - - @info "Downloading $name" - io = Downloads.download(url, IOBuffer(); headers) - zip = ZipFile.Reader(io) - - if length(zip.files) > 1 - error("GitHub workflow artifact contains more than one file:\n$(join(zip.files, '\n'))") - end - - file = only(zip.files) - write(joinpath(tarball_dir, name), read(file)) - end - - return nothing -end - -function download_tarballs() - token = github_token() - - commit_sha = git_head_sha() - @info "Retrieving CI built tarballs for $commit_sha..." - download_ray_julia_artifacts(; commit_sha, token) - - return nothing -end - -if abspath(PROGRAM_FILE) == @__FILE__ - download_tarballs() -end diff --git a/build/upload_tarballs.jl b/build/upload_tarballs.jl deleted file mode 100644 index 5d925af2..00000000 --- a/build/upload_tarballs.jl +++ /dev/null @@ -1,75 +0,0 @@ -using URIs: URI -using ghr_jll: ghr - -include("common.jl") - -function upload_to_github_release(archive_path::AbstractString, - archive_url::AbstractString, - commit; - kwargs...) - return upload_to_github_release(archive_path, parse(URI, archive_url), commit; - kwargs...) -end - -# TODO: Does this work properly with directories? -function upload_to_github_release(archive_path::AbstractString, archive_uri::URI, commit; - kwargs...) - # uri = parse(URI, artifact_url) - if archive_uri.host != "github.com" - throw(ArgumentError("Artifact URL is not for github.com: $(archive_uri)")) - end - - m = match(GH_RELEASE_ASSET_PATH_REGEX, archive_uri.path) - if m === nothing - throw(ArgumentError("Artifact URL is not a GitHub release asset path: $(archive_uri)")) - end - - upload_to_github_release(m[:owner], m[:repo_name], commit, m[:tag], archive_path; - kwargs...) - - return nothing -end - -function upload_to_github_release(owner, repo_name, commit, tag, path; token) - - # Based on: https://github.com/JuliaPackaging/BinaryBuilder.jl/blob/d40ec617d131a1787851559ef1a9f04efce19f90/src/AutoBuild.jl#L487 - # TODO: Passing in a directory path uploads multiple assets - # TODO: Would be nice to perform parallel uploads - cmd = ``` - $(ghr()) \ - -owner $owner \ - -repository $repo_name \ - -commitish $commit \ - -token $token \ - -n $tag \ - -prerelease \ - $tag $path - ``` - - try - run(cmd) - catch e - # ghr() already prints an error message with diagnosis - @debug "Caught exception $(sprint(showerror, e, catch_backtrace()))" - end - - return nothing -end - -if abspath(PROGRAM_FILE) == @__FILE__ - isdir(TARBALL_DIR) || error("$TARBALL_DIR does not exist") - token = github_token() - - # check that contents are tarballs with correct filename - for t in readdir(TARBALL_DIR) - m = match(TARBALL_REGEX, t) - !isnothing(m) || error("Unexpected file found: tarballs/$t") - "v$(m[:jll_version])" == TAG || error("Unexpected JLL version: tarballs/$t") - end - - artifact_url = gen_artifact_url(; repo_url=REPO_HTTPS_URL, tag=TAG, filename="") - branch = LibGit2.with(LibGit2.branch, LibGit2.GitRepo(REPO_PATH)) - - @info "Uploading tarballs to $artifact_url" - upload_to_github_release(TARBALL_DIR, artifact_url, branch; token) -end From 484cb9f99fb97b6e74f70dc6ba1ab90752c7aed9 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 13:42:19 +0100 Subject: [PATCH 45/90] add step at end of artifacts_CI to convert pre-release to release --- .github/workflows/artifacts_CI.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/artifacts_CI.yml b/.github/workflows/artifacts_CI.yml index 89aaa2f8..8b2b5e5c 100644 --- a/.github/workflows/artifacts_CI.yml +++ b/.github/workflows/artifacts_CI.yml @@ -61,3 +61,14 @@ jobs: - uses: julia-actions/julia-runtest@v1 with: annotate: true + release: + name: "Publish Release" + runs-on: ubuntu-latest + needs: [test] + steps: + - name: Publish Release + uses: softprops/action-gh-release@v1 + with: + # Any info keys (i.e. prerelease) that are not explicitly set will retain the original + # value if there is already an existing release for the tag. + prerelease: false From 93af57a2521bfe95d05e585e8c768eedea5d7ae1 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 13:51:09 +0100 Subject: [PATCH 46/90] update docs --- docs/src/building-artifacts.md | 58 ++++------------------------------ 1 file changed, 6 insertions(+), 52 deletions(-) diff --git a/docs/src/building-artifacts.md b/docs/src/building-artifacts.md index 99c6df51..280aa127 100644 --- a/docs/src/building-artifacts.md +++ b/docs/src/building-artifacts.md @@ -8,57 +8,11 @@ Follow [the instructions](https://github.com/beacon-biosignals/ray/blob/beacon-m ### Artifacts -The `ray_julia` artifacts are hosted via [GitHub releases](https://github.com/beacon-biosignals/Ray.jl/releases) and will be downloaded automatically for any supported platform which installs Ray.jl. +The `ray_julia` artifacts are hosted via [GitHub releases](https://github.com/beacon-biosignals/Ray.jl/releases) and will be downloaded automatically for any supported platform which installs Ray.jl from a tag. -To update the Ray.jl artifacts for a new release perform the following steps: +However, the artifacts are only associated with tagged releases of Ray.jl. +If you are working off the `main` branch, or developing Ray.jl locally, you will need to [build Ray.jl](./developer-guide.md#build-rayjl) yourself. -1. Create a new branch (based off of `origin/HEAD`) and bump the Ray.jl version in the `Project.toml` file. Commit and push this change to a new PR. The Julia CI jobs created by this PR will build the artifacts required to make a new release. - -2. Wait for the CI workflows to complete for the created PR. - -3. Navigate to the `build` directory - -4. Run the `download_tarballs.jl` script to fetch the GitHub workflow tarballs for all of the required platforms and Julia versions. - - ```sh - julia --project -e 'using Pkg; Pkg.instantiate()' - - # Cleanup any tarballs from previous builds - rm -rf tarballs - - # Fetches the tarballs from GitHub Actions artifacts (may need to wait on CI) - read -s GITHUB_TOKEN - export GITHUB_TOKEN - julia --project download_tarballs.jl - ``` - -5. Run the `upload_tarballs.jl` script to publish the tarballs as assets of a GitHub pre-release. Re-running this script will only upload new tarballs and skip any that have already been published. - - ```sh - read -s GITHUB_TOKEN - export GITHUB_TOKEN - julia --project upload_tarballs.jl - ``` - -6. Run `bind_artifacts.jl` to modify local `Artifacts.toml` with the artifacts associated with the Ray.jl version specified in the `Project.toml`. After running this you should commit and push the changes to the PR you created in Step 1. - - ```sh - julia --project bind_artifacts.jl - - git commit -m "Update Artifacts.toml" -- Artifacts.toml - git push origin - ``` - -7. Merge the PR. If the PR becomes out of date with the default branch then you will need to repeat steps 3-6 to ensure that the tarballs include the current changes. In some scenarios re-building the tarballs may be unnecessary such as a documentation only change. If in doubt re-build the tarballs. - -8. After the PR is merged, delete the existing tag (which will convert the release to a draft) and create a new one (with the same version) from the commit you just merged. Then update the GitHub release to point to the new tag. - - ```sh - git tag -d $tag - git push origin :$tag - git tag $tag - - # Update GitHub Release to point to the updated tag - ``` - -9. Register the new tag as normal with JuliaRegistrator. +To update the Ray.jl artifacts for a new release simply open a PR which bumps the Ray.jl version in the `Project.toml` and includes any other required changes. +The Julia CI jobs created by this PR will build the artifacts required to make a new release. +Upon merging the PR with `main` a new tag will be created and the artifacts will be uploaded to a GitHub Pre-Release, which will be converted as a full Release once the CI for the tag passes. From b61cc15b9446e57c26625bbf177be13ec408fa59 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 13:54:30 +0100 Subject: [PATCH 47/90] Delete Artifacts.toml --- Artifacts.toml | 38 -------------------------------------- 1 file changed, 38 deletions(-) delete mode 100644 Artifacts.toml diff --git a/Artifacts.toml b/Artifacts.toml deleted file mode 100644 index 39f8f81b..00000000 --- a/Artifacts.toml +++ /dev/null @@ -1,38 +0,0 @@ -[[ray_julia]] -arch = "x86_64" -git-tree-sha1 = "88eb04f4e4a87d0a369288ceb6fb605f4ca4ff03" -julia_version = "1.8.0" -libc = "glibc" -os = "linux" - - [[ray_julia.download]] - sha256 = "03bc890efc63375f901325d8f67a2f3403c7d1067658ed693390276ab37f6958" - url = "https://github.com/beacon-biosignals/Ray.jl/releases/download/v0.0.1/ray_julia.v0.0.1.x86_64-linux-gnu-julia_version+1.8.0.tar.gz" -[[ray_julia]] -arch = "aarch64" -git-tree-sha1 = "a24e4bb69ffa0baf615e27f3ae6d00084556881a" -julia_version = "1.8.0" -os = "macos" - - [[ray_julia.download]] - sha256 = "2d5ee4e6290e1207250a02e434d588af3321d7f978fe7d57e13d4827ca7aa51b" - url = "https://github.com/beacon-biosignals/Ray.jl/releases/download/v0.0.1/ray_julia.v0.0.1.aarch64-apple-darwin-julia_version+1.8.0.tar.gz" -[[ray_julia]] -arch = "x86_64" -git-tree-sha1 = "32c3aaecc5a1e2aed7e9ce6d9a95bef741d1ef9d" -julia_version = "1.9.0" -libc = "glibc" -os = "linux" - - [[ray_julia.download]] - sha256 = "1f96eaf5f35796f60b04172728caa29fc442ffcc30e8ffabfe147791dd711fc8" - url = "https://github.com/beacon-biosignals/Ray.jl/releases/download/v0.0.1/ray_julia.v0.0.1.x86_64-linux-gnu-julia_version+1.9.0.tar.gz" -[[ray_julia]] -arch = "aarch64" -git-tree-sha1 = "81cc7a633a617a01f1fc87674258d880baa4cafb" -julia_version = "1.9.0" -os = "macos" - - [[ray_julia.download]] - sha256 = "5969006375abafacf2fff0a1d499b492d1a883d1529e2a20fe69d3a4908f2729" - url = "https://github.com/beacon-biosignals/Ray.jl/releases/download/v0.0.1/ray_julia.v0.0.1.aarch64-apple-darwin-julia_version+1.9.0.tar.gz" From 9f5e29bcf847acfdb662b9ab949c4d052e913229 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 14:02:33 +0100 Subject: [PATCH 48/90] don't delete Overrides.toml in CI --- .github/workflows/artifacts_CI.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/artifacts_CI.yml b/.github/workflows/artifacts_CI.yml index 8b2b5e5c..30e7327e 100644 --- a/.github/workflows/artifacts_CI.yml +++ b/.github/workflows/artifacts_CI.yml @@ -52,11 +52,6 @@ jobs: # Verify Ray CLI works ray --version - - name: Delete Overrides.toml - shell: julia --color=yes --project {0} - run: | - overrides_toml = joinpath(first(Base.DEPOT_PATH), "artifacts", "Overrides.toml") - isfile(overrides_toml) && rm(overrides_toml) - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 with: From 43ff28d634dc4151c109732e94929e0734424418 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 14:13:39 +0100 Subject: [PATCH 49/90] Revert "don't delete Overrides.toml in CI" This reverts commit 9f5e29bcf847acfdb662b9ab949c4d052e913229. --- .github/workflows/artifacts_CI.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/artifacts_CI.yml b/.github/workflows/artifacts_CI.yml index 30e7327e..8b2b5e5c 100644 --- a/.github/workflows/artifacts_CI.yml +++ b/.github/workflows/artifacts_CI.yml @@ -52,6 +52,11 @@ jobs: # Verify Ray CLI works ray --version + - name: Delete Overrides.toml + shell: julia --color=yes --project {0} + run: | + overrides_toml = joinpath(first(Base.DEPOT_PATH), "artifacts", "Overrides.toml") + isfile(overrides_toml) && rm(overrides_toml) - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 with: From f0258733ae2189bc769a996783e8946fe122e95b Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 14:15:52 +0100 Subject: [PATCH 50/90] log when writing to Overrides.toml --- build/build_library.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/build/build_library.jl b/build/build_library.jl index 9db9d398..8cbe440a 100644 --- a/build/build_library.jl +++ b/build/build_library.jl @@ -46,6 +46,7 @@ function build_library(; override::Bool=true) # Add entry to depot Overrides.toml if override + @info "Adding entry to Overrides.toml..." pkg_uuid = TOML.parsefile(PROJECT_TOML)["uuid"] overrides_toml = joinpath(first(DEPOT_PATH), "artifacts", "Overrides.toml") From 72f6df34a7b2288254ec123e73d24f6adb55a39e Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 14:53:42 +0100 Subject: [PATCH 51/90] Add dummy Artifacts.toml --- Artifacts.toml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Artifacts.toml diff --git a/Artifacts.toml b/Artifacts.toml new file mode 100644 index 00000000..b6e9ceea --- /dev/null +++ b/Artifacts.toml @@ -0,0 +1,2 @@ +[ray_julia] +git-tree-sha1 = "0000000000000000000000000000000000000000" From 66e6813096431138d21fcc033f59160ba7fafd11 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 14:56:51 +0100 Subject: [PATCH 52/90] add line to docs --- docs/src/building-artifacts.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/src/building-artifacts.md b/docs/src/building-artifacts.md index 280aa127..d7cda1d2 100644 --- a/docs/src/building-artifacts.md +++ b/docs/src/building-artifacts.md @@ -12,6 +12,7 @@ The `ray_julia` artifacts are hosted via [GitHub releases](https://github.com/be However, the artifacts are only associated with tagged releases of Ray.jl. If you are working off the `main` branch, or developing Ray.jl locally, you will need to [build Ray.jl](./developer-guide.md#build-rayjl) yourself. +This will also update your `Overrides.toml` to reference the binaries you have built. To update the Ray.jl artifacts for a new release simply open a PR which bumps the Ray.jl version in the `Project.toml` and includes any other required changes. The Julia CI jobs created by this PR will build the artifacts required to make a new release. From 82895fc8fcaf08caa20de373374952abe88c0785 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 19:13:17 +0100 Subject: [PATCH 53/90] build ray_julia_library in Documenter.yml --- .github/workflows/Documenter.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/Documenter.yml b/.github/workflows/Documenter.yml index 5384d37d..c38a7149 100644 --- a/.github/workflows/Documenter.yml +++ b/.github/workflows/Documenter.yml @@ -17,6 +17,8 @@ concurrency: # Cancel intermediate builds: only if it is a pull request build. group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} +env: + build_dir: ./build jobs: docs: name: Build @@ -29,6 +31,14 @@ jobs: with: version: "1" # Use a separate step for instantiation to get more fine grained timing of steps + - name: Build ray_julia library + id: ray_julia + shell: julia --color=yes --project {0} + run: | + using Pkg + Pkg.instantiate() + include(joinpath(pwd(), "build_library.jl")) + working-directory: ${{ env.build_dir }} - name: Configure doc environment shell: julia --project=docs --color=yes {0} run: | From ee8d29a92e7f3572fb519dd4357e168d080d6c2a Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 19:22:03 +0100 Subject: [PATCH 54/90] actually run the build_library command --- .github/workflows/Documenter.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/Documenter.yml b/.github/workflows/Documenter.yml index c38a7149..0271f35a 100644 --- a/.github/workflows/Documenter.yml +++ b/.github/workflows/Documenter.yml @@ -38,6 +38,7 @@ jobs: using Pkg Pkg.instantiate() include(joinpath(pwd(), "build_library.jl")) + build_library(; override=true) working-directory: ${{ env.build_dir }} - name: Configure doc environment shell: julia --project=docs --color=yes {0} From 580ffab487af9ac35983f4f5268a77ce4ee5919e Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 19:33:03 +0100 Subject: [PATCH 55/90] conditionally run release job only if Project version changes --- .github/workflows/CI.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 96edb27e..ccac5f52 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -28,6 +28,8 @@ jobs: test: name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} runs-on: ${{ matrix.os }} + outputs: + src: ${{ steps.version_check.outputs.make_release }} strategy: fail-fast: false matrix: @@ -183,11 +185,22 @@ jobs: with: key: ${{ steps.build-cache.outputs.cache-primary-key }} path: ~/.cache + - name: Check Project Version + id: version_check + shell: julia --color=yes --project {0} + run: | + using Pkg.Types + project = read_project("Project.toml") + latest_tag = parse(VersionNumber, readchomp(`git describe --tags --abbrev=0`)) + open(ENV["GITHUB_OUTPUT"], "a") do io + println(io, "make_release=$(project.version > latest_tag)") + end release: name: "Publish Pre-Release" runs-on: ubuntu-latest needs: [test] + if: ${{ needs.test.outputs.version_check.make_release == 'true' }} steps: - uses: actions/checkout@v4 - name: Retrieve ray_julia libraries From 4c7ebb8bcb0566a5d7ff9629663c8ac7bc0857e7 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 19:54:31 +0100 Subject: [PATCH 56/90] use LibGit2 to check for latest tag --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ccac5f52..240ee891 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -191,7 +191,7 @@ jobs: run: | using Pkg.Types project = read_project("Project.toml") - latest_tag = parse(VersionNumber, readchomp(`git describe --tags --abbrev=0`)) + latest_tag = parse(VersionNumber, last(sort!(LibGit2.tag_list(LibGit2.GitRepo(@__DIR__))))) open(ENV["GITHUB_OUTPUT"], "a") do io println(io, "make_release=$(project.version > latest_tag)") end From 220589ddc7d1b379a27fd4e130f1c94269917da4 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 19:54:49 +0100 Subject: [PATCH 57/90] cache julia_jll build in Documenter.yml --- .github/workflows/Documenter.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/Documenter.yml b/.github/workflows/Documenter.yml index 0271f35a..fc169c34 100644 --- a/.github/workflows/Documenter.yml +++ b/.github/workflows/Documenter.yml @@ -30,6 +30,12 @@ jobs: - uses: julia-actions/setup-julia@v1 with: version: "1" + - name: Restore build cache + uses: actions/cache/restore@v3 + id: build-cache + with: + key: build-cache.ray-jl.documenter + path: ~/.cache # Use a separate step for instantiation to get more fine grained timing of steps - name: Build ray_julia library id: ray_julia @@ -63,3 +69,11 @@ jobs: env: repo: ${{ github.repository }} PR: ${{ github.event.number }} + + # https://github.com/actions/cache/tree/main/save#always-save-cache + - name: Save build cache + uses: actions/cache/save@v3 + if: always() && steps.build-cache.outputs.cache-hit != 'true' + with: + key: ${{ steps.build-cache.outputs.cache-primary-key }} + path: ~/.cache From 093c5c1ca65a8291cd0699c13da7e212fe81059d Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 19:55:31 +0100 Subject: [PATCH 58/90] load LibGit2 --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 240ee891..e4ba9a6c 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -189,7 +189,7 @@ jobs: id: version_check shell: julia --color=yes --project {0} run: | - using Pkg.Types + using Pkg.Types, LibGit2 project = read_project("Project.toml") latest_tag = parse(VersionNumber, last(sort!(LibGit2.tag_list(LibGit2.GitRepo(@__DIR__))))) open(ENV["GITHUB_OUTPUT"], "a") do io From 2f44e5a1cbe95c03734a7bd7613c3c0dd78bd1ee Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 20:29:05 +0100 Subject: [PATCH 59/90] fetch tags and run check from github workspace --- .github/workflows/CI.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index e4ba9a6c..ec53a4c9 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -49,6 +49,7 @@ jobs: arch: x64 steps: - uses: actions/checkout@v3 + fetch-tags: true # we need to know the latest tag to check if we should make a release - name: Determine Ray commit id: ray-commit run: | @@ -195,6 +196,7 @@ jobs: open(ENV["GITHUB_OUTPUT"], "a") do io println(io, "make_release=$(project.version > latest_tag)") end + working-directory: ${{ github.workspace }} release: name: "Publish Pre-Release" From c80aff1a946e051333f35123643b7ea1b09fb3c4 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 20:43:19 +0100 Subject: [PATCH 60/90] fix indent --- .github/workflows/CI.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ec53a4c9..75ba211b 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -49,7 +49,8 @@ jobs: arch: x64 steps: - uses: actions/checkout@v3 - fetch-tags: true # we need to know the latest tag to check if we should make a release + with: + fetch-tags: true # we need to know the latest tag to check if we should make a release - name: Determine Ray commit id: ray-commit run: | From 4baecf75f4f427279d400dbd61eec0d762268283 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 21:02:39 +0100 Subject: [PATCH 61/90] print dir --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 75ba211b..eb28f617 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -192,12 +192,12 @@ jobs: shell: julia --color=yes --project {0} run: | using Pkg.Types, LibGit2 + @info @__DIR__ project = read_project("Project.toml") latest_tag = parse(VersionNumber, last(sort!(LibGit2.tag_list(LibGit2.GitRepo(@__DIR__))))) open(ENV["GITHUB_OUTPUT"], "a") do io println(io, "make_release=$(project.version > latest_tag)") end - working-directory: ${{ github.workspace }} release: name: "Publish Pre-Release" From b7fcc90348219ec804c91fff4f88a01959d64ee5 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 21:13:01 +0100 Subject: [PATCH 62/90] set working-directory --- .github/workflows/CI.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index eb28f617..70927744 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -194,10 +194,12 @@ jobs: using Pkg.Types, LibGit2 @info @__DIR__ project = read_project("Project.toml") + @info project latest_tag = parse(VersionNumber, last(sort!(LibGit2.tag_list(LibGit2.GitRepo(@__DIR__))))) open(ENV["GITHUB_OUTPUT"], "a") do io println(io, "make_release=$(project.version > latest_tag)") end + working-directory: ${{ github.workspace }} release: name: "Publish Pre-Release" From f9d6b46c6d00eca65f0983ad9b803ce076bf8e59 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 21:21:03 +0100 Subject: [PATCH 63/90] try pipeline --- .github/workflows/CI.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 70927744..7fe84f14 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -195,11 +195,10 @@ jobs: @info @__DIR__ project = read_project("Project.toml") @info project - latest_tag = parse(VersionNumber, last(sort!(LibGit2.tag_list(LibGit2.GitRepo(@__DIR__))))) + latest_tag = parse(VersionNumber, readchomp(pipeline(`git fetch --tags`, `git describe --tags --abbrev=0`))) open(ENV["GITHUB_OUTPUT"], "a") do io println(io, "make_release=$(project.version > latest_tag)") end - working-directory: ${{ github.workspace }} release: name: "Publish Pre-Release" From d4c7bfb3565fcdd66a75bdae4a69cc3934c23261 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 21:31:04 +0100 Subject: [PATCH 64/90] don't use describe? --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 7fe84f14..ec2501d3 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -195,7 +195,7 @@ jobs: @info @__DIR__ project = read_project("Project.toml") @info project - latest_tag = parse(VersionNumber, readchomp(pipeline(`git fetch --tags`, `git describe --tags --abbrev=0`))) + latest_tag = parse(VersionNumber, readchomp(pipeline(`git fetch --tags`, `git tag --sort=-version:refname`, `head -n1`))) open(ENV["GITHUB_OUTPUT"], "a") do io println(io, "make_release=$(project.version > latest_tag)") end From c25e3a322fc3cfb0e69989a7c3d3b06a2639d782 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 21:40:40 +0100 Subject: [PATCH 65/90] just move on.. --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ec2501d3..07440db0 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -195,7 +195,7 @@ jobs: @info @__DIR__ project = read_project("Project.toml") @info project - latest_tag = parse(VersionNumber, readchomp(pipeline(`git fetch --tags`, `git tag --sort=-version:refname`, `head -n1`))) + latest_tag = VersionNumber(0, 0, 1) # TODO: replace with command open(ENV["GITHUB_OUTPUT"], "a") do io println(io, "make_release=$(project.version > latest_tag)") end From 786df9caea9a25097e6cc5d5f08c9e96d9c33a19 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 21:51:22 +0100 Subject: [PATCH 66/90] try another GHA --- .github/workflows/CI.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 07440db0..0274c31d 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -50,7 +50,7 @@ jobs: steps: - uses: actions/checkout@v3 with: - fetch-tags: true # we need to know the latest tag to check if we should make a release + fetch-depth: 0 - name: Determine Ray commit id: ray-commit run: | @@ -187,15 +187,20 @@ jobs: with: key: ${{ steps.build-cache.outputs.cache-primary-key }} path: ~/.cache + - name: Get Latest Tag + id: latest_tag + uses: "WyriHaximus/github-action-get-previous-tag@v1" - name: Check Project Version id: version_check shell: julia --color=yes --project {0} + env: + LATEST_TAG: ${{ steps.latest_tag.outputs.tag }} run: | using Pkg.Types, LibGit2 @info @__DIR__ project = read_project("Project.toml") @info project - latest_tag = VersionNumber(0, 0, 1) # TODO: replace with command + latest_tag = parse(VersionNumber, ENV["LATEST_TAG"]) open(ENV["GITHUB_OUTPUT"], "a") do io println(io, "make_release=$(project.version > latest_tag)") end From 08e96b6edae0da076928753c21b1f8e346f03eb4 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 21:59:11 +0100 Subject: [PATCH 67/90] cleanup --- .github/workflows/CI.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 0274c31d..70deabc9 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -197,9 +197,7 @@ jobs: LATEST_TAG: ${{ steps.latest_tag.outputs.tag }} run: | using Pkg.Types, LibGit2 - @info @__DIR__ project = read_project("Project.toml") - @info project latest_tag = parse(VersionNumber, ENV["LATEST_TAG"]) open(ENV["GITHUB_OUTPUT"], "a") do io println(io, "make_release=$(project.version > latest_tag)") From 068a38e4be4d6dfc66adb9eb52093ea2959736fa Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Fri, 13 Oct 2023 22:49:53 +0100 Subject: [PATCH 68/90] fix how outputs are passed --- .github/workflows/CI.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 70deabc9..3e501593 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -29,7 +29,7 @@ jobs: name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} runs-on: ${{ matrix.os }} outputs: - src: ${{ steps.version_check.outputs.make_release }} + make_release: ${{ steps.version_check.outputs.make_release }} strategy: fail-fast: false matrix: @@ -196,18 +196,21 @@ jobs: env: LATEST_TAG: ${{ steps.latest_tag.outputs.tag }} run: | - using Pkg.Types, LibGit2 + using Pkg.Types project = read_project("Project.toml") latest_tag = parse(VersionNumber, ENV["LATEST_TAG"]) + @info "Project version: project.version" + @info "Latest tag: latest_tag" + make_release = project.version > latest_tag open(ENV["GITHUB_OUTPUT"], "a") do io - println(io, "make_release=$(project.version > latest_tag)") + println(io, "make_release=$make_release") end release: name: "Publish Pre-Release" runs-on: ubuntu-latest needs: [test] - if: ${{ needs.test.outputs.version_check.make_release == 'true' }} + if: ${{ needs.test.outputs.make_release == 'true' }} steps: - uses: actions/checkout@v4 - name: Retrieve ray_julia libraries From 534db9c980af5ed2e0abf84c4ea32e6e179fdae4 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 19 Oct 2023 13:23:43 +0100 Subject: [PATCH 69/90] reapply review comments --- .github/workflows/CI.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 3e501593..ae106182 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -29,7 +29,7 @@ jobs: name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} runs-on: ${{ matrix.os }} outputs: - make_release: ${{ steps.version_check.outputs.make_release }} + make_release: ${{ steps.project_toml.outputs.make_release }} strategy: fail-fast: false matrix: @@ -50,7 +50,7 @@ jobs: steps: - uses: actions/checkout@v3 with: - fetch-depth: 0 + fetch-depth: 0 # Needed determine latest tag - name: Determine Ray commit id: ray-commit run: | @@ -189,9 +189,9 @@ jobs: path: ~/.cache - name: Get Latest Tag id: latest_tag - uses: "WyriHaximus/github-action-get-previous-tag@v1" - - name: Check Project Version - id: version_check + uses: WyriHaximus/github-action-get-previous-tag@v1 + - name: Check for updated Project version + id: project_toml shell: julia --color=yes --project {0} env: LATEST_TAG: ${{ steps.latest_tag.outputs.tag }} @@ -203,7 +203,7 @@ jobs: @info "Latest tag: latest_tag" make_release = project.version > latest_tag open(ENV["GITHUB_OUTPUT"], "a") do io - println(io, "make_release=$make_release") + println(io, "make_release=$make_release") end release: @@ -224,14 +224,14 @@ jobs: arch: "x64" - uses: julia-actions/cache@v1 - name: Check Project version - id: version_check + id: project_toml shell: julia --color=yes --project {0} run: | using Pkg.Types project = read_project("Project.toml") open(ENV["GITHUB_OUTPUT"], "a") do io - println(io, "version=$(project.version)") - println(io, "tag=v$(project.version)") + println(io, "version=$(project.version)") + println(io, "tag=v$(project.version)") end - name: Update Artifacts.toml shell: julia --color=yes --project {0} @@ -249,22 +249,22 @@ jobs: git config user.name beacon-buddy git config user.email beacon-buddy@beacon.bio - msg="Update Artifacts.toml to use ${{ steps.version_check.outputs.version }} assets" + msg="Update Artifacts.toml to use ${{ steps.project_toml.outputs.version }} assets" git checkout --detach git commit -m "$msg" -- Artifacts.toml - echo "commit=$(git rev-parse HEAD)" | tee -a "$GITHUB_OUTPUT" + echo "sha=$(git rev-parse HEAD)" | tee -a "$GITHUB_OUTPUT" - name: Tag and push if: ${{ github.ref == 'refs/heads/main' }} run: | - tag=${{ steps.version_check.outputs.tag }} - git tag $tag ${{ steps.commit.outputs.commit }} + tag=${{ steps.project_toml.outputs.tag }} + git tag $tag ${{ steps.commit.outputs.sha }} git push origin $tag - name: Publish Pre-Release uses: softprops/action-gh-release@v1 if: ${{ github.ref == 'refs/heads/main' }} with: prerelease: true - tag_name: ${{ steps.version_check.outputs.tag }} - target_commitish: ${{ steps.commit.outputs.commit }} + tag_name: ${{ steps.project_toml.outputs.tag }} + target_commitish: ${{ steps.commit.outputs.sha }} generate_release_notes: true files: ${{ env.tarballs_dir }}/*.tar.gz From 9f42ae9cfa43118c1d15bab5464b635fbdb28832 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 19 Oct 2023 14:40:05 +0100 Subject: [PATCH 70/90] REVERT: test making a tag/release --- .github/workflows/CI.yml | 4 ++-- Project.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ae106182..f6553e4d 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -254,14 +254,14 @@ jobs: git commit -m "$msg" -- Artifacts.toml echo "sha=$(git rev-parse HEAD)" | tee -a "$GITHUB_OUTPUT" - name: Tag and push - if: ${{ github.ref == 'refs/heads/main' }} + # if: ${{ github.ref == 'refs/heads/main' }} run: | tag=${{ steps.project_toml.outputs.tag }} git tag $tag ${{ steps.commit.outputs.sha }} git push origin $tag - name: Publish Pre-Release uses: softprops/action-gh-release@v1 - if: ${{ github.ref == 'refs/heads/main' }} + # if: ${{ github.ref == 'refs/heads/main' }} with: prerelease: true tag_name: ${{ steps.project_toml.outputs.tag }} diff --git a/Project.toml b/Project.toml index f5016854..355c3562 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Ray" uuid = "3f779ece-f0b6-4c4f-a81a-0cb2add9eb95" authors = ["Beacon Biosignals, Inc"] -version = "0.0.1" +version = "0.0.2" [deps] ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63" From ba60edafec9bc5e20c591ee67f73760687b4be06 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 19 Oct 2023 17:00:21 +0100 Subject: [PATCH 71/90] Use RELEASE_KEY when creating a tag --- .github/workflows/CI.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f6553e4d..95acd730 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -254,6 +254,8 @@ jobs: git commit -m "$msg" -- Artifacts.toml echo "sha=$(git rev-parse HEAD)" | tee -a "$GITHUB_OUTPUT" - name: Tag and push + env: + GITHUB_TOKEN: ${{ secrets.RELEASE_KEY }} # needed so that artifacts_CI runs after tag created # if: ${{ github.ref == 'refs/heads/main' }} run: | tag=${{ steps.project_toml.outputs.tag }} From 1fb3f88846dd6ca13c93d24bef3c1b26a0205ed5 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 19 Oct 2023 17:42:13 +0100 Subject: [PATCH 72/90] Put RELEASE_KEY at head of workflow --- .github/workflows/CI.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 95acd730..2cba304f 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -24,6 +24,7 @@ env: build_dir: ./build ray_dir: ./build/ray tarballs_dir: ./build/tarballs + GITHUB_TOKEN: ${{ secrets.RELEASE_KEY }} # needed so that artifacts_CI runs after tag created jobs: test: name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} @@ -254,8 +255,6 @@ jobs: git commit -m "$msg" -- Artifacts.toml echo "sha=$(git rev-parse HEAD)" | tee -a "$GITHUB_OUTPUT" - name: Tag and push - env: - GITHUB_TOKEN: ${{ secrets.RELEASE_KEY }} # needed so that artifacts_CI runs after tag created # if: ${{ github.ref == 'refs/heads/main' }} run: | tag=${{ steps.project_toml.outputs.tag }} From f61dc03bcc77257b2bf7bc215ab31c6d45320661 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 19 Oct 2023 18:19:23 +0100 Subject: [PATCH 73/90] delete RELEASE_KEY --- .github/workflows/CI.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 2cba304f..f6553e4d 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -24,7 +24,6 @@ env: build_dir: ./build ray_dir: ./build/ray tarballs_dir: ./build/tarballs - GITHUB_TOKEN: ${{ secrets.RELEASE_KEY }} # needed so that artifacts_CI runs after tag created jobs: test: name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} From ec7d91b69d98b33390ac89d5f3432da5583676ed Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 19 Oct 2023 18:42:00 +0100 Subject: [PATCH 74/90] Split out manual Release job --- .github/workflows/CI.yml | 81 ------------------------------- .github/workflows/Release.yml | 91 +++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 81 deletions(-) create mode 100644 .github/workflows/Release.yml diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f6553e4d..8ae85ad9 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -187,84 +187,3 @@ jobs: with: key: ${{ steps.build-cache.outputs.cache-primary-key }} path: ~/.cache - - name: Get Latest Tag - id: latest_tag - uses: WyriHaximus/github-action-get-previous-tag@v1 - - name: Check for updated Project version - id: project_toml - shell: julia --color=yes --project {0} - env: - LATEST_TAG: ${{ steps.latest_tag.outputs.tag }} - run: | - using Pkg.Types - project = read_project("Project.toml") - latest_tag = parse(VersionNumber, ENV["LATEST_TAG"]) - @info "Project version: project.version" - @info "Latest tag: latest_tag" - make_release = project.version > latest_tag - open(ENV["GITHUB_OUTPUT"], "a") do io - println(io, "make_release=$make_release") - end - - release: - name: "Publish Pre-Release" - runs-on: ubuntu-latest - needs: [test] - if: ${{ needs.test.outputs.make_release == 'true' }} - steps: - - uses: actions/checkout@v4 - - name: Retrieve ray_julia libraries - uses: actions/download-artifact@v3 - with: - name: ray_julia_libraries - path: ${{ env.tarballs_dir }} - - uses: julia-actions/setup-julia@v1 - with: - version: "1.8" - arch: "x64" - - uses: julia-actions/cache@v1 - - name: Check Project version - id: project_toml - shell: julia --color=yes --project {0} - run: | - using Pkg.Types - project = read_project("Project.toml") - open(ENV["GITHUB_OUTPUT"], "a") do io - println(io, "version=$(project.version)") - println(io, "tag=v$(project.version)") - end - - name: Update Artifacts.toml - shell: julia --color=yes --project {0} - run: | - using Pkg - Pkg.instantiate() - include(joinpath(pwd(), "bind_artifacts.jl")) - bind_artifacts() - working-directory: ${{ env.build_dir }} - - name: Commit Changes - id: commit - run: | - # Alternatively environment variables can be used but both author/committer need to be set - # https://git-scm.com/book/en/v2/Git-Internals-Environment-Variables#_committing - git config user.name beacon-buddy - git config user.email beacon-buddy@beacon.bio - - msg="Update Artifacts.toml to use ${{ steps.project_toml.outputs.version }} assets" - git checkout --detach - git commit -m "$msg" -- Artifacts.toml - echo "sha=$(git rev-parse HEAD)" | tee -a "$GITHUB_OUTPUT" - - name: Tag and push - # if: ${{ github.ref == 'refs/heads/main' }} - run: | - tag=${{ steps.project_toml.outputs.tag }} - git tag $tag ${{ steps.commit.outputs.sha }} - git push origin $tag - - name: Publish Pre-Release - uses: softprops/action-gh-release@v1 - # if: ${{ github.ref == 'refs/heads/main' }} - with: - prerelease: true - tag_name: ${{ steps.project_toml.outputs.tag }} - target_commitish: ${{ steps.commit.outputs.sha }} - generate_release_notes: true - files: ${{ env.tarballs_dir }}/*.tar.gz diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml new file mode 100644 index 00000000..dff8c5fc --- /dev/null +++ b/.github/workflows/Release.yml @@ -0,0 +1,91 @@ +name: "Publish Release" +on: + workflow_dispatch: + inputs: + branch: + description: 'Name of the branch to create the tag from' + required: true + default: 'main' +env: + tarballs_dir: ./build/tarballs +jobs: + release: + name: "Publish Pre-Release" + runs-on: ubuntu-latest + steps: + - name: Get Latest Tag + id: latest_tag + uses: WyriHaximus/github-action-get-previous-tag@v1 + # TODO: in future we might need to perform backports, at which point we'll need to check + # that the project.version has not already been tagged + - name: Check for updated Project version + id: project_toml + shell: julia --color=yes --project {0} + env: + LATEST_TAG: ${{ steps.latest_tag.outputs.tag }} + run: | + using Pkg.Types + project = read_project("Project.toml") + latest_tag = parse(VersionNumber, ENV["LATEST_TAG"]) + @info "Project version: project.version" + @info "Latest tag: latest_tag" + make_release = project.version > latest_tag + open(ENV["GITHUB_OUTPUT"], "a") do io + println(io, "make_release=$make_release") + end + - uses: actions/checkout@v4 + - name: Download artifacts + id: download-artifact + uses: dawidd6/action-download-artifact@v2 + with: + branch: $BRANCH + name: ray_julia_libraries + path: ${{ env.tarballs_dir }} + - uses: julia-actions/setup-julia@v1 + with: + version: "1.8" + arch: "x64" + - uses: julia-actions/cache@v1 + - name: Check Project version + id: project_toml + shell: julia --color=yes --project {0} + run: | + using Pkg.Types + project = read_project("Project.toml") + open(ENV["GITHUB_OUTPUT"], "a") do io + println(io, "version=$(project.version)") + println(io, "tag=v$(project.version)") + end + - name: Update Artifacts.toml + shell: julia --color=yes --project {0} + run: | + using Pkg + Pkg.instantiate() + include(joinpath(pwd(), "bind_artifacts.jl")) + bind_artifacts() + working-directory: ${{ env.build_dir }} + - name: Commit Changes + id: commit + run: | + # Alternatively environment variables can be used but both author/committer need to be set + # https://git-scm.com/book/en/v2/Git-Internals-Environment-Variables#_committing + git config user.name beacon-buddy + git config user.email beacon-buddy@beacon.bio + + msg="Update Artifacts.toml to use ${{ steps.project_toml.outputs.version }} assets" + git checkout --detach + git commit -m "$msg" -- Artifacts.toml + echo "sha=$(git rev-parse HEAD)" | tee -a "$GITHUB_OUTPUT" + - name: Tag and push + run: | + tag=${{ steps.project_toml.outputs.tag }} + git tag $tag ${{ steps.commit.outputs.sha }} + git push origin $tag + - name: Publish Pre-Release + uses: softprops/action-gh-release@v1 + with: + prerelease: true + tag_name: ${{ steps.project_toml.outputs.tag }} + target_commitish: ${{ steps.commit.outputs.sha }} + generate_release_notes: true + files: ${{ env.tarballs_dir }}/*.tar.gz From 2b953d8f7bfc29850a592171ac265af6ee6cef43 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 19 Oct 2023 18:44:47 +0100 Subject: [PATCH 75/90] error if we can't make a release --- .github/workflows/Release.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml index dff8c5fc..82c8b6c2 100644 --- a/.github/workflows/Release.yml +++ b/.github/workflows/Release.yml @@ -29,10 +29,7 @@ jobs: latest_tag = parse(VersionNumber, ENV["LATEST_TAG"]) @info "Project version: project.version" @info "Latest tag: latest_tag" - make_release = project.version > latest_tag - open(ENV["GITHUB_OUTPUT"], "a") do io - println(io, "make_release=$make_release") - end + project.version > latest_tag || exit(1) # error if can't make a tag - uses: actions/checkout@v4 - name: Download artifacts id: download-artifact From 3a3cad939294e974b5cedfcc6b2ccdcf1405c61c Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 19 Oct 2023 18:51:50 +0100 Subject: [PATCH 76/90] checkout branch in Release.yml --- .github/workflows/CI.yml | 2 -- .github/workflows/Release.yml | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 8ae85ad9..f90c1fb7 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -49,8 +49,6 @@ jobs: arch: x64 steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Needed determine latest tag - name: Determine Ray commit id: ray-commit run: | diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml index 82c8b6c2..eaf2e5eb 100644 --- a/.github/workflows/Release.yml +++ b/.github/workflows/Release.yml @@ -13,6 +13,9 @@ jobs: name: "Publish Pre-Release" runs-on: ubuntu-latest steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Needed determine latest tag - name: Get Latest Tag id: latest_tag uses: WyriHaximus/github-action-get-previous-tag@v1 From 6cb05d23d36c18565ecd52fb5dbdbfb5b50383b8 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 19 Oct 2023 18:52:02 +0100 Subject: [PATCH 77/90] REVERT: test on push --- .github/workflows/Release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml index eaf2e5eb..6821528b 100644 --- a/.github/workflows/Release.yml +++ b/.github/workflows/Release.yml @@ -6,6 +6,7 @@ on: description: 'Name of the branch to create the tag from' required: true default: 'main' + push: env: tarballs_dir: ./build/tarballs jobs: From 50ed57d02a30410480c1613b897bf92ccbec1209 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 19 Oct 2023 18:53:30 +0100 Subject: [PATCH 78/90] Fix Release.yml --- .github/workflows/Release.yml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml index 6821528b..bd715202 100644 --- a/.github/workflows/Release.yml +++ b/.github/workflows/Release.yml @@ -34,6 +34,10 @@ jobs: @info "Project version: project.version" @info "Latest tag: latest_tag" project.version > latest_tag || exit(1) # error if can't make a tag + open(ENV["GITHUB_OUTPUT"], "a") do io + println(io, "version=$(project.version)") + println(io, "tag=v$(project.version)") + end - uses: actions/checkout@v4 - name: Download artifacts id: download-artifact @@ -47,16 +51,6 @@ jobs: version: "1.8" arch: "x64" - uses: julia-actions/cache@v1 - - name: Check Project version - id: project_toml - shell: julia --color=yes --project {0} - run: | - using Pkg.Types - project = read_project("Project.toml") - open(ENV["GITHUB_OUTPUT"], "a") do io - println(io, "version=$(project.version)") - println(io, "tag=v$(project.version)") - end - name: Update Artifacts.toml shell: julia --color=yes --project {0} run: | From 3c6371818738ff89e162b8e5508a315b40fdf8ca Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 19 Oct 2023 18:54:35 +0100 Subject: [PATCH 79/90] Set branch to main --- .github/workflows/Release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml index bd715202..77bd381f 100644 --- a/.github/workflows/Release.yml +++ b/.github/workflows/Release.yml @@ -43,7 +43,7 @@ jobs: id: download-artifact uses: dawidd6/action-download-artifact@v2 with: - branch: $BRANCH + branch: main name: ray_julia_libraries path: ${{ env.tarballs_dir }} - uses: julia-actions/setup-julia@v1 From a0888de3930a3336e9bff9abfb340e354bbcb983 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 19 Oct 2023 18:57:57 +0100 Subject: [PATCH 80/90] get artifacts from CI --- .github/workflows/Release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml index 77bd381f..08279145 100644 --- a/.github/workflows/Release.yml +++ b/.github/workflows/Release.yml @@ -46,6 +46,7 @@ jobs: branch: main name: ray_julia_libraries path: ${{ env.tarballs_dir }} + workflow: CI.yml - uses: julia-actions/setup-julia@v1 with: version: "1.8" From e4fd0c69782e82324bd5d7161ae9929ff411d1f8 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 19 Oct 2023 18:59:38 +0100 Subject: [PATCH 81/90] set tag to gm/tag_gha --- .github/workflows/Release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml index 08279145..2ceb94e6 100644 --- a/.github/workflows/Release.yml +++ b/.github/workflows/Release.yml @@ -43,7 +43,7 @@ jobs: id: download-artifact uses: dawidd6/action-download-artifact@v2 with: - branch: main + branch: gm/tag_gha name: ray_julia_libraries path: ${{ env.tarballs_dir }} workflow: CI.yml From 0c922be69bd73abdb8365ec97c122202a012356b Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 19 Oct 2023 19:01:11 +0100 Subject: [PATCH 82/90] specify build_dir --- .github/workflows/Release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml index 2ceb94e6..d8058d7d 100644 --- a/.github/workflows/Release.yml +++ b/.github/workflows/Release.yml @@ -8,6 +8,7 @@ on: default: 'main' push: env: + build_dir: ./build tarballs_dir: ./build/tarballs jobs: release: From a92e41ce6f61639f65a066e9f6a173d4f981903c Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 19 Oct 2023 19:07:38 +0100 Subject: [PATCH 83/90] only run Release.yml manually for main branch --- .github/workflows/Release.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml index d8058d7d..9f182e9c 100644 --- a/.github/workflows/Release.yml +++ b/.github/workflows/Release.yml @@ -1,12 +1,6 @@ name: "Publish Release" on: workflow_dispatch: - inputs: - branch: - description: 'Name of the branch to create the tag from' - required: true - default: 'main' - push: env: build_dir: ./build tarballs_dir: ./build/tarballs @@ -44,7 +38,7 @@ jobs: id: download-artifact uses: dawidd6/action-download-artifact@v2 with: - branch: gm/tag_gha + branch: main name: ray_julia_libraries path: ${{ env.tarballs_dir }} workflow: CI.yml From 758fb3e3720147b96714a5a329028c2023ac3b1d Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 19 Oct 2023 19:23:27 +0100 Subject: [PATCH 84/90] remove make_release output --- .github/workflows/CI.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f90c1fb7..9cd2a8f7 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -28,8 +28,6 @@ jobs: test: name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} runs-on: ${{ matrix.os }} - outputs: - make_release: ${{ steps.project_toml.outputs.make_release }} strategy: fail-fast: false matrix: From a84bdd0bf9bdd23781db3dbae6b39f5449e833e3 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 19 Oct 2023 19:31:14 +0100 Subject: [PATCH 85/90] update docs --- docs/src/building-artifacts.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/src/building-artifacts.md b/docs/src/building-artifacts.md index d7cda1d2..4fcf37ce 100644 --- a/docs/src/building-artifacts.md +++ b/docs/src/building-artifacts.md @@ -14,6 +14,11 @@ However, the artifacts are only associated with tagged releases of Ray.jl. If you are working off the `main` branch, or developing Ray.jl locally, you will need to [build Ray.jl](./developer-guide.md#build-rayjl) yourself. This will also update your `Overrides.toml` to reference the binaries you have built. -To update the Ray.jl artifacts for a new release simply open a PR which bumps the Ray.jl version in the `Project.toml` and includes any other required changes. -The Julia CI jobs created by this PR will build the artifacts required to make a new release. -Upon merging the PR with `main` a new tag will be created and the artifacts will be uploaded to a GitHub Pre-Release, which will be converted as a full Release once the CI for the tag passes. +To update the Ray.jl artifacts for a new release simply open a PR which bumps the Ray.jl version in the `Project.toml` and includes any other required changes. +Once the PR has been merged into `main` and the CI has passed you must trigger the ["Publish Release" GitHub Action](https://github.com/beacon-biosignals/Ray.jl/actions/workflows/Release.yml) and the following actions will be run in the workflow: +1. Download the various artifacts for supported platforms from `main`. +2. Generate the `Artifacts.toml` bindings and commit them. +3. Create a new tag and GitHub pre-release which includes the artifacts as assets. + +The creation of the tag will trigger the "Artifacts CI" workflow which is responsible for verifying the generated artifacts work. +It will then promote the GitHub pre-release to a full release once those checks pass. From c524bf7564e3848cc070cbbd1b84d35e5fb02131 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 19 Oct 2023 20:06:57 +0100 Subject: [PATCH 86/90] pull artifacts for latest commit on main --- .github/workflows/Release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml index 9f182e9c..36883635 100644 --- a/.github/workflows/Release.yml +++ b/.github/workflows/Release.yml @@ -38,7 +38,7 @@ jobs: id: download-artifact uses: dawidd6/action-download-artifact@v2 with: - branch: main + commit: ${{ github.sha }} # pull the artifacts for the last commit on main name: ray_julia_libraries path: ${{ env.tarballs_dir }} workflow: CI.yml From c0a661a18199f8d647668f6899d801cc52720980 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 19 Oct 2023 20:19:36 +0100 Subject: [PATCH 87/90] check main explicitly --- .github/workflows/Release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml index 36883635..133fc137 100644 --- a/.github/workflows/Release.yml +++ b/.github/workflows/Release.yml @@ -12,6 +12,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 # Needed determine latest tag + ref: main - name: Get Latest Tag id: latest_tag uses: WyriHaximus/github-action-get-previous-tag@v1 From c374f9dbf4bcd1620e4efb9197c1f144433b4478 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 19 Oct 2023 20:26:26 +0100 Subject: [PATCH 88/90] Apply suggestions from code review Co-authored-by: Curtis Vogt --- .github/workflows/Release.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml index 133fc137..07087041 100644 --- a/.github/workflows/Release.yml +++ b/.github/workflows/Release.yml @@ -1,6 +1,10 @@ name: "Publish Release" on: workflow_dispatch: +# Force a single workflow to run at a time. This ensures we don't accidentally +# try to perform concurrent publishing of releases. +concurrency: publish-release + env: build_dir: ./build tarballs_dir: ./build/tarballs @@ -9,9 +13,10 @@ jobs: name: "Publish Pre-Release" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 # Needed determine latest tag + persist-credentials: true ref: main - name: Get Latest Tag id: latest_tag @@ -34,7 +39,6 @@ jobs: println(io, "version=$(project.version)") println(io, "tag=v$(project.version)") end - - uses: actions/checkout@v4 - name: Download artifacts id: download-artifact uses: dawidd6/action-download-artifact@v2 @@ -48,6 +52,8 @@ jobs: version: "1.8" arch: "x64" - uses: julia-actions/cache@v1 + with: + cache-name: "${{ github.workflow }}-${{ github.job }}" - name: Update Artifacts.toml shell: julia --color=yes --project {0} run: | From 1683edc7209cf46e9b183d4fe3743cbf26c20eb7 Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 19 Oct 2023 20:28:17 +0100 Subject: [PATCH 89/90] rename Release -> Pre_release --- .github/workflows/{Release.yml => Pre_release.yml} | 2 +- docs/src/building-artifacts.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename .github/workflows/{Release.yml => Pre_release.yml} (99%) diff --git a/.github/workflows/Release.yml b/.github/workflows/Pre_release.yml similarity index 99% rename from .github/workflows/Release.yml rename to .github/workflows/Pre_release.yml index 07087041..2d209029 100644 --- a/.github/workflows/Release.yml +++ b/.github/workflows/Pre_release.yml @@ -1,4 +1,4 @@ -name: "Publish Release" +name: "Publish Pre-Release" on: workflow_dispatch: # Force a single workflow to run at a time. This ensures we don't accidentally diff --git a/docs/src/building-artifacts.md b/docs/src/building-artifacts.md index 4fcf37ce..a44ea950 100644 --- a/docs/src/building-artifacts.md +++ b/docs/src/building-artifacts.md @@ -15,7 +15,7 @@ If you are working off the `main` branch, or developing Ray.jl locally, you will This will also update your `Overrides.toml` to reference the binaries you have built. To update the Ray.jl artifacts for a new release simply open a PR which bumps the Ray.jl version in the `Project.toml` and includes any other required changes. -Once the PR has been merged into `main` and the CI has passed you must trigger the ["Publish Release" GitHub Action](https://github.com/beacon-biosignals/Ray.jl/actions/workflows/Release.yml) and the following actions will be run in the workflow: +Once the PR has been merged into `main` and the CI has passed you must trigger the ["Publish Release" GitHub Action](https://github.com/beacon-biosignals/Ray.jl/actions/workflows/Pre_release.yml) and the following actions will be run in the workflow: 1. Download the various artifacts for supported platforms from `main`. 2. Generate the `Artifacts.toml` bindings and commit them. 3. Create a new tag and GitHub pre-release which includes the artifacts as assets. From 1eb72292d0b6acfc3883df029094af4ac8629a0d Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Thu, 19 Oct 2023 20:29:33 +0100 Subject: [PATCH 90/90] rename Release -> Pre_release --- docs/src/building-artifacts.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/building-artifacts.md b/docs/src/building-artifacts.md index a44ea950..4e3916b0 100644 --- a/docs/src/building-artifacts.md +++ b/docs/src/building-artifacts.md @@ -15,7 +15,7 @@ If you are working off the `main` branch, or developing Ray.jl locally, you will This will also update your `Overrides.toml` to reference the binaries you have built. To update the Ray.jl artifacts for a new release simply open a PR which bumps the Ray.jl version in the `Project.toml` and includes any other required changes. -Once the PR has been merged into `main` and the CI has passed you must trigger the ["Publish Release" GitHub Action](https://github.com/beacon-biosignals/Ray.jl/actions/workflows/Pre_release.yml) and the following actions will be run in the workflow: +Once the PR has been merged into `main` and the CI has passed you must trigger the ["Publish Pre-Release" GitHub Action](https://github.com/beacon-biosignals/Ray.jl/actions/workflows/Pre_release.yml) and the following actions will be run in the workflow: 1. Download the various artifacts for supported platforms from `main`. 2. Generate the `Artifacts.toml` bindings and commit them. 3. Create a new tag and GitHub pre-release which includes the artifacts as assets.