GHA workflow to automate creation of GH releases & tags

.github/workflows/CI.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches:
       - main
-    tags: ["*"]  # Triggers from tags ignore the `paths` filter: https://github.com/orgs/community/discussions/26273
     paths:
       - "src/**"
       - "test/**"
@@ -21,6 +20,10 @@ concurrency:
   # Cancel intermediate builds: only if it is a pull request build.
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+env:
+  build_dir: ./build
+  ray_dir: ./build/ray
+  tarballs_dir: ./build/tarballs
 jobs:
   test:
     name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
@@ -42,9 +45,6 @@ jobs:
             arch: aarch64
           - os: macos-latest-xlarge
             arch: x64
-    env:
-      build_dir: ./build
-      ray_dir: ./build/ray
     steps:
       - uses: actions/checkout@v3
       - name: Determine Ray commit
@@ -148,7 +148,6 @@ jobs:
           tarball_path = build_host_tarball()
           open(ENV["GITHUB_OUTPUT"], "a") do io
               println(io, "tarball_path=$tarball_path")
-              println(io, "tarball_filename=$(basename(tarball_path))")
           end
         working-directory: ${{ env.build_dir }}
       - uses: julia-actions/julia-buildpkg@v1
@@ -162,10 +161,12 @@ jobs:
           file: lcov.info
           flags: Ray.jl
 
+      # Add the ray_julia library from each job to the same artifact
+      # https://github.com/actions/upload-artifact#uploading-to-the-same-artifact
       - name: Save ray_julia library
         uses: actions/upload-artifact@v3
         with:
-          name: ${{ steps.ray_julia.outputs.tarball_filename }}
+          name: ray_julia_libraries
           path: ${{ steps.ray_julia.outputs.tarball_path }}
 
       - name: Save Ray logs

.github/workflows/Documenter.yml

@@ -17,6 +17,8 @@ concurrency:
   # Cancel intermediate builds: only if it is a pull request build.
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+env:
+  build_dir: ./build
 jobs:
   docs:
     name: Build
@@ -28,7 +30,22 @@ jobs:
       - uses: julia-actions/setup-julia@v1
         with:
           version: "1"
+      - name: Restore build cache
+        uses: actions/cache/restore@v3
+        id: build-cache
+        with:
+          key: build-cache.ray-jl.documenter
+          path: ~/.cache
       # Use a separate step for instantiation to get more fine grained timing of steps
+      - name: Build ray_julia library
+        id: ray_julia
+        shell: julia --color=yes --project {0}
+        run: |
+          using Pkg
+          Pkg.instantiate()
+          include(joinpath(pwd(), "build_library.jl"))
+          build_library(; override=true)
+        working-directory: ${{ env.build_dir }}
       - name: Configure doc environment
         shell: julia --project=docs --color=yes {0}
         run: |
@@ -52,3 +69,11 @@ jobs:
         env:
           repo: ${{ github.repository }}
           PR: ${{ github.event.number }}
+
+      # https://github.com/actions/cache/tree/main/save#always-save-cache
+      - name: Save build cache
+        uses: actions/cache/save@v3
+        if: always() && steps.build-cache.outputs.cache-hit != 'true'
+        with:
+          key: ${{ steps.build-cache.outputs.cache-primary-key }}
+          path: ~/.cache

.github/workflows/Release.yml

@@ -0,0 +1,89 @@
+name: "Publish Release"
+on:
+  workflow_dispatch:
+# Force a single workflow to run at a time. This ensures we don't accidentally
+# try to perform concurrent publishing of releases.
+concurrency: publish-release
+
+env:
+  build_dir: ./build
+  tarballs_dir: ./build/tarballs
+jobs:
+  release:
+    name: "Publish Pre-Release"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Needed determine latest tag
+          persist-credentials: true
+          ref: main
+      - name: Get Latest Tag
+        id: latest_tag
+        uses: WyriHaximus/github-action-get-previous-tag@v1
+        # TODO: in future we might need to perform backports, at which point we'll need to check
+        # that the project.version has not already been tagged 
+      - name: Check for updated Project version
+        id: project_toml
+        shell: julia --color=yes --project {0}
+        env:
+          LATEST_TAG: ${{ steps.latest_tag.outputs.tag }}
+        run: |
+          using Pkg.Types
+          project = read_project("Project.toml")
+          latest_tag = parse(VersionNumber, ENV["LATEST_TAG"])
+          @info "Project version: project.version"
+          @info "Latest tag: latest_tag"
+          project.version > latest_tag || exit(1)  # error if can't make a tag
+          open(ENV["GITHUB_OUTPUT"], "a") do io
+              println(io, "version=$(project.version)")
+              println(io, "tag=v$(project.version)")
+          end
+      - name: Download artifacts
+        id: download-artifact
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          commit: ${{ github.sha }}  # pull the artifacts for the last commit on main
+          name: ray_julia_libraries
+          path: ${{ env.tarballs_dir }}
+          workflow: CI.yml
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: "1.8"
+          arch: "x64"
+      - uses: julia-actions/cache@v1
+        with:
+          cache-name: "${{ github.workflow }}-${{ github.job }}"
+      - name: Update Artifacts.toml
+        shell: julia --color=yes --project {0}
+        run: |
+          using Pkg
+          Pkg.instantiate()
+          include(joinpath(pwd(), "bind_artifacts.jl"))
+          bind_artifacts()
+        working-directory: ${{ env.build_dir }}
+      - name: Commit Changes
+        id: commit
+        run: |
+          # Alternatively environment variables can be used but both author/committer need to be set
+          # https://git-scm.com/book/en/v2/Git-Internals-Environment-Variables#_committing
+          git config user.name beacon-buddy
+          git config user.email beacon-buddy@beacon.bio
+
+          msg="Update Artifacts.toml to use ${{ steps.project_toml.outputs.version }} assets"
+          git checkout --detach
+          git commit -m "$msg" -- Artifacts.toml
+          echo "sha=$(git rev-parse HEAD)" | tee -a "$GITHUB_OUTPUT"
+      - name: Tag and push
+        run: |
+          tag=${{ steps.project_toml.outputs.tag }}
+          git tag $tag ${{ steps.commit.outputs.sha }}
+          git push origin $tag
+      - name: Publish Pre-Release
+        uses: softprops/action-gh-release@v1
+        with:
+          prerelease: true
+          tag_name: ${{ steps.project_toml.outputs.tag }}
+          target_commitish: ${{ steps.commit.outputs.sha }}
+          generate_release_notes: true
+          files: ${{ env.tarballs_dir }}/*.tar.gz

.github/workflows/artifacts_CI.yml

@@ -2,16 +2,7 @@ name: Artifacts CI
 on:
   workflow_dispatch:
   push:
-    branches:
-      - main
     tags: ["*"]  # Triggers from tags ignore the `paths` filter: https://github.com/orgs/community/discussions/26273
-    paths:
-      - "Artifacts.toml"
-      - ".github/workflows/artifacts_CI.yml"
-  pull_request:
-    paths:
-      - "Artifacts.toml"
-      - ".github/workflows/artifacts_CI.yml"
 concurrency:
   # Skip intermediate builds: always.
   # Cancel intermediate builds: only if it is a pull request build.
@@ -70,3 +61,14 @@ jobs:
         - uses: julia-actions/julia-runtest@v1
           with:
             annotate: true
+  release:
+    name: "Publish Release"
+    runs-on: ubuntu-latest
+    needs: [test]
+    steps:
+      - name: Publish Release
+        uses: softprops/action-gh-release@v1
+        with:
+          # Any info keys (i.e. prerelease) that are not explicitly set will retain the original
+          # value if there is already an existing release for the tag.
+          prerelease: false

Artifacts.toml

@@ -1,38 +1,2 @@
-[[ray_julia]]
-arch = "x86_64"
-git-tree-sha1 = "88eb04f4e4a87d0a369288ceb6fb605f4ca4ff03"
-julia_version = "1.8.0"
-libc = "glibc"
-os = "linux"
-
-    [[ray_julia.download]]
-    sha256 = "03bc890efc63375f901325d8f67a2f3403c7d1067658ed693390276ab37f6958"
-    url = "https://github.com/beacon-biosignals/Ray.jl/releases/download/v0.0.1/ray_julia.v0.0.1.x86_64-linux-gnu-julia_version+1.8.0.tar.gz"
-[[ray_julia]]
-arch = "aarch64"
-git-tree-sha1 = "a24e4bb69ffa0baf615e27f3ae6d00084556881a"
-julia_version = "1.8.0"
-os = "macos"
-
-    [[ray_julia.download]]
-    sha256 = "2d5ee4e6290e1207250a02e434d588af3321d7f978fe7d57e13d4827ca7aa51b"
-    url = "https://github.com/beacon-biosignals/Ray.jl/releases/download/v0.0.1/ray_julia.v0.0.1.aarch64-apple-darwin-julia_version+1.8.0.tar.gz"
-[[ray_julia]]
-arch = "x86_64"
-git-tree-sha1 = "32c3aaecc5a1e2aed7e9ce6d9a95bef741d1ef9d"
-julia_version = "1.9.0"
-libc = "glibc"
-os = "linux"
-
-    [[ray_julia.download]]
-    sha256 = "1f96eaf5f35796f60b04172728caa29fc442ffcc30e8ffabfe147791dd711fc8"
-    url = "https://github.com/beacon-biosignals/Ray.jl/releases/download/v0.0.1/ray_julia.v0.0.1.x86_64-linux-gnu-julia_version+1.9.0.tar.gz"
-[[ray_julia]]
-arch = "aarch64"
-git-tree-sha1 = "81cc7a633a617a01f1fc87674258d880baa4cafb"
-julia_version = "1.9.0"
-os = "macos"
-
-    [[ray_julia.download]]
-    sha256 = "5969006375abafacf2fff0a1d499b492d1a883d1529e2a20fe69d3a4908f2729"
-    url = "https://github.com/beacon-biosignals/Ray.jl/releases/download/v0.0.1/ray_julia.v0.0.1.aarch64-apple-darwin-julia_version+1.9.0.tar.gz"
+[ray_julia]
+git-tree-sha1 = "0000000000000000000000000000000000000000"

Project.toml

@@ -1,7 +1,7 @@
 name = "Ray"
 uuid = "3f779ece-f0b6-4c4f-a81a-0cb2add9eb95"
 authors = ["Beacon Biosignals, Inc"]
-version = "0.0.1"
+version = "0.0.2"
 
 [deps]
 ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63"

build/Project.toml

@@ -9,8 +9,6 @@ Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
 URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
-ZipFile = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea"
-ghr_jll = "07c12ed4-43bc-5495-8a2a-d5838ef8d533"
 
 [compat]
 CxxWrap = "0.13.4"

build/bind_artifacts.jl

@@ -23,7 +23,7 @@ function sha256sum(tarball_path)
     end
 end
 
-if abspath(PROGRAM_FILE) == @__FILE__
+function bind_artifacts()
     # Start with a clean Artifacts.toml so that unsupported platforms are removed
     isfile(ARTIFACTS_TOML) && rm(ARTIFACTS_TOML)
 
@@ -32,9 +32,8 @@ if abspath(PROGRAM_FILE) == @__FILE__
         artifact_url = gen_artifact_url(; repo_url=REPO_HTTPS_URL, tag=TAG,
                                         filename=artifact_name)
 
-        @info "Dowloading artifact $artifact_url"
         artifact_path = joinpath(TARBALL_DIR, artifact_name)
-        Downloads.download(artifact_url, artifact_path)
+        isfile(artifact_path) || error("No such file $artifact_path")
 
         @info "Adding artifact for $(triplet(platform))"
         bind_artifact!(ARTIFACTS_TOML,
@@ -44,3 +43,7 @@ if abspath(PROGRAM_FILE) == @__FILE__
                        download_info=[(artifact_url, sha256sum(artifact_path))])
     end
 end
+
+if abspath(PROGRAM_FILE) == @__FILE__
+    bind_artifacts()
+end

build/build_library.jl

@@ -46,6 +46,7 @@ function build_library(; override::Bool=true)
 
     # Add entry to depot Overrides.toml
     if override
+        @info "Adding entry to Overrides.toml..."
         pkg_uuid = TOML.parsefile(PROJECT_TOML)["uuid"]
 
         overrides_toml = joinpath(first(DEPOT_PATH), "artifacts", "Overrides.toml")

build/common.jl

@@ -6,19 +6,6 @@ using URIs: URI
 const TARBALL_DIR = joinpath(@__DIR__, "tarballs")
 const SO_FILE = "julia_core_worker_lib.so"
 
-const TARBALL_REGEX = r"""
-    ^ray_julia\.v(?<jll_version>[0-9]+(\.[0-9]+){2})\.
-    (?<triplet>[a-z0-9_-]+)-
-    julia_version\+(?<julia_version>[0-9]+(\.[0-9]+){2})\.
-    tar\.gz$
-    """x
-
-const GH_RELEASE_ASSET_PATH_REGEX = r"""
-    ^/(?<owner>[^/]+)/(?<repo_name>[^/]+)/
-    releases/download/
-    (?<tag>[^/]+)/?$
-    """x
-
 const REQUIRED_BASE_TRIPLETS = ("x86_64-linux-gnu", "aarch64-apple-darwin")
 const REQUIRED_JULIA_VERSIONS = (v"1.8", v"1.9")
 const REQUIRED_PLATFORMS = let
@@ -36,24 +23,6 @@ function remote_url(repo_root::AbstractString, name::AbstractString="origin")
     end
 end
 
-function git_head_sha(repo_root::AbstractString=REPO_PATH)
-    return LibGit2.with(LibGit2.GitRepo(repo_root)) do repo
-        ref = LibGit2.head(repo)
-        commit = LibGit2.peel(LibGit2.GitCommit, ref)
-        return string(LibGit2.GitHash(commit))
-    end
-end
-
-function github_token()
-    return get(ENV, "GITHUB_TOKEN") do
-        s = Base.getpass("GitHub PAT")
-        println()
-        token = read(s, String)
-        Base.shred!(s)
-        return token
-    end
-end
-
 function convert_to_https_url(url)
     m = match(LibGit2.URL_REGEX, url)
     if m === nothing
@@ -83,12 +52,8 @@ end
 
 const REPO_PATH = abspath(joinpath(@__DIR__, ".."))
 const REPO_HTTPS_URL = convert_to_https_url(remote_url(REPO_PATH))
-const REPO_ORG = split(parse(URI, REPO_HTTPS_URL).path, '/')[2]
-const REPO_NAME = split(parse(URI, REPO_HTTPS_URL).path, '/')[3]
 const COMPILED_DIR = joinpath(REPO_PATH, "build", "bazel-bin")
-
 const ARTIFACTS_TOML = joinpath(REPO_PATH, "Artifacts.toml")
-const ARTIFACTS_WORKFLOW_NAME = "CI"
 
 const TAG = let
     project_toml = joinpath(REPO_PATH, "Project.toml")