ci: Reuse docker cache

To allow docker cache reuse across images, we need to build the template workspace first, then build the other workspaces. This can be done by using the `workflow_run` event to trigger the build of the other workspaces. Since `workflow_run` does not natively support path matching, the `paths-filter` action is used to identify changed files and determine whether a workspace build is needed. Unfortunately, the paths-filter action does not function as intended when used with the workflow_run event due to the loss of commit information. To address this limitation, a workaround involving the use of artifacts between workflows is implemented to transfer the missing information. Since transferring artifacts across workflows is not natively supported, the community-provided download-artifact action variant is utilized. To allow the use of remote build cache, a non-default builder is required, which can be enabled by `docker/setup-buildx-action`. We can then use `--cache-from` and `--cache-to` flags to specify the cache. The dedicated cache registry and `max` mode seems to be required instead of the `inline` mode due to the use of multi-platform images and multi-stage builds. I'm not 100% sure if this is required though. References: * dorny/paths-filter * https://github.com/dorny/paths-filter?tab=readme-ov-file#examples * https://stackoverflow.com/a/70711156 * workflow_run * https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#onworkflow_runbranchesbranches-ignore * https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#workflow_run * docker build cache * https://docs.docker.com/build/cache/backends/ * https://docs.docker.com/build/cache/backends/inline/ * https://docs.docker.com/build/cache/backends/registry/ * docker builder * https://docs.docker.com/build/builders/drivers/ * https://yuki-nakamura.com/2024/01/20/use-buildkit-from-docker-compose/ * https://docs.docker.com/reference/cli/docker/buildx/create/ * https://github.com/docker/buildx/blob/master/README.md#set-buildx-as-the-default-builder * docker/buildx#2142 (reply in thread)
j3soon · Dec 13, 2024 · 024f52a · 024f52a
1 parent 0bba42e
commit 024f52a
Show file tree

Hide file tree

Showing 11 changed files with 657 additions and 156 deletions.
diff --git a/.github/workflows/build-aloha-ws.yaml b/.github/workflows/build-aloha-ws.yaml
@@ -1,22 +1,51 @@
 name: Build Docker Image for aloha-ws
 
 on:
-  push:
-    branches:
-      - "main"
-    tags:
-      - v*
-    paths:
-      - .github/workflows/build-aloha-ws.yaml
-      - aloha_ws/docker/Dockerfile
-      - aloha_ws/docker/.dockerignore
-      - aloha_ws/docker/.bashrc
-      - aloha_ws/docker/script/**
-      - aloha_ws/docker/udev_rules/**
+  workflow_run:
+    workflows: ["Build Docker Image for template-ws"]
+    types: [completed]
 
 jobs:
+  paths-filter:
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    runs-on: ubuntu-latest
+    outputs:
+      results: ${{ steps.filter.outputs.results }}
+    steps:
+    # Ref: https://github.com/dorny/paths-filter/issues/147#issuecomment-1287800590
+    - name: Download a single artifact
+      uses: dawidd6/action-download-artifact@v7
+      with:
+        workflow: build-template-ws.yaml
+        name: original-refs
+        workflow_conclusion: success
+    - name: set REF_BASE to env
+      run: |
+        echo "BASE=$(cat base.txt)" >> $GITHUB_ENV
+        echo "CURRENT_BRANCH=$(cat current-branch.txt)" >> $GITHUB_ENV
+    - uses: actions/checkout@v4
+    - uses: dorny/paths-filter@v3
+      id: filter
+      with:
+        base: ${{ env.BASE }}
+        ref: ${{ env.CURRENT_BRANCH }}
+        filters: |
+          results:
+            - .github/workflows/build-aloha-ws.yaml
+            - aloha_ws/docker/Dockerfile
+            - aloha_ws/docker/.dockerignore
+            - aloha_ws/docker/.bashrc
+            - aloha_ws/docker/script/**
+            - aloha_ws/docker/udev_rules/**
+    - name: Changes matched
+      if: steps.filter.outputs.results == 'true'
+      run: echo "Changes matched, will build image"
+    - name: Changes didn't match
+      if: steps.filter.outputs.results != 'true'
+      run: echo "Changes didn't match, will NOT build image"
   docker:
-    if: github.repository == 'j3soon/ros2-essentials'
+    needs: paths-filter
+    if: ${{ needs.paths-filter.outputs.results == 'true' }}
     runs-on: ubuntu-latest
     steps:
       - name: Maximize build space
@@ -31,7 +60,7 @@ jobs:
       - name: Restart docker
         run: sudo service docker restart
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
@@ -48,25 +77,54 @@ jobs:
           # Output the environment variable
           # Ref: https://stackoverflow.com/a/57989070
           echo "VERSION=$VERSION" >> $GITHUB_ENV
+          # Print for debugging purpose
+          echo "VERSION=$VERSION"
       - name: Docker meta
         id: meta
         # Ref: https://github.com/docker/metadata-action
         uses: docker/metadata-action@v5
         with:
           # Link: https://hub.docker.com/repository/docker/j3soon/ros2-aloha-ws/tags
-          images: ${{ secrets.DOCKERHUB_USERNAME }}/ros2-aloha-ws
+          images: j3soon/ros2-aloha-ws
           tags: |
             type=raw,value={{date 'YYYYMMDD'}}
             type=raw,value=${{ env.VERSION }}
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          username: j3soon
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Pull j3soon/ros2-template-ws (amd64)
+        run: docker pull --platform linux/amd64 j3soon/ros2-template-ws
+      - name: Pull j3soon/ros2-template-ws (arm64)
+        run: docker pull --platform linux/arm64 j3soon/ros2-template-ws
       - name: Build and push
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v6
         with:
           context: aloha_ws/docker
+          cache-from: |
+            j3soon/ros2-template-ws:buildcache-amd64
+            j3soon/ros2-template-ws:buildcache-arm64
           platforms: linux/amd64,linux/arm64
           push: true
           tags: ${{ steps.meta.outputs.tags }}
+      - name: Push amd64 cache
+        uses: docker/build-push-action@v6
+        with:
+          context: aloha_ws/docker
+          cache-from: |
+            j3soon/ros2-template-ws:buildcache-amd64
+            j3soon/ros2-template-ws:buildcache-arm64
+          cache-to: type=registry,ref=j3soon/ros2-aloha-ws:buildcache-amd64,mode=max
+          platforms: linux/amd64
+          tags: ${{ steps.meta.outputs.tags }}
+      - name: Push arm64 cache
+        uses: docker/build-push-action@v6
+        with:
+          context: aloha_ws/docker
+          cache-from: |
+            j3soon/ros2-template-ws:buildcache-amd64
+            j3soon/ros2-template-ws:buildcache-arm64
+          cache-to: type=registry,ref=j3soon/ros2-aloha-ws:buildcache-arm64,mode=max
+          platforms: linux/arm64
+          tags: ${{ steps.meta.outputs.tags }}
diff --git a/.github/workflows/build-cartographer-ws.yaml b/.github/workflows/build-cartographer-ws.yaml
@@ -1,20 +1,49 @@
 name: Build Docker Image for cartographer-ws
 
 on:
-  push:
-    branches:
-      - "main"
-    tags:
-      - v*
-    paths:
-      - .github/workflows/build-cartographer-ws.yaml
-      - cartographer_ws/docker/Dockerfile
-      - cartographer_ws/docker/.dockerignore
-      - cartographer_ws/docker/.bashrc
+  workflow_run:
+    workflows: ["Build Docker Image for template-ws"]
+    types: [completed]
 
 jobs:
+  paths-filter:
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    runs-on: ubuntu-latest
+    outputs:
+      results: ${{ steps.filter.outputs.results }}
+    steps:
+    # Ref: https://github.com/dorny/paths-filter/issues/147#issuecomment-1287800590
+    - name: Download a single artifact
+      uses: dawidd6/action-download-artifact@v7
+      with:
+        workflow: build-template-ws.yaml
+        name: original-refs
+        workflow_conclusion: success
+    - name: set REF_BASE to env
+      run: |
+        echo "BASE=$(cat base.txt)" >> $GITHUB_ENV
+        echo "CURRENT_BRANCH=$(cat current-branch.txt)" >> $GITHUB_ENV
+    - uses: actions/checkout@v4
+    - uses: dorny/paths-filter@v3
+      id: filter
+      with:
+        base: ${{ env.BASE }}
+        ref: ${{ env.CURRENT_BRANCH }}
+        filters: |
+          results:
+            - .github/workflows/build-cartographer-ws.yaml
+            - cartographer_ws/docker/Dockerfile
+            - cartographer_ws/docker/.dockerignore
+            - cartographer_ws/docker/.bashrc
+    - name: Changes matched
+      if: steps.filter.outputs.results == 'true'
+      run: echo "Changes matched, will build image"
+    - name: Changes didn't match
+      if: steps.filter.outputs.results != 'true'
+      run: echo "Changes didn't match, will NOT build image"
   docker:
-    if: github.repository == 'j3soon/ros2-essentials'
+    needs: paths-filter
+    if: ${{ needs.paths-filter.outputs.results == 'true' }}
     runs-on: ubuntu-latest
     steps:
       - name: Maximize build space
@@ -29,7 +58,7 @@ jobs:
       - name: Restart docker
         run: sudo service docker restart
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
@@ -46,25 +75,54 @@ jobs:
           # Output the environment variable
           # Ref: https://stackoverflow.com/a/57989070
           echo "VERSION=$VERSION" >> $GITHUB_ENV
+          # Print for debugging purpose
+          echo "VERSION=$VERSION"
       - name: Docker meta
         id: meta
         # Ref: https://github.com/docker/metadata-action
         uses: docker/metadata-action@v5
         with:
           # Link: https://hub.docker.com/repository/docker/j3soon/ros2-cartographer-ws/tags
-          images: ${{ secrets.DOCKERHUB_USERNAME }}/ros2-cartographer-ws
+          images: j3soon/ros2-cartographer-ws
           tags: |
             type=raw,value={{date 'YYYYMMDD'}}
             type=raw,value=${{ env.VERSION }}
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          username: j3soon
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Pull j3soon/ros2-template-ws (amd64)
+        run: docker pull --platform linux/amd64 j3soon/ros2-template-ws
+      - name: Pull j3soon/ros2-template-ws (arm64)
+        run: docker pull --platform linux/arm64 j3soon/ros2-template-ws
       - name: Build and push
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v6
         with:
           context: cartographer_ws/docker
+          cache-from: |
+            j3soon/ros2-template-ws:buildcache-amd64
+            j3soon/ros2-template-ws:buildcache-arm64
           platforms: linux/amd64,linux/arm64
           push: true
           tags: ${{ steps.meta.outputs.tags }}
+      - name: Push amd64 cache
+        uses: docker/build-push-action@v6
+        with:
+          context: cartographer_ws/docker
+          cache-from: |
+            j3soon/ros2-template-ws:buildcache-amd64
+            j3soon/ros2-template-ws:buildcache-arm64
+          cache-to: type=registry,ref=j3soon/ros2-cartographer-ws:buildcache-amd64,mode=max
+          platforms: linux/amd64
+          tags: ${{ steps.meta.outputs.tags }}
+      - name: Push arm64 cache
+        uses: docker/build-push-action@v6
+        with:
+          context: cartographer_ws/docker
+          cache-from: |
+            j3soon/ros2-template-ws:buildcache-amd64
+            j3soon/ros2-template-ws:buildcache-arm64
+          cache-to: type=registry,ref=j3soon/ros2-cartographer-ws:buildcache-arm64,mode=max
+          platforms: linux/arm64
+          tags: ${{ steps.meta.outputs.tags }}
diff --git a/.github/workflows/build-docs.yaml b/.github/workflows/build-docs.yaml
@@ -14,14 +14,14 @@ jobs:
     if: github.repository == 'j3soon/ros2-essentials'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         # Ref: https://github.com/timvink/mkdocs-git-revision-date-localized-plugin#note-when-using-build-environments
         with:
           fetch-depth: '0'
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.x
-      - uses: actions/cache@v2
+      - uses: actions/cache@v4
         with:
           key: ${{ github.ref }}
           path: .cache