Merge pull request #1 from stateless-minds/sync-with-upstream-v0.24.3

Sync with upstream v0.24.3

.github/workflows/gateway-conformance.yml

@@ -1,63 +1,211 @@
 name: Gateway Conformance
+# This workflow runs https://github.com/ipfs/gateway-conformance
+# against different backend implementations of boxo/gateway
 
 on:
   push:
     branches:
       - main
   pull_request:
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
   cancel-in-progress: true
 
 jobs:
-  gateway-conformance:
+  # This test uses a static CAR file as a local blockstore,
+  # allowing us to test conformance against BlocksBackend (gateway/backend_blocks.go)
+  # which is used by implementations like Kubo
+  local-block-backend:
     runs-on: ubuntu-latest
     steps:
       # 1. Download the gateway-conformance fixtures
       - name: Download gateway-conformance fixtures
-        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.2
+        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.6
         with:
           output: fixtures
           merged: true
 
-      # 2. Build the car-gateway
+      # 2. Build the gateway binary
+      - name: Checkout boxo
+        uses: actions/checkout@v4
+        with:
+          path: boxo
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'boxo/examples/go.mod'
+          cache-dependency-path: "boxo/**/*.sum"
+      - name: Build test-gateway
+        run: go build -o test-gateway
+        working-directory: boxo/examples/gateway/car-file
+
+      # 3. Start the gateway binary
+      - name: Start test-gateway
+        run: boxo/examples/gateway/car-file/test-gateway -c fixtures/fixtures.car -p 8040 &
+
+      # 4. Run the gateway-conformance tests
+      - name: Run gateway-conformance tests
+        uses: ipfs/gateway-conformance/.github/actions/test@v0.6
         with:
-          go-version: 1.19.x
+          gateway-url: http://127.0.0.1:8040
+          subdomain-url: http://example.net:8040
+          specs: -trustless-ipns-gateway,-path-ipns-gateway,-subdomain-ipns-gateway,-dnslink-gateway
+          json: output.json
+          xml: output.xml
+          html: output.html
+          markdown: output.md
+
+      # 5. Upload the results
+      - name: Upload MD summary
+        if: failure() || success()
+        run: cat output.md >> $GITHUB_STEP_SUMMARY
+      - name: Upload HTML report
+        if: failure() || success()
+        uses: actions/upload-artifact@v4
+        with:
+          name: gateway-conformance_local-block-backend.html
+          path: output.html
+      - name: Upload JSON report
+        if: failure() || success()
+        uses: actions/upload-artifact@v4
+        with:
+          name: gateway-conformance_local-block-backend.json
+          path: output.json
+
+  # This test uses remote block gateway (?format=raw) as a remote blockstore,
+  # allowing us to test conformance against RemoteBlocksBackend
+  # (gateway/backend_blocks.go) which is used by implementations like
+  # rainbow configured to use with remote block backend
+  # Ref. https://specs.ipfs.tech/http-gateways/trustless-gateway/#block-responses-application-vnd-ipld-raw
+  remote-block-backend:
+    runs-on: ubuntu-latest
+    steps:
+      # 1. Download the gateway-conformance fixtures
+      - name: Download gateway-conformance fixtures
+        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.6
+        with:
+          output: fixtures
+          merged: true
+
+      # 2. Build the gateway binaries
       - name: Checkout boxo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: boxo
-      - name: Build car-gateway
-        run: go build -o car-gateway
-        working-directory: boxo/examples/gateway/car
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'boxo/examples/go.mod'
+          cache-dependency-path: "boxo/**/*.sum"
+      - name: Build remote-block-backend # it will act as a trustless CAR gateway
+        run: go build -o remote-block-backend
+        working-directory: boxo/examples/gateway/car-file
+      - name: Build test-gateway # this one will be used for tests, it will use previous one as its remote block backend
+        run: go build -o test-gateway
+        working-directory: boxo/examples/gateway/proxy-blocks
 
-      # 3. Start the car-gateway
-      - name: Start car-gateway
-        run: boxo/examples/gateway/car/car-gateway -c fixtures/fixtures.car -p 8040 &
+      # 3. Start the gateway binaries
+      - name: Start remote HTTP backend that serves application/vnd.ipld.raw
+        run: boxo/examples/gateway/car-file/remote-block-backend -c fixtures/fixtures.car -p 8030 & # this endpoint will respond to application/vnd.ipld.car requests
+      - name: Start gateway that uses the remote block backend
+        run: boxo/examples/gateway/proxy-blocks/test-gateway -g http://127.0.0.1:8030 -p 8040 &
 
       # 4. Run the gateway-conformance tests
       - name: Run gateway-conformance tests
-        uses: ipfs/gateway-conformance/.github/actions/test@v0.2
+        uses: ipfs/gateway-conformance/.github/actions/test@v0.6
         with:
-          gateway-url: http://127.0.0.1:8040
+          gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote block gateway
+          subdomain-url: http://example.net:8040
+          specs: -trustless-ipns-gateway,-path-ipns-gateway,-subdomain-ipns-gateway,-dnslink-gateway
+          args: -skip 'TestGatewayCache/.*_for_%2Fipfs%2F_with_only-if-cached_succeeds_when_in_local_datastore'
           json: output.json
           xml: output.xml
           html: output.html
           markdown: output.md
-          subdomain-url: http://example.net
+
+      # 5. Upload the results
+      - name: Upload MD summary
+        if: failure() || success()
+        run: cat output.md >> $GITHUB_STEP_SUMMARY
+      - name: Upload HTML report
+        if: failure() || success()
+        uses: actions/upload-artifact@v4
+        with:
+          name: gateway-conformance_remote-block-backend.html
+          path: output.html
+      - name: Upload JSON report
+        if: failure() || success()
+        uses: actions/upload-artifact@v4
+        with:
+          name: gateway-conformance_remote-block-backend.json
+          path: output.json
+
+  # This test uses remote CAR gateway (?format=car, IPIP-402)
+  # allowing us to test conformance against remote CarFetcher backend.
+  # (gateway/backend_car_fetcher.go) which is used by implementations like
+  # rainbow configured to use with remote car backend
+  # Ref. https://specs.ipfs.tech/http-gateways/trustless-gateway/#car-responses-application-vnd-ipld-car
+  remote-car-backend:
+    runs-on: ubuntu-latest
+    steps:
+      # 1. Download the gateway-conformance fixtures
+      - name: Download gateway-conformance fixtures
+        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.6
+        with:
+          output: fixtures
+          merged: true
+
+      # 2. Build the gateway binaries
+      - name: Checkout boxo
+        uses: actions/checkout@v4
+        with:
+          path: boxo
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'boxo/examples/go.mod'
+          cache-dependency-path: "boxo/**/*.sum"
+      - name: Build remote-car-backend # it will act as a trustless CAR gateway
+        run: go build -o remote-car-backend
+        working-directory: boxo/examples/gateway/car-file
+      - name: Build test-gateway # this one will be used for tests, it will use previous one as its remote CAR backend
+        run: go build -o test-gateway
+        working-directory: boxo/examples/gateway/proxy-car
+
+      # 3. Start the gateway binaries
+      - name: Start remote HTTP backend that serves application/vnd.ipld.car (IPIP-402)
+        run: boxo/examples/gateway/car-file/remote-car-backend -c fixtures/fixtures.car -p 8030 & # this endpoint will respond to application/vnd.ipld.raw requests
+      - name: Start gateway that uses the remote CAR backend
+        run: boxo/examples/gateway/proxy-car/test-gateway -g http://127.0.0.1:8030 -p 8040 &
+
+      # 4. Run the gateway-conformance tests
+      - name: Run gateway-conformance tests
+        uses: ipfs/gateway-conformance/.github/actions/test@v0.6
+        with:
+          gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote car gateway
+          subdomain-url: http://example.net:8040
           specs: -trustless-ipns-gateway,-path-ipns-gateway,-subdomain-ipns-gateway,-dnslink-gateway
-          args: -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length'
+          args: -skip 'TestGatewayCache/.*_for_%2Fipfs%2F_with_only-if-cached_succeeds_when_in_local_datastore'
+          json: output.json
+          xml: output.xml
+          html: output.html
+          markdown: output.md
 
       # 5. Upload the results
       - name: Upload MD summary
         if: failure() || success()
         run: cat output.md >> $GITHUB_STEP_SUMMARY
       - name: Upload HTML report
         if: failure() || success()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: gateway-conformance.html
+          name: gateway-conformance_remote-car-backend.html
           path: output.html
+      - name: Upload JSON report
+        if: failure() || success()
+        uses: actions/upload-artifact@v4
+        with:
+          name: gateway-conformance_remote-car-backend.json
+          path: output.json

.github/workflows/gateway-sharness.yml

@@ -16,9 +16,9 @@ jobs:
         shell: bash
     steps:
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.19.1
+          go-version: 1.23.x
       - name: Checkout boxo
         uses: actions/checkout@v3
         with:
@@ -30,18 +30,15 @@ jobs:
           path: kubo
       - name: Install Missing Tools
         run: sudo apt install -y socat net-tools fish libxml2-utils
-      - name: Restore Go Cache
-        uses: protocol/cache-go-action@v1
-        with:
-          name: ${{ github.job }}
       - name: Replace boxo in Kubo go.mod
         run: |
           go mod edit -replace=github.com/stateless-minds/boxo=../boxo
-          go mod tidy
-          cat go.mod
+          make mod_tidy
         working-directory: kubo
       - name: Install sharness dependencies
-        run: make test_sharness_deps
+        run: |
+          find . -name go.mod -execdir go mod tidy \;
+          make test_sharness_deps
         working-directory: kubo
       - name: Run Kubo Sharness Tests
         run: find . -maxdepth 1 -name "*gateway*.sh" -print0 | xargs -0 -I {} bash -c "echo {}; {}"

.github/workflows/go-check.yml

@@ -1,74 +1,18 @@
-# File managed by web3-bot. DO NOT EDIT.
-# See https://github.com/protocol/.github/ for details.
+name: Go Checks
 
 on:
   pull_request:
   push:
     branches: ["main","release"]
-name: Go Checks
+  workflow_dispatch:
+
+permissions:
+  contents: read
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
   cancel-in-progress: true
 
 jobs:
-  unit:
-    runs-on: ubuntu-latest
-    name: All
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: recursive
-      - id: config
-        uses: protocol/.github/.github/actions/read-config@master
-      - uses: actions/setup-go@v3
-        with:
-          go-version: 1.20.x
-      - name: Run repo-specific setup
-        uses: ./.github/actions/go-check-setup
-        if: hashFiles('./.github/actions/go-check-setup') != ''
-      - name: Install staticcheck
-        run: go install honnef.co/go/tools/cmd/staticcheck@4970552d932f48b71485287748246cf3237cebdf # 2023.1 (v0.4.0)
-      - name: Check that go.mod is tidy
-        uses: protocol/multiple-go-modules@v1.2
-        with:
-          run: |
-            go mod tidy
-            if [[ -n $(git ls-files --other --exclude-standard --directory -- go.sum) ]]; then
-              echo "go.sum was added by go mod tidy"
-              exit 1
-            fi
-            git diff --exit-code -- go.sum go.mod
-      - name: gofmt
-        if: success() || failure() # run this step even if the previous one failed
-        run: |
-          out=$(gofmt -s -l .)
-          if [[ -n "$out" ]]; then
-            echo $out | awk '{print "::error file=" $0 ",line=0,col=0::File is not gofmt-ed."}'
-            exit 1
-          fi
-      - name: go vet
-        if: success() || failure() # run this step even if the previous one failed
-        uses: protocol/multiple-go-modules@v1.2
-        with:
-          run: go vet ./...
-      - name: staticcheck
-        if: success() || failure() # run this step even if the previous one failed
-        uses: protocol/multiple-go-modules@v1.2
-        with:
-          run: |
-            set -o pipefail
-            staticcheck ./... | sed -e 's@\(.*\)\.go@./\1.go@g'
-      - name: go generate
-        uses: protocol/multiple-go-modules@v1.2
-        if: (success() || failure()) && fromJSON(steps.config.outputs.json).gogenerate == true
-        with:
-          run: |
-            git clean -fd # make sure there aren't untracked files / directories
-            go generate -x ./...
-            # check if go generate modified or added any files
-            if ! $(git add . && git diff-index HEAD --exit-code --quiet); then
-              echo "go generated caused changes to the repository:"
-              git status --short
-              exit 1
-            fi
+  go-check:
+    uses: ipdxco/unified-github-workflows/.github/workflows/go-check.yml@v1.0