Merge pull request #251 from wtsi-npg/devel

Merge to master for release 2.0.0

.github/dependabot.yml

@@ -4,3 +4,8 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/create-release.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     defaults:
       run:
         shell: bash -l -e -o pipefail {0}
@@ -22,8 +22,8 @@ jobs:
           # iRODS 4.2.11 clients vs 4.2.11 server
           - go: "1.17"
             irods: "4.2.11"
-            server_image: "wsinpg/ub-18.04-irods-4.2.11:latest"
-            baton: "3.2.0"
+            server_image: "ghcr.io/wtsi-npg/ub-18.04-irods-4.2.11:latest"
+            baton: "4.0.0"
 
     services:
       irods:
@@ -32,10 +32,10 @@ jobs:
           - 1247:1247
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: "Set up Go ${{ matrix.go }}"
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go }}
 
@@ -82,7 +82,7 @@ jobs:
           baton-do --version
 
       - name: "Cache Go modules"
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: |
             "$HOME/go/pkg/mod"
@@ -109,11 +109,15 @@ jobs:
 
           echo VALET_VERSION="version" >> $GITHUB_ENV
 
+      - name: "Get master shasum"
+        run: |
+          echo MASTER_SHA=$(git rev-parse origin/master) >> $GITHUB_ENV
+
       - name: "Create release"
-        uses: ncipollo/release-action@v1.10.0
+        uses: ncipollo/release-action@v1.14.0
         with:
           name: ${{ env.VALET_VERSION }}
-          prerelease: ${{ !(github.ref == 'refs/heads/master') }}
+          prerelease: ${{ !(github.sha == env.MASTER_SHA) }}
           artifacts: "*.tar.bz2,*.tar.bz2.sha256"
           removeArtifacts: true
           artifactErrorsFailBuild: true

.github/workflows/run-tests.yml

@@ -4,82 +4,102 @@ on: [push, pull_request]
 
 jobs:
   build:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     continue-on-error: ${{ matrix.experimental }} # Experimental builds may fail
 
     defaults:
       run:
         shell: bash -l -e -o pipefail {0}
 
     env:
-      WSI_CONDA_CHANNEL: "https://dnap.cog.sanger.ac.uk/npg/conda/devel/generic"
-      CONDA_TEST_ENVIRONMENT: "testenv"
+      SINGULARITY_VERSION: "3.11.1"
 
     strategy:
       matrix:
         include:
-          # iRODS 4.2.10 clients vs 4.2.7 server
-          - go: "1.17"
-            irods: "4.2.10"
-            server_image: "wsinpg/ub-16.04-irods-4.2.7:latest"
-            baton: "3.2.0"
+          - irods: "4.2.7"
+            client_image: "ghcr.io/wtsi-npg/ub-16.04-irods-clients-4.2.7:latest"
+            server_image: "ghcr.io/wtsi-npg/ub-16.04-irods-4.2.7:latest"
             experimental: false
-          # iRODS 4.2.10 clients vs 4.2.10 server
-          - go: "1.17"
-            irods: "4.2.10"
-            server_image: "wsinpg/ub-18.04-irods-4.2.10:latest"
-            baton: "3.2.0"
+          - irods: "4.2.11"
+            client_image: "ghcr.io/wtsi-npg/ub-18.04-irods-clients-4.2.11:latest"
+            server_image: "ghcr.io/wtsi-npg/ub-18.04-irods-4.2.11:latest"
             experimental: false
-          # iRODS 4.2.11 clients vs 4.2.7 server
-          - go: "1.17"
-            irods: "4.2.11"
-            server_image: "wsinpg/ub-16.04-irods-4.2.7:latest"
-            baton: "3.2.0"
+          - irods: "4.2.12"
+            client_image: "ghcr.io/wtsi-npg/ub-18.04-irods-clients-4.2.12:latest"
+            server_image: "ghcr.io/wtsi-npg/ub-18.04-irods-4.2.12:latest"
             experimental: false
-          # iRODS 4.2.11 clients vs 4.2.11 server
-          - go: "1.17"
-            irods: "4.2.11"
-            server_image: "wsinpg/ub-18.04-irods-4.2.11:latest"
-            baton: "3.2.0"
+          - irods: "4.3.1"
+            client_image: "ghcr.io/wtsi-npg/ub-22.04-irods-clients-4.3.1:latest"
+            server_image: "ghcr.io/wtsi-npg/ub-22.04-irods-4.3.1:latest"
             experimental: false
+          - irods: "4.3.0"
+            client_image: "ghcr.io/wtsi-npg/ub-22.04-irods-clients-4.3-nightly:latest"
+            server_image: "ghcr.io/wtsi-npg/ub-22.04-irods-4.3-nightly:latest"
+            experimental: true
 
     services:
-      irods:
+      irods-server:
         image: ${{ matrix.server_image }}
         ports:
-          - 1247:1247
+          - "1247:1247"
+          - "20000-20199:20000-20199"
+        volumes:
+          - /dev/shm:/dev/shm
+        options: >-
+          --health-cmd "nc -z -v localhost 1247"
+          --health-start-period 60s
+          --health-interval 10s
+          --health-timeout 20s
+          --health-retries 6
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
-    - name: "Set up Go ${{ matrix.go }}"
-      uses: actions/setup-go@v2
+    - name: "Set up Go"
+      uses: actions/setup-go@v5
       with:
-        go-version: ${{ matrix.go }}
+        go-version-file: "go.mod"
+        cache: true
 
-    - name: "Initialize Miniconda"
+    - name: "Install Singularity"
       run: |
-        echo 'source $CONDA/etc/profile.d/conda.sh' >> "$HOME/.bash_profile"
-
-    - name: "Configure Miniconda"
-      run: |
-        conda config --prepend pkgs_dirs "$HOME/conda/pkgs"
-        conda config --prepend envs_dirs "$HOME/conda/envs"
-
-        conda config --set auto_update_conda False
-        conda config --prepend channels "$WSI_CONDA_CHANNEL"
-        conda info
-
-    - name: "Install iRODS clients"
+        pkg="singularity-ce_${SINGULARITY_VERSION}-$(lsb_release -sc)_amd64.deb"
+        curl -sSL -O "https://github.com/sylabs/singularity/releases/download/v${SINGULARITY_VERSION}/$pkg"
+        curl -sSL -O "https://github.com/sylabs/singularity/releases/download/v${SINGULARITY_VERSION}/sha256sums"
+        grep "$os_release" sha256sums | sha256sum
+        sudo apt-get install -q -y "./$pkg"
+
+        # Avoid the runner's loop devices being exhausted
+        sudo singularity config global --set "max loop devices" 128
+
+        # Enable DBus for this user, so that rootless cgroups works and we can create
+        # Singularity service instances. See https://github.com/containers/podman/issues/5443
+        systemctl --user enable --now dbus.socket
+        echo DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$(id -u)/bus" >> $GITHUB_ENV
+
+    - name: "Cache Singularity images"
+      uses: actions/cache@v4
+      with:
+        path: ${{ github.workspace }}/.singularity-cache
+        key: ${{ runner.os }}-singularity
+
+    - name: "Install iRODS client wrappers"
+      env:
+        DOCKER_IMAGE: ${{ matrix.client_image }}
+        PREFIX: ${{ github.workspace }}
+        SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity-cache
       run: |
-        conda create -y -n "$CONDA_TEST_ENVIRONMENT"
-        conda install -y -n "$CONDA_TEST_ENVIRONMENT" "irods-icommands=${{ matrix.irods }}"
-        conda install -y -n "$CONDA_TEST_ENVIRONMENT" "baton=${{ matrix.baton }}"
+        # Use -s option for the wrappers to enable re-usable service instances
+        singularity exec docker://$DOCKER_IMAGE singularity-wrapper -s -p $PREFIX install
+        echo "$PREFIX/bin" >> $GITHUB_PATH
 
     - name: "Configure iRODS clients"
+      env:
+        DOCKER_IMAGE: ${{ matrix.client_image }}
+        PREFIX: ${{ github.workspace }}
+        SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity-cache
       run: |
-        conda activate "$CONDA_TEST_ENVIRONMENT"
-
         mkdir -p "$HOME/.irods"
         cat <<'EOF' > "$HOME/.irods/irods_environment.json"
         {
@@ -96,25 +116,15 @@ jobs:
         echo "irods" | script -q -c "iinit" /dev/null
         ienv
         ils
+        ilsresc
 
         baton-do --version
 
-    - name: "Cache Go modules"
-      uses: actions/cache@v2
-      with:
-        path: |
-          "$HOME/go/pkg/mod"
-          "$HOME/.cache/go-build"
-        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: |
-          ${{ runner.os }}-go-
-
     - name: "Install test runner"
       run: |
         go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo
         go get github.com/onsi/gomega/...
 
     - name: "Run tests"
       run: |
-        conda activate "$CONDA_TEST_ENVIRONMENT"
         make test

CHANGELOG.md

@@ -10,6 +10,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+ - Update valet_archive_create script to remove reference to conda
+ - Update CI to use ubuntu latest
+ - Update github actions test workflow baton to 4.0.0
+
+### Removed
+
+ - Remove irods 4.2.10 from github actions test workflow
+
 ### [1.8.0] - 2022-06-20
 
 ### Added

Makefile

@@ -19,10 +19,10 @@ lint:
 check: test
 
 test:
-	ginkgo -r -slow-spec-threshold=60s -race
+	ginkgo -r --race
 
 coverage:
-	ginkgo -r -slow-spec-threshold=60s -cover -coverprofile=coverage.out
+	ginkgo -r --cover -coverprofile=coverage.out
 
 dist: build test
 	cp README.md COPYING ${build_path}

go.mod

@@ -1,36 +1,41 @@
 module github.com/wtsi-npg/valet
 
-go 1.17
+go 1.22
 
 require (
-	github.com/klauspost/pgzip v1.2.5
-	github.com/onsi/ginkgo/v2 v2.1.4
-	github.com/onsi/gomega v1.19.0
+	github.com/klauspost/pgzip v1.2.6
+	github.com/onsi/ginkgo/v2 v2.17.2
+	github.com/onsi/gomega v1.33.1
 	github.com/pkg/errors v0.9.1
-	github.com/rs/zerolog v1.26.1
-	github.com/spf13/cobra v1.4.0
-	github.com/stretchr/testify v1.7.1
-	github.com/wtsi-npg/extendo/v2 v2.4.0
+	github.com/rs/zerolog v1.32.0
+	github.com/spf13/cobra v1.8.0
+	github.com/stretchr/testify v1.9.0
+	github.com/wtsi-npg/extendo/v2 v2.6.1
 	github.com/wtsi-npg/fsnotify v1.4.8-0.20190705153444-45ca73e9793a
-	github.com/wtsi-npg/logshim v1.3.0
-	github.com/wtsi-npg/logshim-zerolog v1.3.0
-	golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898
+	github.com/wtsi-npg/logshim v1.4.0
+	github.com/wtsi-npg/logshim-zerolog v1.4.0
+	golang.org/x/crypto v0.22.0
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/klauspost/compress v1.15.4 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	golang.org/x/net v0.0.0-20220517181318-183a9ca12b87 // indirect
-	golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e // indirect
-	golang.org/x/term v0.0.0-20220411215600-e5f449aeb171 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	google.golang.org/protobuf v1.26.0 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99 // indirect
+	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
+	golang.org/x/term v0.19.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/tools v0.20.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 // replace github.com/wtsi-npg/extendo/v2 => ../extendo