Merge branch 'devel'

.dockerignore

@@ -0,0 +1,10 @@
+.idea/**
+
+.DS_Store
+.env
+
+build
+
+# iRODS environment files should always be mounted into a container
+irods_environment.json
+.irodsA

.github/workflows/create-release.yml

@@ -12,113 +12,57 @@ jobs:
       run:
         shell: bash -l -e -o pipefail {0}
 
-    env:
-      WSI_CONDA_CHANNEL: "https://dnap.cog.sanger.ac.uk/npg/conda/devel/generic"
-      CONDA_TEST_ENVIRONMENT: "testenv"
-
-    strategy:
-      matrix:
-        include:
-          # iRODS 4.2.11 clients vs 4.2.11 server
-          - go: "1.17"
-            irods: "4.2.11"
-            server_image: "ghcr.io/wtsi-npg/ub-18.04-irods-4.2.11:latest"
-            baton: "4.0.0"
-
-    services:
-      irods:
-        image: ${{ matrix.server_image }}
-        ports:
-          - 1247:1247
-
     steps:
       - uses: actions/checkout@v4
 
-      - name: "Set up Go ${{ matrix.go }}"
-        uses: actions/setup-go@v5
-        with:
-          go-version: ${{ matrix.go }}
-
-      - name: "Initialize Miniconda"
-        run: |
-          echo 'source $CONDA/etc/profile.d/conda.sh' >> "$HOME/.bash_profile"
-
-      - name: "Configure Miniconda"
-        run: |
-          conda config --prepend pkgs_dirs "$HOME/conda/pkgs"
-          conda config --prepend envs_dirs "$HOME/conda/envs"
-
-          conda config --set auto_update_conda False
-          conda config --prepend channels "$WSI_CONDA_CHANNEL"
-          conda info
-
-      - name: "Install iRODS clients"
-        run: |
-          conda create -y -n "$CONDA_TEST_ENVIRONMENT"
-          conda install -y -n "$CONDA_TEST_ENVIRONMENT" "irods-icommands=${{ matrix.irods }}"
-          conda install -y -n "$CONDA_TEST_ENVIRONMENT" "baton=${{ matrix.baton }}"
-
-      - name: "Configure iRODS clients"
-        run: |
-          conda activate "$CONDA_TEST_ENVIRONMENT"
-
-          mkdir -p "$HOME/.irods"
-          cat <<'EOF' > "$HOME/.irods/irods_environment.json"
-          {
-              "irods_host": "localhost",
-              "irods_port": 1247,
-              "irods_user_name": "irods",
-              "irods_zone_name": "testZone",
-              "irods_home": "/testZone/home/irods",
-              "irods_default_resource": "replResc",
-              "irods_default_hash_scheme": "MD5"
-          }
-          EOF
-
-          echo "irods" | script -q -c "iinit" /dev/null
-          ienv
-          ils
+      - name: "Fetch Tags"
+        # Workaround for https://github.com/actions/checkout/issues/290
+        run: git fetch --tags --force
 
-          baton-do --version
-
-      - name: "Cache Go modules"
-        uses: actions/cache@v4
+      - name: "Set up Go"
+        uses: actions/setup-go@v5
         with:
-          path: |
-            "$HOME/go/pkg/mod"
-            "$HOME/.cache/go-build"
-          key: ${{ runner.os }}-key1-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-key1-go-
+          go-version-file: "go.mod"
 
-      - name: "Install test runner"
+      - name: "Get release variables"
         run: |
-          go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo
-          go get github.com/onsi/gomega/...
+          echo 'RELEASE_VERSION='$(git describe --always --tags --dirty) >> $GITHUB_ENV
+          echo 'MASTER_SHA='$(git rev-parse origin/master) >> $GITHUB_ENV
+          
+          echo 'GIT_URL='$(git remote get-url origin) >> $GITHUB_ENV
+          echo 'GIT_COMMIT='$(git log --pretty=format:'%H' -n 1) >> $GITHUB_ENV
 
-      - name: "Build distribution"
+      - name: "Build executables"
         run: |
-          conda activate "$CONDA_TEST_ENVIRONMENT"
           make dist
 
-      - name: "Get release version"
-        run: |
-          git status
-          version=$(git describe --always --tags --dirty)
-          echo "$version" | grep dirty && git diff
-
-          echo VALET_VERSION="version" >> $GITHUB_ENV
-
-      - name: "Get master shasum"
-        run: |
-          echo MASTER_SHA=$(git rev-parse origin/master) >> $GITHUB_ENV
-
-      - name: "Create release"
+      - name: "Create Release"
         uses: ncipollo/release-action@v1.14.0
         with:
-          name: ${{ env.VALET_VERSION }}
+          name: ${{ env.RELEASE_VERSION }}
           prerelease: ${{ !(github.sha == env.MASTER_SHA) }}
-          artifacts: "*.tar.bz2,*.tar.bz2.sha256"
+          artifacts: "build/*.tar.bz2,build/*.tar.bz2.sha256"
           removeArtifacts: true
           artifactErrorsFailBuild: true
           generateReleaseNotes: true
+
+      - name: "Set up Docker Buildx"
+        uses: docker/setup-buildx-action@v3
+        with:
+          install: true
+
+      - name: "Login to Docker registry"
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: "Build Docker image"
+        run: |
+          make docker GITHUB_ACTIONS=${GITHUB_ACTIONS} GITHUB_REPOSITORY_OWNER=${{ github.repository_owner }}
+
+      - name: "Push Docker image"
+        run: |
+          docker images
+          make push GITHUB_ACTIONS=${GITHUB_ACTIONS} GITHUB_REPOSITORY_OWNER=${{ github.repository_owner }}

.github/workflows/run-tests.yml

@@ -33,9 +33,9 @@ jobs:
             client_image: "ghcr.io/wtsi-npg/ub-22.04-irods-clients-4.3.1:latest"
             server_image: "ghcr.io/wtsi-npg/ub-22.04-irods-4.3.1:latest"
             experimental: false
-          - irods: "4.3.0"
-            client_image: "ghcr.io/wtsi-npg/ub-22.04-irods-clients-4.3-nightly:latest"
-            server_image: "ghcr.io/wtsi-npg/ub-22.04-irods-4.3-nightly:latest"
+          - irods: "4.3.2"
+            client_image: "ghcr.io/wtsi-npg/ub-22.04-irods-clients-4.3.2:latest"
+            server_image: "ghcr.io/wtsi-npg/ub-22.04-irods-4.3.2:latest"
             experimental: true
 
     services:

.gitignore

@@ -1 +1,13 @@
-build/**
+.DS_store
+
+# Local environment
+.env
+
+# IntelliJ / GoLand
+.idea/
+
+# Local outputs
+build
+
+# Local iRODS auth files
+.irodsA

Dockerfile

@@ -0,0 +1,24 @@
+FROM golang:1.22.4 AS builder
+
+WORKDIR /app
+
+COPY go.mod go.sum /app/
+
+RUN go mod download && go mod verify
+
+COPY . /app
+
+# Build the binary with CGO disabled because the builder image has a newer GLIBC
+# than the iRODS client image.
+RUN make build CGO_ENABLED=0 && \
+    make install CGO_ENABLED=0 GOBIN=/usr/local/bin/
+
+FROM ghcr.io/wtsi-npg/ub-18.04-baton-irods-4.2.11:latest
+
+COPY --from=builder /usr/local/bin/ /usr/local/bin/
+
+USER appuser
+
+ENTRYPOINT ["/usr/local/bin/valet"]
+
+CMD ["--version"]

Dockerfile.dev

@@ -0,0 +1,60 @@
+FROM ghcr.io/wtsi-npg/ub-18.04-baton-irods-4.2.11:latest
+
+ARG GO_VERSION=1.22.4
+
+USER root
+
+RUN apt-get update && \
+    apt-get install -q -y --no-install-recommends \
+    apt-transport-https \
+    apt-utils \
+    build-essential \
+    ca-certificates \
+    curl \
+    gcc \
+    git \
+    make
+
+# Install the iRODS icommands package because it's useful for interactions with \
+# the server during development
+RUN echo "deb [arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" |\
+    tee /etc/apt/sources.list.d/renci-irods.list && \
+    apt-get update && \
+    apt-get install -q -y --no-install-recommends \
+    irods-icommands="4.2.11-1~$(lsb_release -sc)"
+
+# For development we want to be able to compile and run tests in the presence of iRODS
+# clients, it's more practical to use an iRODS client image and install recent Go than
+# to build from a recent Go image and install iRODS clients.
+#
+# For production we can use a standard Go image as a build stage and copy the compiled
+# binary into an iRODS client image.
+RUN curl -sSL -O "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz"
+
+RUN tar -C /usr/local -xzf "go${GO_VERSION}.linux-amd64.tar.gz" && \
+    rm "go${GO_VERSION}.linux-amd64.tar.gz"
+
+ENV GOPATH="/home/appuser/go"
+ENV PATH="$GOPATH/bin:/usr/local/go/bin:$PATH"
+
+RUN mkdir -p "$GOPATH"
+
+WORKDIR /app
+
+COPY go.mod go.sum /app/
+
+RUN go mod download && go mod verify
+
+COPY . /app
+
+RUN go install github.com/onsi/ginkgo/v2/ginkgo && \
+    go get github.com/onsi/gomega/... && \
+    ginkgo version
+
+RUN chown -R appuser:appuser /app /home/appuser
+
+USER appuser
+
+ENTRYPOINT ["/app/scripts/docker-entrypoint.sh"]
+
+CMD ["/bin/bash", "-c", "sleep infinity"]

Makefile

@@ -1,17 +1,43 @@
 VERSION := $(shell git describe --always --tags --dirty)
 ldflags := "-X github.com/wtsi-npg/valet/valet.Version=${VERSION}"
+build_args := -a -v -ldflags ${ldflags}
 build_path = "build/valet-${VERSION}"
 
-.PHONY: build coverage dist install lint test check clean
+CGO_ENABLED := 1
+GOARCH := amd64
+GOOS := linux
 
-all: build
+ifeq ($(GITHUB_ACTIONS),true)
+DOCKER_REGISTRY?=ghcr.io
+DOCKER_USER?=$(GITHUB_REPOSITORY_OWNER)
+else
+DOCKER_REGISTRY?=docker.io
+DOCKER_USER?=wsinpg
+endif
+TAG=${VERSION}
 
-install:
-	go install -ldflags ${ldflags}
+NOW=$(shell date --utc --iso-8601=seconds)
+
+DOCKER_PREFIX?=$(DOCKER_REGISTRY)/$(DOCKER_USER)
+DOCKER_ARGS?=--platform linux/amd64 --progress=plain --rm
+
+image_names := valet
+images := $(addsuffix .$(TAG), $(image_names))
+remote := $(addsuffix .$(TAG).pushed, $(image_names))
+
+git_url=$(shell git remote get-url origin)
+git_commit=$(shell git log --pretty=format:'%H' -n 1)
+
+.PHONY: build check clean coverage dist docker install lint push test
+
+all: build
 
 build:
 	mkdir -p ${build_path}
-	go build -v -ldflags ${ldflags} -o ${build_path}/valet github.com/wtsi-npg/valet
+	go build ${build_args} -o ${build_path}/valet-${GOARCH} ./main.go
+
+install:
+	go install ${build_args}
 
 lint:
 	golangci-lint run ./...
@@ -24,13 +50,35 @@ test:
 coverage:
 	ginkgo -r --cover -coverprofile=coverage.out
 
-dist: build test
+dist: build
 	cp README.md COPYING ${build_path}
-	mkdir ${build_path}/scripts
+	mkdir -p ${build_path}/scripts
 	cp scripts/valet_archive_create.sh ${build_path}/scripts/
-	tar -C ./build -cvj -f valet-${VERSION}.tar.bz2 valet-${VERSION}
-	shasum -a 256 valet-${VERSION}.tar.bz2 > valet-${VERSION}.tar.bz2.sha256
+	tar -C ./build -cvj -f ./build/valet-${VERSION}.tar.bz2 valet-${VERSION}
+	shasum -a 256 ./build/valet-${VERSION}.tar.bz2 > ./build/valet-${VERSION}.tar.bz2.sha256
+
+docker: $(images)
+
+valet.$(TAG): Dockerfile
+	docker buildx build ${DOCKER_ARGS} \
+    --label org.opencontainers.image.title="valet ${VERSION} Linux ${GOARCH}" \
+    --label org.opencontainers.image.source=$(git_url) \
+    --label org.opencontainers.image.revision=$(git_commit) \
+    --label org.opencontainers.image.version=$(TAG) \
+    --label org.opencontainers.image.created=$(NOW) \
+    --tag $(DOCKER_PREFIX)/valet:$(VERSION) \
+    --tag $(DOCKER_PREFIX)/valet:latest \
+    --file $< .
+	touch $@
+
+push: $(remote)
+
+%.$(TAG).pushed: %.$(TAG)
+	echo docker push $(DOCKER_PREFIX)/$*:$(TAG)
+	echo docker push $(DOCKER_PREFIX)/$*:latest
+	touch $@
 
 clean:
 	go clean
-	rm -rf build/*
+	$(RM) -r ./build
+	rm -f $(foreach image_name,$(image_names), $(image_name).*)

config/valet.service

@@ -1,11 +1,10 @@
 [Unit]
 Description=The valet data manager service
-Documentation=https://github.com/kjsanger/valet
+Documentation=https://github.com/wtsi-npg/valet
 StartLimitBurst=3
 
 [Service]
 Environment=PATH=/home/{{ valet_user }}/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin
-Environment=CONDA_ENV={{ valet_conda_env }}
 Environment=INSTRUMENT_MODEL={{ valet_instrument }}
 Environment=INTERVAL={{ valet_interval }}
 Environment=MAX_PROC={{ valet_max_proc }}