From d38760a777be191d2176f5c9d310c6f538beb3a5 Mon Sep 17 00:00:00 2001 From: famosab Date: Thu, 14 Nov 2024 15:31:10 +0100 Subject: [PATCH 01/14] port to nf-test --- .../nf-core/parabricks/deepvariant/main.nf | 13 +- .../parabricks/deepvariant/tests/copy.config | 7 + .../parabricks/deepvariant/tests/gvcf.config | 8 + .../parabricks/deepvariant/tests/main.nf.test | 171 ++++++++++++++++++ .../deepvariant/tests/main.nf.test.snap | 106 +++++++++++ 5 files changed, 297 insertions(+), 8 deletions(-) create mode 100644 modules/nf-core/parabricks/deepvariant/tests/copy.config create mode 100644 modules/nf-core/parabricks/deepvariant/tests/gvcf.config create mode 100644 modules/nf-core/parabricks/deepvariant/tests/main.nf.test create mode 100644 modules/nf-core/parabricks/deepvariant/tests/main.nf.test.snap diff --git a/modules/nf-core/parabricks/deepvariant/main.nf b/modules/nf-core/parabricks/deepvariant/main.nf index ac18b72f3ad..b9eeef84069 100644 --- a/modules/nf-core/parabricks/deepvariant/main.nf +++ b/modules/nf-core/parabricks/deepvariant/main.nf @@ -2,7 +2,7 @@ process PARABRICKS_DEEPVARIANT { tag "$meta.id" label 'process_high' - container "nvcr.io/nvidia/clara/clara-parabricks:4.3.0-1" + container "nvcr.io/nvidia/clara/clara-parabricks:4.4.0-1" input: tuple val(meta), path(input), path(input_index), path(interval_file) @@ -16,25 +16,23 @@ process PARABRICKS_DEEPVARIANT { task.ext.when == null || task.ext.when script: - // Exit if running this module with -profile conda / -profile mamba if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { exit 1, "Parabricks module does not support Conda. Please use Docker / Singularity / Podman instead." } - def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def output_file = args =~ "gvcf" ? "${prefix}.g.vcf" : "${prefix}.vcf" + def output_file = ("--gvcf" =~ task.ext.args)? "${prefix}.g.vcf" : "${prefix}.vcf" def interval_file_command = interval_file ? interval_file.collect{"--interval-file $it"}.join(' ') : "" + def num_gpus = task.accelerator ? "--num-gpus $task.accelerator.request" : '' """ - pbrun \\ deepvariant \\ --ref $fasta \\ --in-bam $input \\ --out-variants $output_file \\ $interval_file_command \\ - --num-gpus $task.accelerator.request \\ + $num_gpus \\ $args cat <<-END_VERSIONS > versions.yml @@ -44,9 +42,8 @@ process PARABRICKS_DEEPVARIANT { """ stub: - def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def output_file = args =~ "gvcf" ? "${prefix}.g.vcf" : "${prefix}.vcf" + def output_file = ("--gvcf" =~ task.ext.args)? "${prefix}.g.vcf" : "${prefix}.vcf" """ touch $output_file diff --git a/modules/nf-core/parabricks/deepvariant/tests/copy.config b/modules/nf-core/parabricks/deepvariant/tests/copy.config new file mode 100644 index 00000000000..71a3ba9e457 --- /dev/null +++ b/modules/nf-core/parabricks/deepvariant/tests/copy.config @@ -0,0 +1,7 @@ +process { + + withName: 'PARABRICKS_DEEPVARIANT' { + stageInMode = 'copy' + } + +} \ No newline at end of file diff --git a/modules/nf-core/parabricks/deepvariant/tests/gvcf.config b/modules/nf-core/parabricks/deepvariant/tests/gvcf.config new file mode 100644 index 00000000000..eac2f317a11 --- /dev/null +++ b/modules/nf-core/parabricks/deepvariant/tests/gvcf.config @@ -0,0 +1,8 @@ +process { + + withName: 'PARABRICKS_DEEPVARIANT' { + ext.args = '--gvcf' + stageInMode = 'copy' + } + +} \ No newline at end of file diff --git a/modules/nf-core/parabricks/deepvariant/tests/main.nf.test b/modules/nf-core/parabricks/deepvariant/tests/main.nf.test new file mode 100644 index 00000000000..c28583007c4 --- /dev/null +++ b/modules/nf-core/parabricks/deepvariant/tests/main.nf.test @@ -0,0 +1,171 @@ +nextflow_process { + + name "Test Process PARABRICKS_DEEPVARIANT" + script "../main.nf" + process "PARABRICKS_DEEPVARIANT" + + tag "modules" + tag "modules_nfcore" + tag "parabricks" + tag "parabricks/deepvariant" + + test("human - bam") { + + when { + process { + """ + input[0] = [ + [ id:'test'], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test2.paired_end.recalibrated.sorted.bam', checkIfExists: true), + [], // bam.bai not needed unless using intervals + [] + ] + input[1] = [ + [ id:'test'], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/chr21/sequence/genome.fasta', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( + path(process.out.vcf[0][1]).vcf.variantsMD5, + process.out.versions + ).match() } + ) + } + + } + + test("human - bam - intervals") { + + config './copy.config' + + when { + process { + """ + input[0] = [ + [ id:'test'], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test2.paired_end.recalibrated.sorted.bam', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test2.paired_end.recalibrated.sorted.bam.bai', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/chr21/sequence/multi_intervals.bed', checkIfExists: true) + ] + input[1] = [ + [ id:'ref'], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/chr21/sequence/genome.fasta', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( + path(process.out.vcf[0][1]).vcf.variantsMD5, + process.out.versions + ).match() } + ) + } + + } + + test("human - bam - intervals - gvcf") { + + config './gvcf.config' + + when { + process { + """ + input[0] = [ + [ id:'test'], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test2.paired_end.recalibrated.sorted.bam', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test2.paired_end.recalibrated.sorted.bam.bai', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/chr21/sequence/multi_intervals.bed', checkIfExists: true) + ] + input[1] = [ + [ id:'ref'], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/chr21/sequence/genome.fasta', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( + process.out.versions, + // file(process.out.vcf[0][1]).name // fails with groovy.lang.MissingMethodException: No signature of method: static com.askimed.nf.test.lang.extensions.GlobalMethods.file() is applicable for argument types: (ArrayList) values: + ).match() + } + ) + } + + } + + test("human - bam - stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ + [ id:'test'], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test2.paired_end.recalibrated.sorted.bam', checkIfExists: true), + [], // bam.bai not needed unless using intervals + [] + ] + input[1] = [ + [ id:'test'], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/chr21/sequence/genome.fasta', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + + } + + test("human - bam - intervals - gvcf - stub") { + + config './gvcf.config' + options "-stub" + + when { + process { + """ + input[0] = [ + [ id:'test'], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test2.paired_end.recalibrated.sorted.bam', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test2.paired_end.recalibrated.sorted.bam.bai', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/chr21/sequence/multi_intervals.bed', checkIfExists: true) + ] + input[1] = [ + [ id:'test'], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/chr21/sequence/genome.fasta', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( process.out ).match() } + ) + } + + } + +} diff --git a/modules/nf-core/parabricks/deepvariant/tests/main.nf.test.snap b/modules/nf-core/parabricks/deepvariant/tests/main.nf.test.snap new file mode 100644 index 00000000000..bb19b848414 --- /dev/null +++ b/modules/nf-core/parabricks/deepvariant/tests/main.nf.test.snap @@ -0,0 +1,106 @@ +{ + "human - bam - intervals - gvcf": { + "content": [ + [ + "versions.yml:md5,c7bcf80d609e2951eb99f0b774cd2f6d" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-14T15:26:48.481295634" + }, + "human - bam - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.vcf:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + "versions.yml:md5,c7bcf80d609e2951eb99f0b774cd2f6d" + ], + "vcf": [ + [ + { + "id": "test" + }, + "test.vcf:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,c7bcf80d609e2951eb99f0b774cd2f6d" + ] + } + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-14T15:05:28.449273519" + }, + "human - bam - intervals": { + "content": [ + "27097afc7aae62918a3e26790cc189c3", + [ + "versions.yml:md5,c7bcf80d609e2951eb99f0b774cd2f6d" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-14T15:06:52.854870182" + }, + "human - bam": { + "content": [ + "8b526a232f93a5036176336415abf7d1", + [ + "versions.yml:md5,c7bcf80d609e2951eb99f0b774cd2f6d" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-14T15:06:37.651354312" + }, + "human - bam - intervals - gvcf - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.g.vcf:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + "versions.yml:md5,c7bcf80d609e2951eb99f0b774cd2f6d" + ], + "vcf": [ + [ + { + "id": "test" + }, + "test.g.vcf:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,c7bcf80d609e2951eb99f0b774cd2f6d" + ] + } + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-14T15:12:11.438654866" + } +} \ No newline at end of file From 94d726f487f9600fc016b47f314bad971f431b7d Mon Sep 17 00:00:00 2001 From: famosab Date: Thu, 14 Nov 2024 15:31:30 +0100 Subject: [PATCH 02/14] remove --- tests/config/nf-test.config | 4 +- tests/config/pytest_modules.yml | 3 - .../nf-core/parabricks/deepvariant/main.nf | 55 ------------------- .../parabricks/deepvariant/nextflow.config | 8 --- .../nf-core/parabricks/deepvariant/test.yml | 29 ---------- .../deepvariant/test_GPU_config.txt | 14 ----- .../parabricks/deepvariant/test_GPU_yml.txt | 29 ---------- 7 files changed, 2 insertions(+), 140 deletions(-) delete mode 100644 tests/modules/nf-core/parabricks/deepvariant/main.nf delete mode 100644 tests/modules/nf-core/parabricks/deepvariant/nextflow.config delete mode 100644 tests/modules/nf-core/parabricks/deepvariant/test.yml delete mode 100644 tests/modules/nf-core/parabricks/deepvariant/test_GPU_config.txt delete mode 100644 tests/modules/nf-core/parabricks/deepvariant/test_GPU_yml.txt diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index fa21cb2edd6..1918fa26258 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -30,8 +30,8 @@ profiles { } docker { docker.enabled = true - docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' - } +docker.runOptions = '-u $(id -u):$(id -g) --gpus device=0 --platform=linux/amd64' +} docker_self_hosted { docker.enabled = true docker.fixOwnership = true diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index 0bd8e7cfa63..bec1555a6bf 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -416,9 +416,6 @@ oncocnv: parabricks/dbsnp: - modules/nf-core/parabricks/dbsnp/** - tests/modules/nf-core/parabricks/dbsnp/** -parabricks/deepvariant: - - modules/nf-core/parabricks/deepvariant/** - - tests/modules/nf-core/parabricks/deepvariant/** parabricks/genotypegvcf: - modules/nf-core/parabricks/genotypegvcf/** - tests/modules/nf-core/parabricks/genotypegvcf/** diff --git a/tests/modules/nf-core/parabricks/deepvariant/main.nf b/tests/modules/nf-core/parabricks/deepvariant/main.nf deleted file mode 100644 index 9023189bbb9..00000000000 --- a/tests/modules/nf-core/parabricks/deepvariant/main.nf +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env nextflow - -nextflow.enable.dsl = 2 - -include { PARABRICKS_DEEPVARIANT } from '../../../../../modules/nf-core/parabricks/deepvariant/main.nf' -include { PARABRICKS_DEEPVARIANT as PARABRICKS_DEEPVARIANT_GVCF } from '../../../../../modules/nf-core/parabricks/deepvariant/main.nf' - -workflow test_parabricks_deepvariant { - - input = [ - [ id:'test'], - file(params.test_data['homo_sapiens']['illumina']['test2_paired_end_recalibrated_sorted_bam'], checkIfExists: true), - [], // bam not needed unless using intervals - [] - ] - fasta = [ - [ id:'test'], - file(params.test_data['homo_sapiens']['genome']['genome_21_fasta'], checkIfExists: true) - ] - - PARABRICKS_DEEPVARIANT ( input, fasta ) -} - -workflow test_parabricks_deepvariant_intervals { - - input = [ - [ id:'test'], - file(params.test_data['homo_sapiens']['illumina']['test2_paired_end_recalibrated_sorted_bam'], checkIfExists: true), - file(params.test_data['homo_sapiens']['illumina']['test2_paired_end_recalibrated_sorted_bam_bai'], checkIfExists: true), - file(params.test_data['homo_sapiens']['genome']['genome_21_multi_interval_bed'], checkIfExists: true) - ] - fasta = [ - [ id:'test'], - file(params.test_data['homo_sapiens']['genome']['genome_21_fasta'], checkIfExists: true) - ] - - - PARABRICKS_DEEPVARIANT ( input, fasta ) -} - -workflow test_parabricks_deepvariant_gvcf { - - input = [ - [ id:'test'], - file(params.test_data['homo_sapiens']['illumina']['test2_paired_end_recalibrated_sorted_bam'], checkIfExists: true), - file(params.test_data['homo_sapiens']['illumina']['test2_paired_end_recalibrated_sorted_bam_bai'], checkIfExists: true), - file(params.test_data['homo_sapiens']['genome']['genome_21_multi_interval_bed'], checkIfExists: true) - ] - fasta = [ - [ id:'test'], - file(params.test_data['homo_sapiens']['genome']['genome_21_fasta'], checkIfExists: true) - ] - - PARABRICKS_DEEPVARIANT_GVCF ( input, fasta ) -} \ No newline at end of file diff --git a/tests/modules/nf-core/parabricks/deepvariant/nextflow.config b/tests/modules/nf-core/parabricks/deepvariant/nextflow.config deleted file mode 100644 index 9fbafc63d37..00000000000 --- a/tests/modules/nf-core/parabricks/deepvariant/nextflow.config +++ /dev/null @@ -1,8 +0,0 @@ -process { - - publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - - withName: PARABRICKS_DEEPVARIANT_GVCF { - ext.args = "--gvcf" - } -} diff --git a/tests/modules/nf-core/parabricks/deepvariant/test.yml b/tests/modules/nf-core/parabricks/deepvariant/test.yml deleted file mode 100644 index 9d1db86ba61..00000000000 --- a/tests/modules/nf-core/parabricks/deepvariant/test.yml +++ /dev/null @@ -1,29 +0,0 @@ -- name: parabricks deepvariant test_parabricks_deepvariant - command: nextflow run ./tests/modules/nf-core/parabricks/deepvariant -entry test_parabricks_deepvariant -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/parabricks/deepvariant/nextflow.config -stub-run - tags: - - parabricks/deepvariant - - parabricks - files: - - path: output/parabricks/test.vcf - should_exist: true - - path: output/parabricks/versions.yml - -- name: parabricks deepvariant test_parabricks_deepvariant_intervals - command: nextflow run ./tests/modules/nf-core/parabricks/deepvariant -entry test_parabricks_deepvariant_intervals -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/parabricks/deepvariant/nextflow.config -stub-run - tags: - - parabricks/deepvariant - - parabricks - files: - - path: output/parabricks/test.vcf - should_exist: true - - path: output/parabricks/versions.yml - -- name: parabricks deepvariant test_parabricks_deepvariant_gvcf - command: nextflow run ./tests/modules/nf-core/parabricks/deepvariant -entry test_parabricks_deepvariant_gvcf -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/parabricks/deepvariant/nextflow.config -stub-run - tags: - - parabricks/deepvariant - - parabricks - files: - - path: output/parabricks/test.g.vcf - should_exist: true - - path: output/parabricks/versions.yml diff --git a/tests/modules/nf-core/parabricks/deepvariant/test_GPU_config.txt b/tests/modules/nf-core/parabricks/deepvariant/test_GPU_config.txt deleted file mode 100644 index f64deb68675..00000000000 --- a/tests/modules/nf-core/parabricks/deepvariant/test_GPU_config.txt +++ /dev/null @@ -1,14 +0,0 @@ -process { - - publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - memory = "15.GB" - cpus = 4 - accelerator = 1 - - withName: PARABRICKS_DEEPVARIANT_GVCF { - ext.args = "--gvcf" - } -} - -docker.runOptions = "--gpus all" -singularity.runOptions = "--nv" diff --git a/tests/modules/nf-core/parabricks/deepvariant/test_GPU_yml.txt b/tests/modules/nf-core/parabricks/deepvariant/test_GPU_yml.txt deleted file mode 100644 index 8718ea4ceb8..00000000000 --- a/tests/modules/nf-core/parabricks/deepvariant/test_GPU_yml.txt +++ /dev/null @@ -1,29 +0,0 @@ -- name: parabricks deepvariant test_parabricks_deepvariant - command: nextflow run ./tests/modules/nf-core/parabricks/deepvariant -entry test_parabricks_deepvariant -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/parabricks/deepvariant/nextflow.config - tags: - - parabricks/deepvariant - - parabricks - files: - - path: output/parabricks/test.vcf - should_exist: true - - path: output/parabricks/versions.yml - -- name: parabricks deepvariant test_parabricks_deepvariant_intervals - command: nextflow run ./tests/modules/nf-core/parabricks/deepvariant -entry test_parabricks_deepvariant_intervals -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/parabricks/deepvariant/nextflow.config - tags: - - parabricks/deepvariant - - parabricks - files: - - path: output/parabricks/test.vcf - should_exist: true - - path: output/parabricks/versions.yml - -- name: parabricks deepvariant test_parabricks_deepvariant_gvcf - command: nextflow run ./tests/modules/nf-core/parabricks/deepvariant -entry test_parabricks_deepvariant_gvcf -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/parabricks/deepvariant/nextflow.config - tags: - - parabricks/deepvariant - - parabricks - files: - - path: output/parabricks/test.g.vcf - should_exist: true - - path: output/parabricks/versions.yml From 3acb8095f7eb0506e9c5b3858041d2282029a134 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Mon, 18 Nov 2024 11:26:04 +0100 Subject: [PATCH 03/14] update test yaml --- .github/workflows/test.yml | 748 +++++++++++++++++++++++++++++++++++++ 1 file changed, 748 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000000..b6f044cb26c --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,748 @@ +name: Run tests +on: + push: + branches: + # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging + - "renovate/**" # branches Renovate creates + pull_request: + branches: [master] + merge_group: + types: [checks_requested] + branches: [master] + workflow_dispatch: + inputs: + runners: + description: "Runners to test on" + type: choice + options: + - "ubuntu-latest" + - "self-hosted" + default: "self-hosted" + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity + NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + with: + python-version: "3.11" + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 + # FIXME Flip this off once we get to less than a couple hundred. Adding + # this so it will only run against changed files. It'll make it much + # easier to fix these as they come up rather than everything at once. + with: + extra_args: "--all-files" + + prettier: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + + - name: Install NodeJS + uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4 + with: + node-version: "20" + + - name: Install Prettier + run: npm install -g prettier@3.2.5 + + - name: Run Prettier --check + run: prettier --check . + + editorconfig: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + + - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4 + with: + node-version: "20" + + - name: Install editorconfig-checker + run: npm install -g editorconfig-checker + + - name: Run ECLint check + run: editorconfig-checker -exclude README.md $(git ls-files | grep -v test) + + pytest-changes: + name: pytest-changes + runs-on: ubuntu-latest + outputs: + tags: ${{ steps.filter.outputs.changes }} + modules: ${{ steps.tags.outputs.modules }} + subworkflows: ${{ steps.tags.outputs.subworkflows }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + fetch-depth: 2 # To retrieve the preceding commit. + + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 + id: filter + with: + filters: "tests/config/pytest_modules.yml" + token: "" + + - name: Fetch module tags + id: tags + run: | + echo modules=$(echo '${{ steps.filter.outputs.changes }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/"; ""))') >> $GITHUB_OUTPUT + echo subworkflows=$(echo '${{ steps.filter.outputs.changes }}' | jq '. | map(select(contains("subworkflow"))) | map(gsub("subworkflows/"; ""))') >> $GITHUB_OUTPUT + + - name: debug + run: | + echo ${{ steps.tags.outputs.modules }} + echo ${{ steps.tags.outputs.subworkflows }} + + nf-test-changes: + name: nf-test-changes + runs-on: ubuntu-latest + outputs: + # Expose detected tags as 'modules' and 'workflows' output variables + paths: ${{ steps.list.outputs.components }} + modules: ${{ steps.outputs.outputs.modules }} + subworkflows: ${{ steps.outputs.outputs.subworkflows}} + # Prod for version bumping + + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + fetch-depth: 0 + + - name: List nf-test files + id: list + uses: adamrtalbot/detect-nf-test-changes@7c8be3ffd0d6538312b363c8c949dbbf5f26c3dd # v0.0.4 + with: + head: ${{ github.sha }} + base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} + n_parents: 2 + + - name: Separate modules and subworkflows + id: outputs + run: | + echo modules=$(echo '${{ steps.list.outputs.components }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/nf-core/"; ""))') >> $GITHUB_OUTPUT + echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT + + - name: debug + run: | + echo ${{ steps.filter.outputs.components }} + echo ${{ steps.outputs.outputs.modules }} + echo ${{ steps.outputs.outputs.subworkflows }} + + nf-core-lint-modules: + runs-on: ${{ github.event.inputs.runners || 'ubuntu-latest' }} + name: nf-core-lint-modules + needs: [pytest-changes, nf-test-changes] + if: ${{ (needs.pytest-changes.outputs.modules != '[]') || ( needs.nf-test-changes.outputs.modules != '[]') }} + strategy: + fail-fast: false + matrix: + tags: + [ + "${{ fromJson(needs.pytest-changes.outputs.modules) }}", + "${{ fromJson(needs.nf-test-changes.outputs.modules) }}", + ] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + + - name: Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + with: + python-version: "3.11" + + - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4 + id: cache-pip + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip + restore-keys: | + ${{ runner.os }}-pip + + - name: Install pip + run: python -m pip install --upgrade pip + + - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 + with: + distribution: "temurin" + java-version: "17" + + - name: Setup Nextflow + uses: nf-core/setup-nextflow@v2 + + - name: Install nf-core tools development version + run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev + + - name: Lint module ${{ matrix.tags }} + run: nf-core modules lint ${{ matrix.tags }} + + nf-core-lint-subworkflows: + runs-on: ubuntu-latest + name: nf-core-lint-modules + needs: [pytest-changes, nf-test-changes] + if: ${{ (needs.pytest-changes.outputs.subworkflows != '[]') || ( needs.nf-test-changes.outputs.subworkflows != '[]') }} + strategy: + fail-fast: false + matrix: + tags: + [ + "${{ fromJson(needs.pytest-changes.outputs.subworkflows) }}", + "${{ fromJson(needs.nf-test-changes.outputs.subworkflows) }}", + ] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + + - name: Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + with: + python-version: "3.11" + + - name: Install pip + run: python -m pip install --upgrade pip + + - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 + with: + distribution: "temurin" + java-version: "17" + + - name: Setup Nextflow + uses: nf-core/setup-nextflow@561fcfc7146dcb12e3871909b635ab092a781f34 # v2 + + - name: Install nf-core tools development version + run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev + + - name: Lint module ${{ matrix.tags }} + run: nf-core subworkflows lint ${{ matrix.tags }} + + pytest: + runs-on: ${{ github.event.inputs.runners || 'ubuntu-latest' }} + name: pytest + needs: [pytest-changes] + if: needs.pytest-changes.outputs.tags != '[]' + strategy: + fail-fast: false + matrix: + tags: ["${{ fromJson(needs.pytest-changes.outputs.tags) }}"] + profile: [conda, docker, singularity] + exclude: + - tags: nf-test + - profile: conda + tags: backsub + - profile: conda + tags: bases2fastq + - profile: singularity + tags: bases2fastq + - profile: conda + tags: basicpy + - profile: conda + tags: bcl2fastq + - profile: conda + tags: bclconvert + - profile: conda + tags: bwa/aln + - profile: conda + tags: bwa/index + - profile: conda + tags: bwa/mem + - profile: conda + tags: bwa/sampe + - profile: conda + tags: bwa/samse + - profile: conda + tags: cellpose + - profile: conda + tags: cellrangerarc/count + - profile: conda + tags: cellrangerarc/mkfastq + - profile: conda + tags: cellrangerarc/mkref + - profile: conda + tags: cellrangeratac/count + - profile: conda + tags: cellrangeratac/mkfastq + - profile: conda + tags: cellrangeratac/mkref + - profile: conda + tags: checkm2/databasedownload + - profile: conda + tags: checkm2/predict + - profile: conda + tags: controlfreec/makegraph2 + - profile: conda + tags: coreograph + - profile: conda + tags: deepcell/mesmer + - profile: conda + tags: deepvariant + - profile: conda + tags: fastani + - profile: conda + tags: fastk/fastk + - profile: conda + tags: fastk/histex + - profile: conda + tags: fastk/merge + - profile: conda + tags: fcs/fcsadaptor + - profile: conda + tags: fcs/fcsgx + - profile: conda + tags: gatk4/cnnscorevariants + - profile: conda + tags: gatk4/determinegermlinecontigploidy + - profile: singularity + tags: gatk4/determinegermlinecontigploidy + - profile: conda + tags: gatk4/germlinecnvcaller + - profile: conda + tags: gatk4/postprocessgermlinecnvcalls + - profile: conda + tags: genescopefk + - profile: conda + tags: happy/sompy + - profile: conda + tags: hlala/preparegraph + - profile: conda + tags: ilastik/multicut + - profile: conda + tags: ilastik/pixelclassification + - profile: conda + tags: imputeme/vcftoprs + - profile: conda + tags: islandpath + - profile: conda + tags: manta/convertinversion + - profile: conda + tags: mcstaging/imc2mc + - profile: conda + tags: mcquant + - profile: conda + tags: medaka + - profile: conda + tags: merquryfk/katcomp + - profile: conda + tags: merquryfk/katgc + - profile: conda + tags: merquryfk/merquryfk + - profile: conda + tags: merquryfk/ploidyplot + - profile: conda + tags: minimap2/align + - profile: conda + tags: mitohifi/findmitoreference + - profile: conda + tags: mitohifi/mitohifi + - profile: conda + tags: nanoplot + - profile: conda + tags: ncbitools/vecscreen + - profile: conda + tags: parabricks/dbsnp + - profile: conda + tags: parabricks/deepvariant + - profile: conda + tags: parabricks/genotypegvcf + - profile: conda + tags: parabricks/haplotypecaller + - profile: conda + tags: parabricks/indexgvcf + - profile: conda + tags: parabricks/mutectcaller + - profile: conda + tags: picard/collecthsmetrics + - profile: conda + tags: picard/collectwgsmetrics + - profile: conda + tags: sentieon/applyvarcal + - profile: conda + tags: sentieon/datametrics + - profile: conda + tags: sentieon/dnamodelapply + - profile: conda + tags: sentieon/dnascope + - profile: conda + tags: sentieon/readwriter + - profile: conda + tags: sentieon/tnfilter + - profile: conda + tags: sentieon/tnhaplotyper2 + - profile: conda + tags: sentieon/tnscope + - profile: conda + tags: sentieon/varcal + - profile: conda + tags: sentieon/wgsmetrics + - profile: conda + tags: subworkflows/bam_qc_picard + - profile: conda + tags: subworkflows/bcl_demultiplex + - profile: conda + tags: subworkflows/fasta_clean_fcs + - profile: conda + tags: svanalyzer/svbenchmark + - profile: conda + tags: universc + - profile: singularity + tags: universc + - profile: conda + tags: vt/decompose + env: + NXF_ANSI_LOG: false + + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + + - name: Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + with: + python-version: "3.11" + + - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4 + id: cache-pip-pytest + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-pytest + restore-keys: | + ${{ runner.os }}-pip-pytest + + - name: Install Python dependencies + run: python -m pip install --upgrade pip pytest-workflow cryptography + + - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 + with: + distribution: "temurin" + java-version: "17" + - name: Setup Nextflow ${{ matrix.NXF_VER }} + uses: nf-core/setup-nextflow@v2 + with: + version: "${{ matrix.NXF_VER }}" + + - name: Setup apptainer + if: matrix.profile == 'singularity' + uses: eWaterCycle/setup-apptainer@main + + - name: Set up Singularity + if: matrix.profile == 'singularity' + run: | + mkdir -p $NXF_SINGULARITY_CACHEDIR + mkdir -p $NXF_SINGULARITY_LIBRARYDIR + + - name: Set up miniconda + uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3 + with: + miniconda-version: "latest" + channels: conda-forge,bioconda + python-version: ${{ matrix.python-version }} + conda-remove-defaults: true + + - name: Conda setup + run: | + conda clean -a + conda install -n base conda-libmamba-solver + conda config --set solver libmamba + echo $(realpath $CONDA)/condabin >> $GITHUB_PATH + echo $(realpath python) >> $GITHUB_PATH + + # Test the module + - name: Run pytest-workflow + # only use one thread for pytest-workflow to avoid race condition on conda cache. + run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes + + - name: Output log on failure + if: failure() + run: | + sudo apt-get update > /dev/null + sudo apt-get install bat > /dev/null + batcat --decorations=always --color=always /home/ubuntu/pytest_workflow_*/*/log.{out,err} + + - name: Setting global variables + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + id: parsed + with: + script: | + return '${{ matrix.tags }}'.toLowerCase().replaceAll(/\//g, '-').trim('-').trim('"') + result-encoding: string + + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4 + with: + name: logs-${{ matrix.profile }}-${{ steps.parsed.outputs.result }} + path: | + /home/ubuntu/pytest_workflow_*/*/.nextflow.log + /home/ubuntu/pytest_workflow_*/*/log.out + /home/ubuntu/pytest_workflow_*/*/log.err + /home/ubuntu/pytest_workflow_*/*/work + !/home/ubuntu/pytest_workflow_*/*/work/conda + !/home/ubuntu/pytest_workflow_*/*/work/singularity + !${{ github.workspace }}/.singularity + + nf-test: + runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} + name: nf-test + needs: [nf-test-changes] + if: ( needs.nf-test-changes.outputs.paths != '[]' ) + strategy: + fail-fast: false + matrix: + path: ["${{ fromJson(needs.nf-test-changes.outputs.paths) }}"] + profile: [conda, docker_self_hosted, singularity] + exclude: + - path: modules/nf-core/nf-test + - profile: conda + path: modules/nf-core/angsd/gl + - profile: conda + path: modules/nf-core/annotsv/installannotations + - profile: conda + path: modules/nf-core/happy/sompy + - profile: conda + path: modules/nf-core/backsub + - profile: conda + path: modules/nf-core/bakta/bakta + - profile: conda + path: modules/nf-core/bakta/baktadbdownload + - profile: conda + path: modules/nf-core/bases2fastq + - profile: conda + path: modules/nf-core/bcl2fastq + - profile: conda + path: modules/nf-core/bclconvert + - profile: conda + path: modules/nf-core/celesta + - profile: conda + path: modules/nf-core/cellpose + - profile: conda + path: modules/nf-core/cellranger/count + - profile: conda + path: modules/nf-core/cellranger/mkfastq + - profile: conda + path: modules/nf-core/cellranger/mkgtf + - profile: conda + path: modules/nf-core/cellranger/mkref + - profile: conda + path: modules/nf-core/cellranger/mkvdjref + - profile: conda + path: modules/nf-core/cellranger/multi + - profile: conda + path: modules/nf-core/cellranger/vdj + - profile: conda + path: modules/nf-core/checkqc + - profile: conda + path: modules/nf-core/custom/dumpsoftwareversions + - profile: conda + path: modules/nf-core/deepcell/mesmer + - profile: conda + path: modules/nf-core/deepsomatic + - profile: singularity + path: modules/nf-core/deepsomatic + - profile: conda + path: modules/nf-core/deepvariant + - profile: conda + path: modules/nf-core/deepvariant/callvariants + - profile: conda + path: modules/nf-core/deepvariant/makeexamples + - profile: conda + path: modules/nf-core/deepvariant/postprocessvariants + - profile: conda + path: modules/nf-core/deepvariant/rundeepvariant + - profile: conda + path: modules/nf-core/deepvariant/vcfstatsreport + - profile: conda + path: modules/nf-core/doubletdetection + - profile: conda + path: modules/nf-core/ensemblvep/vep + - profile: conda + path: modules/nf-core/fastk/fastk + - profile: conda + path: modules/nf-core/cellrangerarc/mkgtf + - profile: conda + path: modules/nf-core/fastk/histex + - profile: conda + path: modules/nf-core/fastk/merge + - profile: conda + path: modules/nf-core/fcs/fcsadaptor + - profile: conda + path: modules/nf-core/fcs/fcsgx + - profile: conda + path: modules/nf-core/ganon/buildcustom + - profile: conda + path: modules/nf-core/ganon/classify + - profile: conda + path: modules/nf-core/ganon/report + - profile: conda + path: modules/nf-core/ganon/table + - profile: conda + path: modules/nf-core/gatk4/cnnscorevariants + - profile: conda + path: modules/nf-core/gatk4/determinegermlinecontigploidy + - profile: conda + path: modules/nf-core/genescopefk + - profile: conda + path: modules/nf-core/ilastik/multicut + - profile: conda + path: modules/nf-core/ilastik/pixelclassification + - profile: conda + path: modules/nf-core/imputeme/vcftoprs + - profile: conda + path: modules/nf-core/mcstaging/imc2mc + - profile: conda + path: modules/nf-core/mcquant + - profile: conda + path: modules/nf-core/mcstaging/phenoimager2mc + - profile: conda + path: modules/nf-core/merquryfk/katcomp + - profile: conda + path: modules/nf-core/merquryfk/katgc + - profile: conda + path: modules/nf-core/merquryfk/merquryfk + - profile: conda + path: modules/nf-core/merquryfk/ploidyplot + - profile: conda + path: modules/nf-core/molkartgarage/clahe + - profile: conda + path: modules/nf-core/quartonotebook + - profile: conda + path: modules/nf-core/scimap/spatiallda + - profile: conda + path: modules/nf-core/sentieon/bwaindex + - profile: conda + path: modules/nf-core/sentieon/bwamem + - profile: conda + path: modules/nf-core/sentieon/datametrics + - profile: conda + path: modules/nf-core/sentieon/dedup + - profile: conda + path: modules/nf-core/sentieon/qualcal + - profile: conda + path: modules/nf-core/spaceranger/mkgtf + - profile: conda + path: modules/nf-core/spaceranger/mkref + - profile: conda + path: modules/nf-core/spaceranger/count + - profile: conda + path: modules/nf-core/spotiflow + - profile: conda + path: modules/nf-core/svanalyzer/svbenchmark + - profile: conda + path: modules/nf-core/universc + - profile: singularity + path: modules/nf-core/universc + - profile: conda + path: modules/nf-core/vt/decompose + - profile: singularity + path: modules/nf-core/bases2fastq + - profile: conda + path: modules/nf-core/wittyer + - profile: conda + path: modules/nf-core/islandpath + - profile: conda + path: modules/nf-core/scimap/mcmicro + - profile: conda + path: subworkflows/nf-core/vcf_annotate_ensemblvep + - profile: conda + path: subworkflows/nf-core/bcl_demultiplex + - profile: conda + path: subworkflows/nf-core/deepvariant + - profile: conda + path: subworkflows/nf-core/fastq_align_bamcmp_bwa + - profile: conda + path: subworkflows/nf-core/fastq_align_bwa + - profile: conda + path: subworkflows/nf-core/fasta_newick_epang_gappa + - path: modules/nf-core/parabricks/fq2bammeth + - path: modules/nf-core/parabricks/fq2bam + - path: modules/nf-core/parabricks/applybqsr + - path: modules/nf-core/parabricks/deepvariant + - profile: conda + path: modules/nf-core/xeniumranger/relabel + - profile: conda + path: modules/nf-core/xeniumranger/rename + - profile: conda + path: modules/nf-core/xeniumranger/resegment + - profile: conda + path: modules/nf-core/xeniumranger/import-segmentation + + env: + NXF_ANSI_LOG: false + + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + - name: Run nf-test Action + uses: ./.github/actions/nf-test-action + with: + path: ${{ matrix.path }} + profile: ${{ matrix.profile }} + + nf-test-gpu: + runs-on: "gpu" + name: nf-test-gpu + needs: [nf-test-changes] + if: ( needs.nf-test-changes.outputs.paths != '[]' && contains(needs.nf-test-changes.outputs.paths, 'modules/nf-core/parabricks') ) + strategy: + fail-fast: false + matrix: + include: + - path: modules/nf-core/parabricks/applybqsr + profile: [docker_self_hosted, singularity] + - path: modules/nf-core/parabricks/fq2bam + profile: [docker_self_hosted, singularity] + - path: modules/nf-core/parabricks/fq2bammeth + profile: [docker_self_hosted, singularity] + - path: modules/nf-core/parabricks/deepvariant + profile: [docker_self_hosted, singularity] + env: + NXF_ANSI_LOG: false + + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 + - name: Run nf-test Action + uses: ./.github/actions/nf-test-action + env: + SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} + SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} + SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} + SENTIEON_AUTH_MECH: "GitHub Actions - token" + with: + path: ${{ matrix.path }} + profile: ${{ matrix.profile }},gpu + + confirm-pass: + runs-on: ubuntu-latest + needs: + [ + prettier, + editorconfig, + pytest-changes, + nf-core-lint-modules, + nf-core-lint-subworkflows, + pytest, + nf-test-changes, + nf-test, + nf-test-gpu, + ] + if: always() + steps: + - name: All tests ok + if: ${{ success() || !contains(needs.*.result, 'failure') }} + run: exit 0 + - name: One or more tests failed + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 + + - name: debug-print + if: always() + run: | + echo "toJSON(needs) = ${{ toJSON(needs) }}" + echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" From f87d2d93784cc293022be4374136d7c5f821aaa9 Mon Sep 17 00:00:00 2001 From: famosab Date: Thu, 21 Nov 2024 16:57:43 +0100 Subject: [PATCH 04/14] swap to nextflow config --- .../nf-core/parabricks/deepvariant/tests/gvcf.config | 8 -------- .../parabricks/deepvariant/tests/main.nf.test | 12 +++++++++--- .../tests/{copy.config => nextflow.config} | 1 + 3 files changed, 10 insertions(+), 11 deletions(-) delete mode 100644 modules/nf-core/parabricks/deepvariant/tests/gvcf.config rename modules/nf-core/parabricks/deepvariant/tests/{copy.config => nextflow.config} (70%) diff --git a/modules/nf-core/parabricks/deepvariant/tests/gvcf.config b/modules/nf-core/parabricks/deepvariant/tests/gvcf.config deleted file mode 100644 index eac2f317a11..00000000000 --- a/modules/nf-core/parabricks/deepvariant/tests/gvcf.config +++ /dev/null @@ -1,8 +0,0 @@ -process { - - withName: 'PARABRICKS_DEEPVARIANT' { - ext.args = '--gvcf' - stageInMode = 'copy' - } - -} \ No newline at end of file diff --git a/modules/nf-core/parabricks/deepvariant/tests/main.nf.test b/modules/nf-core/parabricks/deepvariant/tests/main.nf.test index c28583007c4..75c0f3784f2 100644 --- a/modules/nf-core/parabricks/deepvariant/tests/main.nf.test +++ b/modules/nf-core/parabricks/deepvariant/tests/main.nf.test @@ -42,7 +42,7 @@ nextflow_process { test("human - bam - intervals") { - config './copy.config' + config './nextflow.config' when { process { @@ -75,9 +75,12 @@ nextflow_process { test("human - bam - intervals - gvcf") { - config './gvcf.config' + config './nextflow.config' when { + params { + module_args = '--gvcf' + } process { """ input[0] = [ @@ -139,10 +142,13 @@ nextflow_process { test("human - bam - intervals - gvcf - stub") { - config './gvcf.config' + config './nextflow.config' options "-stub" when { + params { + module_args = '--gvcf' + } process { """ input[0] = [ diff --git a/modules/nf-core/parabricks/deepvariant/tests/copy.config b/modules/nf-core/parabricks/deepvariant/tests/nextflow.config similarity index 70% rename from modules/nf-core/parabricks/deepvariant/tests/copy.config rename to modules/nf-core/parabricks/deepvariant/tests/nextflow.config index 71a3ba9e457..0a1d8b58204 100644 --- a/modules/nf-core/parabricks/deepvariant/tests/copy.config +++ b/modules/nf-core/parabricks/deepvariant/tests/nextflow.config @@ -1,6 +1,7 @@ process { withName: 'PARABRICKS_DEEPVARIANT' { + ext.args = params.module_args stageInMode = 'copy' } From 32e24facaab63d12b69c3677b744894a983b5bf2 Mon Sep 17 00:00:00 2001 From: famosab Date: Thu, 21 Nov 2024 16:59:27 +0100 Subject: [PATCH 05/14] finalize --- .github/workflows/test.yml | 748 ------------------------------------- 1 file changed, 748 deletions(-) delete mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index b6f044cb26c..00000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,748 +0,0 @@ -name: Run tests -on: - push: - branches: - # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging - - "renovate/**" # branches Renovate creates - pull_request: - branches: [master] - merge_group: - types: [checks_requested] - branches: [master] - workflow_dispatch: - inputs: - runners: - description: "Runners to test on" - type: choice - options: - - "ubuntu-latest" - - "self-hosted" - default: "self-hosted" - -# Cancel if a newer run is started -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -env: - NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity - NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - with: - python-version: "3.11" - - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 - # FIXME Flip this off once we get to less than a couple hundred. Adding - # this so it will only run against changed files. It'll make it much - # easier to fix these as they come up rather than everything at once. - with: - extra_args: "--all-files" - - prettier: - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - - name: Install NodeJS - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4 - with: - node-version: "20" - - - name: Install Prettier - run: npm install -g prettier@3.2.5 - - - name: Run Prettier --check - run: prettier --check . - - editorconfig: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4 - with: - node-version: "20" - - - name: Install editorconfig-checker - run: npm install -g editorconfig-checker - - - name: Run ECLint check - run: editorconfig-checker -exclude README.md $(git ls-files | grep -v test) - - pytest-changes: - name: pytest-changes - runs-on: ubuntu-latest - outputs: - tags: ${{ steps.filter.outputs.changes }} - modules: ${{ steps.tags.outputs.modules }} - subworkflows: ${{ steps.tags.outputs.subworkflows }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - with: - fetch-depth: 2 # To retrieve the preceding commit. - - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 - id: filter - with: - filters: "tests/config/pytest_modules.yml" - token: "" - - - name: Fetch module tags - id: tags - run: | - echo modules=$(echo '${{ steps.filter.outputs.changes }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/"; ""))') >> $GITHUB_OUTPUT - echo subworkflows=$(echo '${{ steps.filter.outputs.changes }}' | jq '. | map(select(contains("subworkflow"))) | map(gsub("subworkflows/"; ""))') >> $GITHUB_OUTPUT - - - name: debug - run: | - echo ${{ steps.tags.outputs.modules }} - echo ${{ steps.tags.outputs.subworkflows }} - - nf-test-changes: - name: nf-test-changes - runs-on: ubuntu-latest - outputs: - # Expose detected tags as 'modules' and 'workflows' output variables - paths: ${{ steps.list.outputs.components }} - modules: ${{ steps.outputs.outputs.modules }} - subworkflows: ${{ steps.outputs.outputs.subworkflows}} - # Prod for version bumping - - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - with: - fetch-depth: 0 - - - name: List nf-test files - id: list - uses: adamrtalbot/detect-nf-test-changes@7c8be3ffd0d6538312b363c8c949dbbf5f26c3dd # v0.0.4 - with: - head: ${{ github.sha }} - base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} - n_parents: 2 - - - name: Separate modules and subworkflows - id: outputs - run: | - echo modules=$(echo '${{ steps.list.outputs.components }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/nf-core/"; ""))') >> $GITHUB_OUTPUT - echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT - - - name: debug - run: | - echo ${{ steps.filter.outputs.components }} - echo ${{ steps.outputs.outputs.modules }} - echo ${{ steps.outputs.outputs.subworkflows }} - - nf-core-lint-modules: - runs-on: ${{ github.event.inputs.runners || 'ubuntu-latest' }} - name: nf-core-lint-modules - needs: [pytest-changes, nf-test-changes] - if: ${{ (needs.pytest-changes.outputs.modules != '[]') || ( needs.nf-test-changes.outputs.modules != '[]') }} - strategy: - fail-fast: false - matrix: - tags: - [ - "${{ fromJson(needs.pytest-changes.outputs.modules) }}", - "${{ fromJson(needs.nf-test-changes.outputs.modules) }}", - ] - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - - name: Set up Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - with: - python-version: "3.11" - - - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4 - id: cache-pip - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip - restore-keys: | - ${{ runner.os }}-pip - - - name: Install pip - run: python -m pip install --upgrade pip - - - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 - with: - distribution: "temurin" - java-version: "17" - - - name: Setup Nextflow - uses: nf-core/setup-nextflow@v2 - - - name: Install nf-core tools development version - run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev - - - name: Lint module ${{ matrix.tags }} - run: nf-core modules lint ${{ matrix.tags }} - - nf-core-lint-subworkflows: - runs-on: ubuntu-latest - name: nf-core-lint-modules - needs: [pytest-changes, nf-test-changes] - if: ${{ (needs.pytest-changes.outputs.subworkflows != '[]') || ( needs.nf-test-changes.outputs.subworkflows != '[]') }} - strategy: - fail-fast: false - matrix: - tags: - [ - "${{ fromJson(needs.pytest-changes.outputs.subworkflows) }}", - "${{ fromJson(needs.nf-test-changes.outputs.subworkflows) }}", - ] - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - - name: Set up Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - with: - python-version: "3.11" - - - name: Install pip - run: python -m pip install --upgrade pip - - - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 - with: - distribution: "temurin" - java-version: "17" - - - name: Setup Nextflow - uses: nf-core/setup-nextflow@561fcfc7146dcb12e3871909b635ab092a781f34 # v2 - - - name: Install nf-core tools development version - run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev - - - name: Lint module ${{ matrix.tags }} - run: nf-core subworkflows lint ${{ matrix.tags }} - - pytest: - runs-on: ${{ github.event.inputs.runners || 'ubuntu-latest' }} - name: pytest - needs: [pytest-changes] - if: needs.pytest-changes.outputs.tags != '[]' - strategy: - fail-fast: false - matrix: - tags: ["${{ fromJson(needs.pytest-changes.outputs.tags) }}"] - profile: [conda, docker, singularity] - exclude: - - tags: nf-test - - profile: conda - tags: backsub - - profile: conda - tags: bases2fastq - - profile: singularity - tags: bases2fastq - - profile: conda - tags: basicpy - - profile: conda - tags: bcl2fastq - - profile: conda - tags: bclconvert - - profile: conda - tags: bwa/aln - - profile: conda - tags: bwa/index - - profile: conda - tags: bwa/mem - - profile: conda - tags: bwa/sampe - - profile: conda - tags: bwa/samse - - profile: conda - tags: cellpose - - profile: conda - tags: cellrangerarc/count - - profile: conda - tags: cellrangerarc/mkfastq - - profile: conda - tags: cellrangerarc/mkref - - profile: conda - tags: cellrangeratac/count - - profile: conda - tags: cellrangeratac/mkfastq - - profile: conda - tags: cellrangeratac/mkref - - profile: conda - tags: checkm2/databasedownload - - profile: conda - tags: checkm2/predict - - profile: conda - tags: controlfreec/makegraph2 - - profile: conda - tags: coreograph - - profile: conda - tags: deepcell/mesmer - - profile: conda - tags: deepvariant - - profile: conda - tags: fastani - - profile: conda - tags: fastk/fastk - - profile: conda - tags: fastk/histex - - profile: conda - tags: fastk/merge - - profile: conda - tags: fcs/fcsadaptor - - profile: conda - tags: fcs/fcsgx - - profile: conda - tags: gatk4/cnnscorevariants - - profile: conda - tags: gatk4/determinegermlinecontigploidy - - profile: singularity - tags: gatk4/determinegermlinecontigploidy - - profile: conda - tags: gatk4/germlinecnvcaller - - profile: conda - tags: gatk4/postprocessgermlinecnvcalls - - profile: conda - tags: genescopefk - - profile: conda - tags: happy/sompy - - profile: conda - tags: hlala/preparegraph - - profile: conda - tags: ilastik/multicut - - profile: conda - tags: ilastik/pixelclassification - - profile: conda - tags: imputeme/vcftoprs - - profile: conda - tags: islandpath - - profile: conda - tags: manta/convertinversion - - profile: conda - tags: mcstaging/imc2mc - - profile: conda - tags: mcquant - - profile: conda - tags: medaka - - profile: conda - tags: merquryfk/katcomp - - profile: conda - tags: merquryfk/katgc - - profile: conda - tags: merquryfk/merquryfk - - profile: conda - tags: merquryfk/ploidyplot - - profile: conda - tags: minimap2/align - - profile: conda - tags: mitohifi/findmitoreference - - profile: conda - tags: mitohifi/mitohifi - - profile: conda - tags: nanoplot - - profile: conda - tags: ncbitools/vecscreen - - profile: conda - tags: parabricks/dbsnp - - profile: conda - tags: parabricks/deepvariant - - profile: conda - tags: parabricks/genotypegvcf - - profile: conda - tags: parabricks/haplotypecaller - - profile: conda - tags: parabricks/indexgvcf - - profile: conda - tags: parabricks/mutectcaller - - profile: conda - tags: picard/collecthsmetrics - - profile: conda - tags: picard/collectwgsmetrics - - profile: conda - tags: sentieon/applyvarcal - - profile: conda - tags: sentieon/datametrics - - profile: conda - tags: sentieon/dnamodelapply - - profile: conda - tags: sentieon/dnascope - - profile: conda - tags: sentieon/readwriter - - profile: conda - tags: sentieon/tnfilter - - profile: conda - tags: sentieon/tnhaplotyper2 - - profile: conda - tags: sentieon/tnscope - - profile: conda - tags: sentieon/varcal - - profile: conda - tags: sentieon/wgsmetrics - - profile: conda - tags: subworkflows/bam_qc_picard - - profile: conda - tags: subworkflows/bcl_demultiplex - - profile: conda - tags: subworkflows/fasta_clean_fcs - - profile: conda - tags: svanalyzer/svbenchmark - - profile: conda - tags: universc - - profile: singularity - tags: universc - - profile: conda - tags: vt/decompose - env: - NXF_ANSI_LOG: false - - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - - name: Set up Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - with: - python-version: "3.11" - - - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4 - id: cache-pip-pytest - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-pytest - restore-keys: | - ${{ runner.os }}-pip-pytest - - - name: Install Python dependencies - run: python -m pip install --upgrade pip pytest-workflow cryptography - - - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 - with: - distribution: "temurin" - java-version: "17" - - name: Setup Nextflow ${{ matrix.NXF_VER }} - uses: nf-core/setup-nextflow@v2 - with: - version: "${{ matrix.NXF_VER }}" - - - name: Setup apptainer - if: matrix.profile == 'singularity' - uses: eWaterCycle/setup-apptainer@main - - - name: Set up Singularity - if: matrix.profile == 'singularity' - run: | - mkdir -p $NXF_SINGULARITY_CACHEDIR - mkdir -p $NXF_SINGULARITY_LIBRARYDIR - - - name: Set up miniconda - uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3 - with: - miniconda-version: "latest" - channels: conda-forge,bioconda - python-version: ${{ matrix.python-version }} - conda-remove-defaults: true - - - name: Conda setup - run: | - conda clean -a - conda install -n base conda-libmamba-solver - conda config --set solver libmamba - echo $(realpath $CONDA)/condabin >> $GITHUB_PATH - echo $(realpath python) >> $GITHUB_PATH - - # Test the module - - name: Run pytest-workflow - # only use one thread for pytest-workflow to avoid race condition on conda cache. - run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes - - - name: Output log on failure - if: failure() - run: | - sudo apt-get update > /dev/null - sudo apt-get install bat > /dev/null - batcat --decorations=always --color=always /home/ubuntu/pytest_workflow_*/*/log.{out,err} - - - name: Setting global variables - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 - id: parsed - with: - script: | - return '${{ matrix.tags }}'.toLowerCase().replaceAll(/\//g, '-').trim('-').trim('"') - result-encoding: string - - - name: Upload logs on failure - if: failure() - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4 - with: - name: logs-${{ matrix.profile }}-${{ steps.parsed.outputs.result }} - path: | - /home/ubuntu/pytest_workflow_*/*/.nextflow.log - /home/ubuntu/pytest_workflow_*/*/log.out - /home/ubuntu/pytest_workflow_*/*/log.err - /home/ubuntu/pytest_workflow_*/*/work - !/home/ubuntu/pytest_workflow_*/*/work/conda - !/home/ubuntu/pytest_workflow_*/*/work/singularity - !${{ github.workspace }}/.singularity - - nf-test: - runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} - name: nf-test - needs: [nf-test-changes] - if: ( needs.nf-test-changes.outputs.paths != '[]' ) - strategy: - fail-fast: false - matrix: - path: ["${{ fromJson(needs.nf-test-changes.outputs.paths) }}"] - profile: [conda, docker_self_hosted, singularity] - exclude: - - path: modules/nf-core/nf-test - - profile: conda - path: modules/nf-core/angsd/gl - - profile: conda - path: modules/nf-core/annotsv/installannotations - - profile: conda - path: modules/nf-core/happy/sompy - - profile: conda - path: modules/nf-core/backsub - - profile: conda - path: modules/nf-core/bakta/bakta - - profile: conda - path: modules/nf-core/bakta/baktadbdownload - - profile: conda - path: modules/nf-core/bases2fastq - - profile: conda - path: modules/nf-core/bcl2fastq - - profile: conda - path: modules/nf-core/bclconvert - - profile: conda - path: modules/nf-core/celesta - - profile: conda - path: modules/nf-core/cellpose - - profile: conda - path: modules/nf-core/cellranger/count - - profile: conda - path: modules/nf-core/cellranger/mkfastq - - profile: conda - path: modules/nf-core/cellranger/mkgtf - - profile: conda - path: modules/nf-core/cellranger/mkref - - profile: conda - path: modules/nf-core/cellranger/mkvdjref - - profile: conda - path: modules/nf-core/cellranger/multi - - profile: conda - path: modules/nf-core/cellranger/vdj - - profile: conda - path: modules/nf-core/checkqc - - profile: conda - path: modules/nf-core/custom/dumpsoftwareversions - - profile: conda - path: modules/nf-core/deepcell/mesmer - - profile: conda - path: modules/nf-core/deepsomatic - - profile: singularity - path: modules/nf-core/deepsomatic - - profile: conda - path: modules/nf-core/deepvariant - - profile: conda - path: modules/nf-core/deepvariant/callvariants - - profile: conda - path: modules/nf-core/deepvariant/makeexamples - - profile: conda - path: modules/nf-core/deepvariant/postprocessvariants - - profile: conda - path: modules/nf-core/deepvariant/rundeepvariant - - profile: conda - path: modules/nf-core/deepvariant/vcfstatsreport - - profile: conda - path: modules/nf-core/doubletdetection - - profile: conda - path: modules/nf-core/ensemblvep/vep - - profile: conda - path: modules/nf-core/fastk/fastk - - profile: conda - path: modules/nf-core/cellrangerarc/mkgtf - - profile: conda - path: modules/nf-core/fastk/histex - - profile: conda - path: modules/nf-core/fastk/merge - - profile: conda - path: modules/nf-core/fcs/fcsadaptor - - profile: conda - path: modules/nf-core/fcs/fcsgx - - profile: conda - path: modules/nf-core/ganon/buildcustom - - profile: conda - path: modules/nf-core/ganon/classify - - profile: conda - path: modules/nf-core/ganon/report - - profile: conda - path: modules/nf-core/ganon/table - - profile: conda - path: modules/nf-core/gatk4/cnnscorevariants - - profile: conda - path: modules/nf-core/gatk4/determinegermlinecontigploidy - - profile: conda - path: modules/nf-core/genescopefk - - profile: conda - path: modules/nf-core/ilastik/multicut - - profile: conda - path: modules/nf-core/ilastik/pixelclassification - - profile: conda - path: modules/nf-core/imputeme/vcftoprs - - profile: conda - path: modules/nf-core/mcstaging/imc2mc - - profile: conda - path: modules/nf-core/mcquant - - profile: conda - path: modules/nf-core/mcstaging/phenoimager2mc - - profile: conda - path: modules/nf-core/merquryfk/katcomp - - profile: conda - path: modules/nf-core/merquryfk/katgc - - profile: conda - path: modules/nf-core/merquryfk/merquryfk - - profile: conda - path: modules/nf-core/merquryfk/ploidyplot - - profile: conda - path: modules/nf-core/molkartgarage/clahe - - profile: conda - path: modules/nf-core/quartonotebook - - profile: conda - path: modules/nf-core/scimap/spatiallda - - profile: conda - path: modules/nf-core/sentieon/bwaindex - - profile: conda - path: modules/nf-core/sentieon/bwamem - - profile: conda - path: modules/nf-core/sentieon/datametrics - - profile: conda - path: modules/nf-core/sentieon/dedup - - profile: conda - path: modules/nf-core/sentieon/qualcal - - profile: conda - path: modules/nf-core/spaceranger/mkgtf - - profile: conda - path: modules/nf-core/spaceranger/mkref - - profile: conda - path: modules/nf-core/spaceranger/count - - profile: conda - path: modules/nf-core/spotiflow - - profile: conda - path: modules/nf-core/svanalyzer/svbenchmark - - profile: conda - path: modules/nf-core/universc - - profile: singularity - path: modules/nf-core/universc - - profile: conda - path: modules/nf-core/vt/decompose - - profile: singularity - path: modules/nf-core/bases2fastq - - profile: conda - path: modules/nf-core/wittyer - - profile: conda - path: modules/nf-core/islandpath - - profile: conda - path: modules/nf-core/scimap/mcmicro - - profile: conda - path: subworkflows/nf-core/vcf_annotate_ensemblvep - - profile: conda - path: subworkflows/nf-core/bcl_demultiplex - - profile: conda - path: subworkflows/nf-core/deepvariant - - profile: conda - path: subworkflows/nf-core/fastq_align_bamcmp_bwa - - profile: conda - path: subworkflows/nf-core/fastq_align_bwa - - profile: conda - path: subworkflows/nf-core/fasta_newick_epang_gappa - - path: modules/nf-core/parabricks/fq2bammeth - - path: modules/nf-core/parabricks/fq2bam - - path: modules/nf-core/parabricks/applybqsr - - path: modules/nf-core/parabricks/deepvariant - - profile: conda - path: modules/nf-core/xeniumranger/relabel - - profile: conda - path: modules/nf-core/xeniumranger/rename - - profile: conda - path: modules/nf-core/xeniumranger/resegment - - profile: conda - path: modules/nf-core/xeniumranger/import-segmentation - - env: - NXF_ANSI_LOG: false - - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - - name: Run nf-test Action - uses: ./.github/actions/nf-test-action - with: - path: ${{ matrix.path }} - profile: ${{ matrix.profile }} - - nf-test-gpu: - runs-on: "gpu" - name: nf-test-gpu - needs: [nf-test-changes] - if: ( needs.nf-test-changes.outputs.paths != '[]' && contains(needs.nf-test-changes.outputs.paths, 'modules/nf-core/parabricks') ) - strategy: - fail-fast: false - matrix: - include: - - path: modules/nf-core/parabricks/applybqsr - profile: [docker_self_hosted, singularity] - - path: modules/nf-core/parabricks/fq2bam - profile: [docker_self_hosted, singularity] - - path: modules/nf-core/parabricks/fq2bammeth - profile: [docker_self_hosted, singularity] - - path: modules/nf-core/parabricks/deepvariant - profile: [docker_self_hosted, singularity] - env: - NXF_ANSI_LOG: false - - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - - name: Run nf-test Action - uses: ./.github/actions/nf-test-action - env: - SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} - SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} - SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} - SENTIEON_AUTH_MECH: "GitHub Actions - token" - with: - path: ${{ matrix.path }} - profile: ${{ matrix.profile }},gpu - - confirm-pass: - runs-on: ubuntu-latest - needs: - [ - prettier, - editorconfig, - pytest-changes, - nf-core-lint-modules, - nf-core-lint-subworkflows, - pytest, - nf-test-changes, - nf-test, - nf-test-gpu, - ] - if: always() - steps: - - name: All tests ok - if: ${{ success() || !contains(needs.*.result, 'failure') }} - run: exit 0 - - name: One or more tests failed - if: ${{ contains(needs.*.result, 'failure') }} - run: exit 1 - - - name: debug-print - if: always() - run: | - echo "toJSON(needs) = ${{ toJSON(needs) }}" - echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" From 3f7444d98b0b314c74b01e97a12e10b22554f186 Mon Sep 17 00:00:00 2001 From: famosab Date: Fri, 22 Nov 2024 10:35:52 +0100 Subject: [PATCH 06/14] revert changes to nf-test config --- tests/config/nf-test.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index 1918fa26258..6f4c169f3c8 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -30,7 +30,7 @@ profiles { } docker { docker.enabled = true -docker.runOptions = '-u $(id -u):$(id -g) --gpus device=0 --platform=linux/amd64' + docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' } docker_self_hosted { docker.enabled = true From 220f99d7df1596cc3ef7f6da90722b58c030d7e2 Mon Sep 17 00:00:00 2001 From: famosab Date: Fri, 22 Nov 2024 10:36:25 +0100 Subject: [PATCH 07/14] revert changes to nf-test config --- tests/config/nf-test.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index 6f4c169f3c8..fa21cb2edd6 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -30,8 +30,8 @@ profiles { } docker { docker.enabled = true - docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' -} + docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' + } docker_self_hosted { docker.enabled = true docker.fixOwnership = true From 35eab5d9471f7af4931481f483d9ab8ff544c7dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Mon, 2 Dec 2024 13:46:31 +0100 Subject: [PATCH 08/14] add to conda_skip --- .github/conda_skip.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/conda_skip.yml b/.github/conda_skip.yml index 2ef7216da3e..1a0cb83fae7 100644 --- a/.github/conda_skip.yml +++ b/.github/conda_skip.yml @@ -182,3 +182,5 @@ exclude: path: modules/nf-core/xeniumranger/resegment - profile: conda path: modules/nf-core/xeniumranger/import-segmentation + - profile: conda + path: modules/nf-core/parabricks/deepvariant From 053c1d3fc7849ea7d50dab377f86aa04e8e41203 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Mon, 2 Dec 2024 13:53:18 +0100 Subject: [PATCH 09/14] add tag gpu --- modules/nf-core/parabricks/deepvariant/tests/main.nf.test | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/parabricks/deepvariant/tests/main.nf.test b/modules/nf-core/parabricks/deepvariant/tests/main.nf.test index 75c0f3784f2..8fe993d815e 100644 --- a/modules/nf-core/parabricks/deepvariant/tests/main.nf.test +++ b/modules/nf-core/parabricks/deepvariant/tests/main.nf.test @@ -8,6 +8,7 @@ nextflow_process { tag "modules_nfcore" tag "parabricks" tag "parabricks/deepvariant" + tag "gpu" test("human - bam") { @@ -100,10 +101,10 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot( + { assert snapshot( process.out.versions, // file(process.out.vcf[0][1]).name // fails with groovy.lang.MissingMethodException: No signature of method: static com.askimed.nf.test.lang.extensions.GlobalMethods.file() is applicable for argument types: (ArrayList) values: - ).match() + ).match() } ) } From 13420bb27d7edbdfdcee3c8217fdc4b053eb0ef2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20B=C3=A4uerle?= <45968370+famosab@users.noreply.github.com> Date: Wed, 11 Dec 2024 14:22:29 +0100 Subject: [PATCH 10/14] Update modules/nf-core/parabricks/deepvariant/tests/nextflow.config --- modules/nf-core/parabricks/deepvariant/tests/nextflow.config | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/nf-core/parabricks/deepvariant/tests/nextflow.config b/modules/nf-core/parabricks/deepvariant/tests/nextflow.config index 0a1d8b58204..c21ce810e0d 100644 --- a/modules/nf-core/parabricks/deepvariant/tests/nextflow.config +++ b/modules/nf-core/parabricks/deepvariant/tests/nextflow.config @@ -2,7 +2,6 @@ process { withName: 'PARABRICKS_DEEPVARIANT' { ext.args = params.module_args - stageInMode = 'copy' } } \ No newline at end of file From dc182102295d308832f70160f95188585ca86755 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20B=C3=A4uerle?= <45968370+famosab@users.noreply.github.com> Date: Wed, 11 Dec 2024 14:22:34 +0100 Subject: [PATCH 11/14] Update modules/nf-core/parabricks/deepvariant/main.nf --- modules/nf-core/parabricks/deepvariant/main.nf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/nf-core/parabricks/deepvariant/main.nf b/modules/nf-core/parabricks/deepvariant/main.nf index b9eeef84069..1389079c47b 100644 --- a/modules/nf-core/parabricks/deepvariant/main.nf +++ b/modules/nf-core/parabricks/deepvariant/main.nf @@ -1,6 +1,8 @@ process PARABRICKS_DEEPVARIANT { tag "$meta.id" label 'process_high' + label 'process_gpu' + stageInMode 'copy' // needed by the module to work properly can be removed when fixed upstream container "nvcr.io/nvidia/clara/clara-parabricks:4.4.0-1" From d13179d06d6e4254e7207a92ec862625c0e7cea8 Mon Sep 17 00:00:00 2001 From: famosab Date: Mon, 16 Dec 2024 11:15:07 +0100 Subject: [PATCH 12/14] fix tests and add myself as maintainer to other modules --- modules/nf-core/parabricks/applybqsr/meta.yml | 1 + modules/nf-core/parabricks/dbsnp/meta.yml | 2 + .../nf-core/parabricks/deepvariant/main.nf | 11 +-- .../nf-core/parabricks/deepvariant/meta.yml | 2 + .../parabricks/deepvariant/tests/main.nf.test | 59 ++++++++++-- .../deepvariant/tests/main.nf.test.snap | 90 +++++++++++++++---- modules/nf-core/parabricks/fq2bam/meta.yml | 1 + .../nf-core/parabricks/genotypegvcf/meta.yml | 2 + modules/nf-core/parabricks/indexgvcf/meta.yml | 2 + .../nf-core/parabricks/mutectcaller/meta.yml | 2 + 10 files changed, 143 insertions(+), 29 deletions(-) diff --git a/modules/nf-core/parabricks/applybqsr/meta.yml b/modules/nf-core/parabricks/applybqsr/meta.yml index 22f0c4c2067..3bf25c6832f 100644 --- a/modules/nf-core/parabricks/applybqsr/meta.yml +++ b/modules/nf-core/parabricks/applybqsr/meta.yml @@ -88,3 +88,4 @@ authors: - "@bsiranosian" maintainers: - "@bsiranosian" + - "@famosab" \ No newline at end of file diff --git a/modules/nf-core/parabricks/dbsnp/meta.yml b/modules/nf-core/parabricks/dbsnp/meta.yml index 095110cf45e..fe363cdecf8 100644 --- a/modules/nf-core/parabricks/dbsnp/meta.yml +++ b/modules/nf-core/parabricks/dbsnp/meta.yml @@ -50,3 +50,5 @@ output: pattern: "versions.yml" authors: - "@Furentsu" +maintainers: + - "@famosab" \ No newline at end of file diff --git a/modules/nf-core/parabricks/deepvariant/main.nf b/modules/nf-core/parabricks/deepvariant/main.nf index 1389079c47b..d390e6c860d 100644 --- a/modules/nf-core/parabricks/deepvariant/main.nf +++ b/modules/nf-core/parabricks/deepvariant/main.nf @@ -11,8 +11,9 @@ process PARABRICKS_DEEPVARIANT { tuple val(ref_meta), path(fasta) output: - tuple val(meta), path("*.vcf"), emit: vcf - path "versions.yml", emit: versions + tuple val(meta), path("*.vcf"), optional: true, emit: vcf + tuple val(meta), path("*.g.vcf.gz"), optional: true, emit: gvcf + path "versions.yml", emit: versions when: task.ext.when == null || task.ext.when @@ -24,7 +25,7 @@ process PARABRICKS_DEEPVARIANT { } def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def output_file = ("--gvcf" =~ task.ext.args)? "${prefix}.g.vcf" : "${prefix}.vcf" + def output_file = ("--gvcf" =~ task.ext.args)? "${prefix}.g.vcf.gz" : "${prefix}.vcf" def interval_file_command = interval_file ? interval_file.collect{"--interval-file $it"}.join(' ') : "" def num_gpus = task.accelerator ? "--num-gpus $task.accelerator.request" : '' """ @@ -45,9 +46,9 @@ process PARABRICKS_DEEPVARIANT { stub: def prefix = task.ext.prefix ?: "${meta.id}" - def output_file = ("--gvcf" =~ task.ext.args)? "${prefix}.g.vcf" : "${prefix}.vcf" + def output_cmd = ("--gvcf" =~ task.ext.args)? "echo '' | gzip > ${prefix}.g.vcf.gz" : "touch ${prefix}.vcf" """ - touch $output_file + $output_cmd cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/parabricks/deepvariant/meta.yml b/modules/nf-core/parabricks/deepvariant/meta.yml index 55998ea65e9..9147167a700 100644 --- a/modules/nf-core/parabricks/deepvariant/meta.yml +++ b/modules/nf-core/parabricks/deepvariant/meta.yml @@ -62,3 +62,5 @@ output: pattern: "versions.yml" authors: - "@bsiranosian" +maintainers: + - "@famosab" diff --git a/modules/nf-core/parabricks/deepvariant/tests/main.nf.test b/modules/nf-core/parabricks/deepvariant/tests/main.nf.test index 8fe993d815e..b0c6309228b 100644 --- a/modules/nf-core/parabricks/deepvariant/tests/main.nf.test +++ b/modules/nf-core/parabricks/deepvariant/tests/main.nf.test @@ -34,7 +34,8 @@ nextflow_process { { assert process.success }, { assert snapshot( path(process.out.vcf[0][1]).vcf.variantsMD5, - process.out.versions + process.out.versions, + path(process.out.versions[0]).yaml ).match() } ) } @@ -43,8 +44,6 @@ nextflow_process { test("human - bam - intervals") { - config './nextflow.config' - when { process { """ @@ -67,13 +66,52 @@ nextflow_process { { assert process.success }, { assert snapshot( path(process.out.vcf[0][1]).vcf.variantsMD5, - process.out.versions + process.out.versions, + path(process.out.versions[0]).yaml ).match() } ) } } + test("human - bam - gvcf") { + + config './nextflow.config' + + when { + params { + module_args = '--gvcf' + } + process { + """ + input[0] = [ + [ id:'test'], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/illumina/bam/test2.paired_end.recalibrated.sorted.bam', checkIfExists: true), + [], // bam.bai not needed unless using intervals + [] + ] + input[1] = [ + [ id:'ref'], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/genome/chr21/sequence/genome.fasta', checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( + path(process.out.gvcf[0][1]).vcf.variantsMD5, + process.out.versions, + path(process.out.versions[0]).yaml + ).match() + } + ) + } + + } + test("human - bam - intervals - gvcf") { config './nextflow.config' @@ -102,8 +140,9 @@ nextflow_process { assertAll( { assert process.success }, { assert snapshot( + path(process.out.gvcf[0][1]).vcf.variantsMD5, process.out.versions, - // file(process.out.vcf[0][1]).name // fails with groovy.lang.MissingMethodException: No signature of method: static com.askimed.nf.test.lang.extensions.GlobalMethods.file() is applicable for argument types: (ArrayList) values: + path(process.out.versions[0]).yaml ).match() } ) @@ -135,7 +174,10 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot( + process.out, + path(process.out.versions[0]).yaml + ).match() } ) } @@ -169,7 +211,10 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot( process.out ).match() } + { assert snapshot( + process.out, + path(process.out.versions[0]).yaml + ).match() } ) } diff --git a/modules/nf-core/parabricks/deepvariant/tests/main.nf.test.snap b/modules/nf-core/parabricks/deepvariant/tests/main.nf.test.snap index bb19b848414..451403c4054 100644 --- a/modules/nf-core/parabricks/deepvariant/tests/main.nf.test.snap +++ b/modules/nf-core/parabricks/deepvariant/tests/main.nf.test.snap @@ -1,15 +1,21 @@ { "human - bam - intervals - gvcf": { "content": [ + "8916c6ed7e05a061aa322e70c19f4fd7", [ "versions.yml:md5,c7bcf80d609e2951eb99f0b774cd2f6d" - ] + ], + { + "PARABRICKS_DEEPVARIANT": { + "pbrun": "4.4.0-1" + } + } ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.0" + "nextflow": "24.10.2" }, - "timestamp": "2024-11-14T15:26:48.481295634" + "timestamp": "2024-12-16T11:10:08.45019036" }, "human - bam - stub": { "content": [ @@ -23,7 +29,13 @@ ] ], "1": [ + + ], + "2": [ "versions.yml:md5,c7bcf80d609e2951eb99f0b774cd2f6d" + ], + "gvcf": [ + ], "vcf": [ [ @@ -36,71 +48,115 @@ "versions": [ "versions.yml:md5,c7bcf80d609e2951eb99f0b774cd2f6d" ] + }, + { + "PARABRICKS_DEEPVARIANT": { + "pbrun": "4.4.0-1" + } } ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.0" + "nextflow": "24.10.2" }, - "timestamp": "2024-11-14T15:05:28.449273519" + "timestamp": "2024-12-16T11:12:53.473635656" }, "human - bam - intervals": { "content": [ "27097afc7aae62918a3e26790cc189c3", [ "versions.yml:md5,c7bcf80d609e2951eb99f0b774cd2f6d" - ] + ], + { + "PARABRICKS_DEEPVARIANT": { + "pbrun": "4.4.0-1" + } + } ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.0" + "nextflow": "24.10.2" }, - "timestamp": "2024-11-14T15:06:52.854870182" + "timestamp": "2024-12-16T11:12:14.133679608" }, "human - bam": { "content": [ "8b526a232f93a5036176336415abf7d1", [ "versions.yml:md5,c7bcf80d609e2951eb99f0b774cd2f6d" - ] + ], + { + "PARABRICKS_DEEPVARIANT": { + "pbrun": "4.4.0-1" + } + } ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.0" + "nextflow": "24.10.2" }, - "timestamp": "2024-11-14T15:06:37.651354312" + "timestamp": "2024-12-16T11:11:59.107617915" }, "human - bam - intervals - gvcf - stub": { "content": [ { "0": [ + + ], + "1": [ [ { "id": "test" }, - "test.g.vcf:md5,d41d8cd98f00b204e9800998ecf8427e" + "test.g.vcf.gz:md5,68b329da9893e34099c7d8ad5cb9c940" ] ], - "1": [ + "2": [ "versions.yml:md5,c7bcf80d609e2951eb99f0b774cd2f6d" ], - "vcf": [ + "gvcf": [ [ { "id": "test" }, - "test.g.vcf:md5,d41d8cd98f00b204e9800998ecf8427e" + "test.g.vcf.gz:md5,68b329da9893e34099c7d8ad5cb9c940" ] + ], + "vcf": [ + ], "versions": [ "versions.yml:md5,c7bcf80d609e2951eb99f0b774cd2f6d" ] + }, + { + "PARABRICKS_DEEPVARIANT": { + "pbrun": "4.4.0-1" + } + } + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.2" + }, + "timestamp": "2024-12-16T11:13:01.9854302" + }, + "human - bam - gvcf": { + "content": [ + "be72819d107808dcca17323e64b7b9a2", + [ + "versions.yml:md5,c7bcf80d609e2951eb99f0b774cd2f6d" + ], + { + "PARABRICKS_DEEPVARIANT": { + "pbrun": "4.4.0-1" + } } ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.0" + "nextflow": "24.10.2" }, - "timestamp": "2024-11-14T15:12:11.438654866" + "timestamp": "2024-12-16T11:09:50.417459549" } } \ No newline at end of file diff --git a/modules/nf-core/parabricks/fq2bam/meta.yml b/modules/nf-core/parabricks/fq2bam/meta.yml index b3e2e7e6ee6..af6496c1a8c 100644 --- a/modules/nf-core/parabricks/fq2bam/meta.yml +++ b/modules/nf-core/parabricks/fq2bam/meta.yml @@ -109,3 +109,4 @@ maintainers: - "@bsiranosian" - "@adamrtalbot" - "@gallvp" + - "@famosab" \ No newline at end of file diff --git a/modules/nf-core/parabricks/genotypegvcf/meta.yml b/modules/nf-core/parabricks/genotypegvcf/meta.yml index a876011edd0..a044fa88795 100644 --- a/modules/nf-core/parabricks/genotypegvcf/meta.yml +++ b/modules/nf-core/parabricks/genotypegvcf/meta.yml @@ -53,3 +53,5 @@ output: authors: - "@Furentsu" - "@bsiranosian" +maintainers: + - "@famosab" \ No newline at end of file diff --git a/modules/nf-core/parabricks/indexgvcf/meta.yml b/modules/nf-core/parabricks/indexgvcf/meta.yml index bc0e277660a..16834c645d6 100644 --- a/modules/nf-core/parabricks/indexgvcf/meta.yml +++ b/modules/nf-core/parabricks/indexgvcf/meta.yml @@ -44,3 +44,5 @@ output: authors: - "@Furentsu" - "@bsiranosian" +maintainers: + - "@famosab" \ No newline at end of file diff --git a/modules/nf-core/parabricks/mutectcaller/meta.yml b/modules/nf-core/parabricks/mutectcaller/meta.yml index e3c2a37e0ff..416c286731f 100644 --- a/modules/nf-core/parabricks/mutectcaller/meta.yml +++ b/modules/nf-core/parabricks/mutectcaller/meta.yml @@ -90,3 +90,5 @@ output: pattern: "versions.yml" authors: - "@bsiranosian" +maintainers: + - "@famosab" \ No newline at end of file From e1297f31bfa7c1b1cf1fd8fd9de359c756578ded Mon Sep 17 00:00:00 2001 From: famosab Date: Mon, 16 Dec 2024 11:18:23 +0100 Subject: [PATCH 13/14] fix meta --- modules/nf-core/parabricks/deepvariant/meta.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/modules/nf-core/parabricks/deepvariant/meta.yml b/modules/nf-core/parabricks/deepvariant/meta.yml index 9147167a700..21a55a47ef0 100644 --- a/modules/nf-core/parabricks/deepvariant/meta.yml +++ b/modules/nf-core/parabricks/deepvariant/meta.yml @@ -53,8 +53,18 @@ output: e.g. [ id:'test' ] - "*.vcf": type: file - description: Variant file. + description: vcf file created with deepvariant (does not support .gz for normal vcf), optional pattern: "*.vcf" + - gvcf: + - meta: + type: map + description: | + Groovy Map containing sample information. + e.g. [ id:'test' ] + - "*.g.vcf.gz": + type: file + description: bgzipped gvcf created with deepvariant, optional + pattern: "*.g.vcf.gz" - versions: - versions.yml: type: file From 98b1c2f3341b2d85e06810a3246c626538eb45fa Mon Sep 17 00:00:00 2001 From: famosab Date: Wed, 8 Jan 2025 11:53:12 +0100 Subject: [PATCH 14/14] add isssue to comment --- modules/nf-core/parabricks/deepvariant/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-core/parabricks/deepvariant/main.nf b/modules/nf-core/parabricks/deepvariant/main.nf index d390e6c860d..8c8c1d7dbca 100644 --- a/modules/nf-core/parabricks/deepvariant/main.nf +++ b/modules/nf-core/parabricks/deepvariant/main.nf @@ -2,7 +2,7 @@ process PARABRICKS_DEEPVARIANT { tag "$meta.id" label 'process_high' label 'process_gpu' - stageInMode 'copy' // needed by the module to work properly can be removed when fixed upstream + stageInMode 'copy' // needed by the module to work properly can be removed when fixed upstream - Issue #7226 container "nvcr.io/nvidia/clara/clara-parabricks:4.4.0-1"