From 281b8678ba8fb76596c0d6d9c841ceeb8d64a412 Mon Sep 17 00:00:00 2001 From: benlee0423 Date: Thu, 7 Mar 2024 14:23:51 -0800 Subject: [PATCH 01/14] run build and push as root user --- .github/workflows/singularity_image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/singularity_image.yml b/.github/workflows/singularity_image.yml index b4c1cb4..12751e8 100644 --- a/.github/workflows/singularity_image.yml +++ b/.github/workflows/singularity_image.yml @@ -18,8 +18,8 @@ jobs: - name: Build Singularity Image run: | - singularity build ciroh-ngen-singularity.sif singularity/singularity_ngen.def + sudo singularity build ciroh-ngen-singularity.sif singularity/singularity_ngen.def - name: Push Singularity Image to Library run: | - singularity push -U ciroh-ngen-singularity.sif library://awiciroh/ngiab/ciroh-ngen-singularity:latest + sudo singularity push -U ciroh-ngen-singularity.sif library://awiciroh/ngiab/ciroh-ngen-singularity:latest From 7c8dbff3892954f1a7b8d5d22eaec9ba4db500cf Mon Sep 17 00:00:00 2001 From: benlee0423 Date: Thu, 7 Mar 2024 14:27:28 -0800 Subject: [PATCH 02/14] change directory --- .github/workflows/singularity_image.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/singularity_image.yml b/.github/workflows/singularity_image.yml index 12751e8..572ceec 100644 --- a/.github/workflows/singularity_image.yml +++ b/.github/workflows/singularity_image.yml @@ -18,8 +18,10 @@ jobs: - name: Build Singularity Image run: | - sudo singularity build ciroh-ngen-singularity.sif singularity/singularity_ngen.def + cd singularity + sudo singularity build ciroh-ngen-singularity.sif singularity_ngen.def - name: Push Singularity Image to Library run: | + cd singularity sudo singularity push -U ciroh-ngen-singularity.sif library://awiciroh/ngiab/ciroh-ngen-singularity:latest From 4a08c3da23ae096575cf3b6e809279bcc8ab11e1 Mon Sep 17 00:00:00 2001 From: benlee0423 Date: Thu, 7 Mar 2024 14:53:16 -0800 Subject: [PATCH 03/14] add remote login --- .github/workflows/singularity_image.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/singularity_image.yml b/.github/workflows/singularity_image.yml index 572ceec..42c6f67 100644 --- a/.github/workflows/singularity_image.yml +++ b/.github/workflows/singularity_image.yml @@ -11,17 +11,19 @@ on: jobs: build: runs-on: X86 - + defaults: + run: + shell: bash + working-directory: ./singularity steps: - name: Checkout Repository uses: actions/checkout@v2 - - name: Build Singularity Image - run: | - cd singularity - sudo singularity build ciroh-ngen-singularity.sif singularity_ngen.def + #- name: Build Singularity Image + # run: | + # sudo singularity build ciroh-ngen-singularity.sif singularity_ngen.def - name: Push Singularity Image to Library run: | - cd singularity + echo ${{ secrets.SINGULARITY_TOKEN }} | singularity remote login -u ${{ secrets.SINGULARITY_USERNAME }} --password-stdin sudo singularity push -U ciroh-ngen-singularity.sif library://awiciroh/ngiab/ciroh-ngen-singularity:latest From 2692a976379d8aa97005db90d4ee6ed772d8a35e Mon Sep 17 00:00:00 2001 From: benlee0423 Date: Thu, 7 Mar 2024 15:02:51 -0800 Subject: [PATCH 04/14] add username --- .github/workflows/singularity_image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/singularity_image.yml b/.github/workflows/singularity_image.yml index 42c6f67..c23779c 100644 --- a/.github/workflows/singularity_image.yml +++ b/.github/workflows/singularity_image.yml @@ -25,5 +25,5 @@ jobs: - name: Push Singularity Image to Library run: | - echo ${{ secrets.SINGULARITY_TOKEN }} | singularity remote login -u ${{ secrets.SINGULARITY_USERNAME }} --password-stdin + echo ${{ secrets.SINGULARITY_TOKEN }} | singularity remote login -u awiciroh --password-stdin sudo singularity push -U ciroh-ngen-singularity.sif library://awiciroh/ngiab/ciroh-ngen-singularity:latest From 7408b09bb369781cc127dbe3a7a3e969050b7ec4 Mon Sep 17 00:00:00 2001 From: benlee0423 Date: Thu, 7 Mar 2024 15:11:15 -0800 Subject: [PATCH 05/14] remove login --- .github/workflows/singularity_image.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/singularity_image.yml b/.github/workflows/singularity_image.yml index c23779c..b7e4415 100644 --- a/.github/workflows/singularity_image.yml +++ b/.github/workflows/singularity_image.yml @@ -25,5 +25,4 @@ jobs: - name: Push Singularity Image to Library run: | - echo ${{ secrets.SINGULARITY_TOKEN }} | singularity remote login -u awiciroh --password-stdin sudo singularity push -U ciroh-ngen-singularity.sif library://awiciroh/ngiab/ciroh-ngen-singularity:latest From d814de3080f38a1f61d649805b3d5f35c305b29e Mon Sep 17 00:00:00 2001 From: benlee0423 Date: Thu, 7 Mar 2024 15:28:59 -0800 Subject: [PATCH 06/14] add token as env --- .github/workflows/singularity_image.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/singularity_image.yml b/.github/workflows/singularity_image.yml index b7e4415..a775db4 100644 --- a/.github/workflows/singularity_image.yml +++ b/.github/workflows/singularity_image.yml @@ -26,3 +26,5 @@ jobs: - name: Push Singularity Image to Library run: | sudo singularity push -U ciroh-ngen-singularity.sif library://awiciroh/ngiab/ciroh-ngen-singularity:latest + env: + SINGULARITY_HUB_TOKEN: ${{ secrets.SINGULARITY_HUB_TOKEN }} From e20bb4da029cdb6451c12bc30643ed20854826d3 Mon Sep 17 00:00:00 2001 From: benlee0423 Date: Thu, 7 Mar 2024 16:12:54 -0800 Subject: [PATCH 07/14] remove sudo --- .github/workflows/singularity_image.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/singularity_image.yml b/.github/workflows/singularity_image.yml index a775db4..d5a70e4 100644 --- a/.github/workflows/singularity_image.yml +++ b/.github/workflows/singularity_image.yml @@ -25,6 +25,4 @@ jobs: - name: Push Singularity Image to Library run: | - sudo singularity push -U ciroh-ngen-singularity.sif library://awiciroh/ngiab/ciroh-ngen-singularity:latest - env: - SINGULARITY_HUB_TOKEN: ${{ secrets.SINGULARITY_HUB_TOKEN }} + singularity push -U /home/ubuntu/workspace/Ngen-Singularity/singularity/ciroh-ngen-singularity.sif library://awiciroh/ngiab/ciroh-ngen-singularity:latest \ No newline at end of file From 483bcb5538ba8839063673b24751935b15e8a605 Mon Sep 17 00:00:00 2001 From: benlee0423 Date: Thu, 7 Mar 2024 16:15:51 -0800 Subject: [PATCH 08/14] uncomment build --- .github/workflows/singularity_image.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/singularity_image.yml b/.github/workflows/singularity_image.yml index d5a70e4..5224e4f 100644 --- a/.github/workflows/singularity_image.yml +++ b/.github/workflows/singularity_image.yml @@ -19,9 +19,9 @@ jobs: - name: Checkout Repository uses: actions/checkout@v2 - #- name: Build Singularity Image - # run: | - # sudo singularity build ciroh-ngen-singularity.sif singularity_ngen.def + - name: Build Singularity Image + run: | + sudo singularity build ciroh-ngen-singularity.sif singularity_ngen.def - name: Push Singularity Image to Library run: | From 2b2b33a14457f4e1b09a7b48d1d99117a03a9434 Mon Sep 17 00:00:00 2001 From: benlee0423 Date: Thu, 7 Mar 2024 16:19:20 -0800 Subject: [PATCH 09/14] sif file directory change --- .github/workflows/singularity_image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/singularity_image.yml b/.github/workflows/singularity_image.yml index 5224e4f..9f286ec 100644 --- a/.github/workflows/singularity_image.yml +++ b/.github/workflows/singularity_image.yml @@ -25,4 +25,4 @@ jobs: - name: Push Singularity Image to Library run: | - singularity push -U /home/ubuntu/workspace/Ngen-Singularity/singularity/ciroh-ngen-singularity.sif library://awiciroh/ngiab/ciroh-ngen-singularity:latest \ No newline at end of file + singularity push -U ciroh-ngen-singularity.sif library://awiciroh/ngiab/ciroh-ngen-singularity:latest \ No newline at end of file From 32715f7a21e7ee2240051f7a7eb26d8ab52c046f Mon Sep 17 00:00:00 2001 From: benlee0423 Date: Wed, 20 Mar 2024 12:06:50 -0700 Subject: [PATCH 10/14] add test --- .github/workflows/singularity_image.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/singularity_image.yml b/.github/workflows/singularity_image.yml index 9f286ec..b2ba8b3 100644 --- a/.github/workflows/singularity_image.yml +++ b/.github/workflows/singularity_image.yml @@ -23,6 +23,13 @@ jobs: run: | sudo singularity build ciroh-ngen-singularity.sif singularity_ngen.def + - name: Test + run: | + rm -rf /home/ubuntu/workspace/AWI_09_004/*.csv + rm -rf /home/ubuntu/workspace/AWI_09_004/*.parquet + rm -rf /home/ubuntu/workspace/AWI_09_004/outputs/* + singularity run --bind /home/ubuntu/workspace/AWI_09_004:/ngen/ngen/data ciroh-ngen-singularity.sif /ngen/ngen/data auto + - name: Push Singularity Image to Library run: | singularity push -U ciroh-ngen-singularity.sif library://awiciroh/ngiab/ciroh-ngen-singularity:latest \ No newline at end of file From 901f239af173689c10e6945e9f88a531bcbc6b99 Mon Sep 17 00:00:00 2001 From: benlee0423 Date: Tue, 26 Mar 2024 21:01:51 -0700 Subject: [PATCH 11/14] use commit f91e2ea --- .github/workflows/singularity_image.yml | 1 + singularity/templates/ngen/install_ngen.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/singularity_image.yml b/.github/workflows/singularity_image.yml index b2ba8b3..ef6e0a1 100644 --- a/.github/workflows/singularity_image.yml +++ b/.github/workflows/singularity_image.yml @@ -25,6 +25,7 @@ jobs: - name: Test run: | + rm -rf /home/ubuntu/workspace/AWI_09_004/partitions*.json rm -rf /home/ubuntu/workspace/AWI_09_004/*.csv rm -rf /home/ubuntu/workspace/AWI_09_004/*.parquet rm -rf /home/ubuntu/workspace/AWI_09_004/outputs/* diff --git a/singularity/templates/ngen/install_ngen.sh b/singularity/templates/ngen/install_ngen.sh index 6d95d46..bbbce8e 100755 --- a/singularity/templates/ngen/install_ngen.sh +++ b/singularity/templates/ngen/install_ngen.sh @@ -22,6 +22,7 @@ then fi git clone https://github.com/NOAA-OWP/ngen.git cd ngen +git checkout f91e2ea echo "===========================================================" echo "===========================================================" From 51269d9d98bc8edd397d9815c7dc64155b87e6b1 Mon Sep 17 00:00:00 2001 From: benlee0423 Date: Wed, 27 Mar 2024 21:00:59 -0700 Subject: [PATCH 12/14] add test --- .github/workflows/singularity_image.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/singularity_image.yml b/.github/workflows/singularity_image.yml index ef6e0a1..8c16d81 100644 --- a/.github/workflows/singularity_image.yml +++ b/.github/workflows/singularity_image.yml @@ -30,7 +30,14 @@ jobs: rm -rf /home/ubuntu/workspace/AWI_09_004/*.parquet rm -rf /home/ubuntu/workspace/AWI_09_004/outputs/* singularity run --bind /home/ubuntu/workspace/AWI_09_004:/ngen/ngen/data ciroh-ngen-singularity.sif /ngen/ngen/data auto - + output_count=`ls /home/ubuntu/workspace/AWI_09_004/outputs | wc -l` + if [[ "$output_count" < 1 ]]; then + echo "No outputs found" + exit 1 + else + echo "Test run successfully" + fi + - name: Push Singularity Image to Library run: | singularity push -U ciroh-ngen-singularity.sif library://awiciroh/ngiab/ciroh-ngen-singularity:latest \ No newline at end of file From 92e64ff21c1329decb4f30e34453014831ba636b Mon Sep 17 00:00:00 2001 From: benlee0423 Date: Wed, 27 Mar 2024 21:20:57 -0700 Subject: [PATCH 13/14] use quote --- .github/workflows/singularity_image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/singularity_image.yml b/.github/workflows/singularity_image.yml index 8c16d81..d6376a6 100644 --- a/.github/workflows/singularity_image.yml +++ b/.github/workflows/singularity_image.yml @@ -29,7 +29,7 @@ jobs: rm -rf /home/ubuntu/workspace/AWI_09_004/*.csv rm -rf /home/ubuntu/workspace/AWI_09_004/*.parquet rm -rf /home/ubuntu/workspace/AWI_09_004/outputs/* - singularity run --bind /home/ubuntu/workspace/AWI_09_004:/ngen/ngen/data ciroh-ngen-singularity.sif /ngen/ngen/data auto + singularity run --bind /home/ubuntu/workspace/AWI_09_004:/ngen/ngen/data ciroh-ngen-singularity.sif "/ngen/ngen/data auto" output_count=`ls /home/ubuntu/workspace/AWI_09_004/outputs | wc -l` if [[ "$output_count" < 1 ]]; then echo "No outputs found" From 6ebb6ffd26e6d17111f5da77fe27a67ae8ea322e Mon Sep 17 00:00:00 2001 From: benlee0423 Date: Wed, 27 Mar 2024 21:57:49 -0700 Subject: [PATCH 14/14] use 2 partitions --- .github/workflows/singularity_image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/singularity_image.yml b/.github/workflows/singularity_image.yml index d6376a6..66bb4fe 100644 --- a/.github/workflows/singularity_image.yml +++ b/.github/workflows/singularity_image.yml @@ -29,7 +29,7 @@ jobs: rm -rf /home/ubuntu/workspace/AWI_09_004/*.csv rm -rf /home/ubuntu/workspace/AWI_09_004/*.parquet rm -rf /home/ubuntu/workspace/AWI_09_004/outputs/* - singularity run --bind /home/ubuntu/workspace/AWI_09_004:/ngen/ngen/data ciroh-ngen-singularity.sif "/ngen/ngen/data auto" + singularity run --bind /home/ubuntu/workspace/AWI_09_004:/ngen/ngen/data ciroh-ngen-singularity.sif "/ngen/ngen/data auto 2" output_count=`ls /home/ubuntu/workspace/AWI_09_004/outputs | wc -l` if [[ "$output_count" < 1 ]]; then echo "No outputs found"