Merge branch 'develop' of github.com:opencb/java-common-libs into dev…

…elop
opencb · Oct 3, 2024 · 2122af0 · 2122af0
2 parents e44daec + 0840f50
commit 2122af0
Show file tree

Hide file tree

Showing 10 changed files with 182 additions and 43 deletions.
diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml
@@ -6,11 +6,36 @@ on:
     types: [ submitted ]
 
 jobs:
-  build:
-    uses: ./.github/workflows/build-java-app-workflow.yml
-
+  calculate-xetabase-branch:
+    name: Calculate Xetabase branch
+    runs-on: ubuntu-22.04
+    outputs:
+      xetabase_branch: ${{ steps.get_xetabase_branch.outputs.xetabase_branch }}
+    steps:
+      - name: Clone java-common-libs
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: '10'
+          ## This is important to avoid the error in the next step: "fatal: repository 'https://github.com/zetta-genomics/opencga-enterprise.git/' not found"
+          persist-credentials: false
+      - id: get_xetabase_branch
+        name: "Get current branch for Xetabase from target branch"
+        run: |
+          chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh
+          echo "github.event.pull_request.base.ref: ${{ github.event.pull_request.base.ref }}"
+          echo "github.event.pull_request.head.ref: ${{ github.event.pull_request.head.ref }}"
+          echo "secrets.ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}"
+          xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.base.ref }})
+          echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY}
+          echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT
+        env:
+          ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}
+
   test:
     name: "Run all tests before merging"
-    uses: ./.github/workflows/test-analysis.yml
-    needs: build
+    needs: calculate-xetabase-branch
+    uses: opencb/java-common-libs/.github/workflows/test-xetabase-workflow.yml@develop
+    with:
+      branch: ${{ needs.calculate-xetabase-branch.outputs.xetabase_branch }}
+      task: ${{ github.event.pull_request.head.ref }}
     secrets: inherit
diff --git a/.github/workflows/scripts/get-xetabase-branch.sh b/.github/workflows/scripts/get-xetabase-branch.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+set -e
+set -x
+# Function to calculate the corresponding branch of Xetabase project
+get_xetabase_branch() {
+  # Input parameter (branch name)
+  input_branch="$1"
+
+  # If the branch begins with 'TASK' and exists in the opencga-enterprise repository, I return it
+  if [[ $input_branch == TASK* ]]; then
+    if [ "$(git ls-remote "https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git" "$input_branch" )" ] ; then
+      echo $input_branch;
+      return 0;
+    fi
+  fi
+
+  # Check if the branch name is "develop" in that case return the same branch name
+  if [[ "$input_branch" == "develop" ]]; then
+    echo "develop"
+    return 0
+  fi
+
+  # Check if the branch name starts with "release-" and follows the patterns "release-a.x.x" or "release-a.b.x"
+  if [[ "$input_branch" =~ ^release-([0-9]+)\.x\.x$ ]] || [[ "$input_branch" =~ ^release-([0-9]+)\.([0-9]+)\.x$ ]]; then
+    # Extract the MAJOR part of the branch name
+    MAJOR=${BASH_REMATCH[1]}
+    # Calculate the XETABASE_MAJOR by subtracting 3 from MAJOR
+    XETABASE_MAJOR=$((MAJOR - 3))
+    # Check if the XETABASE_MAJOR is negative
+    if (( XETABASE_MAJOR < 0 )); then
+      echo "Error: 'MAJOR' digit after subtraction results in a negative number."
+      return 1
+    fi
+    # Construct and echo the new branch name
+    echo "release-$XETABASE_MAJOR.${input_branch#release-$MAJOR.}"
+    return 0
+  fi
+
+  # If the branch name does not match any of the expected patterns
+  echo "Error: The branch name is not correct."
+  return 1
+}
+
+
+# Call the function with the input branch name
+get_xetabase_branch "$1"
diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml
@@ -19,33 +19,53 @@ on:
         type: string
         description: 'Branch of opencga-enterprise to be tested and built.'
         required: true
-env:
-  AZCOPY_SPA_CLIENT_SECRET: ${{ secrets.AZCOPY_SPA_CLIENT_SECRET }}
-  AZCOPY_AUTO_LOGIN_TYPE: "SPN"
-  AZCOPY_SPA_APPLICATION_ID: ${{ secrets.AZCOPY_SPA_APPLICATION_ID }}
-  AZCOPY_TENANT_ID: ${{ secrets.AZCOPY_TENANT_ID }}
 
 jobs:
   test:
     name: Execute JUnit and Jacoco tests
     runs-on: ubuntu-22.04
     steps:
-      - name: Clone OpenCGA Enterprise branch '${{ github.event.inputs.branch }}'
+      - name: Retrieve secrets from Keeper
+        id: ksecrets
+        uses: Keeper-Security/ksm-action@master
+        with:
+          keeper-secret-config: ${{ secrets.KEEPER_SM_GH_OPENCB }}
+          secrets: |
+            AZURE_KUBE_CONFIG/field/Secret Value > env:AZURE_KUBE_CONFIG
+            DOCKER_HUB_USER/field/Secret Value > env:DOCKER_HUB_USER
+            DOCKER_HUB_PASSWORD/field/Secret Value > env:DOCKER_HUB_PASSWORD
+            SSH_TESTING_SERVER_HOST/field/Secret Value > env:SSH_HOST
+            SSH_TESTING_SERVER_PORT/field/Secret Value > env:SSH_PORT
+            SSH_TESTING_SERVER_USER/field/Secret Value > env:SSH_USER
+            SSH_TESTING_SERVER_PASSWORD/field/Secret Value > env:SSH_PASS
+      - name: Log inputs
+        run: |
+          echo "__OpenCGA-enterprise branch:__ \"${{ inputs.branch }}\"" | tee -a $GITHUB_STEP_SUMMARY
+          echo "__Task to test:__ \"${{ inputs.task }}\"" | tee -a $GITHUB_STEP_SUMMARY
+      - name: Clone OpenCGA Enterprise branch '${{ inputs.branch }}'
         uses: actions/checkout@v4
         with:
           repository: zetta-genomics/opencga-enterprise
-          ref: ${{ github.event.inputs.branch }}
+          ref: ${{ inputs.branch }}
           token: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}
           path: opencga-enterprise
           fetch-depth: "10"
       - id: get_opencga_branch
-        name: Get OpenCGA branch from 'pom.xml' property
+        name: Get OpenCGA branch
         run: |
-          pwd
-          chmod +x ./opencga-enterprise/.github/workflows/scripts/xetabase-branch.sh
-          opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/xetabase-branch.sh ${{ github.event.inputs.task }})
+          # If the task exists in the opencga repository, this is the branch to be tested
+          if [[ "${{ inputs.task }}" == TASK* ]]; then
+            if [ "$(git ls-remote "https://github.com/opencb/opencga.git" "${{ inputs.task }}")" ]; then
+              opencga_branch="${{ inputs.task }}";
+            fi
+          else
+            chmod +x ./opencga-enterprise/.github/workflows/scripts/get-opencga-xetabase-branch.sh
+            opencga_branch=$(./opencga-enterprise/.github/workflows/scripts/get-opencga-xetabase-branch.sh)
+          fi      
+          echo "opencga_branch=${opencga_branch}"
           echo "opencga_branch=${opencga_branch}" >> $GITHUB_OUTPUT
-      - uses: actions/checkout@v4
+      - name: Clone OpenCGA branch '${{ steps.get_opencga_branch.outputs.opencga_branch }}'
+        uses: actions/checkout@v4
         with:
           repository: opencb/opencga
           ref: ${{ steps.get_opencga_branch.outputs.opencga_branch }}
@@ -70,39 +90,39 @@ jobs:
           chmod +x ./kubectl
           echo "${{ secrets.AZURE_KUBE_CONFIG }}" > admin.conf
           ./kubectl -n cellbase-db port-forward services/cellbase-rs0-svc 27018:27017 --kubeconfig ./admin.conf &
-      - name: Install Azure AZCOPY
-        uses: kheiakiyama/install-azcopy-action@v1
-        with:
-          version: 'v10'
       - name: DockerHub login
         uses: docker/login-action@v3
         with:
-          username: ${{ secrets.DOCKER_HUB_USER }}
-          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          username: ${{ env.DOCKER_HUB_USER }}
+          password: ${{ env.DOCKER_HUB_PASSWORD }}
+      - name: Install sshpass
+        run: sudo apt-get install sshpass
+      - name: Add SSH Host to known_hosts
+        run: |
+          mkdir -p ~/.ssh
+          ssh-keyscan -p ${{ env.SSH_PORT }} ${{ env.SSH_HOST }} >> ~/.ssh/known_hosts
+        env:
+          SSH_HOST: ${{ env.SSH_HOST }}
+          SSH_PORT: ${{ env.SSH_PORT }}
       - name: Run all OpenCB Junit tests, ie. java-common-libs, biodata, cellbase, opencga and opencga-enterprise
         run: |
-          ln -s opencga opencga-enterprise/opencga-home
           cd opencga-enterprise
-          ./build.sh -t -l runShortTests,runMediumTests,runLongTests -b -s -f -T ${{ github.event.inputs.task }} -c localhost:27018 -H hdp3.1
+          ln -s ../opencga opencga-home
+          ./build.sh -t -l runShortTests -b -s -T ${{ inputs.task }} -c localhost:27018 -H hdp3.1
       - name: Upload reports results to Github
         uses: actions/upload-artifact@v4
         with:
           name: report-test
-          path: /home/runner/work/testing-environment/testing-environment/opencga-enterprise/reports/test
+          path: ./opencga-enterprise/reports/test
       - name: Upload log
         uses: actions/upload-artifact@v4
         with:
           name: build-log
-          path: /home/runner/work/testing-environment/testing-environment/opencga-enterprise/build.log
-      - name: Upload junit reports to a remote scp server
-        uses: garygrossgarten/github-action-scp@release
-        with:
-          local: opencga-enterprise/reports/test
-          remote: /var/www/html/reports/xetabase/${{ github.event.inputs.task }}/
-          host: ${{ secrets.SSH_TESTING_SERVER_HOST}}
-          port: ${{ secrets.SSH_TESTING_SERVER_PORT}}
-          username: ${{ secrets.SSH_TESTING_SERVER_USER }}
-          password: ${{ secrets.SSH_TESTING_SERVER_PASSWORD }}
-          concurrency: 2
+          path: ./opencga-enterprise/build.log
+      - name: Log summary
+        run: |
+          cat ./opencga-enterprise/build.log | tee -a $GITHUB_STEP_SUMMARY
+          
+          
 
 
diff --git a/.github/workflows/tmp-test-xetabase-branch.yml b/.github/workflows/tmp-test-xetabase-branch.yml
@@ -0,0 +1,48 @@
+name: TMP test-xetabase-branch
+run-name: 'Pull request approve workflow ${{ github.event.pull_request.head.ref }} -> ${{ github.event.pull_request.base.ref }} by @${{ github.actor }}'
+
+on:
+  push:
+    branches:
+      - TASK-*
+
+
+jobs:
+  calculate-xetabase-branch:
+    name: Calculate Xetabase branch
+    runs-on: ubuntu-22.04
+    outputs:
+      xetabase_branch: ${{ steps.get_xetabase_branch.outputs.xetabase_branch }}
+    steps:
+      - name: Clone java-common-libs
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: '10'
+          ## This is important to avoid the error in the next step: "fatal: repository 'https://github.com/zetta-genomics/opencga-enterprise.git/' not found"
+          persist-credentials: false
+      - id: get_xetabase_branch
+        name: "Get current branch for Xetabase from target branch"
+        run: |
+          if [ "$( git ls-remote https://$ZETTA_REPO_ACCESS_TOKEN@github.com/opencb/opencga.git "TASK-6879" )" ] ; then
+           echo "OPENCGA TASK-6879 branch";
+          fi
+          
+          if [ "$( git ls-remote https://$ZETTA_REPO_ACCESS_TOKEN@github.com/zetta-genomics/opencga-enterprise.git "$input_branch" )" ] ; then
+            echo "Here it is TASK-6807 branch";
+          fi
+          chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh
+          echo "secrets.ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}" | tee -a ${GITHUB_STEP_SUMMARY}
+          xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh "TASK-6879" )
+          echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY}
+          echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT
+        env:
+          ZETTA_REPO_ACCESS_TOKEN: ${{ secrets.ZETTA_REPO_ACCESS_TOKEN }}
+
+  test:
+    name: "Run all tests before merging"
+    needs: calculate-xetabase-branch
+    uses: opencb/java-common-libs/.github/workflows/test-xetabase-workflow.yml@develop
+    with:
+      branch: ${{ needs.calculate-xetabase-branch.outputs.xetabase_branch }}
+      task: TASK-6879
+    secrets: inherit
diff --git a/commons-datastore/commons-datastore-core/pom.xml b/commons-datastore/commons-datastore-core/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.opencb.commons</groupId>
         <artifactId>commons-datastore</artifactId>
-        <version>5.3.0-SNAPSHOT</version>
+        <version>6.0.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/commons-datastore/commons-datastore-mongodb/pom.xml b/commons-datastore/commons-datastore-mongodb/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.opencb.commons</groupId>
         <artifactId>commons-datastore</artifactId>
-        <version>5.3.0-SNAPSHOT</version>
+        <version>6.0.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/commons-datastore/commons-datastore-solr/pom.xml b/commons-datastore/commons-datastore-solr/pom.xml
@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.opencb.commons</groupId>
         <artifactId>commons-datastore</artifactId>
-        <version>5.3.0-SNAPSHOT</version>
+        <version>6.0.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/commons-datastore/pom.xml b/commons-datastore/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.opencb.commons</groupId>
         <artifactId>commons</artifactId>
-        <version>5.3.0-SNAPSHOT</version>
+        <version>6.0.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/commons-lib/pom.xml b/commons-lib/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.opencb.commons</groupId>
         <artifactId>commons</artifactId>
-        <version>5.3.0-SNAPSHOT</version>
+        <version>6.0.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,7 @@
 
     <groupId>org.opencb.commons</groupId>
     <artifactId>commons</artifactId>
-    <version>5.3.0-SNAPSHOT</version>
+    <version>6.0.0-SNAPSHOT</version>
     <packaging>pom</packaging>
 
     <name>OpenCB commons project</name>