From 7bea95bea6a8db985361be6297cd20e406056f93 Mon Sep 17 00:00:00 2001
From: hhasija <106058340+hhasija@users.noreply.github.com>
Date: Wed, 13 Nov 2024 18:32:24 +0530
Subject: [PATCH] fix: Jenkins optimisation update (#1024)

* fix: adding blogpost link for BigQuery to JDBC

* fix: including pom.xml and jenkins file for running test cases

* fix: making changes in UI Jenkins file as well

* fix: changes made for python jenkins file as well

* fix: including requirements.txt file of python as well
---
 java/.ci/Jenkinsfile   |  44 ++++++++--------
 java/.ci/UiJenkinsfile |  20 ++++----
 python/.ci/Jenkinsfile | 112 ++++++++++++++++++++---------------------
 3 files changed, 88 insertions(+), 88 deletions(-)

diff --git a/java/.ci/Jenkinsfile b/java/.ci/Jenkinsfile
index a178f96f..d2dfc4c6 100644
--- a/java/.ci/Jenkinsfile
+++ b/java/.ci/Jenkinsfile
@@ -101,9 +101,9 @@ EOF
         stage('JDBC TO BQ'){
         when {
                 expression {
-                    // Check if there are any changes in Java files in the last commit
+                    // Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
                     return sh(
-                        script: 'git diff --name-only HEAD~1 | grep -q ".java"',
+                        script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
                         returnStatus: true
                     ) == 0
                 }
@@ -130,9 +130,9 @@ EOF
         stage('Parallel Execution 1'){
         when {
                 expression {
-                    // Check if there are any changes in Java files in the last commit
+                    // Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
                     return sh(
-                        script: 'git diff --name-only HEAD~1 | grep -q ".java"',
+                        script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
                         returnStatus: true
                     ) == 0
                 }
@@ -262,9 +262,9 @@ EOF
         stage('Parallel Execution 2'){
         when {
                 expression {
-                    // Check if there are any changes in Java files in the last commit
+                    // Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
                     return sh(
-                        script: 'git diff --name-only HEAD~1 | grep -q ".java"',
+                        script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
                         returnStatus: true
                     ) == 0
                 }
@@ -391,9 +391,9 @@ EOF
         stage('Parallel Execution 3'){
         when {
                 expression {
-                    // Check if there are any changes in Java files in the last commit
+                    // Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
                     return sh(
-                        script: 'git diff --name-only HEAD~1 | grep -q ".java"',
+                        script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
                         returnStatus: true
                     ) == 0
                 }
@@ -494,9 +494,9 @@ EOF
         stage('Parallel Execution 4'){
         when {
                 expression {
-                    // Check if there are any changes in Java files in the last commit
+                    // Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
                     return sh(
-                        script: 'git diff --name-only HEAD~1 | grep -q ".java"',
+                        script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
                         returnStatus: true
                     ) == 0
                 }
@@ -603,9 +603,9 @@ EOF
         stage('Parallel Execution 5'){
         when {
                 expression {
-                    // Check if there are any changes in Java files in the last commit
+                    // Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
                     return sh(
-                        script: 'git diff --name-only HEAD~1 | grep -q ".java"',
+                        script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
                         returnStatus: true
                     ) == 0
                 }
@@ -699,9 +699,9 @@ EOF
         stage('Parallel Execution 6'){
         when {
                 expression {
-                    // Check if there are any changes in Java files in the last commit
+                    // Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
                     return sh(
-                        script: 'git diff --name-only HEAD~1 | grep -q ".java"',
+                        script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
                         returnStatus: true
                     ) == 0
                 }
@@ -802,9 +802,9 @@ EOF
         stage('Parallel Execution 7'){
         when {
                 expression {
-                    // Check if there are any changes in Java files in the last commit
+                    // Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
                     return sh(
-                        script: 'git diff --name-only HEAD~1 | grep -q ".java"',
+                        script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
                         returnStatus: true
                     ) == 0
                 }
@@ -884,9 +884,9 @@ EOF
         stage('Parallel Execution 8'){
         when {
                 expression {
-                    // Check if there are any changes in Java files in the last commit
+                    // Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
                     return sh(
-                        script: 'git diff --name-only HEAD~1 | grep -q ".java"',
+                        script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
                         returnStatus: true
                     ) == 0
                 }
@@ -989,9 +989,9 @@ EOF
         stage('Parallel Execution 9'){
         when {
                 expression {
-                    // Check if there are any changes in Java files in the last commit
+                    // Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
                     return sh(
-                        script: 'git diff --name-only HEAD~1 | grep -q ".java"',
+                        script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
                         returnStatus: true
                     ) == 0
                 }
@@ -1105,9 +1105,9 @@ EOF
         stage('Parallel Execution 10'){
         when {
                 expression {
-                    // Check if there are any changes in Java files in the last commit
+                    // Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
                     return sh(
-                        script: 'git diff --name-only HEAD~1 | grep -q ".java"',
+                        script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
                         returnStatus: true
                     ) == 0
                 }
diff --git a/java/.ci/UiJenkinsfile b/java/.ci/UiJenkinsfile
index 72c3625e..3f70df69 100644
--- a/java/.ci/UiJenkinsfile
+++ b/java/.ci/UiJenkinsfile
@@ -78,9 +78,9 @@ pipeline {
         stage('Parallel Execution 1'){
         when {
                 expression {
-                    // Check if there are any changes in Java files in the last commit
+                    // Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
                     return sh(
-                        script: 'git diff --name-only HEAD~1 | grep -q ".java"',
+                        script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
                         returnStatus: true
                     ) == 0
                 }
@@ -184,9 +184,9 @@ pipeline {
         stage('Parallel Execution 2'){
         when {
                 expression {
-                    // Check if there are any changes in Java files in the last commit
+                    // Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
                     return sh(
-                        script: 'git diff --name-only HEAD~1 | grep -q ".java"',
+                        script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
                         returnStatus: true
                     ) == 0
                 }
@@ -286,9 +286,9 @@ pipeline {
         stage('Parallel Execution 3'){
         when {
                 expression {
-                    // Check if there are any changes in Java files in the last commit
+                    // Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
                     return sh(
-                        script: 'git diff --name-only HEAD~1 | grep -q ".java"',
+                        script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
                         returnStatus: true
                     ) == 0
                 }
@@ -366,9 +366,9 @@ pipeline {
         stage('Parallel Execution 4'){
         when {
                 expression {
-                    // Check if there are any changes in Java files in the last commit
+                    // Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
                     return sh(
-                        script: 'git diff --name-only HEAD~1 | grep -q ".java"',
+                        script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
                         returnStatus: true
                     ) == 0
                 }
@@ -424,9 +424,9 @@ pipeline {
         stage('Parallel Execution 5'){
         when {
                 expression {
-                    // Check if there are any changes in Java files in the last commit
+                    // Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
                     return sh(
-                        script: 'git diff --name-only HEAD~1 | grep -q ".java"',
+                        script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
                         returnStatus: true
                     ) == 0
                 }
diff --git a/python/.ci/Jenkinsfile b/python/.ci/Jenkinsfile
index 98751d96..588d3f47 100644
--- a/python/.ci/Jenkinsfile
+++ b/python/.ci/Jenkinsfile
@@ -69,14 +69,14 @@ pipeline {
         }
         stage('Parallel Execution 1'){
             when {
-                expression {
-                    // Check if there are any changes in Python files in the last commit
-                    return sh(
-                        script: 'git diff --name-only HEAD~1 | grep -q ".py"',
-                        returnStatus: true
-                    ) == 0
+                    expression {
+                        // Check for changes in .py files, python/.ci/Jenkinsfile, or python/requirements.txt
+                        return sh(
+                            script: 'git diff --name-only HEAD~1 | grep -E ".py$|python/.ci/Jenkinsfile|python/requirements.txt"',
+                            returnStatus: true
+                        ) == 0
+                    }
                 }
-            }
             parallel{ 
                 stage('Hive TO GCS') {
                     steps{
@@ -165,14 +165,14 @@ pipeline {
         }
         stage('Parallel Execution 2'){
             when {
-                expression {
-                    // Check if there are any changes in Python files in the last commit
-                    return sh(
-                        script: 'git diff --name-only HEAD~1 | grep -q ".py"',
-                        returnStatus: true
-                    ) == 0
+                    expression {
+                        // Check for changes in .py files, python/.ci/Jenkinsfile, or python/requirements.txt
+                        return sh(
+                            script: 'git diff --name-only HEAD~1 | grep -E ".py$|python/.ci/Jenkinsfile|python/requirements.txt"',
+                            returnStatus: true
+                        ) == 0
+                    }
                 }
-            }
             parallel{
                 stage('JDBC To JDBC') {
                     steps{
@@ -257,14 +257,14 @@ pipeline {
         }
         stage('Parallel Execution 3'){
             when {
-                expression {
-                    // Check if there are any changes in Python files in the last commit
-                    return sh(
-                        script: 'git diff --name-only HEAD~1 | grep -q ".py"',
-                        returnStatus: true
-                    ) == 0
+                    expression {
+                        // Check for changes in .py files, python/.ci/Jenkinsfile, or python/requirements.txt
+                        return sh(
+                            script: 'git diff --name-only HEAD~1 | grep -E ".py$|python/.ci/Jenkinsfile|python/requirements.txt"',
+                            returnStatus: true
+                        ) == 0
+                    }
                 }
-            }
             parallel{
                 stage('GCS TO BIGQUERY') {
                     steps{
@@ -353,14 +353,14 @@ pipeline {
         }
         stage('Parallel Execution 4'){
             when {
-                expression {
-                    // Check if there are any changes in Python files in the last commit
-                    return sh(
-                        script: 'git diff --name-only HEAD~1 | grep -q ".py"',
-                        returnStatus: true
-                    ) == 0
+                    expression {
+                        // Check for changes in .py files, python/.ci/Jenkinsfile, or python/requirements.txt
+                        return sh(
+                            script: 'git diff --name-only HEAD~1 | grep -E ".py$|python/.ci/Jenkinsfile|python/requirements.txt"',
+                            returnStatus: true
+                        ) == 0
+                    }
                 }
-            }
             parallel{
                 stage('MONGO TO GCS') {
                     steps{
@@ -514,14 +514,14 @@ pipeline {
         }
         stage('Parallel Execution 5'){
             when {
-                expression {
-                    // Check if there are any changes in Python files in the last commit
-                    return sh(
-                        script: 'git diff --name-only HEAD~1 | grep -q ".py"',
-                        returnStatus: true
-                    ) == 0
+                    expression {
+                        // Check for changes in .py files, python/.ci/Jenkinsfile, or python/requirements.txt
+                        return sh(
+                            script: 'git diff --name-only HEAD~1 | grep -E ".py$|python/.ci/Jenkinsfile|python/requirements.txt"',
+                            returnStatus: true
+                        ) == 0
+                    }
                 }
-            }
             parallel{
                 stage('JDBC TO GCS') {
                     steps{
@@ -614,14 +614,14 @@ pipeline {
         }  
          stage('Parallel Execution 6'){
              when {
-                 expression {
-                     // Check if there are any changes in Python files in the last commit
-                     return sh(
-                         script: 'git diff --name-only HEAD~1 | grep -q ".py"',
-                         returnStatus: true
-                     ) == 0
+                     expression {
+                         // Check for changes in .py files, python/.ci/Jenkinsfile, or python/requirements.txt
+                         return sh(
+                             script: 'git diff --name-only HEAD~1 | grep -E ".py$|python/.ci/Jenkinsfile|python/requirements.txt"',
+                             returnStatus: true
+                         ) == 0
+                     }
                  }
-             }
             parallel{
                 stage('S3 TO BigQuery (avro)') {    
                     steps{
@@ -659,14 +659,14 @@ pipeline {
         }
         stage('Parallel Execution 7'){
             when {
-                expression {
-                    // Check if there are any changes in Python files in the last commit
-                    return sh(
-                        script: 'git diff --name-only HEAD~1 | grep -q ".py"',
-                        returnStatus: true
-                    ) == 0
+                    expression {
+                        // Check for changes in .py files, python/.ci/Jenkinsfile, or python/requirements.txt
+                        return sh(
+                            script: 'git diff --name-only HEAD~1 | grep -E ".py$|python/.ci/Jenkinsfile|python/requirements.txt"',
+                            returnStatus: true
+                        ) == 0
+                    }
                 }
-            }
             parallel{
                 // stage('Hbase TO GCS (Manual)') {    
                 //     when {
@@ -790,14 +790,14 @@ pipeline {
         }
         stage('Parallel Execution 8'){
             when {
-                expression {
-                    // Check if there are any changes in Python files in the last commit
-                    return sh(
-                        script: 'git diff --name-only HEAD~1 | grep -q ".py"',
-                        returnStatus: true
-                    ) == 0
+                    expression {
+                        // Check for changes in .py files, python/.ci/Jenkinsfile, or python/requirements.txt
+                        return sh(
+                            script: 'git diff --name-only HEAD~1 | grep -E ".py$|python/.ci/Jenkinsfile|python/requirements.txt"',
+                            returnStatus: true
+                        ) == 0
+                    }
                 }
-            }
             parallel{
                 stage('Elasticsearch to GCS'){
                     steps{