Add tests for test parallelization

ballerina-platform · May 3, 2023 · f73ac1f · f73ac1f
1 parent a751e26
commit f73ac1f
Show file tree

Hide file tree

Showing 22 changed files with 1,560 additions and 17 deletions.
diff --git a/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal b/misc/testerina/modules/testerina-core/src/main/ballerina/execute.bal
@@ -78,42 +78,60 @@ function executeTests() returns error? {
 
     }
 
+    int paralalTestExecutionListLength = parallelTestExecutionList.length();
+    int serialTestExecutionListLength = serialTestExecutionList.length();
+
     while true {
-        if (parallelTestExecutionList.length() == 0 && getAvailableWorkerCount() == testWorkers
-            && serialTestExecutionList.length() == 0) {
+
+        TestFunction? testFunction = ();
+        lock {
+            paralalTestExecutionListLength = parallelTestExecutionList.length();
+            serialTestExecutionListLength = serialTestExecutionList.length();
+        }
+
+        if (paralalTestExecutionListLength == 0 && getAvailableWorkerCount() == testWorkers
+            && serialTestExecutionListLength == 0) {
             break;
         }
 
         if (getAvailableWorkerCount() != 0) {
 
-            if parallelTestExecutionList.length() == 0 && serialTestExecutionList.length() == 0 {
+            if paralalTestExecutionListLength == 0 && serialTestExecutionListLength == 0 {
                 runtime:sleep(0.0001);
                 continue;
 
             }
 
-            if (serialTestExecutionList.length() != 0 && getAvailableWorkerCount() == testWorkers) {
+            if (serialTestExecutionListLength != 0 && getAvailableWorkerCount() == testWorkers) {
+                lock {
+                    testFunction = serialTestExecutionList.remove(0);
+                }
 
-                TestFunction testFunction = serialTestExecutionList.remove(0);
-                allocateWorker();
-                future<error?> serialWaiter = start executeTest(testFunction);
-                any _ = check wait serialWaiter;
+                if testFunction is TestFunction {
+                    allocateWorker();
+                    future<error?> serialWaiter = start executeTest(testFunction);
+                    any _ = check wait serialWaiter;
+                }
 
-            } else if parallelTestExecutionList.length() != 0 && serialTestExecutionList.length() == 0 {
-                TestFunction testFunction = parallelTestExecutionList.remove(0);
-                allocateWorker();
-                future<(error?)> parallelWaiter = start executeTest(testFunction);
-                runtime:sleep(0.0001);
-                if isDataDrivenTest(testFunction) {
-                    any _ = check wait parallelWaiter;
+            } else if paralalTestExecutionListLength != 0 && serialTestExecutionListLength == 0 {
+                lock {
+                    testFunction = parallelTestExecutionList.remove(0);
+                }
+                if testFunction is TestFunction {
+                    allocateWorker();
+                    future<(error?)> parallelWaiter = start executeTest(testFunction);
+                    if isDataDrivenTest(testFunction) {
+                        any _ = check wait parallelWaiter;
+                    }
                 }
 
             }
         } else {
             runtime:sleep(0.0001);
         }
+        runtime:sleep(0.0001);
     }
-    // println("Test execution time :" + (currentTimeInMillis() - startTime).toString() + "ms");
+    println("\n\t\tTest execution time :" + (currentTimeInMillis() - startTime).toString() + "ms\n");
 }
 
 function executeTest(TestFunction testFunction) returns error? {
@@ -161,7 +179,6 @@ function executeTest(TestFunction testFunction) returns error? {
     }
     testFunction.dependents.forEach(dependent => checkExecutionReadiness(dependent));
     releaseWorker();
-    // isSerialTestExecution = !isSerialTestExecution && !testFunction.parallelizable;
 }
 
 function checkExecutionReadiness(TestFunction testFunction) {
@@ -227,6 +244,7 @@ function executeDataDrivenTestSet(TestFunction testFunction) returns error? {
             runtime:sleep(0.0001);
             continue;
         }
+        runtime:sleep(0.0001);
     }
     allocateWorker();
 }

diff --git a/...egration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java b/...egration-test/src/test/java/org/ballerinalang/testerina/test/TestparallelizationTest.java
@@ -0,0 +1,156 @@
+package org.ballerinalang.testerina.test;
+
+import org.ballerinalang.test.context.BMainInstance;
+import org.ballerinalang.test.context.BallerinaTestException;
+import org.ballerinalang.testerina.test.utils.AssertionUtils;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+import static org.ballerinalang.testerina.test.utils.CommonUtils.replaceExecutionTime;
+
+public class TestparallelizationTest extends BaseTestCase {
+
+    private BMainInstance balClient;
+    private String projectPath;
+
+    @BeforeClass
+    public void setup() throws BallerinaTestException {
+        balClient = new BMainInstance(balServer);
+        projectPath = projectBasedTestsPath.resolve("parallelisation-test").toString();
+    }
+
+    @Test
+    public void testParallelization() throws BallerinaTestException, IOException {
+        String[] args = mergeCoverageArgs(new String[]{"--workers=30", "parallelisation-simple-test"});
+        String output = balClient.runMainAndReadStdOut("test", args,
+                new HashMap<>(), projectPath, false);
+        Float executionTimeW30 = getTimeForTestExecution(output);
+        output = replaceExecutionTime(output);
+//        AssertionUtils.assertOutput("TestparallelizationTest-testParallelization_w1.txt", output);
+
+
+        args = mergeCoverageArgs(new String[]{"--workers=1", "parallelisation-simple-test", });
+        output = balClient.runMainAndReadStdOut("test", args,
+                new HashMap<>(), projectPath, false);
+        Float executionTimeW1 = getTimeForTestExecution(output);
+        output = replaceExecutionTime(output);
+//        AssertionUtils.assertOutput("TestparallelizationTest-testParallelization_w30.txt", output);
+
+        Assert.assertTrue(executionTimeW1 / 3 > executionTimeW30);
+
+    }
+
+    @Test
+    public void testNonParallelizable() throws BallerinaTestException, IOException {
+        String[] args = mergeCoverageArgs(new String[]{"--workers=30", "parallelisation-parallelizable"});
+        String output = balClient.runMainAndReadStdOut("test", args,
+                new HashMap<>(), projectPath, false);
+        Float executionTimeW30 = getTimeForTestExecution(output);
+        output = replaceExecutionTime(output);
+//        AssertionUtils.assertOutput("TestparallelizationTest-testNonParallelizable_w1.txt", output);
+
+
+        args = mergeCoverageArgs(new String[]{"--workers=1", "parallelisation-parallelizable", });
+        output = balClient.runMainAndReadStdOut("test", args,
+                new HashMap<>(), projectPath, false);
+        Float executionTimeW1 = getTimeForTestExecution(output);
+        output = replaceExecutionTime(output);
+//        AssertionUtils.assertOutput("TestparallelizationTest-testNonParallelizable_w30.txt", output);
+
+        Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000);
+    }
+
+
+    @Test
+    public void testParalallelizableTupleDataProvider() throws BallerinaTestException, IOException {
+        String[] args = mergeCoverageArgs(new String[]{"--workers=30", "parallelisation-tuple-data-provider"});
+        String output = balClient.runMainAndReadStdOut("test", args,
+                new HashMap<>(), projectPath, false);
+        Float executionTimeW30 = getTimeForTestExecution(output);
+        output = replaceExecutionTime(output);
+//        AssertionUtils.assertOutput("TestparallelizationTest-testParalallelizableTupleDataProvider_w1.txt", output);
+
+
+        args = mergeCoverageArgs(new String[]{"--workers=1", "parallelisation-tuple-data-provider", });
+        output = balClient.runMainAndReadStdOut("test", args,
+                new HashMap<>(), projectPath, false);
+        Float executionTimeW1 = getTimeForTestExecution(output);
+        output = replaceExecutionTime(output);
+//        AssertionUtils.assertOutput("TestparallelizationTest-testParalallelizableTupleDataProvider_w30.txt", output);
+
+        Assert.assertTrue(executionTimeW1 / 3 > executionTimeW30);
+    }
+
+    @Test
+    public void testParalallelizableMapDataProvider() throws BallerinaTestException, IOException {
+        String[] args = mergeCoverageArgs(new String[]{"--workers=30", "parallelisation-map-data-provider"});
+        String output = balClient.runMainAndReadStdOut("test", args,
+                new HashMap<>(), projectPath, false);
+        Float executionTimeW30 = getTimeForTestExecution(output);
+        output = replaceExecutionTime(output);
+//        AssertionUtils.assertOutput("TestparallelizationTest-testParalallelizableMapDataProvider_w1.txt", output);
+
+
+        args = mergeCoverageArgs(new String[]{"--workers=1", "parallelisation-map-data-provider", });
+        output = balClient.runMainAndReadStdOut("test", args,
+                new HashMap<>(), projectPath, false);
+        Float executionTimeW1 = getTimeForTestExecution(output);
+        output = replaceExecutionTime(output);
+//        AssertionUtils.assertOutput("TestparallelizationTest-testParalallelizableMapDataProvider_w30.txt", output);
+
+        Assert.assertTrue(executionTimeW1 / 3 > executionTimeW30);
+    }
+
+    @Test
+    public void testNonParalallelizableTupleDataProvider() throws BallerinaTestException, IOException {
+        String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-parallelisation-tuple-data-provider"});
+        String output = balClient.runMainAndReadStdOut("test", args,
+                new HashMap<>(), projectPath, false);
+        Float executionTimeW30 = getTimeForTestExecution(output);
+        output = replaceExecutionTime(output);
+//        AssertionUtils.assertOutput("TestparallelizationTest-testNonParalallelizableTupleDataProvider_w1.txt",
+//        output);
+
+
+        args = mergeCoverageArgs(new String[]{"--workers=1", "non-parallelisation-tuple-data-provider", });
+        output = balClient.runMainAndReadStdOut("test", args,
+                new HashMap<>(), projectPath, false);
+        Float executionTimeW1 = getTimeForTestExecution(output);
+        output = replaceExecutionTime(output);
+//        AssertionUtils.assertOutput("TestparallelizationTest-testNonParalallelizableTupleDataProvider_w30.txt",
+//        output);
+
+        Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000);
+    }
+
+    @Test
+    public void testNonParalallelizableMapDataProvider() throws BallerinaTestException, IOException {
+        String[] args = mergeCoverageArgs(new String[]{"--workers=30", "non-parallelisation-map-data-provider"});
+        String output = balClient.runMainAndReadStdOut("test", args,
+                new HashMap<>(), projectPath, false);
+        Float executionTimeW30 = getTimeForTestExecution(output);
+        output = replaceExecutionTime(output);
+//        AssertionUtils.assertOutput("TestparallelizationTest-testNonParalallelizableMapDataProvider_w1.txt", output);
+
+
+        args = mergeCoverageArgs(new String[]{"--workers=1", "parallelisation-map-data-provider", });
+        output = balClient.runMainAndReadStdOut("test", args,
+                new HashMap<>(), projectPath, false);
+        Float executionTimeW1 = getTimeForTestExecution(output);
+        output = replaceExecutionTime(output);
+//        AssertionUtils.assertOutput("TestparallelizationTest-testNonParalallelizableMapDataProvider_w30.txt", output);
+
+        Assert.assertTrue(executionTimeW1 - executionTimeW30 < 1000);
+    }
+
+    private float getTimeForTestExecution(String output) {
+        int firstPos = output.indexOf("Test execution time :") + ("Test execution time :").length();
+        int lastPos = output.indexOf("ms");
+        String executionTime = output.substring(firstPos, lastPos);
+        return Float.parseFloat(executionTime);
+    }
+}
diff --git a/...na-integration-test/src/test/java/org/ballerinalang/testerina/test/utils/CommonUtils.java b/...na-integration-test/src/test/java/org/ballerinalang/testerina/test/utils/CommonUtils.java
@@ -28,6 +28,10 @@ public class CommonUtils {
     private CommonUtils() {
     }
 
+    public static String replaceExecutionTime(String content) {
+        return replaceVaryingString("Test execution time :", "ms", content);
+    }
+
     public static String replaceVaryingString(String firstString, String endString, String content) {
         String modifiedContent = content;
         int firstPos = modifiedContent.indexOf(firstString);

diff --git a/...ect-based-tests/parallelisation-test/non-parallelisation-map-data-provider/Ballerina.toml b/...ect-based-tests/parallelisation-test/non-parallelisation-map-data-provider/Ballerina.toml
@@ -0,0 +1,8 @@
+[package]
+org = "luheerathan"
+name = "test_parallelisation_test"
+version = "0.1.0"
+distribution = "2201.3.0-rc3"
+
+[build-options]
+observabilityIncluded = true
diff --git a/...s/project-based-tests/parallelisation-test/non-parallelisation-map-data-provider/main.bal b/...s/project-based-tests/parallelisation-test/non-parallelisation-map-data-provider/main.bal
@@ -0,0 +1,55 @@
+import ballerina/io;
+import ballerina/lang.runtime;
+import ballerina/time;
+
+int employedCount = 20000;
+int[] quantities_ = [
+    1,
+    2,
+    3,
+    4,
+    5,
+    6,
+    7,
+    8,
+    9,
+    10,
+    11,
+    12,
+    13,
+    14,
+
+    15,
+    16,
+    17,
+    18,
+    19,
+    20,
+    21,
+    22,
+
+    23,
+    24
+];
+
+public function main() {
+    decimal now = time:monotonicNow();
+    int count = 10;
+
+    while true {
+        if count > 0 {
+            future<()> f = start get_average();
+            count -= 1;
+        }
+        // int avg = get_average();
+        // io:println("Average: ", avg);
+
+    }
+
+}
+
+function get_average() {
+    runtime:sleep(10);
+    io:println("Helllwo");
+
+}
diff --git a/...sts/parallelisation-test/non-parallelisation-map-data-provider/tests/data_driven_test.bal b/...sts/parallelisation-test/non-parallelisation-map-data-provider/tests/data_driven_test.bal
@@ -0,0 +1,60 @@
+
+import ballerina/lang.runtime;
+import ballerina/test;
+
+@test:Config {
+    dataProvider: mapDataProvider,
+    parallelizable: false
+}
+function mapDataProviderTest(int value1, int value2, string fruit) returns error? {
+    test:assertEquals(value1, value2, msg = "The provided values are not equal");
+    runtime:sleep(0.5);
+    test:assertEquals(fruit.length(), 6);
+}
+
+function mapDataProvider() returns map<[int, int, string]>|error {
+    map<[int, int, string]> dataSet = {
+        "banana": [10, 10, "banana"],
+        "cherry": [5, 5, "cherry"],
+        "apple": [5, 5, "apple"],
+        "orange": [5, 5, "orange"],
+        "carrot": [5, 5, "carrot"],
+        "lemon": [5, 5, "lemon"],
+        "tomatto": [5, 5, "tomatto"],
+        "papaya": [5, 5, "papaya"],
+        "grapes": [5, 5, "grapes"],
+        "mango": [5, 5, "mango"],
+        "pineapple": [5, 5, "pineapple"],
+        "watermelon": [5, 5, "watermelon"],
+        "strawberry": [5, 5, "strawberry"],
+        "melon": [5, 5, "melon"],
+        "guava": [5, 5, "guava"],
+        "pomegranate": [5, 5, "pomegranate"],
+        "jackfruit": [5, 5, "jackfruit"],
+        "coconut": [5, 5, "coconut"],
+        "peach": [5, 5, "peach"],
+        "pear": [5, 5, "pear"],
+        "plum": [5, 5, "plum"],
+        "blueberry": [5, 5, "blueberry"],
+        "raspberry": [5, 5, "raspberry"],
+        "kiwi": [5, 5, "kiwi"],
+        "avocado": [5, 5, "avocado"],
+        "cucumber": [5, 5, "cucumber"],
+        "pepper": [5, 5, "pepper"],
+        "onion": [5, 5, "onion"],
+        "potato": [5, 5, "potato"],
+        "tomato": [5, 5, "tomato"],
+        "garlic": [5, 5, "garlic"],
+        "ginger": [5, 5, "ginger"],
+        "spinach": [5, 5, "spinach"],
+        "broccoli": [5, 5, "broccoli"],
+        "cauliflower": [5, 5, "cauliflower"],
+        "cabbage": [5, 5, "cabbage"],
+        "beetroot": [5, 5, "beetroot"],
+        "celery": [5, 5, "celery"],
+        "corn": [5, 5, "corn"],
+        "mushroom": [5, 5, "mushroom"]
+
+    };
+    return dataSet;
+}
diff --git a/...t-based-tests/parallelisation-test/non-parallelisation-tuple-data-provider/Ballerina.toml b/...t-based-tests/parallelisation-test/non-parallelisation-tuple-data-provider/Ballerina.toml
@@ -0,0 +1,8 @@
+[package]
+org = "luheerathan"
+name = "test_parallelisation_test"
+version = "0.1.0"
+distribution = "2201.3.0-rc3"
+
+[build-options]
+observabilityIncluded = true