Merge branch '2201.11.0-stage' into nutcracker

.github/workflows/pull_request_full_build.yml

@@ -6,6 +6,8 @@ on:
       - master
       - java21
       - nutcracker
+      - 2201.[0-9]+.x
+      - 2201.[0-9]+.[0-9]+-stage
 jobs:
   build-lang:
     name: Build Ballerina Lang

.github/workflows/pull_request_windows_build.yml

@@ -22,7 +22,7 @@ jobs:
   windows_build:
     name: Build with some tests on Windows
     runs-on: windows-latest
-    timeout-minutes: 240
+    timeout-minutes: 300
     concurrency: 
        group: ${{ github.head_ref }}-windows
        cancel-in-progress: true

cli/central-client/src/main/java/org/ballerinalang/central/client/CentralAPIClient.java

@@ -51,11 +51,21 @@
 import org.ballerinalang.central.client.model.ToolResolutionCentralResponse;
 import org.ballerinalang.central.client.model.ToolSearchResult;
 
+import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.PrintStream;
 import java.net.Proxy;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
+import java.security.KeyManagementException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.SecureRandom;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateException;
+import java.security.cert.CertificateFactory;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -64,6 +74,9 @@
 import java.util.concurrent.TimeUnit;
 
 import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.X509TrustManager;
 
 import static java.net.HttpURLConnection.HTTP_BAD_GATEWAY;
 import static java.net.HttpURLConnection.HTTP_BAD_REQUEST;
@@ -143,14 +156,16 @@ public class CentralAPIClient {
     private static final MediaType JSON = MediaType.parse("application/json; charset=utf-8");
     private static final MediaType JSON_CONTENT_TYPE = MediaType.parse("application/json");
 
-    // System property name for enabling central verbose
-    public static final String SYS_PROP_CENTRAL_VERBOSE_ENABLED = "CENTRAL_VERBOSE_ENABLED";
     private static final int DEFAULT_CONNECT_TIMEOUT = 60;
     private static final int DEFAULT_READ_TIMEOUT = 60;
     private static final int DEFAULT_WRITE_TIMEOUT = 60;
     private static final int DEFAULT_CALL_TIMEOUT = 0;
     private static final int MAX_RETRY = 1;
     public static final String CONNECTION_RESET = "Connection reset";
+    private static final String ENV_CENTRAL_VERBOSE_ENABLED = "CENTRAL_VERBOSE_ENABLED";
+    private static final String ENV_TRUSTSTORE_PATH = "BALLERINA_CA_BUNDLE";
+    private static final String ENV_TRUSTSTORE_PASSWORD = "BALLERINA_CA_PASSWORD";
+    private static final String ENV_CERT_PATH = "BALLERINA_CA_CERT";
 
     private final String baseUrl;
     private final Proxy proxy;
@@ -164,20 +179,26 @@ public class CentralAPIClient {
     private final int writeTimeout;
     private final int callTimeout;
     private final int maxRetries;
+    private final String trustStorePath;
+    private final String trustStorePassword;
+    private final String singleCertPath;
 
     public CentralAPIClient(String baseUrl, Proxy proxy, String accessToken) {
         this.outStream = System.out;
         this.baseUrl = baseUrl;
         this.proxy = proxy;
         this.accessToken = accessToken;
-        this.verboseEnabled = Boolean.parseBoolean(System.getenv(SYS_PROP_CENTRAL_VERBOSE_ENABLED));
+        this.verboseEnabled = Boolean.parseBoolean(System.getenv(ENV_CENTRAL_VERBOSE_ENABLED));
         this.proxyUsername = "";
         this.proxyPassword = "";
         this.connectTimeout = DEFAULT_CONNECT_TIMEOUT;
         this.readTimeout = DEFAULT_READ_TIMEOUT;
         this.writeTimeout = DEFAULT_WRITE_TIMEOUT;
         this.callTimeout = DEFAULT_CALL_TIMEOUT;
         this.maxRetries = MAX_RETRY;
+        this.trustStorePath = System.getenv(ENV_TRUSTSTORE_PATH);
+        this.trustStorePassword = System.getenv(ENV_TRUSTSTORE_PASSWORD);
+        this.singleCertPath = System.getenv(ENV_CERT_PATH);
     }
 
     public CentralAPIClient(String baseUrl, Proxy proxy, String accessToken, boolean verboseEnabled, int maxRetries,
@@ -194,6 +215,9 @@ public CentralAPIClient(String baseUrl, Proxy proxy, String accessToken, boolean
         this.writeTimeout = DEFAULT_WRITE_TIMEOUT;
         this.callTimeout = DEFAULT_CALL_TIMEOUT;
         this.maxRetries = maxRetries;
+        this.trustStorePath = System.getenv(ENV_TRUSTSTORE_PATH);
+        this.trustStorePassword = System.getenv(ENV_TRUSTSTORE_PASSWORD);
+        this.singleCertPath = System.getenv(ENV_CERT_PATH);
     }
 
     public CentralAPIClient(String baseUrl, Proxy proxy, String proxyUsername, String proxyPassword,
@@ -203,14 +227,17 @@ public CentralAPIClient(String baseUrl, Proxy proxy, String proxyUsername, Strin
         this.baseUrl = baseUrl;
         this.proxy = proxy;
         this.accessToken = accessToken;
-        this.verboseEnabled = Boolean.parseBoolean(System.getenv(SYS_PROP_CENTRAL_VERBOSE_ENABLED));
+        this.verboseEnabled = Boolean.parseBoolean(System.getenv(ENV_CENTRAL_VERBOSE_ENABLED));
         this.proxyUsername = proxyUsername;
         this.proxyPassword = proxyPassword;
         this.connectTimeout = connectionTimeout;
         this.readTimeout = readTimeout;
         this.writeTimeout = writeTimeout;
         this.callTimeout = callTimeout;
         this.maxRetries = maxRetries;
+        this.trustStorePath = System.getenv(ENV_TRUSTSTORE_PATH);
+        this.trustStorePassword = System.getenv(ENV_TRUSTSTORE_PASSWORD);
+        this.singleCertPath = System.getenv(ENV_CERT_PATH);
     }
 
     /**
@@ -1579,17 +1606,58 @@ public JsonObject getConnector(ConnectorInfo connector, String supportedPlatform
      *
      * @return the client
      */
-    protected OkHttpClient getClient() {
-        OkHttpClient okHttpClient = new OkHttpClient.Builder()
+    protected OkHttpClient getClient() throws CentralClientException {
+        OkHttpClient.Builder builder = new OkHttpClient.Builder()
                 .connectTimeout(connectTimeout, TimeUnit.SECONDS)
                 .readTimeout(readTimeout, TimeUnit.SECONDS)
                 .writeTimeout(writeTimeout, TimeUnit.SECONDS)
                 .callTimeout(callTimeout, TimeUnit.SECONDS)
                 .followRedirects(false)
                 .retryOnConnectionFailure(true)
                 .proxy(this.proxy)
-                .addInterceptor(new CustomRetryInterceptor(this.maxRetries))
-                .build();
+                .addInterceptor(new CustomRetryInterceptor(this.maxRetries));
+
+        // Load custom truststore if provided, otherwise use the default truststore
+        try {
+            KeyStore truststore;
+            if (this.trustStorePath != null && this.trustStorePassword != null) {
+                truststore = KeyStore.getInstance(KeyStore.getDefaultType());
+                try (InputStream keys = new FileInputStream(trustStorePath)) {
+                    truststore.load(keys, trustStorePassword.toCharArray());
+                }
+            } else {
+                truststore = KeyStore.getInstance(KeyStore.getDefaultType());
+                try (InputStream defaultKeys = new FileInputStream(System.getProperty("java.home") +
+                        "/lib/security/cacerts")) {
+                    truststore.load(defaultKeys, "changeit".toCharArray()); // Default password for cacerts
+                }
+            }
+
+            // If there's a single certificate to add
+            if (this.singleCertPath != null) {
+                CertificateFactory certificateFactory = CertificateFactory.getInstance("X.509");
+                try (InputStream certInputStream = new FileInputStream(singleCertPath)) {
+                    Certificate certificate = certificateFactory.generateCertificate(certInputStream);
+                    truststore.setCertificateEntry("bal-cert", certificate);
+                }
+            }
+
+            TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(
+                    TrustManagerFactory.getDefaultAlgorithm());
+            trustManagerFactory.init(truststore);
+
+            SSLContext sslContext = SSLContext.getInstance("TLS");
+            sslContext.init(null, trustManagerFactory.getTrustManagers(), new SecureRandom());
+            SSLContext.setDefault(sslContext);
+
+            builder.sslSocketFactory(sslContext.getSocketFactory(),
+                    (X509TrustManager) trustManagerFactory.getTrustManagers()[0]);
+        } catch (CertificateException | KeyStoreException | IOException | NoSuchAlgorithmException |
+                 KeyManagementException e) {
+            throw new CentralClientException(e.getMessage());
+        }
+
+        OkHttpClient okHttpClient = builder.build();
 
         if ((!(this.proxyUsername).isEmpty() && !(this.proxyPassword).isEmpty())) {
             Authenticator proxyAuthenticator = (route, response) -> {

cli/central-client/src/test/java/org/ballerinalang/central/client/TestCustomRetryInterceptor.java

@@ -23,6 +23,7 @@
 import okhttp3.Response;
 import okhttp3.mockwebserver.MockResponse;
 import okhttp3.mockwebserver.MockWebServer;
+import org.ballerinalang.central.client.exceptions.CentralClientException;
 import org.testng.Assert;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
@@ -49,7 +50,7 @@ public class TestCustomRetryInterceptor {
 
 
     @BeforeClass
-    public void setUp() {
+    public void setUp() throws CentralClientException {
         this.console = new ByteArrayOutputStream();
         PrintStream outStream = new PrintStream(this.console);
         CentralAPIClient centralAPIClient = new CentralAPIClient("https://localhost:9090/registry",

compiler/ballerina-lang/src/main/java/io/ballerina/projects/bala/BalaProject.java

@@ -45,6 +45,7 @@
  */
 public class BalaProject extends Project {
     private final String platform;
+    private final String balaVersion;
 
     /**
      * Loads a BalaProject from the provided bala path.
@@ -70,6 +71,7 @@ public static BalaProject loadProject(ProjectEnvironmentBuilder environmentBuild
     private BalaProject(ProjectEnvironmentBuilder environmentBuilder, Path balaPath, BuildOptions buildOptions) {
         super(ProjectKind.BALA_PROJECT, balaPath, environmentBuilder, buildOptions);
         this.platform = BalaFiles.readPackageJson(balaPath).getPlatform();
+        this.balaVersion = BalaFiles.readBalaJson(balaPath).getBala_version();
     }
 
     @Override
@@ -136,6 +138,10 @@ public String platform() {
         return platform;
     }
 
+    public String balaVersion() {
+        return balaVersion;
+    }
+
     private boolean isFilePathInProject(Path filepath) {
         try {
             ProjectPaths.packageRoot(filepath);

compiler/ballerina-lang/src/main/java/io/ballerina/projects/internal/BalaFiles.java

@@ -32,6 +32,7 @@
 import io.ballerina.projects.PlatformLibraryScope;
 import io.ballerina.projects.ProjectException;
 import io.ballerina.projects.internal.bala.BalToolJson;
+import io.ballerina.projects.internal.bala.BalaJson;
 import io.ballerina.projects.internal.bala.CompilerPluginJson;
 import io.ballerina.projects.internal.bala.DependencyGraphJson;
 import io.ballerina.projects.internal.bala.ModuleDependency;
@@ -66,6 +67,7 @@
 import static io.ballerina.projects.internal.ProjectFiles.loadDocuments;
 import static io.ballerina.projects.internal.ProjectFiles.loadResources;
 import static io.ballerina.projects.util.ProjectConstants.BALA_DOCS_DIR;
+import static io.ballerina.projects.util.ProjectConstants.BALA_JSON;
 import static io.ballerina.projects.util.ProjectConstants.BAL_TOOL_JSON;
 import static io.ballerina.projects.util.ProjectConstants.COMPILER_PLUGIN_DIR;
 import static io.ballerina.projects.util.ProjectConstants.COMPILER_PLUGIN_JSON;
@@ -756,6 +758,41 @@ public static PackageJson readPackageJson(Path balaPath) {
         return packageJson;
     }
 
+    public static BalaJson readBalaJson(Path balaPath) {
+        BalaJson balaJson;
+        if (balaPath.toFile().isDirectory()) {
+            Path balaJsonPath = balaPath.resolve(BALA_JSON);
+            if (Files.notExists(balaJsonPath)) {
+                throw new ProjectException("bala.json does not exists in '" + balaPath + "'");
+            }
+            try (BufferedReader bufferedReader = Files.newBufferedReader(balaJsonPath)) {
+                balaJson = gson.fromJson(bufferedReader, BalaJson.class);
+            } catch (JsonSyntaxException e) {
+                throw new ProjectException("Invalid bala.json format in '" + balaPath + "'");
+            } catch (IOException e) {
+                throw new ProjectException("Failed to read the bala.json in '" + balaPath + "'");
+            }
+        } else {
+            URI zipURI = getZipURI(balaPath);
+            try (FileSystem zipFileSystem = FileSystems.newFileSystem(zipURI, new HashMap<>())) {
+                Path balaJsonPath = zipFileSystem.getPath(BALA_JSON);
+                if (Files.notExists(balaJsonPath)) {
+                    throw new ProjectException("package.json does not exists in '" + balaPath + "'");
+                }
+                try (BufferedReader bufferedReader = Files.newBufferedReader(balaJsonPath)) {
+                    balaJson = gson.fromJson(bufferedReader, BalaJson.class);
+                } catch (JsonSyntaxException e) {
+                    throw new ProjectException("Invalid package.json format in '" + balaPath + "'");
+                } catch (IOException e) {
+                    throw new ProjectException("Failed to read the package.json in '" + balaPath + "'");
+                }
+            } catch (IOException e) {
+                throw new ProjectException("Failed to read bala file:" + balaPath);
+            }
+        }
+        return balaJson;
+    }
+
     private static URI getZipURI(Path balaPath) {
         return URI.create("jar:" + balaPath.toAbsolutePath().toUri());
     }

compiler/ballerina-lang/src/main/java/io/ballerina/projects/internal/ManifestBuilder.java

@@ -358,12 +358,14 @@ private List<PackageManifest.Module> getModuleEntries(
                         || !moduleName.contains(DOT)) { // The invalid module name is already handled
                     continue;
                 }
+                String moduleNamepart = moduleName.substring(packageName.toString().length() + 1);
 
                 boolean export = Boolean.TRUE.equals(getBooleanValueFromTomlTableNode(modulesNode, EXPORT));
                 String description = getStringValueFromTomlTableNode(modulesNode, DESCRIPTION, null);
                 String modReadme = getStringValueFromTomlTableNode(modulesNode, README, null);
                 if (modReadme == null) {
-                    Path defaultReadme = modulesRoot.resolve(moduleName).resolve(ProjectConstants.README_MD_FILE_NAME);
+                    Path defaultReadme = modulesRoot.resolve(moduleNamepart)
+                            .resolve(ProjectConstants.README_MD_FILE_NAME);
                     if (Files.exists(defaultReadme)) {
                         modReadme = defaultReadme.toString();
                     }
@@ -375,7 +377,7 @@ private List<PackageManifest.Module> getModuleEntries(
                 PackageManifest.Module module = new PackageManifest.Module(moduleName, export,
                         description, modReadme);
                 moduleList.add(module);
-                moduleDirs.remove(moduleName.split("[.]")[1]);
+                moduleDirs.remove(moduleNamepart);
             }
             // If there are README.mds in other modules, add them
             for (Map.Entry<String, Path> pathEntry : moduleDirs.entrySet()) {

compiler/ballerina-lang/src/main/java/org/wso2/ballerinalang/compiler/bir/codegen/methodgen/LambdaGen.java

@@ -363,6 +363,7 @@ private LambdaDetails populateAsyncLambdaDetails(BIRTerminator.AsyncCall asyncIn
         lambdaDetails.lhsType = asyncIns.lhsOp != null ? asyncIns.lhsOp.variableDcl.type : null;
         lambdaDetails.packageID = asyncIns.calleePkg;
         lambdaDetails.funcName = asyncIns.name.getValue();
+        lambdaDetails.encodedFuncName = Utils.encodeFunctionIdentifier(lambdaDetails.funcName);
         if (!asyncIns.isVirtual) {
             populateLambdaFunctionDetails(lambdaDetails);
         }

distribution/zip/jballerina-tools/build.gradle

@@ -90,6 +90,9 @@ dependencies {
     dist libs.ow2.asm.util
     dist libs.jackson.datatype.jsr310
 
+    // Following dependencies are required for lang-server
+    dist libs.sqlite.jdbc;
+
     // Following dependencies are required for transactions
     dist libs.atomikos.transactions.jta
     dist libs.atomikos.transactions.api

gradle/libs.versions.toml

@@ -86,15 +86,15 @@ mockitoCoreVersion="5.3.1"
 mockitoTestNGVersion="0.5.0"
 mustacheJavaCompilerVersion="0.8.9"
 mvel2Version="2.4.4.Final"
-nettyBufferVersion="4.1.100.Final"
-nettyCodecHttpVersion="4.1.45.Final"
-nettyCodecHttp2Version="4.1.45.Final"
-nettyCommonVersion="4.1.100.Final"
-nettyHandlerVersion="4.1.45.Final"
-nettyResolverVersion="4.1.100.Final"
-nettyTransportNativeEpollVersion="4.1.100.Final"
-nettyTransportNativeKqueueVersion="4.1.100.Final"
-nettyTransportVersion="4.1.100.Final"
+nettyBufferVersion="4.1.115.Final"
+nettyCodecHttpVersion="4.1.115.Final"
+nettyCodecHttp2Version="4.1.115.Final"
+nettyCommonVersion="4.1.115.Final"
+nettyHandlerVersion="4.1.115.Final"
+nettyResolverVersion="4.1.115.Final"
+nettyTransportNativeEpollVersion="4.1.115.Final"
+nettyTransportNativeKqueueVersion="4.1.115.Final"
+nettyTransportVersion="4.1.115.Final"
 okhttpVersion="3.14.0"
 openhftCompilerVersion="2.23ea0"
 openTelemetryApiVersion="1.32.0"
@@ -115,6 +115,7 @@ slf4jJdk14Version="2.0.7"
 slf4jSimpleVersion="2.0.7"
 snakeyamlVersion="2.0"
 sonarqubeGradlePluginVersion="4.0.0.2929"
+sqliteJdbcVersion="3.47.0.0"
 squareupOkioVersion="3.4.0"
 swaggerModelsVersion="2.2.22"
 swaggerParserV2Version="2.1.22"
@@ -239,6 +240,7 @@ slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4jApiVersion"}
 slf4j-jdk14 = { module = "org.slf4j:slf4j-jdk14", version.ref = "slf4jJdk14Version"}
 slf4j-simple = { module = "org.slf4j:slf4j-simple", version.ref = "slf4jSimpleVersion"}
 snakeyaml = { module = "org.yaml:snakeyaml", version.ref = "snakeyamlVersion"}
+sqlite-jdbc = { module = "org.xerial:sqlite-jdbc", version.ref = "sqliteJdbcVersion"}
 squareup-okhttp = { module = "com.squareup.okhttp3:okhttp", version.ref = "okhttpVersion"}
 squareup-mock-webserver = { module = "com.squareup.okhttp3:mockwebserver", version.ref = "okhttpVersion"}
 squareup-okio = { module = "com.squareup.okio:okio", version.ref = "squareupOkioVersion"}

langlib/lang.query/src/main/ballerina/types.bal

@@ -680,9 +680,9 @@ class _GroupByFunction {
     }
 
     private function getKey(_Frame f) returns anydata|error {
-        anydata[] keys = [];
+        record{} keys = {};
         foreach var key in self.keys {
-            keys.push(<anydata> check f[key]);
+            keys[key] = <anydata> check f[key];
         }
         return keys;
     }

misc/debug-adapter/modules/debug-adapter-core/src/main/java/org/ballerinalang/debugadapter/BreakpointProcessor.java

@@ -268,7 +268,8 @@ void activateDynamicBreakPoints(int threadId, DynamicBreakpointMode mode, boolea
      */
     private boolean isWithinSameSource(Location currentLocation, Location prevLocation) {
         try {
-            return Objects.equals(currentLocation.sourcePath(), prevLocation.sourcePath());
+            return Objects.equals(currentLocation.sourcePath(), prevLocation.sourcePath())
+                    && Objects.equals(currentLocation.method().name(), prevLocation.method().name());
         } catch (AbsentInformationException e) {
             return false;
         }