diff --git a/cli/central-client/src/main/java/org/ballerinalang/central/client/CentralAPIClient.java b/cli/central-client/src/main/java/org/ballerinalang/central/client/CentralAPIClient.java index c2307c06caf8..a2e036d963bf 100644 --- a/cli/central-client/src/main/java/org/ballerinalang/central/client/CentralAPIClient.java +++ b/cli/central-client/src/main/java/org/ballerinalang/central/client/CentralAPIClient.java @@ -85,10 +85,13 @@ import static org.ballerinalang.central.client.CentralClientConstants.DEPRECATE_MESSAGE; import static org.ballerinalang.central.client.CentralClientConstants.IDENTITY; import static org.ballerinalang.central.client.CentralClientConstants.IS_DEPRECATED; +import static org.ballerinalang.central.client.CentralClientConstants.DIGEST; import static org.ballerinalang.central.client.CentralClientConstants.LOCATION; import static org.ballerinalang.central.client.CentralClientConstants.ORGANIZATION; import static org.ballerinalang.central.client.CentralClientConstants.PKG_NAME; import static org.ballerinalang.central.client.CentralClientConstants.PLATFORM; +import static org.ballerinalang.central.client.CentralClientConstants.SHA256; +import static org.ballerinalang.central.client.CentralClientConstants.SHA256_ALGORITHM; import static org.ballerinalang.central.client.CentralClientConstants.USER_AGENT; import static org.ballerinalang.central.client.CentralClientConstants.VERSION; import static org.ballerinalang.central.client.Utils.ProgressRequestBody; @@ -97,6 +100,9 @@ import static org.ballerinalang.central.client.Utils.getBearerToken; import static org.ballerinalang.central.client.Utils.getRemoteRepo; import static org.ballerinalang.central.client.Utils.isApplicationJsonContentType; +import static org.ballerinalang.central.client.Utils.checkHash; +import static org.ballerinalang.central.client.Utils.bytesToHex; + /** * {@code CentralAPIClient} is a client for the Central API. * @@ -161,8 +167,8 @@ public CentralAPIClient(String baseUrl, Proxy proxy, String accessToken) { } public CentralAPIClient(String baseUrl, Proxy proxy, String proxyUsername, String proxyPassword, - String accessToken, int connectionTimeout, int readTimeout, int writeTimeout, - int callTimeout) { + String accessToken, int connectionTimeout, int readTimeout, int writeTimeout, + int callTimeout) { this.outStream = System.out; this.baseUrl = baseUrl; this.proxy = proxy; @@ -181,7 +187,8 @@ public CentralAPIClient(String baseUrl, Proxy proxy, String proxyUsername, Strin * * @param orgNamePath The organization name of the package. (required) * @param packageNamePath The name of the package. (required) - * @param version The version or version range of the module. (required) + * @param version The version or version range of the module. + * (required) * @param supportedPlatform The ballerina platform. (required) * @param ballerinaVersion The ballerina version. (required) * @return PackageJsonSchema @@ -228,7 +235,7 @@ public Package getPackage(String orgNamePath, String packageNamePath, String ver throw new NoPackageException(error.getMessage()); } else { throw new CentralClientException(ERR_CANNOT_FIND_PACKAGE + packageSignature + ". reason: " - + error.getMessage()); + + error.getMessage()); } } @@ -239,8 +246,8 @@ public Package getPackage(String orgNamePath, String packageNamePath, String ver // If request sent is wrong or error occurred at remote repository if (getPackageResponse.code() == HTTP_BAD_REQUEST || - getPackageResponse.code() == HTTP_INTERNAL_ERROR || - getPackageResponse.code() == HTTP_UNAVAILABLE) { + getPackageResponse.code() == HTTP_INTERNAL_ERROR || + getPackageResponse.code() == HTTP_UNAVAILABLE) { Error error = new Gson().fromJson(responseBodyContent, Error.class); if (error.getMessage() != null && !"".equals(error.getMessage())) { throw new CentralClientException(error.getMessage()); @@ -252,10 +259,10 @@ public Package getPackage(String orgNamePath, String packageNamePath, String ver throw new CentralClientException(ERR_CANNOT_FIND_PACKAGE + packageSignature); } catch (SocketTimeoutException e) { throw new ConnectionErrorException(ERR_CANNOT_FIND_PACKAGE + packageSignature + ". reason: " + - e.getMessage()); + e.getMessage()); } catch (IOException e) { throw new CentralClientException(ERR_CANNOT_FIND_PACKAGE + packageSignature + ". reason: " + - e.getMessage()); + e.getMessage()); } finally { body.ifPresent(ResponseBody::close); try { @@ -320,17 +327,17 @@ public List getPackageVersions(String orgNamePath, String packageNamePat return new ArrayList<>(); } else { throw new CentralClientException(ERR_CANNOT_FIND_VERSIONS + packageSignature + - ". reason: " + error.getMessage()); + ". reason: " + error.getMessage()); } } // If request sent is wrong or error occurred at remote repository if (getVersionsResponse.code() == HTTP_BAD_REQUEST || - getVersionsResponse.code() == HTTP_INTERNAL_ERROR || - getVersionsResponse.code() == HTTP_UNAVAILABLE) { + getVersionsResponse.code() == HTTP_INTERNAL_ERROR || + getVersionsResponse.code() == HTTP_UNAVAILABLE) { Error error = new Gson().fromJson(responseBodyContent, Error.class); throw new CentralClientException(ERR_CANNOT_FIND_VERSIONS + packageSignature + - ". reason: " + error.getMessage()); + ". reason: " + error.getMessage()); } } } @@ -338,10 +345,10 @@ public List getPackageVersions(String orgNamePath, String packageNamePat throw new CentralClientException(ERR_CANNOT_FIND_VERSIONS + packageSignature + "."); } catch (SocketTimeoutException | UnknownHostException e) { throw new ConnectionErrorException(ERR_CANNOT_FIND_VERSIONS + packageSignature + ". reason: " + - e.getMessage()); + e.getMessage()); } catch (IOException e) { throw new CentralClientException(ERR_CANNOT_FIND_VERSIONS + packageSignature + ". reason: " + - e.getMessage()); + e.getMessage()); } finally { body.ifPresent(ResponseBody::close); try { @@ -355,17 +362,17 @@ public List getPackageVersions(String orgNamePath, String packageNamePat /** * Push a package to registry. * - * @param balaPath The path to the bala file. - * @param org The organization of the package. - * @param name The name of the package. - * @param version The version of the package. - * @param supportedPlatform The supported platform. - * @param ballerinaVersion The ballerina version. + * @param balaPath The path to the bala file. + * @param org The organization of the package. + * @param name The name of the package. + * @param version The version of the package. + * @param supportedPlatform The supported platform. + * @param ballerinaVersion The ballerina version. */ public void pushPackage(Path balaPath, String org, String name, String version, String supportedPlatform, - String ballerinaVersion) throws CentralClientException { - boolean enableOutputStream = - Boolean.parseBoolean(System.getProperty(CentralClientConstants.ENABLE_OUTPUT_STREAM)); + String ballerinaVersion) throws CentralClientException { + boolean enableOutputStream = Boolean + .parseBoolean(System.getProperty(CentralClientConstants.ENABLE_OUTPUT_STREAM)); String packageSignature = org + "/" + name + ":" + version; String url = this.baseUrl + "/" + PACKAGES; Optional body = Optional.empty(); @@ -387,8 +394,11 @@ public void pushPackage(Path balaPath, String org, String name, String version, ProgressRequestBody balaFileReqBodyWithProgressBar = new ProgressRequestBody(balaFileReqBody, packageSignature + " [" + projectRepo + " -> " + remoteRepo + "]", this.outStream); + byte[] hashInBytes = checkHash(balaPath.toString(), SHA256_ALGORITHM); + String digestVal = SHA256 + bytesToHex(hashInBytes); // If OutStream is disabled, then pass `balaFileReqBody` only Request pushRequest = getNewRequest(supportedPlatform, ballerinaVersion) + .addHeader(DIGEST, digestVal) .post(enableOutputStream ? balaFileReqBodyWithProgressBar : balaFileReqBody) .url(url) .build(); @@ -420,16 +430,18 @@ public void pushPackage(Path balaPath, String org, String name, String version, if (body.isPresent()) { Optional contentType = Optional.ofNullable(body.get().contentType()); - if (contentType.isPresent() && isApplicationJsonContentType(contentType.get().toString())) { + if (contentType.isPresent() && isApplicationJsonContentType(contentType.get().toString())) { // When request sent is invalid if (packagePushResponse.code() == HTTP_BAD_REQUEST) { Error error = new Gson().fromJson(responseBodyContent, Error.class); if (error.getMessage() != null && !"".equals(error.getMessage())) { - // Currently this error is returned from central when token is unauthorized. This will later - // be removed with https://github.com/wso2-enterprise/ballerina-registry/issues/745 + // Currently this error is returned from central when token is unauthorized. + // This will later + // be removed with + // https://github.com/wso2-enterprise/ballerina-registry/issues/745 if (error.getMessage().contains("subject claims missing in the user info repsonse")) { error.setMessage("unauthorized access token for organization: '" + org + "'. check " + - "access token set in 'Settings.toml' file."); + "access token set in 'Settings.toml' file."); } throw new CentralClientException(error.getMessage()); } @@ -437,11 +449,11 @@ public void pushPackage(Path balaPath, String org, String name, String version, // When error occurred at remote repository if (packagePushResponse.code() == HTTP_INTERNAL_ERROR || - packagePushResponse.code() == HTTP_UNAVAILABLE) { + packagePushResponse.code() == HTTP_UNAVAILABLE) { Error error = new Gson().fromJson(responseBodyContent, Error.class); if (error.getMessage() != null && !"".equals(error.getMessage())) { throw new CentralClientException(ERR_CANNOT_PUSH + "'" + packageSignature + - "' reason:" + error.getMessage()); + "' reason:" + error.getMessage()); } } } @@ -451,7 +463,7 @@ public void pushPackage(Path balaPath, String org, String name, String version, url + "'."); } catch (IOException e) { throw new CentralClientException(ERR_CANNOT_PUSH + "'" + packageSignature + "' to the remote repository '" + - url + "'. reason: " + e.getMessage()); + url + "'. reason: " + e.getMessage()); } finally { body.ifPresent(ResponseBody::close); try { @@ -465,22 +477,22 @@ public void pushPackage(Path balaPath, String org, String name, String version, /** * Pull a package from central. * - * @param org The organization of the package. - * @param name The name of the package. - * @param version The version of the package. - * @param packagePathInBalaCache The package path in Bala cache. - * @param supportedPlatform The supported platform. - * @param ballerinaVersion The ballerina version. - * @param isBuild If build option is enabled or not. - * @throws CentralClientException Central Client exception. + * @param org The organization of the package. + * @param name The name of the package. + * @param version The version of the package. + * @param packagePathInBalaCache The package path in Bala cache. + * @param supportedPlatform The supported platform. + * @param ballerinaVersion The ballerina version. + * @param isBuild If build option is enabled or not. + * @throws CentralClientException Central Client exception. */ public void pullPackage(String org, String name, String version, Path packagePathInBalaCache, - String supportedPlatform, String ballerinaVersion, boolean isBuild) + String supportedPlatform, String ballerinaVersion, boolean isBuild) throws CentralClientException { String resourceUrl = "/" + PACKAGES + "/" + org + "/" + name; - boolean enableOutputStream = - Boolean.parseBoolean(System.getProperty(CentralClientConstants.ENABLE_OUTPUT_STREAM)); - String packageSignature = org + "/" + name; + boolean enableOutputStream = Boolean + .parseBoolean(System.getProperty(CentralClientConstants.ENABLE_OUTPUT_STREAM)); + String packageSignature = org + "/" + name; String url = this.baseUrl + resourceUrl; // append version to url if available if (null != version && !version.isEmpty()) { @@ -517,16 +529,19 @@ public void pullPackage(String org, String name, String version, Path packagePat } logResponseVerbose(packagePullResponse, pkgPullResBodyContent); - // 302 - Package is found + // 302 - Package is found if (packagePullResponse.code() == HTTP_MOVED_TEMP) { // get redirect url from "location" header field Optional balaUrl = Optional.ofNullable(packagePullResponse.header(LOCATION)); Optional balaFileName = Optional.ofNullable(packagePullResponse.header(CONTENT_DISPOSITION)); Optional deprecationFlag = Optional.ofNullable(packagePullResponse.header(IS_DEPRECATED)); Optional deprecationMsg = Optional.ofNullable(packagePullResponse.header(DEPRECATE_MESSAGE)); + Optional digest = Optional.ofNullable(packagePullResponse.header(DIGEST)); + String digestVal = digest.isPresent() ? digest.get() : ""; boolean isDeprecated = deprecationFlag.isPresent() && Boolean.parseBoolean(deprecationFlag.get()); String deprecationMessage = deprecationMsg.isPresent() ? deprecationMsg.get() : ""; + if (!isBuild && isDeprecated) { outStream.println("WARNING [" + name + "] " + packageSignature + " is deprecated: " + deprecationMessage); @@ -549,7 +564,8 @@ public void pullPackage(String org, String name, String version, Path packagePat boolean isNightlyBuild = ballerinaVersion.contains("SNAPSHOT"); createBalaInHomeRepo(balaDownloadResponse, packagePathInBalaCache, org, name, isNightlyBuild, isDeprecated ? deprecationMessage : null, - balaUrl.get(), balaFileName.get(), enableOutputStream ? outStream : null, logFormatter); + balaUrl.get(), balaFileName.get(), enableOutputStream ? outStream : null, logFormatter, + digestVal); return; } else { String errorMessage = logFormatter.formatLog(ERR_CANNOT_PULL_PACKAGE + "'" + packageSignature + @@ -573,23 +589,22 @@ public void pullPackage(String org, String name, String version, Path packagePat if (contentType.isPresent() && isApplicationJsonContentType(contentType.get().toString())) { // If request sent is invalid or when package is not found if (packagePullResponse.code() == HTTP_BAD_REQUEST || - packagePullResponse.code() == HTTP_NOT_FOUND) { + packagePullResponse.code() == HTTP_NOT_FOUND) { Error error = new Gson().fromJson(pkgPullResBodyContent, Error.class); if (error.getMessage() != null && !"".equals(error.getMessage())) { throw new CentralClientException("error: " + error.getMessage()); } } - // When error occurred at remote repository + // When error occurred at remote repository if (packagePullResponse.code() == HTTP_INTERNAL_ERROR || - packagePullResponse.code() == HTTP_UNAVAILABLE) { + packagePullResponse.code() == HTTP_UNAVAILABLE) { Error error = new Gson().fromJson(pkgPullResBodyContent, Error.class); if (error.getMessage() != null && !"".equals(error.getMessage())) { - String errorMsg = - logFormatter.formatLog(ERR_CANNOT_PULL_PACKAGE + "'" + packageSignature + - "' from" + - " the remote repository '" + url + - "'. reason: " + error.getMessage()); + String errorMsg = logFormatter.formatLog(ERR_CANNOT_PULL_PACKAGE + "'" + packageSignature + + "' from" + + " the remote repository '" + url + + "'. reason: " + error.getMessage()); throw new CentralClientException(errorMsg); } } @@ -614,20 +629,20 @@ public void pullPackage(String org, String name, String version, Path packagePat /** * Pull a tool from central. * - * @param toolId The id of the tool. - * @param version The version of the package. - * @param balaCacheDirPath The package path in Bala cache. - * @param supportedPlatform The supported platform. - * @param ballerinaVersion The ballerina version. - * @param isBuild If build option is enabled or not. + * @param toolId The id of the tool. + * @param version The version of the package. + * @param balaCacheDirPath The package path in Bala cache. + * @param supportedPlatform The supported platform. + * @param ballerinaVersion The ballerina version. + * @param isBuild If build option is enabled or not. * @return An array containing isPulled, organization, package name and version. - * @throws CentralClientException Central Client exception. + * @throws CentralClientException Central Client exception. */ public String[] pullTool(String toolId, String version, Path balaCacheDirPath, String supportedPlatform, - String ballerinaVersion, boolean isBuild) throws CentralClientException { + String ballerinaVersion, boolean isBuild) throws CentralClientException { String resourceUrl = "/" + TOOLS + "/" + toolId; - boolean enableOutputStream = - Boolean.parseBoolean(System.getProperty(CentralClientConstants.ENABLE_OUTPUT_STREAM)); + boolean enableOutputStream = Boolean + .parseBoolean(System.getProperty(CentralClientConstants.ENABLE_OUTPUT_STREAM)); String toolSignature = toolId; String url = this.baseUrl + resourceUrl; @@ -662,7 +677,7 @@ public String[] pullTool(String toolId, String version, Path balaCacheDirPath, S } logResponseVerbose(packagePullResponse, pkgPullResBodyContent); - // 302 - Package is found + // 302 - Package is found if (packagePullResponse.code() == HTTP_OK) { Optional org = Optional.empty(); Optional pkgName = Optional.empty(); @@ -671,7 +686,7 @@ public String[] pullTool(String toolId, String version, Path balaCacheDirPath, S Optional platform = Optional.empty(); if (body.isPresent()) { Optional contentType = Optional.ofNullable(body.get().contentType()); - if (contentType.isPresent() && isApplicationJsonContentType(contentType.get().toString())) { + if (contentType.isPresent() && isApplicationJsonContentType(contentType.get().toString())) { JsonObject jsonContent = new Gson().fromJson(pkgPullResBodyContent, JsonObject.class); org = Optional.ofNullable(jsonContent.get(ORGANIZATION).getAsString()); pkgName = Optional.ofNullable(jsonContent.get(PKG_NAME).getAsString()); @@ -704,11 +719,12 @@ public String[] pullTool(String toolId, String version, Path balaCacheDirPath, S try { createBalaInHomeRepo(balaDownloadResponse, packagePathInBalaCache, org.get(), pkgName.get(), isNightlyBuild, null, balaUrl.get(), balaFileName, - enableOutputStream ? outStream : null, logFormatter); - return new String[]{String.valueOf(true), org.get(), pkgName.get(), latestVersion.get()}; + enableOutputStream ? outStream : null, logFormatter, ""); + return new String[] { String.valueOf(true), org.get(), pkgName.get(), latestVersion.get() }; } catch (PackageAlreadyExistsException ignore) { // package already exists. setting org, name and version fields is enough - return new String[]{String.valueOf(false), org.get(), pkgName.get(), latestVersion.get()}; + return new String[] { String.valueOf(false), org.get(), pkgName.get(), + latestVersion.get() }; } } else { String errorMessage = logFormatter.formatLog(ERR_CANNOT_PULL_PACKAGE + "'" + toolSignature + @@ -739,16 +755,15 @@ public String[] pullTool(String toolId, String version, Path balaCacheDirPath, S } } - // When error occurred at remote repository + // When error occurred at remote repository if (packagePullResponse.code() == HTTP_INTERNAL_ERROR || packagePullResponse.code() == HTTP_UNAVAILABLE) { Error error = new Gson().fromJson(pkgPullResBodyContent, Error.class); if (error.getMessage() != null && !"".equals(error.getMessage())) { - String errorMsg = - logFormatter.formatLog(ERR_CANNOT_PULL_PACKAGE + "'" + toolSignature + - "' from" + - " the remote repository '" + url + - "'. reason: " + error.getMessage()); + String errorMsg = logFormatter.formatLog(ERR_CANNOT_PULL_PACKAGE + "'" + toolSignature + + "' from" + + " the remote repository '" + url + + "'. reason: " + error.getMessage()); throw new CentralClientException(errorMsg); } } @@ -770,7 +785,6 @@ public String[] pullTool(String toolId, String version, Path balaCacheDirPath, S } } - /** * Resolve Package Names of modules. * @@ -778,8 +792,7 @@ public String[] pullTool(String toolId, String version, Path balaCacheDirPath, S * @throws CentralClientException Central Client exception. */ public PackageNameResolutionResponse resolvePackageNames(PackageNameResolutionRequest request, - String supportedPlatform, String ballerinaVersion) - throws CentralClientException { + String supportedPlatform, String ballerinaVersion) throws CentralClientException { String url = this.baseUrl + "/" + PACKAGES + "/" + RESOLVE_MODULES; @@ -853,10 +866,10 @@ public PackageNameResolutionResponse resolvePackageNames(PackageNameResolutionRe /** * Resolve Dependencies from central. * - * @throws CentralClientException Central Client exception. + * @throws CentralClientException Central Client exception. */ public PackageResolutionResponse resolveDependencies(PackageResolutionRequest request, String supportedPlatform, - String ballerinaVersion) + String ballerinaVersion) throws CentralClientException { String url = this.baseUrl + "/" + PACKAGES + "/" + RESOLVE_DEPENDENCIES; @@ -884,7 +897,7 @@ public PackageResolutionResponse resolveDependencies(PackageResolutionRequest re logResponseVerbose(packageResolutionResponse, packageResolutionResponseBody); if (body.isPresent()) { Optional contentType = Optional.ofNullable(body.get().contentType()); - if (contentType.isPresent() && isApplicationJsonContentType(contentType.get().toString())) { + if (contentType.isPresent() && isApplicationJsonContentType(contentType.get().toString())) { // If searching was successful if (packageResolutionResponse.code() == HTTP_OK) { return new Gson().fromJson(packageResolutionResponseBody, PackageResolutionResponse.class); @@ -952,7 +965,7 @@ public PackageSearchResult searchPackage(String query, String supportedPlatform, logResponseVerbose(searchResponse, searchResponseBody); if (body.isPresent()) { Optional contentType = Optional.ofNullable(body.get().contentType()); - if (contentType.isPresent() && isApplicationJsonContentType(contentType.get().toString())) { + if (contentType.isPresent() && isApplicationJsonContentType(contentType.get().toString())) { // If searching was successful if (searchResponse.code() == HTTP_OK) { return new Gson().fromJson(searchResponseBody, PackageSearchResult.class); @@ -973,7 +986,7 @@ public PackageSearchResult searchPackage(String query, String supportedPlatform, // If error occurred at remote repository if (searchResponse.code() == HTTP_INTERNAL_ERROR || - searchResponse.code() == HTTP_UNAVAILABLE) { + searchResponse.code() == HTTP_UNAVAILABLE) { Error error = new Gson().fromJson(searchResponseBody, Error.class); if (error.getMessage() != null && !"".equals(error.getMessage())) { throw new CentralClientException(ERR_CANNOT_SEARCH + "'" + query + "' reason:" + @@ -1022,7 +1035,7 @@ public ToolSearchResult searchTool(String keyword, String supportedPlatform, Str logResponseVerbose(searchResponse, searchResponseBody); if (body.isPresent()) { Optional contentType = Optional.ofNullable(body.get().contentType()); - if (contentType.isPresent() && isApplicationJsonContentType(contentType.get().toString())) { + if (contentType.isPresent() && isApplicationJsonContentType(contentType.get().toString())) { // If searching was successful if (searchResponse.code() == HTTP_OK) { return new Gson().fromJson(searchResponseBody, ToolSearchResult.class); @@ -1070,14 +1083,15 @@ public ToolSearchResult searchTool(String keyword, String supportedPlatform, Str * Deprecate a package in registry. */ public void deprecatePackage(String packageInfo, String deprecationMsg, String supportedPlatform, - String ballerinaVersion, Boolean isUndo) throws CentralClientException { + String ballerinaVersion, Boolean isUndo) throws CentralClientException { // Get existing package details - // PackageInfo is already validated to support the format org-name/package-name:version + // PackageInfo is already validated to support the format + // org-name/package-name:version Package existingPackage = getPackage(packageInfo.split("/")[0], packageInfo.split("/")[1].split(":")[0], packageInfo.split("/")[1].split(":")[1], supportedPlatform, ballerinaVersion); - String packageValue = packageInfo.endsWith(":*") ? packageInfo.substring(0, packageInfo.length() - 2) : - packageInfo; + String packageValue = packageInfo.endsWith(":*") ? packageInfo.substring(0, packageInfo.length() - 2) + : packageInfo; if (isUndo && !existingPackage.getDeprecated()) { this.outStream.println("package " + packageValue + " is not marked as deprecated in central"); return; @@ -1118,7 +1132,7 @@ public void deprecatePackage(String packageInfo, String deprecationMsg, String s logResponseVerbose(deprecationResponse, deprecationResponseBody); if (body.isPresent()) { Optional contentType = Optional.ofNullable(body.get().contentType()); - if (contentType.isPresent() && isApplicationJsonContentType(contentType.get().toString())) { + if (contentType.isPresent() && isApplicationJsonContentType(contentType.get().toString())) { // If deprecation was successful if (deprecationResponse.code() == HTTP_OK) { Package packageResponse = new Gson().fromJson(deprecationResponseBody, Package.class); @@ -1240,7 +1254,6 @@ public JsonElement getPackages(Map params, String supportedPlatf } } - /** * Get connectors with search filters. * @@ -1253,7 +1266,8 @@ public JsonElement getPackages(Map params, String supportedPlatf public JsonElement getConnectors(Map params, String supportedPlatform, String ballerinaVersion) throws CentralClientException { Optional body = Optional.empty(); - // TODO: update this client initiation with default timeouts after fixing central/connectors API. + // TODO: update this client initiation with default timeouts after fixing + // central/connectors API. OkHttpClient client = new OkHttpClient.Builder() .followRedirects(false) .connectTimeout(connectTimeout, TimeUnit.SECONDS) @@ -1472,7 +1486,8 @@ protected void handleResponseErrors(Response response, String msg) throws Centra handleUnauthorizedResponse(body); } - // If error occurred at remote repository or invalid/no response received at the gateway server + // If error occurred at remote repository or invalid/no response received at the + // gateway server if (response.code() == HTTP_INTERNAL_ERROR || response.code() == HTTP_UNAVAILABLE || response.code() == HTTP_BAD_GATEWAY || response.code() == HTTP_GATEWAY_TIMEOUT) { Error error = new Gson().fromJson(body.get().string(), Error.class); @@ -1540,7 +1555,8 @@ public void setAccessToken(String accessToken) { public JsonElement getTriggers(Map params, String supportedPlatform, String ballerinaVersion) throws CentralClientException { Optional body = Optional.empty(); - // TODO: update this client initiation with default timeouts after fixing central/triggers API. + // TODO: update this client initiation with default timeouts after fixing + // central/triggers API. OkHttpClient client = new OkHttpClient.Builder() .followRedirects(false) .connectTimeout(connectTimeout, TimeUnit.SECONDS) @@ -1648,8 +1664,8 @@ public JsonObject getTrigger(String id, String supportedPlatform, String balleri /** * Handle unauthorized response. * - * @param org org name - * @param body response body + * @param org org name + * @param body response body * @param responseBody error message * @throws IOException when accessing response body * @throws CentralClientException with unauthorized error message @@ -1672,7 +1688,7 @@ private void handleUnauthorizedResponse(String org, Optional body, /** * Handle unauthorized response. * - * @param body response body + * @param body response body * @param responseBody error message * @throws IOException when accessing response body * @throws CentralClientException with unauthorized error message diff --git a/cli/central-client/src/main/java/org/ballerinalang/central/client/CentralClientConstants.java b/cli/central-client/src/main/java/org/ballerinalang/central/client/CentralClientConstants.java index 3c94e0e177a2..933ebaa160f2 100644 --- a/cli/central-client/src/main/java/org/ballerinalang/central/client/CentralClientConstants.java +++ b/cli/central-client/src/main/java/org/ballerinalang/central/client/CentralClientConstants.java @@ -47,6 +47,7 @@ private CentralClientConstants() { static final String ANY_PLATFORM = "any"; static final String PKG_NAME = "name"; static final String IS_DEPRECATED = "isdeprecated"; + static final String DIGEST = "digest"; static final String DEPRECATE_MESSAGE = "deprecatemessage"; public static final String ENABLE_OUTPUT_STREAM = "enableOutputStream"; static final String PRODUCTION_REPO = "central.ballerina.io"; @@ -54,5 +55,10 @@ private CentralClientConstants() { static final String DEV_REPO = "dev-central.ballerina.io"; public static final String BALLERINA_STAGE_CENTRAL = "BALLERINA_STAGE_CENTRAL"; public static final String BALLERINA_DEV_CENTRAL = "BALLERINA_DEV_CENTRAL"; + public static final int BYTES_FOR_KB = 1024; + public static final int PROGRESS_BAR_BYTE_THRESHOLD = 5; + public static final int UPDATE_INTERVAL_MILLIS = 1000; + public static final String SHA256 = "sha-256="; + public static final String SHA256_ALGORITHM = "SHA-256"; } diff --git a/cli/central-client/src/main/java/org/ballerinalang/central/client/Utils.java b/cli/central-client/src/main/java/org/ballerinalang/central/client/Utils.java index 8bfa30db8e10..f246d1194c17 100644 --- a/cli/central-client/src/main/java/org/ballerinalang/central/client/Utils.java +++ b/cli/central-client/src/main/java/org/ballerinalang/central/client/Utils.java @@ -40,6 +40,7 @@ import java.io.BufferedWriter; import java.io.File; +import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FileWriter; import java.io.IOException; @@ -53,8 +54,12 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; +import java.security.DigestInputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.HashMap; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; @@ -64,7 +69,12 @@ import static org.ballerinalang.central.client.CentralClientConstants.DEV_REPO; import static org.ballerinalang.central.client.CentralClientConstants.PRODUCTION_REPO; import static org.ballerinalang.central.client.CentralClientConstants.RESOLVED_REQUESTED_URI; +import static org.ballerinalang.central.client.CentralClientConstants.SHA256; +import static org.ballerinalang.central.client.CentralClientConstants.SHA256_ALGORITHM; import static org.ballerinalang.central.client.CentralClientConstants.STAGING_REPO; +import static org.ballerinalang.central.client.CentralClientConstants.BYTES_FOR_KB; +import static org.ballerinalang.central.client.CentralClientConstants.PROGRESS_BAR_BYTE_THRESHOLD; +import static org.ballerinalang.central.client.CentralClientConstants.UPDATE_INTERVAL_MILLIS; /** * Utils class for this package. @@ -76,6 +86,9 @@ public class Utils { System.getenv(BALLERINA_STAGE_CENTRAL)); public static final boolean SET_BALLERINA_DEV_CENTRAL = Boolean.parseBoolean( System.getenv(BALLERINA_DEV_CENTRAL)); + + public static final String ERR_CANNOT_PULL_PACKAGE = "error: failed to pull the package: "; + private Utils() { } @@ -89,21 +102,22 @@ public enum RequestMethod { /** * Create the bala in home repo. * - * @param balaDownloadResponse http response for downloading the bala file - * @param pkgPathInBalaCache package path in bala cache, .ballerina/bala_cache// - * @param pkgOrg package org - * @param pkgName package name - * @param isNightlyBuild is nightly build - * @param deprecationMsg deprecation message for deprecated packages - * @param newUrl new redirect url - * @param contentDisposition content disposition header - * @param outStream Output print stream - * @param logFormatter log formatter + * @param balaDownloadResponse http response for downloading the bala file + * @param pkgPathInBalaCache package path in bala cache, + * .ballerina/bala_cache// + * @param pkgOrg package org + * @param pkgName package name + * @param isNightlyBuild is nightly build + * @param deprecationMsg deprecation message for deprecated packages + * @param newUrl new redirect url + * @param contentDisposition content disposition header + * @param outStream Output print stream + * @param logFormatter log formatter */ public static void createBalaInHomeRepo(Response balaDownloadResponse, Path pkgPathInBalaCache, String pkgOrg, - String pkgName, boolean isNightlyBuild, String deprecationMsg, - String newUrl, String contentDisposition, PrintStream outStream, - LogFormatter logFormatter) + String pkgName, boolean isNightlyBuild, String deprecationMsg, + String newUrl, String contentDisposition, PrintStream outStream, + LogFormatter logFormatter, String trueDigest) throws CentralClientException { long responseContentLength = 0; @@ -130,7 +144,7 @@ public static void createBalaInHomeRepo(Response balaDownloadResponse, Path pkgP String balaFile = getBalaFileName(contentDisposition, uriParts[uriParts.length - 1]); String platform = getPlatformFromBala(balaFile, pkgName, validPkgVersion); Path balaCacheWithPkgPath = pkgPathInBalaCache.resolve(validPkgVersion).resolve(platform); - //.ballerina/bala_cache/// + // .ballerina/bala_cache/// try { if (Files.isDirectory(balaCacheWithPkgPath) && Files.list(balaCacheWithPkgPath).findAny().isPresent()) { @@ -158,14 +172,15 @@ public static void createBalaInHomeRepo(Response balaDownloadResponse, Path pkgP logFormatter.formatLog("error accessing bala : " + balaCacheWithPkgPath.toString())); } - // Create the following temp path bala/// + // Create the following temp path + // bala/// Path tempPath = pkgPathInBalaCache.resolve(validPkgVersion + "_temp").resolve(platform); createBalaFileDirectory(tempPath, logFormatter); // Write balaFiles to tempPath writeBalaFile(balaDownloadResponse, tempPath.resolve(balaFile), pkgOrg + "/" + pkgName + ":" + validPkgVersion, responseContentLength, - outStream, logFormatter, pkgPathInBalaCache.resolve(validPkgVersion)); + outStream, logFormatter, pkgPathInBalaCache.resolve(validPkgVersion), trueDigest); // Once files are written to temp path, rename temp path with platform name try { @@ -240,21 +255,23 @@ private static void createBalaFileDirectory(Path fullPathToStoreBala, LogFormatt /** * Write bala file to the home repo. * - * @param balaDownloadResponse http bala file download response - * @param balaPath path of the bala file - * @param fullPkgName full package name, /: - * @param resContentLength response content length - * @param outStream Output print stream - * @param logFormatter log formatter - * @param homeRepo path of the repo bala file is saved to + * @param balaDownloadResponse http bala file download response + * @param balaPath path of the bala file + * @param fullPkgName full package name, + * /: + * @param resContentLength response content length + * @param outStream Output print stream + * @param logFormatter log formatter + * @param homeRepo path of the repo bala file is saved to */ static void writeBalaFile(Response balaDownloadResponse, Path balaPath, String fullPkgName, long resContentLength, - PrintStream outStream, LogFormatter logFormatter, Path homeRepo) throws CentralClientException { + PrintStream outStream, LogFormatter logFormatter, Path homeRepo, String trueDigest) + throws CentralClientException { Optional body = Optional.ofNullable(balaDownloadResponse.body()); if (body.isPresent()) { try { try (InputStream inputStream = body.get().byteStream(); - FileOutputStream outputStream = new FileOutputStream(balaPath.toString())) { + FileOutputStream outputStream = new FileOutputStream(balaPath.toString())) { if (outStream == null) { writeAndHandleProgressQuietly(inputStream, outputStream); } else { @@ -266,9 +283,9 @@ static void writeBalaFile(Response balaDownloadResponse, Path balaPath, String f logFormatter.formatLog("error occurred copying the bala file: " + e.getMessage())); } try { - extractBala(balaPath, Optional.of(balaPath.getParent()).get()); + extractBala(balaPath, Optional.of(balaPath.getParent()).get(), trueDigest, fullPkgName, outStream); Files.delete(balaPath); - } catch (IOException e) { + } catch (IOException | CentralClientException e) { throw new CentralClientException( logFormatter.formatLog("error occurred extracting the bala file: " + e.getMessage())); } @@ -302,12 +319,12 @@ private static void handleNightlyBuild(boolean isNightlyBuild, Path balaCacheWit /** * Handle package deprecation. * - * @param deprecateMsg deprecated message + * @param deprecateMsg deprecated message * @param balaCacheWithPkgPath bala cache with package path * @param logFormatter log formatter */ private static void handlePackageDeprecation(String deprecateMsg, Path balaCacheWithPkgPath, - LogFormatter logFormatter) throws CentralClientException { + LogFormatter logFormatter) throws CentralClientException { if (deprecateMsg != null) { // If its a deprecated package tag a file to denote as deprecated Path deprecateMsgFile = Paths.get(balaCacheWithPkgPath.toString(), DEPRECATED_META_FILE_NAME); @@ -338,7 +355,7 @@ private static void writeAndHandleProgress(InputStream inputStream, FileOutputSt String remoteRepo = getRemoteRepo(); String progressBarTask = fullPkgName + " [" + remoteRepo + " ->" + homeRepo + "] "; try (ProgressBar progressBar = new ProgressBar(progressBarTask, totalSizeInKB, 1000, - outStream, ProgressBarStyle.ASCII, " KB", 1)) { + outStream, ProgressBarStyle.ASCII, " KB", 1)) { while ((count = inputStream.read(buffer)) > 0) { outputStream.write(buffer, 0, count); progressBar.step(); @@ -385,7 +402,7 @@ private static void writeDeprecatedMsg(Path metaFilePath, LogFormatter logFormat if (metaFilePath.toFile().exists()) { try (FileWriter fileWriter = new FileWriter(metaFilePath.toAbsolutePath().toString(), Charset.defaultCharset()); - BufferedWriter bufferedWriter = new BufferedWriter(fileWriter)) { + BufferedWriter bufferedWriter = new BufferedWriter(fileWriter)) { bufferedWriter.write(message); } catch (IOException e) { throw new CentralClientException( @@ -402,7 +419,8 @@ private static void writeDeprecatedMsg(Path metaFilePath, LogFormatter logFormat * @return converted list of strings */ static List getAsList(String arrayString) { - return new Gson().fromJson(arrayString, new TypeToken>() { }.getType()); + return new Gson().fromJson(arrayString, new TypeToken>() { + }.getType()); } /** @@ -419,9 +437,27 @@ private static String getPlatformFromBala(String balaName, String packageName, S return balaName.split(packageName + "-")[1].split("-" + version)[0]; } - private static void extractBala(Path balaFilePath, Path balaFileDestPath) throws IOException { + private static void extractBala(Path balaFilePath, Path balaFileDestPath, String trueDigest, String packageName, + PrintStream outStream) + throws IOException, CentralClientException { Files.createDirectories(balaFileDestPath); URI zipURI = URI.create("jar:" + balaFilePath.toUri().toString()); + byte[] hashInBytes = checkHash(balaFilePath.toString(), SHA256_ALGORITHM); + + // If the hash value is not matching , throw an exception. + if (Objects.equals((SHA256 + bytesToHex(hashInBytes)), trueDigest)) { + StringBuilder warning = new StringBuilder( + String.format("*************************************************************%n" + + "* WARNING: Certain packages may have originated from sources other than the official distributors. *%n" + + "*************************************************************%n%n" + + "* Verification failed: The hash value of the following package could not be confirmed. %n" + + packageName + + "%n")); + if (outStream != null) { + outStream.println(warning.toString()); + } + } + try (FileSystem zipFileSystem = FileSystems.newFileSystem(zipURI, new HashMap<>())) { Path packageRoot = zipFileSystem.getPath("/"); List paths = Files.walk(packageRoot).filter(path -> path != packageRoot).collect(Collectors.toList()); @@ -436,24 +472,54 @@ private static void extractBala(Path balaFilePath, Path balaFileDestPath) throws } } + public static byte[] checkHash(String filePath, String algorithm) { + MessageDigest md; + try { + md = MessageDigest.getInstance(algorithm); + } catch (NoSuchAlgorithmException e) { + throw new IllegalArgumentException(e); + } + + try (InputStream is = new FileInputStream(filePath); + DigestInputStream dis = new DigestInputStream(is, md)) { + while (dis.read() != -1) { + } + md = dis.getMessageDigest(); + return md.digest(); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new IllegalArgumentException(e); + } + } + + public static String bytesToHex(byte[] bytes) { + StringBuilder sb = new StringBuilder(); + for (byte b : bytes) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } + static String getBearerToken(String accessToken) { return "Bearer " + accessToken; } /** - * Custom request body implementation that indicate the number of bytes written using a progress bar. + * Custom request body implementation that indicate the number of bytes written + * using a progress bar. */ public static class ProgressRequestBody extends RequestBody { private final RequestBody reqBody; private final String task; private final PrintStream out; - + public ProgressRequestBody(RequestBody reqBody, String task, PrintStream out) { this.reqBody = reqBody; this.task = task; this.out = out; } - + @Override public MediaType contentType() { return this.reqBody.contentType(); @@ -469,19 +535,21 @@ public void writeTo(BufferedSink sink) throws IOException { final long totalBytes = contentLength(); long byteConverter; String unitName; - - if (totalBytes < 1024 * 5) { // use bytes for progress bar if payload is less than 5 KB + + if (totalBytes < BYTES_FOR_KB * PROGRESS_BAR_BYTE_THRESHOLD) { + // use bytes for progress bar if payload is less than 5 KB byteConverter = 1; unitName = " B"; - } else if (totalBytes < 1024 * 1024 * 5) { // use kilobytes for progress bar if payload is less than 5 MB - byteConverter = 1024; + } else if (totalBytes < BYTES_FOR_KB * BYTES_FOR_KB * PROGRESS_BAR_BYTE_THRESHOLD) { + // use kilobytes for progress bar if payload is less than 5 MB + byteConverter = BYTES_FOR_KB; unitName = " KB"; } else { // else use megabytes for progress bar. - byteConverter = 1024 * 1024; + byteConverter = BYTES_FOR_KB * BYTES_FOR_KB; unitName = " MB"; } - - ProgressBar progressBar = new ProgressBar(task, contentLength(), 1000, out, + + ProgressBar progressBar = new ProgressBar(task, contentLength(), UPDATE_INTERVAL_MILLIS, out, ProgressBarStyle.ASCII, unitName, byteConverter); CountingSink countingSink = new CountingSink(sink, progressBar); BufferedSink progressSink = Okio.buffer(countingSink); @@ -490,16 +558,16 @@ public void writeTo(BufferedSink sink) throws IOException { progressBar.close(); } } - + private static class CountingSink extends ForwardingSink { private long bytesWritten = 0; private final ProgressBar progressBar; - + public CountingSink(Sink delegate, ProgressBar progressBar) { super(delegate); this.progressBar = progressBar; } - + @Override public void write(Buffer source, long byteCount) throws IOException { super.write(source, byteCount); diff --git a/cli/central-client/src/test/java/org/ballerinalang/central/client/TestCentralApiClient.java b/cli/central-client/src/test/java/org/ballerinalang/central/client/TestCentralApiClient.java index 2f49898bd712..86917090ea7f 100644 --- a/cli/central-client/src/test/java/org/ballerinalang/central/client/TestCentralApiClient.java +++ b/cli/central-client/src/test/java/org/ballerinalang/central/client/TestCentralApiClient.java @@ -63,6 +63,7 @@ import static org.ballerinalang.central.client.CentralClientConstants.APPLICATION_OCTET_STREAM; import static org.ballerinalang.central.client.CentralClientConstants.AUTHORIZATION; import static org.ballerinalang.central.client.CentralClientConstants.CONTENT_DISPOSITION; +import static org.ballerinalang.central.client.CentralClientConstants.DIGEST; import static org.ballerinalang.central.client.CentralClientConstants.IDENTITY; import static org.ballerinalang.central.client.CentralClientConstants.LOCATION; import static org.ballerinalang.central.client.TestUtils.cleanDirectory; @@ -156,6 +157,7 @@ public void testPullPackage() throws IOException, CentralClientException { .code(HttpURLConnection.HTTP_MOVED_TEMP) .addHeader(LOCATION, this.balaUrl) .addHeader(CONTENT_DISPOSITION, balaFileName) + .addHeader(DIGEST, "sha-256=47e043c80d516234b1e6bd93140f126c9d9e79b5c7c0600cc6316d12504c2cf4") .message("") .body(null) .build(); diff --git a/cli/central-client/src/test/java/org/ballerinalang/central/client/TestUtils.java b/cli/central-client/src/test/java/org/ballerinalang/central/client/TestUtils.java index c0de458e58d5..537b34dedef1 100644 --- a/cli/central-client/src/test/java/org/ballerinalang/central/client/TestUtils.java +++ b/cli/central-client/src/test/java/org/ballerinalang/central/client/TestUtils.java @@ -125,7 +125,8 @@ public void testWriteBalaFile() throws IOException, CentralClientException { .build(); writeBalaFile(mockResponse, tempBalaCache.resolve(balaName), "wso2/sf:1.1.0", - 10000, System.out, new LogFormatter(), UTILS_TEST_RESOURCES); + 10000, System.out, new LogFormatter(), UTILS_TEST_RESOURCES, + "sha-256=47e043c80d516234b1e6bd93140f126c9d9e79b5c7c0600cc6316d12504c2cf4"); Assert.assertTrue(tempBalaCache.resolve("package.json").toFile().exists()); Assert.assertTrue(tempBalaCache.resolve("bala.json").toFile().exists()); @@ -159,7 +160,7 @@ public void testCreateBalaInHomeRepo() throws IOException, CentralClientExceptio Path balaFile = UTILS_TEST_RESOURCES.resolve(balaName); File initialFile = new File(String.valueOf(balaFile)); InputStream targetStream = new FileInputStream(initialFile); - + Request mockRequest = new Request.Builder() .get() .url("https://localhost:9090/registry/packages/wso2/sf/*") @@ -173,26 +174,26 @@ public void testCreateBalaInHomeRepo() throws IOException, CentralClientExceptio .message("") .body(ResponseBody.create( MediaType.get(APPLICATION_JSON), - Files.readAllBytes(balaFile) - )) + Files.readAllBytes(balaFile))) .build(); final String balaUrl = "https://fileserver.dev-central.ballerina.io/2.0/wso2/sf/1.3.5/sf-2020r2-any-1.3.5.bala"; createBalaInHomeRepo(mockResponse, tempBalaCache.resolve("wso2").resolve("sf"), - "wso2", "sf", false, null, balaUrl, "", System.out, new LogFormatter()); + "wso2", "sf", false, null, balaUrl, "", System.out, new LogFormatter(), + "sha-256=47e043c80d516234b1e6bd93140f126c9d9e79b5c7c0600cc6316d12504c2cf4"); Assert.assertTrue(tempBalaCache.resolve("wso2").resolve("sf").resolve("1.3.5").toFile().exists()); cleanBalaCache(); } - @Test(description = "Test create bala when same bala exists in the given directory", - dependsOnMethods = "testCreateBalaInHomeRepo") + @Test(description = "Test create bala when same bala exists in the given directory", + dependsOnMethods = "testCreateBalaInHomeRepo") public void testCreateBalaInHomeRepoWhenBalaExists() throws IOException { final String balaName = "sf-any.bala"; Path balaFile = UTILS_TEST_RESOURCES.resolve(balaName); File initialFile = new File(String.valueOf(balaFile)); InputStream targetStream = new FileInputStream(initialFile); - + Request mockRequest = new Request.Builder() .get() .url("https://localhost:9090/registry/packages/wso2/sf/*") @@ -206,22 +207,22 @@ public void testCreateBalaInHomeRepoWhenBalaExists() throws IOException { .message("") .body(ResponseBody.create( MediaType.get(APPLICATION_JSON), - Files.readAllBytes(balaFile) - )) + Files.readAllBytes(balaFile))) .build(); final String balaUrl = "https://fileserver.dev-central.ballerina.io/2.0/wso2/sf/1.3.5/sf-2020r2-any-1.3.5.bala"; try { createBalaInHomeRepo(mockResponse, tempBalaCache.resolve("wso2").resolve("sf"), "wso2", "sf", false, null, - balaUrl, "", System.out, new LogFormatter()); + balaUrl, "", System.out, new LogFormatter(), + "sha-256=47e043c80d516234b1e6bd93140f126c9d9e79b5c7c0600cc6316d12504c2cf4"); } catch (CentralClientException e) { Assert.assertTrue(e.getMessage().contains("package already exists in the home repository:")); } finally { cleanBalaCache(); } } - + @Test public void testJsonContentTypeChecker() { Assert.assertTrue(isApplicationJsonContentType(APPLICATION_JSON));