From e9800d11b6b6a7e8b6490f6d01d0e64e66006152 Mon Sep 17 00:00:00 2001 From: Ian Lynagh Date: Thu, 16 Nov 2023 13:34:14 +0000 Subject: [PATCH 1/3] Kotlin: Build: Refactor version handling We now have a proper class to represent versions, rather than using tuples. The version is passed deeper down, so we can now have version-dependent compilation flags. --- java/kotlin-extractor/build.py | 27 +++++------- .../kotlin_plugin_versions.py | 41 ++++++++++++++----- 2 files changed, 40 insertions(+), 28 deletions(-) diff --git a/java/kotlin-extractor/build.py b/java/kotlin-extractor/build.py index 752a436218fe..6d5bfcea92c5 100755 --- a/java/kotlin-extractor/build.py +++ b/java/kotlin-extractor/build.py @@ -87,7 +87,7 @@ def write_arg_file(arg_file, args): raise Exception('Single quote in argument: ' + arg) f.write("'" + arg.replace('\\', '/') + "'\n") -def compile_to_dir(build_dir, srcs, language_version, classpath, java_classpath, output): +def compile_to_dir(build_dir, srcs, version, classpath, java_classpath, output): # Use kotlinc to compile .kt files: kotlin_arg_file = build_dir + '/kotlin.args' kotlin_args = ['-Werror', @@ -96,7 +96,7 @@ def compile_to_dir(build_dir, srcs, language_version, classpath, java_classpath, '-d', output, '-module-name', 'codeql-kotlin-extractor', '-Xsuppress-version-warnings', - '-language-version', language_version, + '-language-version', version.toLanguageVersionString(), '-no-reflect', '-no-stdlib', '-jvm-target', '1.8', '-classpath', classpath] + srcs @@ -116,14 +116,14 @@ def compile_to_dir(build_dir, srcs, language_version, classpath, java_classpath, run_process([javac, '@' + java_arg_file]) -def compile_to_jar(build_dir, tmp_src_dir, srcs, language_version, classpath, java_classpath, output): +def compile_to_jar(build_dir, tmp_src_dir, srcs, version, classpath, java_classpath, output): class_dir = build_dir + '/classes' if os.path.exists(class_dir): shutil.rmtree(class_dir) os.makedirs(class_dir) - compile_to_dir(build_dir, srcs, language_version, classpath, java_classpath, class_dir) + compile_to_dir(build_dir, srcs, version, classpath, java_classpath, class_dir) run_process(['jar', 'cf', output, '-C', class_dir, '.', @@ -161,7 +161,7 @@ def transform_to_embeddable(srcs): f.write(content) -def compile(jars, java_jars, dependency_folder, transform_to_embeddable, output, build_dir, current_version): +def compile(jars, java_jars, dependency_folder, transform_to_embeddable, output, build_dir, version_str): classpath = bases_to_classpath(dependency_folder, jars) java_classpath = bases_to_classpath(dependency_folder, java_jars) @@ -179,23 +179,16 @@ def compile(jars, java_jars, dependency_folder, transform_to_embeddable, output, with open(resource_dir + '/extractor.name', 'w') as f: f.write(output) - parsed_current_version = kotlin_plugin_versions.version_string_to_tuple( - current_version) + version = kotlin_plugin_versions.version_string_to_version(version_str) - for version in kotlin_plugin_versions.many_versions: - parsed_version = kotlin_plugin_versions.version_string_to_tuple( - version) - if parsed_version[0] < parsed_current_version[0] or \ - (parsed_version[0] == parsed_current_version[0] and parsed_version[1] < parsed_current_version[1]) or \ - (parsed_version[0] == parsed_current_version[0] and parsed_version[1] == parsed_current_version[1] and parsed_version[2] <= parsed_current_version[2]): + for a_version in kotlin_plugin_versions.many_versions_versions_asc: + if a_version.lessThanOrEqual(version): d = tmp_src_dir + '/main/kotlin/utils/versions/v_' + \ - version.replace('.', '_') + a_version.toString().replace('.', '_') if os.path.exists(d): # copy and overwrite files from the version folder to the include folder shutil.copytree(d, include_version_folder, dirs_exist_ok=True) - language_version = str(parsed_current_version[0]) + '.' + str(parsed_current_version[1]) - # remove all version folders: shutil.rmtree(tmp_src_dir + '/main/kotlin/utils/versions') @@ -203,7 +196,7 @@ def compile(jars, java_jars, dependency_folder, transform_to_embeddable, output, transform_to_embeddable(srcs) - compile_to_jar(build_dir, tmp_src_dir, srcs, language_version, classpath, java_classpath, output) + compile_to_jar(build_dir, tmp_src_dir, srcs, version, classpath, java_classpath, output) shutil.rmtree(tmp_src_dir) diff --git a/java/kotlin-extractor/kotlin_plugin_versions.py b/java/kotlin-extractor/kotlin_plugin_versions.py index 1cf8a9cb0d85..da2b1d19b8c5 100755 --- a/java/kotlin-extractor/kotlin_plugin_versions.py +++ b/java/kotlin-extractor/kotlin_plugin_versions.py @@ -14,19 +14,40 @@ def is_windows(): return True return False -def version_tuple_to_string(version): - return f'{version[0]}.{version[1]}.{version[2]}{version[3]}' +class Version: + def __init__(self, major, minor, patch, tag): + self.major = major + self.minor = minor + self.patch = patch + self.tag = tag -def version_string_to_tuple(version): + def toTupleWithTag(self): + return [self.major, self.minor, self.patch, self.tag] + + def toTupleNoTag(self): + return [self.major, self.minor, self.patch] + + def lessThanOrEqual(self, other): + return self.toTupleNoTag() <= other.toTupleNoTag() + + def toString(self): + return f'{self.major}.{self.minor}.{self.patch}{self.tag}' + + def toLanguageVersionString(self): + return f'{self.major}.{self.minor}' + +def version_string_to_version(version): m = re.match(r'([0-9]+)\.([0-9]+)\.([0-9]+)(.*)', version) - return tuple([int(m.group(i)) for i in range(1, 4)] + [m.group(4)]) + return Version(int(m.group(1)), int(m.group(2)), int(m.group(3)), m.group(4)) # Version number used by CI. ci_version = '1.9.0' many_versions = [ '1.5.0', '1.5.10', '1.5.20', '1.5.30', '1.6.0', '1.6.20', '1.7.0', '1.7.20', '1.8.0', '1.9.0-Beta', '1.9.20-Beta' ] -many_versions_tuples = [version_string_to_tuple(v) for v in many_versions] +many_versions_versions = [version_string_to_version(v) for v in many_versions] +many_versions_versions_asc = sorted(many_versions_versions, key = lambda v: v.toTupleWithTag()) +many_versions_versions_desc = reversed(many_versions_versions_asc) class KotlincNotFoundException(Exception): pass @@ -40,13 +61,11 @@ def get_single_version(fakeVersionOutput = None): m = re.match(r'.* kotlinc-jvm ([0-9]+\.[0-9]+\.[0-9]+-?[a-zA-Z]*) .*', versionOutput) if m is None: raise Exception('Cannot detect version of kotlinc (got ' + str(versionOutput) + ')') - current_version = version_string_to_tuple(m.group(1)) - - many_versions_tuples.sort(reverse = True) + current_version = version_string_to_version(m.group(1)) - for version in many_versions_tuples: - if version[0:3] <= current_version[0:3]: - return version_tuple_to_string(version) + for version in many_versions_versions_desc: + if version.lessThanOrEqual(current_version): + return version.toString() raise Exception(f'No suitable kotlinc version found for {current_version} (got {versionOutput}; know about {str(many_versions)})') From cca78ca1908ed07c3fd780b06c01608fd25acad0 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Thu, 16 Nov 2023 19:05:59 +0100 Subject: [PATCH 2/3] C#: Fix the dotnet pack integration test. --- csharp/ql/integration-tests/all-platforms/dotnet_pack/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/csharp/ql/integration-tests/all-platforms/dotnet_pack/test.py b/csharp/ql/integration-tests/all-platforms/dotnet_pack/test.py index b91e73ef11c8..258f5f3b7644 100644 --- a/csharp/ql/integration-tests/all-platforms/dotnet_pack/test.py +++ b/csharp/ql/integration-tests/all-platforms/dotnet_pack/test.py @@ -2,10 +2,10 @@ from create_database_utils import * from diagnostics_test_utils import * -run_codeql_database_create(['dotnet pack'], db=None, lang="csharp") +run_codeql_database_create(['dotnet pack -o nugetpackage'], db=None, lang="csharp") ## Check that the NuGet package is created. -if not os.path.isfile("bin/Debug/dotnet_pack.1.0.0.nupkg"): +if not os.path.isfile("nugetpackage/dotnet_pack.1.0.0.nupkg"): raise Exception("The NuGet package was not created.") check_diagnostics() From d7760de4c6e5c4580147c79aaec27f4f83df1311 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Thu, 16 Nov 2023 19:56:01 +0000 Subject: [PATCH 3/3] C#: Fix dotnet_test_mstest --- .../posix-only/dotnet_test_mstest/dotnet_test_mstest.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/csharp/ql/integration-tests/posix-only/dotnet_test_mstest/dotnet_test_mstest.csproj b/csharp/ql/integration-tests/posix-only/dotnet_test_mstest/dotnet_test_mstest.csproj index 95c7586e04eb..c6d0c87e9dbf 100644 --- a/csharp/ql/integration-tests/posix-only/dotnet_test_mstest/dotnet_test_mstest.csproj +++ b/csharp/ql/integration-tests/posix-only/dotnet_test_mstest/dotnet_test_mstest.csproj @@ -7,6 +7,7 @@ false Exe + false