From fa4161118bcf981a694bd19a32ceab0521393784 Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Fri, 22 Nov 2024 16:29:53 -0700 Subject: [PATCH 01/20] init implementation --- rmake.py | 502 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 272 insertions(+), 230 deletions(-) diff --git a/rmake.py b/rmake.py index 986ad6087..ec38a6597 100644 --- a/rmake.py +++ b/rmake.py @@ -10,246 +10,288 @@ import ctypes import pathlib from fnmatch import fnmatchcase +import platform as pf +import shutil + +class AutoBuilder: + def __init__(self): + self.param = {} + self.parser = argparse.ArgumentParser(description=""" + Checks build arguments + """) + + sysInfo = pf.uname() + + # getting os information + self.OS_info = { + "Machine" : sysInfo.machine, + "Node Name" : sysInfo.node, + "Num Processor" : os.cpu_count(), + "Processor" : sysInfo.processor, + "Release" : sysInfo.release, + "System" : sysInfo.system, + "Version" : sysInfo.version, + } + m = ' System Information ' + + print() + print(f'{m:-^100}') + + for k in self.OS_info: + print(f'\t {k}: {self.OS_info[k]}') + print(f'\n\n') + + + self.lib_dir = os.path.dirname(os.path.abspath(__file__)) + self.toolchain = f'toolchain-linux.cmake' if self.OS_info['System'] == 'Linux' else f'toolchain-windows.cmake' + + def __parse_args__(self): + """Parse command-line arguments""" + default_gpus = 'gfx906:xnack-,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' + + self.parser.add_argument('-g', '--debug', required=False, default=False, action='store_true', + help='Generate Debug build (default: False)') + self.parser.add_argument( '--build_dir', type=str, required=False, default="build", + help='Build directory path (default: build)') + self.parser.add_argument( '--deps_dir', type=str, required=False, default=None, + help='Dependencies directory path (default: build/deps)') + self.parser.add_argument( '--skip_ld_conf_entry', required=False, default=False) + self.parser.add_argument( '--static', required=False, default=False, dest='static_lib', action='store_true', + help='Generate static library build (default: False)') + self.parser.add_argument('-c', '--clients', required=False, default=False, dest='build_clients', action='store_true', + help='Generate all client builds (default: False)') + self.parser.add_argument('-t', '--tests', required=False, default=False, dest='build_tests', action='store_true', + help='Generate unit tests only (default: False)') + self.parser.add_argument('-i', '--install', required=False, default=False, dest='install', action='store_true', + help='Install after build (default: False)') + self.parser.add_argument( '--cmake-darg', required=False, dest='cmake_dargs', action='append', default=[], + help='List of additional cmake defines for builds (e.g. CMAKE_CXX_COMPILER_LAUNCHER=ccache)') + self.parser.add_argument('-a', '--architecture', dest='gpu_architecture', required=False, default=default_gpus, #:sramecc+:xnack-" ) #gfx1030" ) #gfx906" ) # gfx1030" ) + help='Set GPU architectures, e.g. all, gfx000, gfx803, gfx906:xnack-;gfx1030;gfx1100 (optional, default: all)') + self.parser.add_argument('-v', '--verbose', required=False, default=False, action='store_true', + help='Verbose build (default: False)') + + self.args = self.parser.parse_args() + + def __mk_dir__(self, dir_path: str): + if os.path.isabs(dir_path): + full_path = dir_path + else: + full_path = os.path.join(self.lib_dir, dir_path) + + try: + os.mkdir(full_path) + print(f'{full_path} created') + except FileExistsError: + print(f'{full_path} already exists ...') -args = {} -param = {} -OS_info = {} - -def parse_args(): - """Parse command-line arguments""" - parser = argparse.ArgumentParser(description=""" - Checks build arguments - """) - - default_gpus = 'gfx906:xnack-,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' - - parser.add_argument('-g', '--debug', required=False, default=False, action='store_true', - help='Generate Debug build (default: False)') - parser.add_argument( '--build_dir', type=str, required=False, default="build", - help='Build directory path (default: build)') - parser.add_argument( '--deps_dir', type=str, required=False, default=None, - help='Dependencies directory path (default: build/deps)') - parser.add_argument( '--skip_ld_conf_entry', required=False, default=False) - parser.add_argument( '--static', required=False, default=False, dest='static_lib', action='store_true', - help='Generate static library build (default: False)') - parser.add_argument('-c', '--clients', required=False, default=False, dest='build_clients', action='store_true', - help='Generate all client builds (default: False)') - parser.add_argument('-t', '--tests', required=False, default=False, dest='build_tests', action='store_true', - help='Generate unit tests only (default: False)') - parser.add_argument('-i', '--install', required=False, default=False, dest='install', action='store_true', - help='Install after build (default: False)') - parser.add_argument( '--cmake-darg', required=False, dest='cmake_dargs', action='append', default=[], - help='List of additional cmake defines for builds (e.g. CMAKE_CXX_COMPILER_LAUNCHER=ccache)') - parser.add_argument('-a', '--architecture', dest='gpu_architecture', required=False, default=default_gpus, #:sramecc+:xnack-" ) #gfx1030" ) #gfx906" ) # gfx1030" ) - help='Set GPU architectures, e.g. all, gfx000, gfx803, gfx906:xnack-;gfx1030;gfx1100 (optional, default: all)') - parser.add_argument('-v', '--verbose', required=False, default=False, action='store_true', - help='Verbose build (default: False)') - return parser.parse_args() - -def os_detect(): - inf_file = "/etc/os-release" - if os.path.exists(inf_file): - with open(inf_file) as f: - for line in f: - if "=" in line: - k,v = line.strip().split("=") - OS_info[k] = v.replace('"','') - else: - OS_info["ID"] = 'windows' - OS_info["VERSION_ID"] = 10 - OS_info["NUM_PROC"] = os.cpu_count() - print(OS_info) - -def create_dir(dir_path): - if os.path.isabs(dir_path): - full_path = dir_path - else: - fullpath = os.path.join( os.getcwd(), dir_path ) - pathlib.Path(fullpath).mkdir(parents=True, exist_ok=True) - return - -def delete_dir(dir_path) : - if (not os.path.exists(dir_path)): - return - if (OS_info["ID"] == 'windows'): - run_cmd( "RMDIR" , f"/S /Q {dir_path}") - else: - linux_path = pathlib.Path(dir_path).absolute() - run_cmd( "rm" , f"-rf {linux_path}") - -def cmake_path(os_path): - if OS_info["ID"] == "windows": - return os_path.replace("\\", "/") - else: - return os.path.realpath(os_path) - -def config_cmd(): - global args - global OS_info - cwd_path = os.getcwd() - src_path = cwd_path.replace("\\", "/") - - print( f"***************** {src_path}") - cmake_executable = "" - cmake_options = [] - cmake_platform_opts = [] - if (OS_info["ID"] == 'windows'): - # we don't have ROCM on windows but have hip, ROCM can be downloaded if required - # CMAKE_PREFIX_PATH set to rocm_path and HIP_PATH set BY SDK Installer - raw_rocm_path = cmake_path(os.getenv('HIP_PATH', "C:/hip")) - rocm_path = f'"{raw_rocm_path}"' # guard against spaces in path - cmake_executable = "cmake.exe" - toolchain = os.path.join( src_path, "toolchain-windows.cmake" ) - #set CPACK_PACKAGING_INSTALL_PREFIX= defined as blank as it is appended to end of path for archive creation - cmake_platform_opts.append( f"-DWIN32=ON -DCPACK_PACKAGING_INSTALL_PREFIX=") #" -DCPACK_PACKAGING_INSTALL_PREFIX={rocm_path}" - cmake_platform_opts.append( f"-DCMAKE_INSTALL_PREFIX=\"C:/hipSDK\"" ) - - # MSVC requires acknowledgement of using extended aligned storage. - # Before VS 2017 15.8, has non-conforming alignment. VS 2017 15.8 fixes this, but inherently changes layouts of - # aligned storage with extended alignment, and thus binary compatibility with such types. - cmake_platform_opts.append( "-DCMAKE_CXX_FLAGS=\"-D_ENABLE_EXTENDED_ALIGNED_STORAGE\"") - - rocm_cmake_path = '"' + cmake_path(os.getenv("ROCM_CMAKE_PATH", "C:/hipSDK")) + '"' - generator = f"-G Ninja" - # "-G \"Visual Studio 16 2019\" -A x64" # -G NMake ") # - cmake_options.append( generator ) - else: - rocm_path = os.getenv( 'ROCM_PATH', "/opt/rocm") - rocm_cmake_path = '"' + rocm_path + '"' - if (OS_info["ID"] in ['centos', 'rhel']): - cmake_executable = "cmake3" + def __rm_dir__(self, dir_path: str): + if os.path.isabs(dir_path): + full_path = dir_path + else: + full_path = os.path.join(self.lib_dir, dir_path) + print(f'{full_path} deleted') + try: + shutil.rmtree(full_path) + except FileNotFoundError: + print(f'{full_path} does not exists ...') + + def __get_cmake_cmd__(self): + + m = ' Current Working Directory ' + print(f'{m:-^100}\n\t{self.lib_dir}') + + cmake_exe = '' + cmake_options = [f'-DCMAKE_TOOLCHAIN_FILE={self.toolchain}'] + + if self.args.debug: + build_path = os.path.join(self.args.build_dir, 'debug') + cmake_config = 'Debug' else: - cmake_executable = "cmake" - toolchain = "toolchain-linux.cmake" - cmake_platform_opts = [f"-DROCM_DIR:PATH={rocm_path}", f"-DCPACK_PACKAGING_INSTALL_PREFIX={rocm_path}"] + build_path = os.path.join(self.args.build_dir, 'release') + cmake_config = 'Release' + + cmake_options.append(f"-DCMAKE_BUILD_TYPE={cmake_config}") - tools = f"-DCMAKE_TOOLCHAIN_FILE={toolchain}" - cmake_options.append( tools ) + if self.args.deps_dir is None: + deps_dir = os.path.abspath(os.path.join(self.args.build_dir, 'deps')) + else: + deps_dir = self.args.deps_dir - cmake_options.extend( cmake_platform_opts) + if self.OS_info['System'] == 'Linux': + cmake_exe = shutil.which('cmake3') + if cmake_exe is None: + cmake_exe = shutil.which('cmake') + + if cmake_exe is None: + raise(SystemError('Did not find cmake or cmake3 in system')) - # build type - cmake_config = "" - build_dir = args.build_dir - if not args.debug: - build_path = os.path.join(build_dir, "release") - cmake_config="Release" - else: - build_path = os.path.join(build_dir, "debug") - cmake_config="Debug" + rocm_path = os.getenv('ROCM_PATH', '/opt/rocm') + + cmake_options.append(f'-DROCM_DIR:PATH={rocm_path}') + cmake_options.append(f'-DCPACK_PACKAGING_INSTALL_PREFIX={rocm_path}') + cmake_options.append(f'-DROCM_PATH={rocm_path}') + cmake_options.append(f'-DCMAKE_PREFIX_PATH:PATH={rocm_path},{rocm_path}') - cmake_options.append( f"-DCMAKE_BUILD_TYPE={cmake_config}" ) #--build {build_path}" ) - if args.deps_dir is None: - deps_dir = os.path.abspath(os.path.join(build_dir, 'deps')).replace('\\','/') - else: - deps_dir = args.deps_dir - if (OS_info["ID"] == 'windows'): - cmake_base_options = f"-DROCM_PATH={rocm_path} -DCMAKE_PREFIX_PATH:PATH={rocm_path[:-1]};{rocm_cmake_path[1:]}" # -DCMAKE_INSTALL_PREFIX=rocmath-install" #-DCMAKE_INSTALL_LIBDIR= - else: - cmake_base_options = f"-DROCM_PATH={rocm_path} -DCMAKE_PREFIX_PATH:PATH={rocm_path[:-1]},{rocm_cmake_path[1:-1]}" # -DCMAKE_INSTALL_PREFIX=rocmath-install" #-DCMAKE_INSTALL_LIBDIR= + # if (OS_info["ID"] == 'windows'): + # cmake_base_options = f"-DROCM_PATH={rocm_path} -DCMAKE_PREFIX_PATH:PATH={rocm_path[:-1]};{rocm_cmake_path[1:]}" # -DCMAKE_INSTALL_PREFIX=rocmath-install" #-DCMAKE_INSTALL_LIBDIR= + # else: - cmake_options.append( cmake_base_options ) - - print( cmake_options ) - - # clean - delete_dir( build_path ) - - create_dir( os.path.join(build_path, "clients") ) - os.chdir( build_path ) - - # packaging options - cmake_pack_options = f"-DCPACK_SET_DESTDIR=OFF -DCPACK_INCLUDE_TOPLEVEL_DIRECTORY=OFF" - cmake_options.append( cmake_pack_options ) - - if args.static_lib: - cmake_options.append( f"-DBUILD_SHARED_LIBS=OFF" ) - - if args.skip_ld_conf_entry: - cmake_options.append( f"-DROCM_DISABLE_LDCONFIG=ON" ) - - if args.build_tests: - cmake_options.append( f"-DBUILD_TEST=ON -DBUILD_DIR={build_dir}" ) - - if args.build_clients: - cmake_options.append( f"-DBUILD_TEST=ON -DBUILD_BENCHMARK=ON -DBUILD_EXAMPLE=ON -DBUILD_DIR={build_dir}" ) - - cmake_options.append( f"-DAMDGPU_TARGETS={args.gpu_architecture}" ) - - if args.cmake_dargs: - for i in args.cmake_dargs: - cmake_options.append( f"-D{i}" ) - - cmake_options.append( f"{src_path}") - -# case "${ID}" in -# centos|rhel) -# cmake_options="${cmake_options} -DCMAKE_FIND_ROOT_PATH=/usr/lib64/llvm7.0/lib/cmake/" -# ;; -# windows) -# cmake_options="${cmake_options} -DWIN32=ON -DROCM_PATH=${rocm_path} -DROCM_DIR:PATH=${rocm_path} -DCMAKE_PREFIX_PATH:PATH=${rocm_path}" -# cmake_options="${cmake_options} --debug-trycompile -DCMAKE_MAKE_PROGRAM=nmake.exe -DCMAKE_TOOLCHAIN_FILE=toolchain-windows.cmake" -# # -G '"NMake Makefiles JOM"'" -# ;; -# esac - cmd_opts = " ".join(cmake_options) - - return cmake_executable, cmd_opts - - -def make_cmd(): - global args - global OS_info - - make_options = [] - - if (OS_info["ID"] == 'windows'): - make_executable = "cmake.exe --build ." # ninja" - if args.verbose: - make_options.append( "--verbose" ) - make_options.append( "--target all" ) - if args.install: - make_options.append( "--target package --target install" ) - else: - nproc = OS_info["NUM_PROC"] - make_executable = f"make -j {nproc}" - if args.verbose: - make_options.append( "VERBOSE=1" ) - if args.install: - make_options.append( "install" ) - cmd_opts = " ".join(make_options) - - return make_executable, cmd_opts - -def run_cmd(exe, opts): - program = f"{exe} {opts}" - if sys.platform.startswith('win'): - sh = True - else: - sh = True - print(program) - proc = subprocess.run(program, check=True, stderr=subprocess.STDOUT, shell=sh) - #proc = subprocess.Popen(cmd, cwd=os.getcwd()) - #cwd=os.path.join(workingdir,"..",".."), stdout=fout, stderr=fout, - # env=os.environ.copy()) - #proc.wait() - return proc.returncode - -def main(): - global args - os_detect() - args = parse_args() - # configure - exe, opts = config_cmd() - run_cmd(exe, opts) - - # make/build/install - exe, opts = make_cmd() - run_cmd(exe, opts) + + print(cmake_options) + + def run(self): + self.__parse_args__() + self.__get_cmake_cmd__() + + + + +# if (OS_info["ID"] == 'windows'): +# # we don't have ROCM on windows but have hip, ROCM can be downloaded if required +# # CMAKE_PREFIX_PATH set to rocm_path and HIP_PATH set BY SDK Installer +# raw_rocm_path = cmake_path(os.getenv('HIP_PATH', "C:/hip")) +# rocm_path = f'"{raw_rocm_path}"' # guard against spaces in path +# cmake_executable = "cmake.exe" +# toolchain = os.path.join( src_path, "toolchain-windows.cmake" ) +# #set CPACK_PACKAGING_INSTALL_PREFIX= defined as blank as it is appended to end of path for archive creation +# cmake_platform_opts.append( f"-DWIN32=ON -DCPACK_PACKAGING_INSTALL_PREFIX=") #" -DCPACK_PACKAGING_INSTALL_PREFIX={rocm_path}" +# cmake_platform_opts.append( f"-DCMAKE_INSTALL_PREFIX=\"C:/hipSDK\"" ) + +# # MSVC requires acknowledgement of using extended aligned storage. +# # Before VS 2017 15.8, has non-conforming alignment. VS 2017 15.8 fixes this, but inherently changes layouts of +# # aligned storage with extended alignment, and thus binary compatibility with such types. +# cmake_platform_opts.append( "-DCMAKE_CXX_FLAGS=\"-D_ENABLE_EXTENDED_ALIGNED_STORAGE\"") + +# rocm_cmake_path = '"' + cmake_path(os.getenv("ROCM_CMAKE_PATH", "C:/hipSDK")) + '"' +# generator = f"-G Ninja" +# # "-G \"Visual Studio 16 2019\" -A x64" # -G NMake ") # +# cmake_options.append( generator ) + +# if (OS_info["ID"] == 'windows'): +# cmake_base_options = f"-DROCM_PATH={rocm_path} -DCMAKE_PREFIX_PATH:PATH={rocm_path[:-1]};{rocm_cmake_path[1:]}" # -DCMAKE_INSTALL_PREFIX=rocmath-install" #-DCMAKE_INSTALL_LIBDIR= +# else: +# cmake_base_options = f"-DROCM_PATH={rocm_path} -DCMAKE_PREFIX_PATH:PATH={rocm_path[:-1]},{rocm_cmake_path[1:-1]}" # -DCMAKE_INSTALL_PREFIX=rocmath-install" #-DCMAKE_INSTALL_LIBDIR= + +# cmake_options.append( cmake_base_options ) + +# print( cmake_options ) + +# # clean +# delete_dir( build_path ) + +# create_dir( os.path.join(build_path, "clients") ) +# os.chdir( build_path ) + +# # packaging options +# cmake_pack_options = f"-DCPACK_SET_DESTDIR=OFF -DCPACK_INCLUDE_TOPLEVEL_DIRECTORY=OFF" +# cmake_options.append( cmake_pack_options ) + +# if args.static_lib: +# cmake_options.append( f"-DBUILD_SHARED_LIBS=OFF" ) + +# if args.skip_ld_conf_entry: +# cmake_options.append( f"-DROCM_DISABLE_LDCONFIG=ON" ) + +# if args.build_tests: +# cmake_options.append( f"-DBUILD_TEST=ON -DBUILD_DIR={build_dir}" ) + +# if args.build_clients: +# cmake_options.append( f"-DBUILD_TEST=ON -DBUILD_BENCHMARK=ON -DBUILD_EXAMPLE=ON -DBUILD_DIR={build_dir}" ) + +# cmake_options.append( f"-DAMDGPU_TARGETS={args.gpu_architecture}" ) + +# if args.cmake_dargs: +# for i in args.cmake_dargs: +# cmake_options.append( f"-D{i}" ) + +# cmake_options.append( f"{src_path}") + +# # case "${ID}" in +# # centos|rhel) +# # cmake_options="${cmake_options} -DCMAKE_FIND_ROOT_PATH=/usr/lib64/llvm7.0/lib/cmake/" +# # ;; +# # windows) +# # cmake_options="${cmake_options} -DWIN32=ON -DROCM_PATH=${rocm_path} -DROCM_DIR:PATH=${rocm_path} -DCMAKE_PREFIX_PATH:PATH=${rocm_path}" +# # cmake_options="${cmake_options} --debug-trycompile -DCMAKE_MAKE_PROGRAM=nmake.exe -DCMAKE_TOOLCHAIN_FILE=toolchain-windows.cmake" +# # # -G '"NMake Makefiles JOM"'" +# # ;; +# # esac +# cmd_opts = " ".join(cmake_options) + +# return cmake_executable, cmd_opts + +# def run_cmd(exe, opts): +# program = f"{exe} {opts}" +# if sys.platform.startswith('win'): +# sh = True +# else: +# sh = True +# print(program) +# proc = subprocess.run(program, check=True, stderr=subprocess.STDOUT, shell=sh) +# #proc = subprocess.Popen(cmd, cwd=os.getcwd()) +# #cwd=os.path.join(workingdir,"..",".."), stdout=fout, stderr=fout, +# # env=os.environ.copy()) +# #proc.wait() +# return proc.returncode + +# def cmake_path(os_path): +# if OS_info["ID"] == "windows": +# return os_path.replace("\\", "/") +# else: +# return os.path.realpath(os_path) + + + +# def make_cmd(): +# global args +# global OS_info + +# make_options = [] + +# if (OS_info["ID"] == 'windows'): +# make_executable = "cmake.exe --build ." # ninja" +# if args.verbose: +# make_options.append( "--verbose" ) +# make_options.append( "--target all" ) +# if args.install: +# make_options.append( "--target package --target install" ) +# else: +# nproc = OS_info["NUM_PROC"] +# make_executable = f"make -j {nproc}" +# if args.verbose: +# make_options.append( "VERBOSE=1" ) +# if args.install: +# make_options.append( "install" ) +# cmd_opts = " ".join(make_options) + +# return make_executable, cmd_opts + + +# def main(): +# global args +# os_detect() +# args = parse_args() +# # configure +# exe, opts = config_cmd() +# run_cmd(exe, opts) + +# # make/build/install +# exe, opts = make_cmd() +# run_cmd(exe, opts) if __name__ == '__main__': - main() + builder = AutoBuilder() + + builder.run() + + # builder.__mk_dir__(f'build') + # builder.__mk_dir__(f'/home/zenguyen/forks/rocPRIM/build') + # builder.__rm_dir__(f'build') + # builder.__rm_dir__(f'/home/zenguyen/forks/rocPRIM/build') + + From 3b34e005a5e2b77cef3a34976773bbcc14eaed6d Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Fri, 22 Nov 2024 17:17:34 -0700 Subject: [PATCH 02/20] finished linux implementation --- rmake.py | 126 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 71 insertions(+), 55 deletions(-) diff --git a/rmake.py b/rmake.py index ec38a6597..88b0f7281 100644 --- a/rmake.py +++ b/rmake.py @@ -39,7 +39,7 @@ def __init__(self): for k in self.OS_info: print(f'\t {k}: {self.OS_info[k]}') - print(f'\n\n') + print() self.lib_dir = os.path.dirname(os.path.abspath(__file__)) @@ -62,6 +62,8 @@ def __parse_args__(self): help='Generate all client builds (default: False)') self.parser.add_argument('-t', '--tests', required=False, default=False, dest='build_tests', action='store_true', help='Generate unit tests only (default: False)') + self.parser.add_argument('-b', '--benchmarks', required=False, default=False, dest='build_bench', action='store_true', + help='Generate benchmarks only (default: False)') self.parser.add_argument('-i', '--install', required=False, default=False, dest='install', action='store_true', help='Install after build (default: False)') self.parser.add_argument( '--cmake-darg', required=False, dest='cmake_dargs', action='append', default=[], @@ -81,28 +83,32 @@ def __mk_dir__(self, dir_path: str): try: os.mkdir(full_path) - print(f'{full_path} created') except FileExistsError: - print(f'{full_path} already exists ...') + ... def __rm_dir__(self, dir_path: str): if os.path.isabs(dir_path): full_path = dir_path else: full_path = os.path.join(self.lib_dir, dir_path) - print(f'{full_path} deleted') try: shutil.rmtree(full_path) except FileNotFoundError: - print(f'{full_path} does not exists ...') - + ... + def __get_cmake_cmd__(self): m = ' Current Working Directory ' print(f'{m:-^100}\n\t{self.lib_dir}') + print() cmake_exe = '' - cmake_options = [f'-DCMAKE_TOOLCHAIN_FILE={self.toolchain}'] + cmake_options = [ + f'--toolchain={self.toolchain}', + f'-DCPACK_SET_DESTDIR=OFF', + f'-DCPACK_INCLUDE_TOPLEVEL_DIRECTORY=OFF', + f'-DGPU_TARGETS={self.args.gpu_architecture}' + ] if self.args.debug: build_path = os.path.join(self.args.build_dir, 'debug') @@ -111,13 +117,10 @@ def __get_cmake_cmd__(self): build_path = os.path.join(self.args.build_dir, 'release') cmake_config = 'Release' + self.build_path = build_path + cmake_options.append(f"-DCMAKE_BUILD_TYPE={cmake_config}") - if self.args.deps_dir is None: - deps_dir = os.path.abspath(os.path.join(self.args.build_dir, 'deps')) - else: - deps_dir = self.args.deps_dir - if self.OS_info['System'] == 'Linux': cmake_exe = shutil.which('cmake3') @@ -135,20 +138,69 @@ def __get_cmake_cmd__(self): cmake_options.append(f'-DCMAKE_PREFIX_PATH:PATH={rocm_path},{rocm_path}') + if self.args.static_lib: + cmake_options.append( f'-DBUILD_SHARED_LIBS=OFF') - # if (OS_info["ID"] == 'windows'): - # cmake_base_options = f"-DROCM_PATH={rocm_path} -DCMAKE_PREFIX_PATH:PATH={rocm_path[:-1]};{rocm_cmake_path[1:]}" # -DCMAKE_INSTALL_PREFIX=rocmath-install" #-DCMAKE_INSTALL_LIBDIR= - # else: - + if self.args.skip_ld_conf_entry: + cmake_options.append( f'-DROCM_DISABLE_LDCONFIG=ON' ) + + if self.args.build_clients: + self.args.build_tests = True + self.args.build_bench = True + cmake_options.append(f'-DBUILD_EXAMPLE=ON') + + if self.args.build_tests: + cmake_options.append(f'-DBUILD_TEST=ON') + + if self.args.build_bench: + cmake_options.append(f'-DBUILD_BENCHMARK=ON') + + if(self.args.build_clients or self.args.build_tests or self.args.build_bench): + cmake_options.append(f'-DBUILD_DIR={build_path}') + + if self.args.cmake_dargs: + cmake_options += [f'-D{i}' for i in self.args.cmake_dargs] + + command_str = cmake_exe + + m = 'CMAKE Options' + print(f'{m:-^100}') + for op in cmake_options: + print(f'\t{op}') + command_str += f' {op}' + print() - print(cmake_options) + command_str += f' {self.lib_dir}' + m = 'Final Command' + print(f'{m:-^100}') + print(command_str) + print() + return command_str def run(self): self.__parse_args__() - self.__get_cmake_cmd__() - + cmake_command = self.__get_cmake_cmd__() + + self.__rm_dir__(self.build_path) + self.__mk_dir__(self.build_path) + + curr_dir = os.path.abspath(os.curdir) + os.chdir(self.build_path) + + os.system(cmake_command) + + if self.OS_info['System'] == 'Linux': + if self.args.verbose: + v = 'VERBOSE=1' + else: + v = '' + + os.system(f' make -j {self.OS_info["Num Processor"]} {v}') + if self.args.install: + os.system(f'make install') + os.chdir(curr_dir) # if (OS_info["ID"] == 'windows'): # # we don't have ROCM on windows but have hip, ROCM can be downloaded if required @@ -174,9 +226,6 @@ def run(self): # if (OS_info["ID"] == 'windows'): # cmake_base_options = f"-DROCM_PATH={rocm_path} -DCMAKE_PREFIX_PATH:PATH={rocm_path[:-1]};{rocm_cmake_path[1:]}" # -DCMAKE_INSTALL_PREFIX=rocmath-install" #-DCMAKE_INSTALL_LIBDIR= # else: -# cmake_base_options = f"-DROCM_PATH={rocm_path} -DCMAKE_PREFIX_PATH:PATH={rocm_path[:-1]},{rocm_cmake_path[1:-1]}" # -DCMAKE_INSTALL_PREFIX=rocmath-install" #-DCMAKE_INSTALL_LIBDIR= - -# cmake_options.append( cmake_base_options ) # print( cmake_options ) @@ -186,40 +235,7 @@ def run(self): # create_dir( os.path.join(build_path, "clients") ) # os.chdir( build_path ) -# # packaging options -# cmake_pack_options = f"-DCPACK_SET_DESTDIR=OFF -DCPACK_INCLUDE_TOPLEVEL_DIRECTORY=OFF" -# cmake_options.append( cmake_pack_options ) - -# if args.static_lib: -# cmake_options.append( f"-DBUILD_SHARED_LIBS=OFF" ) - -# if args.skip_ld_conf_entry: -# cmake_options.append( f"-DROCM_DISABLE_LDCONFIG=ON" ) - -# if args.build_tests: -# cmake_options.append( f"-DBUILD_TEST=ON -DBUILD_DIR={build_dir}" ) - -# if args.build_clients: -# cmake_options.append( f"-DBUILD_TEST=ON -DBUILD_BENCHMARK=ON -DBUILD_EXAMPLE=ON -DBUILD_DIR={build_dir}" ) - -# cmake_options.append( f"-DAMDGPU_TARGETS={args.gpu_architecture}" ) - -# if args.cmake_dargs: -# for i in args.cmake_dargs: -# cmake_options.append( f"-D{i}" ) - -# cmake_options.append( f"{src_path}") -# # case "${ID}" in -# # centos|rhel) -# # cmake_options="${cmake_options} -DCMAKE_FIND_ROOT_PATH=/usr/lib64/llvm7.0/lib/cmake/" -# # ;; -# # windows) -# # cmake_options="${cmake_options} -DWIN32=ON -DROCM_PATH=${rocm_path} -DROCM_DIR:PATH=${rocm_path} -DCMAKE_PREFIX_PATH:PATH=${rocm_path}" -# # cmake_options="${cmake_options} --debug-trycompile -DCMAKE_MAKE_PROGRAM=nmake.exe -DCMAKE_TOOLCHAIN_FILE=toolchain-windows.cmake" -# # # -G '"NMake Makefiles JOM"'" -# # ;; -# # esac # cmd_opts = " ".join(cmake_options) # return cmake_executable, cmd_opts From 9860fdd17b72582ad9fe770007d2e596551d4cf9 Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Fri, 22 Nov 2024 17:26:24 -0700 Subject: [PATCH 03/20] changed mkdir to makedirs --- rmake.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/rmake.py b/rmake.py index 88b0f7281..e956906b4 100644 --- a/rmake.py +++ b/rmake.py @@ -82,7 +82,7 @@ def __mk_dir__(self, dir_path: str): full_path = os.path.join(self.lib_dir, dir_path) try: - os.mkdir(full_path) + os.makedirs(full_path) except FileExistsError: ... @@ -155,9 +155,6 @@ def __get_cmake_cmd__(self): if self.args.build_bench: cmake_options.append(f'-DBUILD_BENCHMARK=ON') - if(self.args.build_clients or self.args.build_tests or self.args.build_bench): - cmake_options.append(f'-DBUILD_DIR={build_path}') - if self.args.cmake_dargs: cmake_options += [f'-D{i}' for i in self.args.cmake_dargs] From a671ce1fb7cf40d60ef410042f6176fd8bf99cbc Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Fri, 22 Nov 2024 17:33:49 -0700 Subject: [PATCH 04/20] updated remove directory --- rmake.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rmake.py b/rmake.py index e956906b4..a2b231ce5 100644 --- a/rmake.py +++ b/rmake.py @@ -178,7 +178,7 @@ def run(self): self.__parse_args__() cmake_command = self.__get_cmake_cmd__() - self.__rm_dir__(self.build_path) + self.__rm_dir__('build') self.__mk_dir__(self.build_path) curr_dir = os.path.abspath(os.curdir) From 6e6ad9e49da96e8f099d7d2a033a8051422c03f4 Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Mon, 25 Nov 2024 15:19:34 -0700 Subject: [PATCH 05/20] added extra remove --- rmake.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rmake.py b/rmake.py index a2b231ce5..d9819f07b 100644 --- a/rmake.py +++ b/rmake.py @@ -179,6 +179,7 @@ def run(self): cmake_command = self.__get_cmake_cmd__() self.__rm_dir__('build') + self.__rm_dir__(self.build_path) self.__mk_dir__(self.build_path) curr_dir = os.path.abspath(os.curdir) From d1fbacfae4ee90c309a7d37ec569b1e8897d60a9 Mon Sep 17 00:00:00 2001 From: cm chen Date: Mon, 25 Nov 2024 18:31:36 -0700 Subject: [PATCH 06/20] finished windows implementation --- rmake.py | 163 ++++++++++++++++--------------------------------------- 1 file changed, 47 insertions(+), 116 deletions(-) diff --git a/rmake.py b/rmake.py index d9819f07b..b1ec6ec6c 100644 --- a/rmake.py +++ b/rmake.py @@ -131,11 +131,33 @@ def __get_cmake_cmd__(self): raise(SystemError('Did not find cmake or cmake3 in system')) rocm_path = os.getenv('ROCM_PATH', '/opt/rocm') + + cmake_options.append(f"-DROCM_DIR:PATH={rocm_path}") + cmake_options.append(f"-DCPACK_PACKAGING_INSTALL_PREFIX={rocm_path}") + cmake_options.append(f"-DROCM_PATH={rocm_path}") + cmake_options.append(f"-DCMAKE_PREFIX_PATH:PATH={rocm_path}") + + else: + cmake_exe = shutil.which('cmake.exe') + + if cmake_exe is None: + cmake_exe = shutil.which('cmake3.exe') + + if cmake_exe is None: + raise(SystemError('Did not find cmake or cmake3 in system')) - cmake_options.append(f'-DROCM_DIR:PATH={rocm_path}') - cmake_options.append(f'-DCPACK_PACKAGING_INSTALL_PREFIX={rocm_path}') - cmake_options.append(f'-DROCM_PATH={rocm_path}') - cmake_options.append(f'-DCMAKE_PREFIX_PATH:PATH={rocm_path},{rocm_path}') + rocm_path = os.getenv('ROCM_PATH', 'C:/hip') + rocm_cmake_path = os.getenv('ROCM_CMAKE_PATH', r'C:/hipSDK') + + rocm_path.replace('\\', '/') + rocm_cmake_path.replace('\\', '/') + + cmake_options.append(f'-G Ninja') + cmake_options.append( f"-DWIN32=ON -DCPACK_PACKAGING_INSTALL_PREFIX=") #" -DCPACK_PACKAGING_INSTALL_PREFIX={rocm_path}" + cmake_options.append( f'-DCMAKE_INSTALL_PREFIX="C:/hipSDK"' ) + cmake_options.append( f'-DCMAKE_CXX_FLAGS="-D_ENABLE_EXTENDED_ALIGNED_STORAGE"') + cmake_options.append(f'-DROCM_PATH={rocm_path}') + cmake_options.append(f'-DCMAKE_PREFIX_PATH:PATH={rocm_path};{rocm_cmake_path}') if self.args.static_lib: @@ -158,7 +180,11 @@ def __get_cmake_cmd__(self): if self.args.cmake_dargs: cmake_options += [f'-D{i}' for i in self.args.cmake_dargs] - command_str = cmake_exe + # putting '' around paths to avoid white space in pathing + if self.OS_info['System'] == 'Linux': + command_str = f"{cmake_exe}" + else: + command_str = f'"{cmake_exe}"' m = 'CMAKE Options' print(f'{m:-^100}') @@ -167,7 +193,7 @@ def __get_cmake_cmd__(self): command_str += f' {op}' print() - command_str += f' {self.lib_dir}' + command_str += f' "{self.lib_dir}"' m = 'Final Command' print(f'{m:-^100}') print(command_str) @@ -185,127 +211,32 @@ def run(self): curr_dir = os.path.abspath(os.curdir) os.chdir(self.build_path) - os.system(cmake_command) + subprocess.run(cmake_command) + if self.OS_info['System'] == 'Linux': + v = '' if self.args.verbose: v = 'VERBOSE=1' - else: - v = '' - - os.system(f' make -j {self.OS_info["Num Processor"]} {v}') + subprocess.run(f' make -j {self.OS_info["Num Processor"]} {v}') if self.args.install: - os.system(f'make install') + subprocess.run(f'make install') + else: + v, i, = '', '' + if self.args.verbose: + v = '--verbose' - os.chdir(curr_dir) + if self.args.install: + i = f'--target package --target install' -# if (OS_info["ID"] == 'windows'): -# # we don't have ROCM on windows but have hip, ROCM can be downloaded if required -# # CMAKE_PREFIX_PATH set to rocm_path and HIP_PATH set BY SDK Installer -# raw_rocm_path = cmake_path(os.getenv('HIP_PATH', "C:/hip")) -# rocm_path = f'"{raw_rocm_path}"' # guard against spaces in path -# cmake_executable = "cmake.exe" -# toolchain = os.path.join( src_path, "toolchain-windows.cmake" ) -# #set CPACK_PACKAGING_INSTALL_PREFIX= defined as blank as it is appended to end of path for archive creation -# cmake_platform_opts.append( f"-DWIN32=ON -DCPACK_PACKAGING_INSTALL_PREFIX=") #" -DCPACK_PACKAGING_INSTALL_PREFIX={rocm_path}" -# cmake_platform_opts.append( f"-DCMAKE_INSTALL_PREFIX=\"C:/hipSDK\"" ) - -# # MSVC requires acknowledgement of using extended aligned storage. -# # Before VS 2017 15.8, has non-conforming alignment. VS 2017 15.8 fixes this, but inherently changes layouts of -# # aligned storage with extended alignment, and thus binary compatibility with such types. -# cmake_platform_opts.append( "-DCMAKE_CXX_FLAGS=\"-D_ENABLE_EXTENDED_ALIGNED_STORAGE\"") - -# rocm_cmake_path = '"' + cmake_path(os.getenv("ROCM_CMAKE_PATH", "C:/hipSDK")) + '"' -# generator = f"-G Ninja" -# # "-G \"Visual Studio 16 2019\" -A x64" # -G NMake ") # -# cmake_options.append( generator ) - -# if (OS_info["ID"] == 'windows'): -# cmake_base_options = f"-DROCM_PATH={rocm_path} -DCMAKE_PREFIX_PATH:PATH={rocm_path[:-1]};{rocm_cmake_path[1:]}" # -DCMAKE_INSTALL_PREFIX=rocmath-install" #-DCMAKE_INSTALL_LIBDIR= -# else: - -# print( cmake_options ) - -# # clean -# delete_dir( build_path ) - -# create_dir( os.path.join(build_path, "clients") ) -# os.chdir( build_path ) - - -# cmd_opts = " ".join(cmake_options) - -# return cmake_executable, cmd_opts - -# def run_cmd(exe, opts): -# program = f"{exe} {opts}" -# if sys.platform.startswith('win'): -# sh = True -# else: -# sh = True -# print(program) -# proc = subprocess.run(program, check=True, stderr=subprocess.STDOUT, shell=sh) -# #proc = subprocess.Popen(cmd, cwd=os.getcwd()) -# #cwd=os.path.join(workingdir,"..",".."), stdout=fout, stderr=fout, -# # env=os.environ.copy()) -# #proc.wait() -# return proc.returncode - -# def cmake_path(os_path): -# if OS_info["ID"] == "windows": -# return os_path.replace("\\", "/") -# else: -# return os.path.realpath(os_path) - - - -# def make_cmd(): -# global args -# global OS_info - -# make_options = [] - -# if (OS_info["ID"] == 'windows'): -# make_executable = "cmake.exe --build ." # ninja" -# if args.verbose: -# make_options.append( "--verbose" ) -# make_options.append( "--target all" ) -# if args.install: -# make_options.append( "--target package --target install" ) -# else: -# nproc = OS_info["NUM_PROC"] -# make_executable = f"make -j {nproc}" -# if args.verbose: -# make_options.append( "VERBOSE=1" ) -# if args.install: -# make_options.append( "install" ) -# cmd_opts = " ".join(make_options) - -# return make_executable, cmd_opts - - -# def main(): -# global args -# os_detect() -# args = parse_args() -# # configure -# exe, opts = config_cmd() -# run_cmd(exe, opts) - -# # make/build/install -# exe, opts = make_cmd() -# run_cmd(exe, opts) + if self.args.install: + subprocess.run(f'ninja install') + subprocess.run(f'ninja -j {self.OS_info["Num Processor"]} {v} {i}') + os.chdir(curr_dir) if __name__ == '__main__': builder = AutoBuilder() builder.run() - - # builder.__mk_dir__(f'build') - # builder.__mk_dir__(f'/home/zenguyen/forks/rocPRIM/build') - # builder.__rm_dir__(f'build') - # builder.__rm_dir__(f'/home/zenguyen/forks/rocPRIM/build') - - From ccd5a7579760b77634e6456f9f566801240ab6b6 Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Mon, 25 Nov 2024 18:41:05 -0700 Subject: [PATCH 07/20] fixed windows' execution --- rmake.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/rmake.py b/rmake.py index b1ec6ec6c..a7628c92d 100644 --- a/rmake.py +++ b/rmake.py @@ -223,16 +223,12 @@ def run(self): if self.args.install: subprocess.run(f'make install') else: - v, i, = '', '' + v, = '' if self.args.verbose: v = '--verbose' - - if self.args.install: - i = f'--target package --target install' - + subprocess.run(f'ninja -j {self.OS_info["Num Processor"]} {v}') if self.args.install: subprocess.run(f'ninja install') - subprocess.run(f'ninja -j {self.OS_info["Num Processor"]} {v} {i}') os.chdir(curr_dir) From 688efe702a05f371aae21fe63cf4b2dfba076322 Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Mon, 25 Nov 2024 18:41:37 -0700 Subject: [PATCH 08/20] removed unused imports --- rmake.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/rmake.py b/rmake.py index a7628c92d..e5b8dc0cd 100644 --- a/rmake.py +++ b/rmake.py @@ -2,13 +2,9 @@ """ Copyright (c) 2021-2023 Advanced Micro Devices, Inc. All rights reserved. Manage build and installation""" -import re -import sys import os import subprocess import argparse -import ctypes -import pathlib from fnmatch import fnmatchcase import platform as pf import shutil From 69aa8822b664b9ec60bc2c344dce2cfe41b90a9a Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Tue, 26 Nov 2024 09:13:48 -0700 Subject: [PATCH 09/20] fixed syntax error --- rmake.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rmake.py b/rmake.py index e5b8dc0cd..b14be6a93 100644 --- a/rmake.py +++ b/rmake.py @@ -219,7 +219,7 @@ def run(self): if self.args.install: subprocess.run(f'make install') else: - v, = '' + v = '' if self.args.verbose: v = '--verbose' subprocess.run(f'ninja -j {self.OS_info["Num Processor"]} {v}') From 7b5e6df9728a2121f9181acba4f54c1a96476713 Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Tue, 26 Nov 2024 09:26:21 -0700 Subject: [PATCH 10/20] fixed windows 5 error when build dir already exists --- rmake.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/rmake.py b/rmake.py index b14be6a93..f43584775 100644 --- a/rmake.py +++ b/rmake.py @@ -5,7 +5,6 @@ import os import subprocess import argparse -from fnmatch import fnmatchcase import platform as pf import shutil @@ -80,7 +79,7 @@ def __mk_dir__(self, dir_path: str): try: os.makedirs(full_path) except FileExistsError: - ... + ... # file already exists def __rm_dir__(self, dir_path: str): if os.path.isabs(dir_path): @@ -88,9 +87,13 @@ def __rm_dir__(self, dir_path: str): else: full_path = os.path.join(self.lib_dir, dir_path) try: - shutil.rmtree(full_path) + if self.OS_info['System'] == 'Linux': + subprocess.run(f'rm -rf {full_path}') + else: + subprocess.run(f'RMDIR /S /Q {full_path}') + except FileNotFoundError: - ... + ... # no file to remove def __get_cmake_cmd__(self): From 257a93b15ec5bbe0e4dfac7ac15ca7eb8ed717f6 Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Tue, 26 Nov 2024 09:57:56 -0700 Subject: [PATCH 11/20] used shell mode to remove on windows --- rmake.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rmake.py b/rmake.py index f43584775..571c614fa 100644 --- a/rmake.py +++ b/rmake.py @@ -88,9 +88,9 @@ def __rm_dir__(self, dir_path: str): full_path = os.path.join(self.lib_dir, dir_path) try: if self.OS_info['System'] == 'Linux': - subprocess.run(f'rm -rf {full_path}') + subprocess.run(f'rm -rf "{full_path}"') else: - subprocess.run(f'RMDIR /S /Q {full_path}') + subprocess.run(f'RMDIR /S /Q {full_path}', shell=True) except FileNotFoundError: ... # no file to remove @@ -203,8 +203,8 @@ def run(self): self.__parse_args__() cmake_command = self.__get_cmake_cmd__() - self.__rm_dir__('build') self.__rm_dir__(self.build_path) + self.__rm_dir__('build') self.__mk_dir__(self.build_path) curr_dir = os.path.abspath(os.curdir) From 07d6fd84186719b92cdfd0fb229c583f023f851a Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Tue, 26 Nov 2024 13:41:28 -0700 Subject: [PATCH 12/20] added shell=True to all subprocess run, and changed run to __call__ --- rmake.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/rmake.py b/rmake.py index 571c614fa..647b579a0 100644 --- a/rmake.py +++ b/rmake.py @@ -10,7 +10,6 @@ class AutoBuilder: def __init__(self): - self.param = {} self.parser = argparse.ArgumentParser(description=""" Checks build arguments """) @@ -66,7 +65,7 @@ def __parse_args__(self): self.parser.add_argument('-a', '--architecture', dest='gpu_architecture', required=False, default=default_gpus, #:sramecc+:xnack-" ) #gfx1030" ) #gfx906" ) # gfx1030" ) help='Set GPU architectures, e.g. all, gfx000, gfx803, gfx906:xnack-;gfx1030;gfx1100 (optional, default: all)') self.parser.add_argument('-v', '--verbose', required=False, default=False, action='store_true', - help='Verbose build (default: False)') + help='Verbose build (default: False)') self.args = self.parser.parse_args() @@ -88,7 +87,7 @@ def __rm_dir__(self, dir_path: str): full_path = os.path.join(self.lib_dir, dir_path) try: if self.OS_info['System'] == 'Linux': - subprocess.run(f'rm -rf "{full_path}"') + subprocess.run(f'rm -rf "{full_path}"', shell=True) else: subprocess.run(f'RMDIR /S /Q {full_path}', shell=True) @@ -199,7 +198,7 @@ def __get_cmake_cmd__(self): print() return command_str - def run(self): + def __call__(self): self.__parse_args__() cmake_command = self.__get_cmake_cmd__() @@ -211,27 +210,26 @@ def run(self): os.chdir(self.build_path) - subprocess.run(cmake_command) + subprocess.run(cmake_command, shell=True) if self.OS_info['System'] == 'Linux': v = '' if self.args.verbose: - v = 'VERBOSE=1' - subprocess.run(f' make -j {self.OS_info["Num Processor"]} {v}') + v = ' VERBOSE=1' + subprocess.run(f' make -j {self.OS_info["Num Processor"]}{v}', shell=True) if self.args.install: - subprocess.run(f'make install') + subprocess.run(f'make install', shell=True) else: v = '' if self.args.verbose: - v = '--verbose' - subprocess.run(f'ninja -j {self.OS_info["Num Processor"]} {v}') + v = ' --verbose' + subprocess.run(f'ninja -j {self.OS_info["Num Processor"]}{v}', shell=True) if self.args.install: - subprocess.run(f'ninja install') + subprocess.run(f'ninja install', shell=True) os.chdir(curr_dir) if __name__ == '__main__': builder = AutoBuilder() - - builder.run() + builder() From eaf6f80516c294eb772f4a7c24b0528cbea6d8cc Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Tue, 26 Nov 2024 13:41:42 -0700 Subject: [PATCH 13/20] rtest init --- rtest.py | 674 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 353 insertions(+), 321 deletions(-) diff --git a/rtest.py b/rtest.py index 029b5665d..802ffd2f0 100644 --- a/rtest.py +++ b/rtest.py @@ -15,339 +15,371 @@ from xml.dom import minidom import multiprocessing import time +import platform as pf -args = {} -OS_info = {} - -timeout = False -test_proc = None -stop = 0 - -test_script = [ 'cd %IDIR%', '%XML%' ] - -def parse_args(): - """Parse command-line arguments""" - parser = argparse.ArgumentParser(description=""" - Checks build arguments - """) - parser.add_argument('-e', '--emulation', required=False, default="", - help='Test set to run from rtest.xml (optional, eg.smoke). At least one but not both of -e or -t must be set') - parser.add_argument('-t', '--test', required=False, default="", - help='Test set to run from rtest.xml (optional, e.g. osdb). At least one but not both of -e or -t must be set') - parser.add_argument('-g', '--debug', required=False, default=False, action='store_true', - help='Test Debug build (optional, default: false)') - parser.add_argument('-o', '--output', type=str, required=False, default="xml", - help='Test output file (optional, default: test_detail.xml)') - parser.add_argument( '--install_dir', type=str, required=False, default="build", - help='Installation directory where build or release folders are (optional, default: build)') - parser.add_argument( '--fail_test', default=False, required=False, action='store_true', - help='Return as if test failed (optional, default: false)') - # parser.add_argument('-v', '--verbose', required=False, default = False, action='store_true', - # help='Verbose install (optional, default: False)') - return parser.parse_args() - - -def vram_detect(): - global OS_info - OS_info["VRAM"] = 0 - if os.name == "nt": - cmd = "hipinfo.exe" - process = subprocess.run([cmd], stdout=subprocess.PIPE) - for line_in in process.stdout.decode().splitlines(): - if 'totalGlobalMem' in line_in: - OS_info["VRAM"] = float(line_in.split()[1]) - break - else: - cmd = "rocminfo" - process = subprocess.run([cmd], stdout=subprocess.PIPE) - for line_in in process.stdout.decode().splitlines(): - match = re.search(r'.*Size:.*([0-9]+)\(.*\).*KB', line_in, re.IGNORECASE) - if match: - OS_info["VRAM"] = float(match.group(1))/(1024*1024) - break - -def os_detect(): - global OS_info - if os.name == "nt": - OS_info["ID"] = platform.system() - else: - inf_file = "/etc/os-release" - if os.path.exists(inf_file): - with open(inf_file) as f: - for line in f: - if "=" in line: - k,v = line.strip().split("=") - OS_info[k] = v.replace('"','') - OS_info["NUM_PROC"] = os.cpu_count() - vram_detect() - print(OS_info) - - -def create_dir(dir_path): - if os.path.isabs(dir_path): - full_path = dir_path - else: - full_path = os.path.join( os.getcwd(), dir_path ) - return pathlib.Path(full_path).mkdir(parents=True, exist_ok=True) - -def delete_dir(dir_path) : - if (not os.path.exists(dir_path)): - return - if os.name == "nt": - return run_cmd( "RMDIR" , f"/S /Q {dir_path}") - else: - linux_path = pathlib.Path(dir_path).absolute() - return run_cmd( "rm" , f"-rf {linux_path}") - -class TimerProcess(multiprocessing.Process): - - def __init__(self, start, stop, kill_pid): - multiprocessing.Process.__init__(self) - self.quit = multiprocessing.Event() - self.timed_out = multiprocessing.Event() - self.start_time = start - self.max_time = stop - self.kill_pid = kill_pid - - def run(self): - while not self.quit.is_set(): - #print( f'time_stop {self.start_time} limit {self.max_time}') - if (self.max_time == 0): - return - t = time.monotonic() - if ( t - self.start_time > self.max_time ): - print( f'killing {self.kill_pid} t {t}') - if os.name == "nt": - cmd = ['TASKKILL', '/F', '/T', '/PID', str(self.kill_pid)] - proc = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr) - else: - os.kill(self.kill_pid, signal.SIGKILL) - self.timed_out.set() - self.stop() - pass - - def stop(self): - self.quit.set() +class TestRunner(): + def __init__(self): + self.parser = argparse.ArgumentParser(description=""" + Checks build arguments + """) + + sysInfo = pf.uname() + + # getting os information + self.OS_info = { + "Machine" : sysInfo.machine, + "Node Name" : sysInfo.node, + "Num Processor" : os.cpu_count(), + "Processor" : sysInfo.processor, + "Release" : sysInfo.release, + "System" : sysInfo.system, + "Version" : sysInfo.version, + } + m = ' System Information ' + + print() + print(f'{m:-^100}') + + for k in self.OS_info: + print(f'\t {k}: {self.OS_info[k]}') + print() + self.lib_dir = os.path.dirname(os.path.abspath(__file__)) + + def __parse_args__(self): + self.parser.add_argument('-e', '--emulation', required=False, default="", + help='Test set to run from rtest.xml (optional, eg.smoke). At least one but not both of -e or -t must be set') + self.parser.add_argument('-t', '--test', required=False, default="", + help='Test set to run from rtest.xml (optional, e.g. osdb). At least one but not both of -e or -t must be set') + self.parser.add_argument('-g', '--debug', required=False, default=False, action='store_true', + help='Test Debug build (optional, default: false)') + self.parser.add_argument('-o', '--output', type=str, required=False, default="xml", + help='Test output file (optional, default: test_detail.xml)') + self.parser.add_argument( '--install_dir', type=str, required=False, default="build", + help='Installation directory where build or release folders are (optional, default: build)') + self.parser.add_argument( '--fail_test', default=False, required=False, action='store_true', + help='Return as if test failed (optional, default: false)') + + self.args = self.parser.parse_args() + + def __get_vram__(self): + if self.OS_info['System'] == 'Linux': + subprocess.run('rocminfo') + + + def __call__(self): + self.__get_vram__() + + + + +# timeout = False +# test_proc = None +# stop = 0 + +# test_script = [ 'cd %IDIR%', '%XML%' ] + +# def vram_detect(): +# global OS_info +# OS_info["VRAM"] = 0 +# if os.name == "nt": +# cmd = "hipinfo.exe" +# process = subprocess.run([cmd], stdout=subprocess.PIPE) +# for line_in in process.stdout.decode().splitlines(): +# if 'totalGlobalMem' in line_in: +# OS_info["VRAM"] = float(line_in.split()[1]) +# break +# else: +# cmd = "rocminfo" +# process = subprocess.run([cmd], stdout=subprocess.PIPE) +# for line_in in process.stdout.decode().splitlines(): +# match = re.search(r'.*Size:.*([0-9]+)\(.*\).*KB', line_in, re.IGNORECASE) +# if match: +# OS_info["VRAM"] = float(match.group(1))/(1024*1024) +# break + +# def os_detect(): +# global OS_info +# if os.name == "nt": +# OS_info["ID"] = platform.system() +# else: +# inf_file = "/etc/os-release" +# if os.path.exists(inf_file): +# with open(inf_file) as f: +# for line in f: +# if "=" in line: +# k,v = line.strip().split("=") +# OS_info[k] = v.replace('"','') +# OS_info["NUM_PROC"] = os.cpu_count() +# vram_detect() +# print(OS_info) + + +# def create_dir(dir_path): +# if os.path.isabs(dir_path): +# full_path = dir_path +# else: +# full_path = os.path.join( os.getcwd(), dir_path ) +# return pathlib.Path(full_path).mkdir(parents=True, exist_ok=True) + +# def delete_dir(dir_path) : +# if (not os.path.exists(dir_path)): +# return +# if os.name == "nt": +# return run_cmd( "RMDIR" , f"/S /Q {dir_path}") +# else: +# linux_path = pathlib.Path(dir_path).absolute() +# return run_cmd( "rm" , f"-rf {linux_path}") + +# class TimerProcess(multiprocessing.Process): + +# def __init__(self, start, stop, kill_pid): +# multiprocessing.Process.__init__(self) +# self.quit = multiprocessing.Event() +# self.timed_out = multiprocessing.Event() +# self.start_time = start +# self.max_time = stop +# self.kill_pid = kill_pid + +# def run(self): +# while not self.quit.is_set(): +# #print( f'time_stop {self.start_time} limit {self.max_time}') +# if (self.max_time == 0): +# return +# t = time.monotonic() +# if ( t - self.start_time > self.max_time ): +# print( f'killing {self.kill_pid} t {t}') +# if os.name == "nt": +# cmd = ['TASKKILL', '/F', '/T', '/PID', str(self.kill_pid)] +# proc = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr) +# else: +# os.kill(self.kill_pid, signal.SIGKILL) +# self.timed_out.set() +# self.stop() +# pass + +# def stop(self): +# self.quit.set() - def stopped(self): - return self.timed_out.is_set() - - -def time_stop(start, pid): - global timeout, stop - while (True): - print( f'time_stop {start} limit {stop}') - t = time.monotonic() - if (stop == 0): - return - if ( (stop > 0) and (t - start > stop) ): - print( f'killing {pid} t {t}') - if os.name == "nt": - cmd = ['TASKKILL', '/F', '/T', '/PID', str(pid)] - proc = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr) - else: - test_proc.kill() - timeout = True - stop = 0 - time.sleep(0) - -def run_cmd(cmd, test = False, time_limit = 0): - global args - global test_proc, timer_thread - global stop - if (cmd.startswith('cd ')): - return os.chdir(cmd[3:]) - if (cmd.startswith('mkdir ')): - return create_dir(cmd[6:]) - cmdline = f"{cmd}" - print(cmdline) - try: - if not test: - proc = subprocess.run(cmdline, check=True, stderr=subprocess.STDOUT, shell=True) - status = proc.returncode - else: - error = False - timeout = False - test_proc = subprocess.Popen(cmdline, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) - if time_limit > 0: - start = time.monotonic() - #p = multiprocessing.Process(target=time_stop, args=(start, test_proc.pid)) - p = TimerProcess(start, time_limit, test_proc.pid) - p.start() - while True: - output = test_proc.stdout.readline() - if output == '' and test_proc.poll() is not None: - break - elif output: - outstring = output.strip() - print (outstring) - error = error or re.search(r'FAILED', outstring) - status = test_proc.poll() - if time_limit > 0: - p.stop() - p.join() - timeout = p.stopped() - print(f"timeout {timeout}") - if error: - status = 1 - elif timeout: - status = 2 - else: - status = test_proc.returncode - except: - import traceback - exc = traceback.format_exc() - print( "Python Exception: {0}".format(exc) ) - status = 3 - return status - -def batch(script, xml): - global OS_info - global args - # - cwd = pathlib.os.curdir - rtest_cwd_path = os.path.abspath( os.path.join( cwd, 'rtest.xml') ) - - if os.path.isfile(rtest_cwd_path) and os.path.dirname(rtest_cwd_path).endswith( "staging" ): - # if in a staging directory then test locally - test_dir = cwd - else: - # deal with windows pathing - install_dir = '//'.join(args.install_dir.split('\\')) - - if args.debug: - build_type = "debug" - else: - #check if we have a release folder in build - if os.path.isdir(f'{install_dir}//release//test'): - build_type = "release" - else: - build_type = "" +# def stopped(self): +# return self.timed_out.is_set() + + +# def time_stop(start, pid): +# global timeout, stop +# while (True): +# print( f'time_stop {start} limit {stop}') +# t = time.monotonic() +# if (stop == 0): +# return +# if ( (stop > 0) and (t - start > stop) ): +# print( f'killing {pid} t {t}') +# if os.name == "nt": +# cmd = ['TASKKILL', '/F', '/T', '/PID', str(pid)] +# proc = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr) +# else: +# test_proc.kill() +# timeout = True +# stop = 0 +# time.sleep(0) + +# def run_cmd(cmd, test = False, time_limit = 0): +# global args +# global test_proc, timer_thread +# global stop +# if (cmd.startswith('cd ')): +# return os.chdir(cmd[3:]) +# if (cmd.startswith('mkdir ')): +# return create_dir(cmd[6:]) +# cmdline = f"{cmd}" +# print(cmdline) +# try: +# if not test: +# proc = subprocess.run(cmdline, check=True, stderr=subprocess.STDOUT, shell=True) +# status = proc.returncode +# else: +# error = False +# timeout = False +# test_proc = subprocess.Popen(cmdline, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) +# if time_limit > 0: +# start = time.monotonic() +# #p = multiprocessing.Process(target=time_stop, args=(start, test_proc.pid)) +# p = TimerProcess(start, time_limit, test_proc.pid) +# p.start() +# while True: +# output = test_proc.stdout.readline() +# if output == '' and test_proc.poll() is not None: +# break +# elif output: +# outstring = output.strip() +# print (outstring) +# error = error or re.search(r'FAILED', outstring) +# status = test_proc.poll() +# if time_limit > 0: +# p.stop() +# p.join() +# timeout = p.stopped() +# print(f"timeout {timeout}") +# if error: +# status = 1 +# elif timeout: +# status = 2 +# else: +# status = test_proc.returncode +# except: +# import traceback +# exc = traceback.format_exc() +# print( "Python Exception: {0}".format(exc) ) +# status = 3 +# return status + +# def batch(script, xml): +# global OS_info +# global args +# # +# cwd = pathlib.os.curdir +# rtest_cwd_path = os.path.abspath( os.path.join( cwd, 'rtest.xml') ) + +# if os.path.isfile(rtest_cwd_path) and os.path.dirname(rtest_cwd_path).endswith( "staging" ): +# # if in a staging directory then test locally +# test_dir = cwd +# else: +# # deal with windows pathing +# install_dir = '//'.join(args.install_dir.split('\\')) + +# if args.debug: +# build_type = "debug" +# else: +# #check if we have a release folder in build +# if os.path.isdir(f'{install_dir}//release//test'): +# build_type = "release" +# else: +# build_type = "" - if len(build_type) > 0: - test_dir = f"{install_dir}//{build_type}//test" - else: - test_dir = f"{install_dir}//test" - fail = False - for i in range(len(script)): - cmdline = script[i] - xcmd = cmdline.replace('%IDIR%', test_dir) - cmd = xcmd.replace('%ODIR%', args.output) - if cmd.startswith('tdir '): - if pathlib.Path(cmd[5:]).exists(): - return 0 # all further cmds skipped - else: - continue - error = False - if cmd.startswith('%XML%'): - # run the matching tests listed in the xml test file - var_subs = {} - for var in xml.getElementsByTagName('var'): - name = var.getAttribute('name') - val = var.getAttribute('value') - var_subs[name] = val - for test in xml.getElementsByTagName('test'): - sets = test.getAttribute('sets') - runset = sets.split(',') - - A, B = args.test != '', args.emulation != '' - if not (A ^ B): - raise ValueError('At least one but not both of -e/--emulation or -t/--test must be set') - - if args.test in runset: - for run in test.getElementsByTagName('run'): - name = run.getAttribute('name') - vram_limit = run.getAttribute('vram_min') - if vram_limit: - if OS_info["VRAM"] < float(vram_limit): - print( f'***\n*** Skipped: {name} due to VRAM req.\n***') - continue - if name: - print( f'***\n*** Running: {name}\n***') - time_limit = run.getAttribute('time_max') - if time_limit: - timeout = float(time_limit) - else: - timeout = 0 - - raw_cmd = run.firstChild.data - var_cmd = raw_cmd.format_map(var_subs) - error = run_cmd(var_cmd, True, timeout) - if (error == 2): - print( f'***\n*** Timed out when running: {name}\n***') +# if len(build_type) > 0: +# test_dir = f"{install_dir}//{build_type}//test" +# else: +# test_dir = f"{install_dir}//test" +# fail = False +# for i in range(len(script)): +# cmdline = script[i] +# xcmd = cmdline.replace('%IDIR%', test_dir) +# cmd = xcmd.replace('%ODIR%', args.output) +# if cmd.startswith('tdir '): +# if pathlib.Path(cmd[5:]).exists(): +# return 0 # all further cmds skipped +# else: +# continue +# error = False +# if cmd.startswith('%XML%'): +# # run the matching tests listed in the xml test file +# var_subs = {} +# for var in xml.getElementsByTagName('var'): +# name = var.getAttribute('name') +# val = var.getAttribute('value') +# var_subs[name] = val +# for test in xml.getElementsByTagName('test'): +# sets = test.getAttribute('sets') +# runset = sets.split(',') + +# A, B = args.test != '', args.emulation != '' +# if not (A ^ B): +# raise ValueError('At least one but not both of -e/--emulation or -t/--test must be set') + +# if args.test in runset: +# for run in test.getElementsByTagName('run'): +# name = run.getAttribute('name') +# vram_limit = run.getAttribute('vram_min') +# if vram_limit: +# if OS_info["VRAM"] < float(vram_limit): +# print( f'***\n*** Skipped: {name} due to VRAM req.\n***') +# continue +# if name: +# print( f'***\n*** Running: {name}\n***') +# time_limit = run.getAttribute('time_max') +# if time_limit: +# timeout = float(time_limit) +# else: +# timeout = 0 + +# raw_cmd = run.firstChild.data +# var_cmd = raw_cmd.format_map(var_subs) +# error = run_cmd(var_cmd, True, timeout) +# if (error == 2): +# print( f'***\n*** Timed out when running: {name}\n***') - if args.emulation in runset: - for run in test.getElementsByTagName('run'): - name = run.getAttribute('name') - vram_limit = run.getAttribute('vram_min') - if vram_limit: - if OS_info["VRAM"] < float(vram_limit): - print( f'***\n*** Skipped: {name} due to VRAM req.\n***') - continue - if name: - print( f'***\n*** Running: {name}\n***') - time_limit = run.getAttribute('time_max') - if time_limit: - timeout = float(time_limit) - else: - timeout = 0 - - raw_cmd = run.firstChild.data - var_cmd = raw_cmd.format_map(var_subs) - error = run_cmd(var_cmd, True, timeout) - if (error == 2): - print( f'***\n*** Timed out when running: {name}\n***') - else: - error = run_cmd(cmd) - fail = fail or error - if (fail): - if (cmd == "%XML%"): - print(f"FAILED xml test suite!") - else: - print(f"ERROR running: {cmd}") - if (os.curdir != cwd): - os.chdir( cwd ) - return 1 - if (os.curdir != cwd): - os.chdir( cwd ) +# if args.emulation in runset: +# for run in test.getElementsByTagName('run'): +# name = run.getAttribute('name') +# vram_limit = run.getAttribute('vram_min') +# if vram_limit: +# if OS_info["VRAM"] < float(vram_limit): +# print( f'***\n*** Skipped: {name} due to VRAM req.\n***') +# continue +# if name: +# print( f'***\n*** Running: {name}\n***') +# time_limit = run.getAttribute('time_max') +# if time_limit: +# timeout = float(time_limit) +# else: +# timeout = 0 + +# raw_cmd = run.firstChild.data +# var_cmd = raw_cmd.format_map(var_subs) +# error = run_cmd(var_cmd, True, timeout) +# if (error == 2): +# print( f'***\n*** Timed out when running: {name}\n***') +# else: +# error = run_cmd(cmd) +# fail = fail or error +# if (fail): +# if (cmd == "%XML%"): +# print(f"FAILED xml test suite!") +# else: +# print(f"ERROR running: {cmd}") +# if (os.curdir != cwd): +# os.chdir( cwd ) +# return 1 +# if (os.curdir != cwd): +# os.chdir( cwd ) - return 0 +# return 0 -def run_tests(): - global test_script - global xmlDoc +# def run_tests(): +# global test_script +# global xmlDoc - # install - cwd = os.curdir +# # install +# cwd = os.curdir - xmlPath = os.path.join( cwd, 'rtest.xml') - xmlDoc = minidom.parse( xmlPath ) +# xmlPath = os.path.join( cwd, 'rtest.xml') +# xmlDoc = minidom.parse( xmlPath ) - scripts = [] - scripts.append( test_script ) - for i in scripts: - if (batch(i, xmlDoc)): - #print("Failure in script. ABORTING") - if (os.curdir != cwd): - os.chdir( cwd ) - return 1 - if (os.curdir != cwd): - os.chdir( cwd ) - return 0 +# scripts = [] +# scripts.append( test_script ) +# for i in scripts: +# if (batch(i, xmlDoc)): +# #print("Failure in script. ABORTING") +# if (os.curdir != cwd): +# os.chdir( cwd ) +# return 1 +# if (os.curdir != cwd): +# os.chdir( cwd ) +# return 0 -def main(): - global args - global timer_thread +# def main(): +# global args +# global timer_thread - os_detect() - args = parse_args() +# os_detect() +# args = parse_args() - status = run_tests() +# status = run_tests() - if args.fail_test: - status = 1 - if (status): - sys.exit(status) +# if args.fail_test: +# status = 1 +# if (status): +# sys.exit(status) if __name__ == '__main__': - main() \ No newline at end of file + runner = TestRunner() + runner() \ No newline at end of file From cede9f301777d72c8082a8ce594bf24975b42753 Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Tue, 26 Nov 2024 16:51:47 -0700 Subject: [PATCH 14/20] implemented linux --- rtest.py | 401 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 212 insertions(+), 189 deletions(-) diff --git a/rtest.py b/rtest.py index 802ffd2f0..19b9dfefe 100644 --- a/rtest.py +++ b/rtest.py @@ -2,6 +2,7 @@ """Copyright (c) 2021-2024 Advanced Micro Devices, Inc. All rights reserved. Run tests on build""" +from multiprocessing import process import re import os import sys @@ -17,11 +18,49 @@ import time import platform as pf +#TODO Implement time outs when its added to the xml +#TODO Implement VRAM limit when its added to the xml + class TestRunner(): + def __get_vram__(self): + if self.OS_info['System'] == 'Linux': + process = subprocess.run('rocm-smi --showmeminfo vram', shell=True, stdout=subprocess.PIPE) + gpu_id = os.getenv('HIP_VISIBLE_DEVICES', '0') + + for l in process.stdout.decode().splitlines(): + if 'Total Memory' in l and f'GPU[{gpu_id}]' in l: + self.OS_info['VRAM'] = float(l.split()[-1]) / (1024 ** 3) + break + + def __parse_args__(self): + self.parser.add_argument('-e', '--emulation', required=False, default='', + help='Test set to run from rtest.xml (optional, eg.smoke). At least one but not both of -e or -t must be set') + self.parser.add_argument('-t', '--test', required=False, default='', + help='Test set to run from rtest.xml (optional, e.g. osdb). At least one but not both of -e or -t must be set') + self.parser.add_argument('-g', '--debug', required=False, default=False, action='store_true', + help='Test Debug build (optional, default: false)') + self.parser.add_argument('-o', '--output', type=str, required=False, default=None, + help='Test output file (optional, default: None [output to stdout])') + self.parser.add_argument( '--install_dir', type=str, required=False, default="build", + help='Installation directory where build or release folders are (optional, default: build)') + self.parser.add_argument( '--fail_test', default=False, required=False, action='store_true', + help='Return as if test failed (optional, default: false)') + self.args = self.parser.parse_args() + def __init__(self): self.parser = argparse.ArgumentParser(description=""" Checks build arguments """) + + self.__parse_args__() + + if (self.args.emulation != '') ^ (self.args.test != ''): + if self.args.emulation != '': + self.test_choice = self.args.emulation + else: + self.test_choice = self.args.test + else: + raise ValueError('At least one but not both of -e/--emulation or -t/--test must be set') sysInfo = pf.uname() @@ -35,6 +74,9 @@ def __init__(self): "System" : sysInfo.system, "Version" : sysInfo.version, } + + self.__get_vram__() + m = ' System Information ' print() @@ -43,32 +85,183 @@ def __init__(self): for k in self.OS_info: print(f'\t {k}: {self.OS_info[k]}') print() - self.lib_dir = os.path.dirname(os.path.abspath(__file__)) - - def __parse_args__(self): - self.parser.add_argument('-e', '--emulation', required=False, default="", - help='Test set to run from rtest.xml (optional, eg.smoke). At least one but not both of -e or -t must be set') - self.parser.add_argument('-t', '--test', required=False, default="", - help='Test set to run from rtest.xml (optional, e.g. osdb). At least one but not both of -e or -t must be set') - self.parser.add_argument('-g', '--debug', required=False, default=False, action='store_true', - help='Test Debug build (optional, default: false)') - self.parser.add_argument('-o', '--output', type=str, required=False, default="xml", - help='Test output file (optional, default: test_detail.xml)') - self.parser.add_argument( '--install_dir', type=str, required=False, default="build", - help='Installation directory where build or release folders are (optional, default: build)') - self.parser.add_argument( '--fail_test', default=False, required=False, action='store_true', - help='Return as if test failed (optional, default: false)') - self.args = self.parser.parse_args() + self.lib_dir = os.path.dirname(os.path.abspath(__file__)) + self.xml_path = os.path.join(self.lib_dir, r'rtest.xml') - def __get_vram__(self): if self.OS_info['System'] == 'Linux': - subprocess.run('rocminfo') + + # find the test dir with default install dir + if self.args.install_dir == 'build': + + # if its debug mode + if self.args.debug: + self.test_dir = os.path.join(self.lib_dir, f'build/debug/test') + # if its release mode + elif os.path.isdir(os.path.join(self.lib_dir, f'build/release/test')): + self.test_dir = os.path.join(self.lib_dir, f'build/release/test') + else: + self.test_dir = os.path.join(self.lib_dir, f'build/test') + + + else: + # if its an actual directory AND it has test directory + if os.path.isdir(os.path.join(self.args.install_dir, f'test')): + self.test_dir = os.path.join(self.args.install_dir, f'test') + else: + raise ValueError(f'{self.args.install_dir} is not a valid install directory!') + + if self.args.output: + self.output = open(os.path.abspath(self.args.output), 'w') + self.output_path = os.path.abspath(self.args.output) + else: + self.output = None + self.output_path = None + + m = ' Current Paths' + print(f'{m:-^100}') + print(f'Working Directory: {self.lib_dir}') + print(f'rtest.xml: {self.xml_path}') + print(f'Test Directory: {self.test_dir}') + print(f'Output File: {self.output_path}') + + print() + def __call__(self): - self.__get_vram__() + xml_file = minidom.parse(self.xml_path) + + curr_dir = os.curdir + + os.chdir(self.test_dir) + + cmd_values = {} + for var in xml_file.getElementsByTagName('var'): + name, val = var.getAttribute('name'), var.getAttribute('value') + cmd_values[name] = val + + noMatch = True + for test in xml_file.getElementsByTagName('test'): + sets = test.getAttribute('sets') + if self.test_choice == sets: + + for run in test.getElementsByTagName('run'): + temp = run.firstChild.data + temp = temp.replace('{', '') + temp = temp.replace('}', '') + cmd_list = temp.split() + + cmd_str = '' + for var in cmd_list: + cmd_str += cmd_values[var] + + m = 'Final Command' + print(f'{m:-^100}') + print(cmd_str) + print() + + subprocess.run(cmd_str, shell=True, stdout=self.output) + noMatch = False + break + + os.chdir(curr_dir) + if noMatch: + raise ValueError(f'Test value passed in: "{self.test_choice}" does not match any known test suite') + + if self.args.fail_test: + sys.exit(1) + + + +# def batch(script, xml): +# global OS_info +# global args +# # +# fail = False +# for i in range(len(script)): +# cmdline = script[i] +# xcmd = cmdline.replace('%IDIR%', test_dir) +# cmd = xcmd.replace('%ODIR%', args.output) +# if cmd.startswith('tdir '): +# if pathlib.Path(cmd[5:]).exists(): +# return 0 # all further cmds skipped +# else: +# continue +# error = False +# if cmd.startswith('%XML%'): +# # run the matching tests listed in the xml test file +# var_subs = {} +# for var in xml.getElementsByTagName('var'): +# name = var.getAttribute('name') +# val = var.getAttribute('value') +# var_subs[name] = val +# for test in xml.getElementsByTagName('test'): +# sets = test.getAttribute('sets') +# runset = sets.split(',') + +# A, B = args.test != '', args.emulation != '' +# if not (A ^ B): +# raise ValueError('At least one but not both of -e/--emulation or -t/--test must be set') + +# if args.test in runset: +# for run in test.getElementsByTagName('run'): +# name = run.getAttribute('name') +# vram_limit = run.getAttribute('vram_min') +# if vram_limit: +# if OS_info["VRAM"] < float(vram_limit): +# print( f'***\n*** Skipped: {name} due to VRAM req.\n***') +# continue +# if name: +# print( f'***\n*** Running: {name}\n***') +# time_limit = run.getAttribute('time_max') +# if time_limit: +# timeout = float(time_limit) +# else: +# timeout = 0 + +# raw_cmd = run.firstChild.data +# var_cmd = raw_cmd.format_map(var_subs) +# error = run_cmd(var_cmd, True, timeout) +# if (error == 2): +# print( f'***\n*** Timed out when running: {name}\n***') + +# if args.emulation in runset: +# for run in test.getElementsByTagName('run'): +# name = run.getAttribute('name') +# vram_limit = run.getAttribute('vram_min') +# if vram_limit: +# if OS_info["VRAM"] < float(vram_limit): +# print( f'***\n*** Skipped: {name} due to VRAM req.\n***') +# continue +# if name: +# print( f'***\n*** Running: {name}\n***') +# time_limit = run.getAttribute('time_max') +# if time_limit: +# timeout = float(time_limit) +# else: +# timeout = 0 +# raw_cmd = run.firstChild.data +# var_cmd = raw_cmd.format_map(var_subs) +# error = run_cmd(var_cmd, True, timeout) +# if (error == 2): +# print( f'***\n*** Timed out when running: {name}\n***') +# else: +# error = run_cmd(cmd) +# fail = fail or error +# if (fail): +# if (cmd == "%XML%"): +# print(f"FAILED xml test suite!") +# else: +# print(f"ERROR running: {cmd}") +# if (os.curdir != cwd): +# os.chdir( cwd ) +# return 1 +# if (os.curdir != cwd): +# os.chdir( cwd ) + +# return 0 @@ -88,47 +281,6 @@ def __call__(self): # if 'totalGlobalMem' in line_in: # OS_info["VRAM"] = float(line_in.split()[1]) # break -# else: -# cmd = "rocminfo" -# process = subprocess.run([cmd], stdout=subprocess.PIPE) -# for line_in in process.stdout.decode().splitlines(): -# match = re.search(r'.*Size:.*([0-9]+)\(.*\).*KB', line_in, re.IGNORECASE) -# if match: -# OS_info["VRAM"] = float(match.group(1))/(1024*1024) -# break - -# def os_detect(): -# global OS_info -# if os.name == "nt": -# OS_info["ID"] = platform.system() -# else: -# inf_file = "/etc/os-release" -# if os.path.exists(inf_file): -# with open(inf_file) as f: -# for line in f: -# if "=" in line: -# k,v = line.strip().split("=") -# OS_info[k] = v.replace('"','') -# OS_info["NUM_PROC"] = os.cpu_count() -# vram_detect() -# print(OS_info) - - -# def create_dir(dir_path): -# if os.path.isabs(dir_path): -# full_path = dir_path -# else: -# full_path = os.path.join( os.getcwd(), dir_path ) -# return pathlib.Path(full_path).mkdir(parents=True, exist_ok=True) - -# def delete_dir(dir_path) : -# if (not os.path.exists(dir_path)): -# return -# if os.name == "nt": -# return run_cmd( "RMDIR" , f"/S /Q {dir_path}") -# else: -# linux_path = pathlib.Path(dir_path).absolute() -# return run_cmd( "rm" , f"-rf {linux_path}") # class TimerProcess(multiprocessing.Process): @@ -164,24 +316,6 @@ def __call__(self): # return self.timed_out.is_set() -# def time_stop(start, pid): -# global timeout, stop -# while (True): -# print( f'time_stop {start} limit {stop}') -# t = time.monotonic() -# if (stop == 0): -# return -# if ( (stop > 0) and (t - start > stop) ): -# print( f'killing {pid} t {t}') -# if os.name == "nt": -# cmd = ['TASKKILL', '/F', '/T', '/PID', str(pid)] -# proc = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr) -# else: -# test_proc.kill() -# timeout = True -# stop = 0 -# time.sleep(0) - # def run_cmd(cmd, test = False, time_limit = 0): # global args # global test_proc, timer_thread @@ -232,117 +366,6 @@ def __call__(self): # status = 3 # return status -# def batch(script, xml): -# global OS_info -# global args -# # -# cwd = pathlib.os.curdir -# rtest_cwd_path = os.path.abspath( os.path.join( cwd, 'rtest.xml') ) - -# if os.path.isfile(rtest_cwd_path) and os.path.dirname(rtest_cwd_path).endswith( "staging" ): -# # if in a staging directory then test locally -# test_dir = cwd -# else: -# # deal with windows pathing -# install_dir = '//'.join(args.install_dir.split('\\')) - -# if args.debug: -# build_type = "debug" -# else: -# #check if we have a release folder in build -# if os.path.isdir(f'{install_dir}//release//test'): -# build_type = "release" -# else: -# build_type = "" - -# if len(build_type) > 0: -# test_dir = f"{install_dir}//{build_type}//test" -# else: -# test_dir = f"{install_dir}//test" -# fail = False -# for i in range(len(script)): -# cmdline = script[i] -# xcmd = cmdline.replace('%IDIR%', test_dir) -# cmd = xcmd.replace('%ODIR%', args.output) -# if cmd.startswith('tdir '): -# if pathlib.Path(cmd[5:]).exists(): -# return 0 # all further cmds skipped -# else: -# continue -# error = False -# if cmd.startswith('%XML%'): -# # run the matching tests listed in the xml test file -# var_subs = {} -# for var in xml.getElementsByTagName('var'): -# name = var.getAttribute('name') -# val = var.getAttribute('value') -# var_subs[name] = val -# for test in xml.getElementsByTagName('test'): -# sets = test.getAttribute('sets') -# runset = sets.split(',') - -# A, B = args.test != '', args.emulation != '' -# if not (A ^ B): -# raise ValueError('At least one but not both of -e/--emulation or -t/--test must be set') - -# if args.test in runset: -# for run in test.getElementsByTagName('run'): -# name = run.getAttribute('name') -# vram_limit = run.getAttribute('vram_min') -# if vram_limit: -# if OS_info["VRAM"] < float(vram_limit): -# print( f'***\n*** Skipped: {name} due to VRAM req.\n***') -# continue -# if name: -# print( f'***\n*** Running: {name}\n***') -# time_limit = run.getAttribute('time_max') -# if time_limit: -# timeout = float(time_limit) -# else: -# timeout = 0 - -# raw_cmd = run.firstChild.data -# var_cmd = raw_cmd.format_map(var_subs) -# error = run_cmd(var_cmd, True, timeout) -# if (error == 2): -# print( f'***\n*** Timed out when running: {name}\n***') - -# if args.emulation in runset: -# for run in test.getElementsByTagName('run'): -# name = run.getAttribute('name') -# vram_limit = run.getAttribute('vram_min') -# if vram_limit: -# if OS_info["VRAM"] < float(vram_limit): -# print( f'***\n*** Skipped: {name} due to VRAM req.\n***') -# continue -# if name: -# print( f'***\n*** Running: {name}\n***') -# time_limit = run.getAttribute('time_max') -# if time_limit: -# timeout = float(time_limit) -# else: -# timeout = 0 - -# raw_cmd = run.firstChild.data -# var_cmd = raw_cmd.format_map(var_subs) -# error = run_cmd(var_cmd, True, timeout) -# if (error == 2): -# print( f'***\n*** Timed out when running: {name}\n***') -# else: -# error = run_cmd(cmd) -# fail = fail or error -# if (fail): -# if (cmd == "%XML%"): -# print(f"FAILED xml test suite!") -# else: -# print(f"ERROR running: {cmd}") -# if (os.curdir != cwd): -# os.chdir( cwd ) -# return 1 -# if (os.curdir != cwd): -# os.chdir( cwd ) - -# return 0 # def run_tests(): # global test_script From 309e48d246536412190660a763b08922641571ef Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Tue, 26 Nov 2024 16:52:54 -0700 Subject: [PATCH 15/20] removed old comments and unused imports --- rtest.py | 240 ------------------------------------------------------- 1 file changed, 240 deletions(-) diff --git a/rtest.py b/rtest.py index 19b9dfefe..7b2b12734 100644 --- a/rtest.py +++ b/rtest.py @@ -2,20 +2,11 @@ """Copyright (c) 2021-2024 Advanced Micro Devices, Inc. All rights reserved. Run tests on build""" -from multiprocessing import process -import re import os import sys import subprocess -import shlex import argparse -import pathlib -import platform -from genericpath import exists -from fnmatch import fnmatchcase from xml.dom import minidom -import multiprocessing -import time import platform as pf #TODO Implement time outs when its added to the xml @@ -172,237 +163,6 @@ def __call__(self): if self.args.fail_test: sys.exit(1) - - -# def batch(script, xml): -# global OS_info -# global args -# # -# fail = False -# for i in range(len(script)): -# cmdline = script[i] -# xcmd = cmdline.replace('%IDIR%', test_dir) -# cmd = xcmd.replace('%ODIR%', args.output) -# if cmd.startswith('tdir '): -# if pathlib.Path(cmd[5:]).exists(): -# return 0 # all further cmds skipped -# else: -# continue -# error = False -# if cmd.startswith('%XML%'): -# # run the matching tests listed in the xml test file -# var_subs = {} -# for var in xml.getElementsByTagName('var'): -# name = var.getAttribute('name') -# val = var.getAttribute('value') -# var_subs[name] = val -# for test in xml.getElementsByTagName('test'): -# sets = test.getAttribute('sets') -# runset = sets.split(',') - -# A, B = args.test != '', args.emulation != '' -# if not (A ^ B): -# raise ValueError('At least one but not both of -e/--emulation or -t/--test must be set') - -# if args.test in runset: -# for run in test.getElementsByTagName('run'): -# name = run.getAttribute('name') -# vram_limit = run.getAttribute('vram_min') -# if vram_limit: -# if OS_info["VRAM"] < float(vram_limit): -# print( f'***\n*** Skipped: {name} due to VRAM req.\n***') -# continue -# if name: -# print( f'***\n*** Running: {name}\n***') -# time_limit = run.getAttribute('time_max') -# if time_limit: -# timeout = float(time_limit) -# else: -# timeout = 0 - -# raw_cmd = run.firstChild.data -# var_cmd = raw_cmd.format_map(var_subs) -# error = run_cmd(var_cmd, True, timeout) -# if (error == 2): -# print( f'***\n*** Timed out when running: {name}\n***') - -# if args.emulation in runset: -# for run in test.getElementsByTagName('run'): -# name = run.getAttribute('name') -# vram_limit = run.getAttribute('vram_min') -# if vram_limit: -# if OS_info["VRAM"] < float(vram_limit): -# print( f'***\n*** Skipped: {name} due to VRAM req.\n***') -# continue -# if name: -# print( f'***\n*** Running: {name}\n***') -# time_limit = run.getAttribute('time_max') -# if time_limit: -# timeout = float(time_limit) -# else: -# timeout = 0 - -# raw_cmd = run.firstChild.data -# var_cmd = raw_cmd.format_map(var_subs) -# error = run_cmd(var_cmd, True, timeout) -# if (error == 2): -# print( f'***\n*** Timed out when running: {name}\n***') -# else: -# error = run_cmd(cmd) -# fail = fail or error -# if (fail): -# if (cmd == "%XML%"): -# print(f"FAILED xml test suite!") -# else: -# print(f"ERROR running: {cmd}") -# if (os.curdir != cwd): -# os.chdir( cwd ) -# return 1 -# if (os.curdir != cwd): -# os.chdir( cwd ) - -# return 0 - - - -# timeout = False -# test_proc = None -# stop = 0 - -# test_script = [ 'cd %IDIR%', '%XML%' ] - -# def vram_detect(): -# global OS_info -# OS_info["VRAM"] = 0 -# if os.name == "nt": -# cmd = "hipinfo.exe" -# process = subprocess.run([cmd], stdout=subprocess.PIPE) -# for line_in in process.stdout.decode().splitlines(): -# if 'totalGlobalMem' in line_in: -# OS_info["VRAM"] = float(line_in.split()[1]) -# break - -# class TimerProcess(multiprocessing.Process): - -# def __init__(self, start, stop, kill_pid): -# multiprocessing.Process.__init__(self) -# self.quit = multiprocessing.Event() -# self.timed_out = multiprocessing.Event() -# self.start_time = start -# self.max_time = stop -# self.kill_pid = kill_pid - -# def run(self): -# while not self.quit.is_set(): -# #print( f'time_stop {self.start_time} limit {self.max_time}') -# if (self.max_time == 0): -# return -# t = time.monotonic() -# if ( t - self.start_time > self.max_time ): -# print( f'killing {self.kill_pid} t {t}') -# if os.name == "nt": -# cmd = ['TASKKILL', '/F', '/T', '/PID', str(self.kill_pid)] -# proc = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr) -# else: -# os.kill(self.kill_pid, signal.SIGKILL) -# self.timed_out.set() -# self.stop() -# pass - -# def stop(self): -# self.quit.set() - -# def stopped(self): -# return self.timed_out.is_set() - - -# def run_cmd(cmd, test = False, time_limit = 0): -# global args -# global test_proc, timer_thread -# global stop -# if (cmd.startswith('cd ')): -# return os.chdir(cmd[3:]) -# if (cmd.startswith('mkdir ')): -# return create_dir(cmd[6:]) -# cmdline = f"{cmd}" -# print(cmdline) -# try: -# if not test: -# proc = subprocess.run(cmdline, check=True, stderr=subprocess.STDOUT, shell=True) -# status = proc.returncode -# else: -# error = False -# timeout = False -# test_proc = subprocess.Popen(cmdline, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) -# if time_limit > 0: -# start = time.monotonic() -# #p = multiprocessing.Process(target=time_stop, args=(start, test_proc.pid)) -# p = TimerProcess(start, time_limit, test_proc.pid) -# p.start() -# while True: -# output = test_proc.stdout.readline() -# if output == '' and test_proc.poll() is not None: -# break -# elif output: -# outstring = output.strip() -# print (outstring) -# error = error or re.search(r'FAILED', outstring) -# status = test_proc.poll() -# if time_limit > 0: -# p.stop() -# p.join() -# timeout = p.stopped() -# print(f"timeout {timeout}") -# if error: -# status = 1 -# elif timeout: -# status = 2 -# else: -# status = test_proc.returncode -# except: -# import traceback -# exc = traceback.format_exc() -# print( "Python Exception: {0}".format(exc) ) -# status = 3 -# return status - - -# def run_tests(): -# global test_script -# global xmlDoc - -# # install -# cwd = os.curdir - -# xmlPath = os.path.join( cwd, 'rtest.xml') -# xmlDoc = minidom.parse( xmlPath ) - -# scripts = [] -# scripts.append( test_script ) -# for i in scripts: -# if (batch(i, xmlDoc)): -# #print("Failure in script. ABORTING") -# if (os.curdir != cwd): -# os.chdir( cwd ) -# return 1 -# if (os.curdir != cwd): -# os.chdir( cwd ) -# return 0 - -# def main(): -# global args -# global timer_thread - -# os_detect() -# args = parse_args() - -# status = run_tests() - -# if args.fail_test: -# status = 1 -# if (status): -# sys.exit(status) - if __name__ == '__main__': runner = TestRunner() runner() \ No newline at end of file From feb60c08b2a3ce5e916f975e1afc1112b687c8c8 Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Wed, 27 Nov 2024 15:03:09 -0700 Subject: [PATCH 16/20] got rid of some whitespaces --- rmake.py | 7 +++---- rtest.py | 1 - 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/rmake.py b/rmake.py index 647b579a0..aa94b5577 100644 --- a/rmake.py +++ b/rmake.py @@ -151,9 +151,9 @@ def __get_cmake_cmd__(self): rocm_cmake_path.replace('\\', '/') cmake_options.append(f'-G Ninja') - cmake_options.append( f"-DWIN32=ON -DCPACK_PACKAGING_INSTALL_PREFIX=") #" -DCPACK_PACKAGING_INSTALL_PREFIX={rocm_path}" - cmake_options.append( f'-DCMAKE_INSTALL_PREFIX="C:/hipSDK"' ) - cmake_options.append( f'-DCMAKE_CXX_FLAGS="-D_ENABLE_EXTENDED_ALIGNED_STORAGE"') + cmake_options.append(f"-DWIN32=ON -DCPACK_PACKAGING_INSTALL_PREFIX=") #" -DCPACK_PACKAGING_INSTALL_PREFIX={rocm_path}" + cmake_options.append(f'-DCMAKE_INSTALL_PREFIX="C:/hipSDK"') + cmake_options.append(f'-DCMAKE_CXX_FLAGS="-D_ENABLE_EXTENDED_ALIGNED_STORAGE"') cmake_options.append(f'-DROCM_PATH={rocm_path}') cmake_options.append(f'-DCMAKE_PREFIX_PATH:PATH={rocm_path};{rocm_cmake_path}') @@ -209,7 +209,6 @@ def __call__(self): curr_dir = os.path.abspath(os.curdir) os.chdir(self.build_path) - subprocess.run(cmake_command, shell=True) if self.OS_info['System'] == 'Linux': diff --git a/rtest.py b/rtest.py index 7b2b12734..79d38000f 100644 --- a/rtest.py +++ b/rtest.py @@ -84,7 +84,6 @@ def __init__(self): # find the test dir with default install dir if self.args.install_dir == 'build': - # if its debug mode if self.args.debug: self.test_dir = os.path.join(self.lib_dir, f'build/debug/test') From d6d6e3a8b27c3662d661ebbb588f8faf99e141ec Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Wed, 27 Nov 2024 15:23:56 -0700 Subject: [PATCH 17/20] implemented for windows and added --test_dir option --- rtest.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/rtest.py b/rtest.py index 79d38000f..2cf3ecab4 100644 --- a/rtest.py +++ b/rtest.py @@ -34,6 +34,8 @@ def __parse_args__(self): help='Test output file (optional, default: None [output to stdout])') self.parser.add_argument( '--install_dir', type=str, required=False, default="build", help='Installation directory where build or release folders are (optional, default: build)') + self.parser.add_argument( '--test_dir', type=str, required=False, default=None, + help='Test directory where rocprim tests are (optinal, default=None)') self.parser.add_argument( '--fail_test', default=False, required=False, action='store_true', help='Return as if test failed (optional, default: false)') self.args = self.parser.parse_args() @@ -80,8 +82,12 @@ def __init__(self): self.lib_dir = os.path.dirname(os.path.abspath(__file__)) self.xml_path = os.path.join(self.lib_dir, r'rtest.xml') - if self.OS_info['System'] == 'Linux': - + if self.args.test_dir: + if not os.path.isdir(self.args.test_dir): + raise ValueError(f'Value {self.args.test_dir} is not a directory!') + + self.test_dir = self.args.test_dir + else: # find the test dir with default install dir if self.args.install_dir == 'build': # if its debug mode @@ -90,10 +96,10 @@ def __init__(self): # if its release mode elif os.path.isdir(os.path.join(self.lib_dir, f'build/release/test')): self.test_dir = os.path.join(self.lib_dir, f'build/release/test') - else: + elif os.path.isdir(os.path.join(self.lib_dir, f'build/test')): self.test_dir = os.path.join(self.lib_dir, f'build/test') - - + else: + raise FileNotFoundError(f'Did not find test directory') else: # if its an actual directory AND it has test directory if os.path.isdir(os.path.join(self.args.install_dir, f'test')): @@ -107,6 +113,16 @@ def __init__(self): else: self.output = None self.output_path = None + + if self.OS_info['System'] == 'Windows': + self.lib_dir = self.lib_dir.replace('\\', '/') + self.xml_path = self.xml_path.replace('\\', '/') + self.test_dir = self.test_dir.replace('\\', '/') + + if self.output_path: + self.output_path = self.output_path.replace('\\', '/') + + self.output = open(os.path.abspath(self.output_path), 'w') m = ' Current Paths' print(f'{m:-^100}') @@ -116,8 +132,6 @@ def __init__(self): print(f'Output File: {self.output_path}') print() - - def __call__(self): xml_file = minidom.parse(self.xml_path) From aef9ed2b5672eaeffa57c968de308cb0915470f4 Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Thu, 28 Nov 2024 14:45:37 -0700 Subject: [PATCH 18/20] made sure there can be no duplicate arguments --- rmake.py | 108 ++++++++++++++++++++++++++++++++++--------------------- rtest.py | 1 - 2 files changed, 67 insertions(+), 42 deletions(-) diff --git a/rmake.py b/rmake.py index aa94b5577..06880cbe5 100644 --- a/rmake.py +++ b/rmake.py @@ -9,36 +9,6 @@ import shutil class AutoBuilder: - def __init__(self): - self.parser = argparse.ArgumentParser(description=""" - Checks build arguments - """) - - sysInfo = pf.uname() - - # getting os information - self.OS_info = { - "Machine" : sysInfo.machine, - "Node Name" : sysInfo.node, - "Num Processor" : os.cpu_count(), - "Processor" : sysInfo.processor, - "Release" : sysInfo.release, - "System" : sysInfo.system, - "Version" : sysInfo.version, - } - m = ' System Information ' - - print() - print(f'{m:-^100}') - - for k in self.OS_info: - print(f'\t {k}: {self.OS_info[k]}') - print() - - - self.lib_dir = os.path.dirname(os.path.abspath(__file__)) - self.toolchain = f'toolchain-linux.cmake' if self.OS_info['System'] == 'Linux' else f'toolchain-windows.cmake' - def __parse_args__(self): """Parse command-line arguments""" default_gpus = 'gfx906:xnack-,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' @@ -69,6 +39,39 @@ def __parse_args__(self): self.args = self.parser.parse_args() + def __init__(self): + self.parser = argparse.ArgumentParser(description="Checks build arguments") + self.__parse_args__() + + sysInfo = pf.uname() + + # getting os information + self.OS_info = { + "Machine" : sysInfo.machine, + "Node Name" : sysInfo.node, + "Num Processor" : os.cpu_count(), + "Processor" : sysInfo.processor, + "Release" : sysInfo.release, + "System" : sysInfo.system, + "Version" : sysInfo.version, + } + m = ' System Information ' + + print() + print(f'{m:-^100}') + + for k in self.OS_info: + print(f'\t {k}: {self.OS_info[k]}') + print() + + self.custom_cmake_args = set() + + for item in self.args.cmake_dargs: + self.custom_cmake_args.add(item.split('=')[0]) # get the argument name + + self.lib_dir = os.path.dirname(os.path.abspath(__file__)) + self.toolchain = f'toolchain-linux.cmake' if self.OS_info['System'] == 'Linux' else f'toolchain-windows.cmake' + def __mk_dir__(self, dir_path: str): if os.path.isabs(dir_path): full_path = dir_path @@ -130,10 +133,20 @@ def __get_cmake_cmd__(self): rocm_path = os.getenv('ROCM_PATH', '/opt/rocm') - cmake_options.append(f"-DROCM_DIR:PATH={rocm_path}") - cmake_options.append(f"-DCPACK_PACKAGING_INSTALL_PREFIX={rocm_path}") - cmake_options.append(f"-DROCM_PATH={rocm_path}") - cmake_options.append(f"-DCMAKE_PREFIX_PATH:PATH={rocm_path}") + #These ifs will make sure there are no duplicates arguments + if "ROCM_DIR:PATH" not in self.custom_cmake_args: + cmake_options.append(f"-DROCM_DIR:PATH={rocm_path}") + + if "CPACK_PACKAGING_INSTALL_PREFIX" not in self.custom_cmake_args: + cmake_options.append(f"-DCPACK_PACKAGING_INSTALL_PREFIX={rocm_path}") + if "ROCM_PATH" not in self.custom_cmake_args: + cmake_options.append(f"-DROCM_PATH={rocm_path}") + + if "CMAKE_PREFIX_PATH:PATH" not in self.custom_cmake_args: + cmake_options.append(f"-DCMAKE_PREFIX_PATH:PATH={rocm_path}") + + if "CMAKE_CXX_FLAGS" not in self.custom_cmake_args: + cmake_options.append(f'-DCMAKE_CXX_FLAGS="-w"') else: cmake_exe = shutil.which('cmake.exe') @@ -150,12 +163,26 @@ def __get_cmake_cmd__(self): rocm_path.replace('\\', '/') rocm_cmake_path.replace('\\', '/') - cmake_options.append(f'-G Ninja') - cmake_options.append(f"-DWIN32=ON -DCPACK_PACKAGING_INSTALL_PREFIX=") #" -DCPACK_PACKAGING_INSTALL_PREFIX={rocm_path}" - cmake_options.append(f'-DCMAKE_INSTALL_PREFIX="C:/hipSDK"') - cmake_options.append(f'-DCMAKE_CXX_FLAGS="-D_ENABLE_EXTENDED_ALIGNED_STORAGE"') - cmake_options.append(f'-DROCM_PATH={rocm_path}') - cmake_options.append(f'-DCMAKE_PREFIX_PATH:PATH={rocm_path};{rocm_cmake_path}') + if '-G Ninja' not in self.custom_cmake_args: + cmake_options.append(f'-G Ninja') + + if 'WIN32' not in self.custom_cmake_args: + cmake_options.append(f"-DWIN32=ON") + + if 'CPACK_PACKAGING_INSTALL_PREFIX' not in self.custom_cmake_args: + cmake_options.append(f"-DCPACK_PACKAGING_INSTALL_PREFIX=") + + if 'CMAKE_INSTALL_PREFIX' not in self.custom_cmake_args: + cmake_options.append(f'-DCMAKE_INSTALL_PREFIX="C:/hipSDK"') + + if 'CMAKE_CXX_FLAGS' not in self.custom_cmake_args: + cmake_options.append(f'-DCMAKE_CXX_FLAGS="-D_ENABLE_EXTENDED_ALIGNED_STORAGE -w"') + + if 'ROCM_PATH' not in self.custom_cmake_args: + cmake_options.append(f'-DROCM_PATH={rocm_path}') + + if 'CMAKE_PREFIX_PATH:PATH' not in self.custom_cmake_args: + cmake_options.append(f'-DCMAKE_PREFIX_PATH:PATH={rocm_path};{rocm_cmake_path}') if self.args.static_lib: @@ -199,7 +226,6 @@ def __get_cmake_cmd__(self): return command_str def __call__(self): - self.__parse_args__() cmake_command = self.__get_cmake_cmd__() self.__rm_dir__(self.build_path) diff --git a/rtest.py b/rtest.py index 2cf3ecab4..4d1426e8b 100644 --- a/rtest.py +++ b/rtest.py @@ -121,7 +121,6 @@ def __init__(self): if self.output_path: self.output_path = self.output_path.replace('\\', '/') - self.output = open(os.path.abspath(self.output_path), 'w') m = ' Current Paths' From 77a7f7d48ff7f1023ef586a21efa98384e765146 Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Thu, 28 Nov 2024 16:29:14 -0700 Subject: [PATCH 19/20] implemented insert_cmake_args function --- rmake.py | 87 ++++++++++++++++++++------------------------------------ 1 file changed, 31 insertions(+), 56 deletions(-) diff --git a/rmake.py b/rmake.py index 06880cbe5..9df0e3878 100644 --- a/rmake.py +++ b/rmake.py @@ -65,6 +65,7 @@ def __init__(self): print() self.custom_cmake_args = set() + self.cmake_options = [] for item in self.args.cmake_dargs: self.custom_cmake_args.add(item.split('=')[0]) # get the argument name @@ -96,6 +97,11 @@ def __rm_dir__(self, dir_path: str): except FileNotFoundError: ... # no file to remove + + def __insert_cmake_args__(self, arg, val): + # This is to prevent the case of having duplicate cmake args from the argument --cmake-darg + if arg not in self.custom_cmake_args: + self.cmake_options.append(f'-D{arg}={val}') def __get_cmake_cmd__(self): @@ -103,14 +109,12 @@ def __get_cmake_cmd__(self): print(f'{m:-^100}\n\t{self.lib_dir}') print() - cmake_exe = '' - cmake_options = [ - f'--toolchain={self.toolchain}', - f'-DCPACK_SET_DESTDIR=OFF', - f'-DCPACK_INCLUDE_TOPLEVEL_DIRECTORY=OFF', - f'-DGPU_TARGETS={self.args.gpu_architecture}' - ] + self.__insert_cmake_args__('CMAKE_TOOLCHAIN_FILE', self.toolchain) + self.__insert_cmake_args__('CPACK_INCLUDE_TOPLEVEL_DIRECTORY', 'OFF') + self.__insert_cmake_args__('CPACK_SET_DESTDIR', 'OFF') + self.__insert_cmake_args__('GPU_TARGETS', self.args.gpu_architecture) + cmake_exe = '' if self.args.debug: build_path = os.path.join(self.args.build_dir, 'debug') cmake_config = 'Debug' @@ -120,7 +124,7 @@ def __get_cmake_cmd__(self): self.build_path = build_path - cmake_options.append(f"-DCMAKE_BUILD_TYPE={cmake_config}") + self.cmake_options.append(f"-DCMAKE_BUILD_TYPE={cmake_config}") if self.OS_info['System'] == 'Linux': cmake_exe = shutil.which('cmake3') @@ -133,21 +137,10 @@ def __get_cmake_cmd__(self): rocm_path = os.getenv('ROCM_PATH', '/opt/rocm') - #These ifs will make sure there are no duplicates arguments - if "ROCM_DIR:PATH" not in self.custom_cmake_args: - cmake_options.append(f"-DROCM_DIR:PATH={rocm_path}") - - if "CPACK_PACKAGING_INSTALL_PREFIX" not in self.custom_cmake_args: - cmake_options.append(f"-DCPACK_PACKAGING_INSTALL_PREFIX={rocm_path}") - if "ROCM_PATH" not in self.custom_cmake_args: - cmake_options.append(f"-DROCM_PATH={rocm_path}") - - if "CMAKE_PREFIX_PATH:PATH" not in self.custom_cmake_args: - cmake_options.append(f"-DCMAKE_PREFIX_PATH:PATH={rocm_path}") - - if "CMAKE_CXX_FLAGS" not in self.custom_cmake_args: - cmake_options.append(f'-DCMAKE_CXX_FLAGS="-w"') - + self.__insert_cmake_args__('CMAKE_CXX_FLAGS', f'"-w"') + self.__insert_cmake_args__('CMAKE_PREFIX_PATH:PATH', rocm_path) + self.__insert_cmake_args__('CPACK_PACKAGING_INSTALL_PREFIX', rocm_path) + self.__insert_cmake_args__('ROCM_DIR:PATH', rocm_path) else: cmake_exe = shutil.which('cmake.exe') @@ -164,46 +157,28 @@ def __get_cmake_cmd__(self): rocm_cmake_path.replace('\\', '/') if '-G Ninja' not in self.custom_cmake_args: - cmake_options.append(f'-G Ninja') - - if 'WIN32' not in self.custom_cmake_args: - cmake_options.append(f"-DWIN32=ON") - - if 'CPACK_PACKAGING_INSTALL_PREFIX' not in self.custom_cmake_args: - cmake_options.append(f"-DCPACK_PACKAGING_INSTALL_PREFIX=") - - if 'CMAKE_INSTALL_PREFIX' not in self.custom_cmake_args: - cmake_options.append(f'-DCMAKE_INSTALL_PREFIX="C:/hipSDK"') + self.cmake_options.append(f'-G Ninja') + self.__insert_cmake_args__('WIN32', 'ON') + self.__insert_cmake_args__('CPACK_PACKAGING_INSTALL_PREFIX', '') + self.__insert_cmake_args__('CMAKE_INSTALL_PREFIX', rocm_cmake_path) + self.__insert_cmake_args__('CMAKE_CXX_FLAGS', f'"-D_ENABLE_EXTENDED_ALIGNED_STORAGE -w"') + self.__insert_cmake_args__('CMAKE_PREFIX_PATH:PATH', f'{rocm_path};{rocm_cmake_path}') - if 'CMAKE_CXX_FLAGS' not in self.custom_cmake_args: - cmake_options.append(f'-DCMAKE_CXX_FLAGS="-D_ENABLE_EXTENDED_ALIGNED_STORAGE -w"') - - if 'ROCM_PATH' not in self.custom_cmake_args: - cmake_options.append(f'-DROCM_PATH={rocm_path}') - - if 'CMAKE_PREFIX_PATH:PATH' not in self.custom_cmake_args: - cmake_options.append(f'-DCMAKE_PREFIX_PATH:PATH={rocm_path};{rocm_cmake_path}') - - - if self.args.static_lib: - cmake_options.append( f'-DBUILD_SHARED_LIBS=OFF') - - if self.args.skip_ld_conf_entry: - cmake_options.append( f'-DROCM_DISABLE_LDCONFIG=ON' ) + self.__insert_cmake_args__('ROCM_PATH', rocm_path) + if self.args.static_lib: self.__insert_cmake_args__('BUILD_SHARED_LIBS', 'OFF') + if self.args.skip_ld_conf_entry: self.__insert_cmake_args__('ROCM_DISABLE_LDCONFIG', 'ON') + if self.args.build_clients: self.args.build_tests = True self.args.build_bench = True - cmake_options.append(f'-DBUILD_EXAMPLE=ON') + self.__insert_cmake_args__('BUILD_EXAMPLE', 'ON') - if self.args.build_tests: - cmake_options.append(f'-DBUILD_TEST=ON') - - if self.args.build_bench: - cmake_options.append(f'-DBUILD_BENCHMARK=ON') + if self.args.build_tests: self.__insert_cmake_args__('BUILD_TEST', 'ON') + if self.args.build_bench: self.__insert_cmake_args__('BUILD_BENCHMARK', 'ON') if self.args.cmake_dargs: - cmake_options += [f'-D{i}' for i in self.args.cmake_dargs] + self.cmake_options += [f'-D{i}' for i in self.args.cmake_dargs] # putting '' around paths to avoid white space in pathing if self.OS_info['System'] == 'Linux': @@ -213,7 +188,7 @@ def __get_cmake_cmd__(self): m = 'CMAKE Options' print(f'{m:-^100}') - for op in cmake_options: + for op in self.cmake_options: print(f'\t{op}') command_str += f' {op}' print() From 5dd7c1a184d2f05cddca088656166c72ec7ba892 Mon Sep 17 00:00:00 2001 From: NguyenNhuDi Date: Mon, 2 Dec 2024 12:05:39 -0700 Subject: [PATCH 20/20] reformated code, and updated changelog --- CHANGELOG.md | 2 ++ rmake.py | 50 +++++++++++++++++++++++++------------------------- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 033a2ad5a..be75336ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ Full documentation for rocPRIM is available at [https://rocm.docs.amd.com/projec ### Added +* Added `-b` option to `rmake.py`. This option will allow users to build benchmarks only instead of having +to build tests as well with `-c` option. * Added extended tests to `rtest.py`. These tests are extra tests that did not fit the criteria of smoke and regression tests. These tests will take much longer to run relative to smoke and regression tests. * Use `python rtest.py [--emulation|-e|--test|-t]=extended` to run these tests. * Added regression tests to `rtest.py`. Regression tests are a subset of tests that caused hardware problems for past emulation environments. diff --git a/rmake.py b/rmake.py index 9df0e3878..dd7de4c26 100644 --- a/rmake.py +++ b/rmake.py @@ -11,31 +11,31 @@ class AutoBuilder: def __parse_args__(self): """Parse command-line arguments""" - default_gpus = 'gfx906:xnack-,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' + self.default_gpus = 'gfx906:xnack-,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201' + self.parser.add_argument('-a', '--architecture', dest='gpu_architecture', required=False, default=self.default_gpus, + help='Set GPU architectures, e.g. all, gfx000, gfx803, gfx906:xnack-;gfx1030;gfx1100 (optional, default: all)') + self.parser.add_argument('-b', '--benchmarks', required=False, default=False, dest='build_bench', action='store_true', + help='Generate benchmarks only (default: False)') + self.parser.add_argument('-c', '--clients', required=False, default=False, dest='build_clients', action='store_true', + help='Generate all client builds (default: False)') self.parser.add_argument('-g', '--debug', required=False, default=False, action='store_true', help='Generate Debug build (default: False)') + self.parser.add_argument('-i', '--install', required=False, default=False, dest='install', action='store_true', + help='Install after build (default: False)') + self.parser.add_argument('-t', '--tests', required=False, default=False, dest='build_tests', action='store_true', + help='Generate unit tests only (default: False)') + self.parser.add_argument('-v', '--verbose', required=False, default=False, action='store_true', + help='Verbose build (default: False)') self.parser.add_argument( '--build_dir', type=str, required=False, default="build", help='Build directory path (default: build)') + self.parser.add_argument( '--cmake-darg', required=False, dest='cmake_dargs', action='append', default=[], + help='List of additional cmake defines for builds (e.g. CMAKE_CXX_COMPILER_LAUNCHER=ccache)') self.parser.add_argument( '--deps_dir', type=str, required=False, default=None, help='Dependencies directory path (default: build/deps)') self.parser.add_argument( '--skip_ld_conf_entry', required=False, default=False) self.parser.add_argument( '--static', required=False, default=False, dest='static_lib', action='store_true', help='Generate static library build (default: False)') - self.parser.add_argument('-c', '--clients', required=False, default=False, dest='build_clients', action='store_true', - help='Generate all client builds (default: False)') - self.parser.add_argument('-t', '--tests', required=False, default=False, dest='build_tests', action='store_true', - help='Generate unit tests only (default: False)') - self.parser.add_argument('-b', '--benchmarks', required=False, default=False, dest='build_bench', action='store_true', - help='Generate benchmarks only (default: False)') - self.parser.add_argument('-i', '--install', required=False, default=False, dest='install', action='store_true', - help='Install after build (default: False)') - self.parser.add_argument( '--cmake-darg', required=False, dest='cmake_dargs', action='append', default=[], - help='List of additional cmake defines for builds (e.g. CMAKE_CXX_COMPILER_LAUNCHER=ccache)') - self.parser.add_argument('-a', '--architecture', dest='gpu_architecture', required=False, default=default_gpus, #:sramecc+:xnack-" ) #gfx1030" ) #gfx906" ) # gfx1030" ) - help='Set GPU architectures, e.g. all, gfx000, gfx803, gfx906:xnack-;gfx1030;gfx1100 (optional, default: all)') - self.parser.add_argument('-v', '--verbose', required=False, default=False, action='store_true', - help='Verbose build (default: False)') self.args = self.parser.parse_args() @@ -55,14 +55,6 @@ def __init__(self): "System" : sysInfo.system, "Version" : sysInfo.version, } - m = ' System Information ' - - print() - print(f'{m:-^100}') - - for k in self.OS_info: - print(f'\t {k}: {self.OS_info[k]}') - print() self.custom_cmake_args = set() self.cmake_options = [] @@ -186,7 +178,7 @@ def __get_cmake_cmd__(self): else: command_str = f'"{cmake_exe}"' - m = 'CMAKE Options' + m = ' CMake Options ' print(f'{m:-^100}') for op in self.cmake_options: print(f'\t{op}') @@ -194,13 +186,21 @@ def __get_cmake_cmd__(self): print() command_str += f' "{self.lib_dir}"' - m = 'Final Command' + m = ' Final Command ' print(f'{m:-^100}') print(command_str) print() return command_str def __call__(self): + m = ' System Information ' + print() + print(f'{m:-^100}') + + for k in self.OS_info: + print(f'\t {k}: {self.OS_info[k]}') + print() + cmake_command = self.__get_cmake_cmd__() self.__rm_dir__(self.build_path)