From e510e32b63b141538320bea25f39382055e0f069 Mon Sep 17 00:00:00 2001 From: fjwillemsen Date: Fri, 2 Jun 2023 08:26:30 +0200 Subject: [PATCH] Deleted unused experiment files, updated README for versioning --- .gitignore | 3 + README.md | 6 + experiment_files/schema.json | 138 -------- experiment_files/template.json | 37 -- experiment_files/test_henk.json | 114 ------- experiment_files/test_hyperparams.json | 112 ------ experiment_files/test_hyperparams_ei.json | 162 --------- experiment_files/test_hyperparams_new.json | 321 ------------------ .../test_hyperparams_synthetic.json | 71 ---- experiment_files/test_methodology.json | 89 ----- experiment_files/test_methodology_quick.json | 50 --- .../test_methodology_quicker.json | 29 -- experiment_files/test_methodology_random.json | 56 --- experiment_files/test_output_file_writer.json | 63 ---- experiment_files/test_profile.json | 42 --- pyproject.toml | 3 +- 16 files changed, 11 insertions(+), 1285 deletions(-) delete mode 100755 experiment_files/schema.json delete mode 100755 experiment_files/template.json delete mode 100755 experiment_files/test_henk.json delete mode 100755 experiment_files/test_hyperparams.json delete mode 100755 experiment_files/test_hyperparams_ei.json delete mode 100755 experiment_files/test_hyperparams_new.json delete mode 100755 experiment_files/test_hyperparams_synthetic.json delete mode 100755 experiment_files/test_methodology.json delete mode 100644 experiment_files/test_methodology_quick.json delete mode 100755 experiment_files/test_methodology_quicker.json delete mode 100644 experiment_files/test_methodology_random.json delete mode 100644 experiment_files/test_output_file_writer.json delete mode 100755 experiment_files/test_profile.json diff --git a/.gitignore b/.gitignore index 9728607..269bfe7 100755 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,9 @@ cached_data_used/kernels/* cached_data_used/visualizations/* cached_data_used/last_run/* +# ignore specific experiment files +experiment_files/milo.json + # ignore mypy mypy.ini diff --git a/README.md b/README.md index ee0eb96..2869aae 100644 --- a/README.md +++ b/README.md @@ -67,3 +67,9 @@ Upon submitting a pull request or pushing to main, these same checks will be ran ### Publishing For publising the package to PyPI (the Python Package Index), we use [Flit](https://flit.pypa.io) and the [to-pypi-using-flit](https://github.com/AsifArmanRahman/to-pypi-using-flit/tree/v1/) GitHub Action to automate this. + +[Semantic version numbering](https://semver.org) is used as follows: `MAJOR.Minor.patch`. +`MAJOR` version for incompatible API changes. +`Minor` version for functionality in a backward compatible manner. +`patch` version for backward compatible bug fixes. +In addition, [PEP 440](https://peps.python.org/pep-0440/) is adhered to, specifically for [pre-release versioning](https://packaging.python.org/en/latest/guides/distributing-packages-using-setuptools/#id62). diff --git a/experiment_files/schema.json b/experiment_files/schema.json deleted file mode 100755 index 85c6f5d..0000000 --- a/experiment_files/schema.json +++ /dev/null @@ -1,138 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://example.com/schemas/experiments/v0.1.1.schema.json", - "version": "0.1.2", - "title": "Experiment", - "description": "An experiment configuration file", - "type": "object", - "properties": { - "version": { - "description": "Version number of the experiment file standard", - "type": "string" - }, - "name": { - "description": "Name of the experiment", - "type": "string" - }, - "folder_id": { - "description": "Unique ID of the folder to store the results of this experiment in", - "type": "string" - }, - "kernels": { - "description": "Kernels to optimize", - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1, - "uniqueItems": true - }, - "GPUs": { - "description": "GPUs to optimize on", - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1, - "uniqueItems": true - }, - "minimization": { - "description": "Direction of optimization (minimize or maximize)", - "type": "boolean", - "default": true - }, - "resolution": { - "description": "The resolution of the time range", - "type": "integer", - "minimum": 2 - }, - "cutoff_percentile": { - "description": "Fraction of difference between median and absolute optimum at which to stop the time range", - "type": "number", - "exclusiveMinimum": 0, - "maximum": 1 - }, - "cutoff_percentile_start": { - "description": "Fraction of difference between median and absolute optimum at which to start the time range", - "type": "number", - "minimum": 0, - "exclusiveMaximum": 1 - }, - "cutoff_type": { - "description": "Whether to base the cutoff on function evaluations or time", - "type": "string", - "enum": [ - "fevals", - "time" - ] - }, - "plot": { - "type": "object", - "properties": { - "plot_x_value_types": { - "description": "Types of value on the x-axis", - "type": "array", - "items": { - "type": "string", - "enum": [ - "fevals", - "time", - "aggregated" - ] - }, - "minItems": 1, - "uniqueItems": true - }, - "plot_y_value_types": { - "description": "Types of value on the y-axis (absolute values, median-absolute normalized, improvement over baseline)", - "type": "array", - "items": { - "type": "string", - "enum": [ - "absolute", - "scatter", - "normalized", - "baseline" - ] - }, - "minItems": 1, - "uniqueItems": true - }, - "confidence_level": { - "type": [ - "number", - "null" - ], - "exclusiveMinimum": 0, - "maximum": 1 - }, - "compare_baselines": { - "type": "boolean", - "default": false - }, - "compare_split_times": { - "type": "boolean", - "default": false - } - }, - "required": [ - "plot_x_value_types", - "plot_y_value_types", - "confidence_level" - ] - } - }, - "required": [ - "version", - "name", - "folder_id", - "kernels", - "GPUs", - "minimization", - "resolution", - "cutoff_percentile", - "cutoff_percentile_start", - "cutoff_type", - "plot" - ] -} diff --git a/experiment_files/template.json b/experiment_files/template.json deleted file mode 100755 index 6e64114..0000000 --- a/experiment_files/template.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "name": "Compare search method", - "kernels": [ - "GEMM" - ], - "GPUs": [ - "GTX_TITAN_X" - ], - "strategy_defaults": { - "repeats": 35, - "nums_of_evaluations": [ - 20, - 40, - 60, - 80, - 100, - 120, - 140, - 160, - 180, - 200, - 220 - ], - "record_data": [ - "time", - "GFLOP/s" - ] - }, - "strategies": [ - { - "name": "", - "strategy": "", - "display_name": "", - "options": {} - } - ] -} diff --git a/experiment_files/test_henk.json b/experiment_files/test_henk.json deleted file mode 100755 index a3d2b93..0000000 --- a/experiment_files/test_henk.json +++ /dev/null @@ -1,114 +0,0 @@ -{ - "name": "Compare optimization methods for Peak Memory problem", - "kernels": [ - "predict_peak_mem" - ], - "GPUs": [ - "generator" - ], - "strategy_defaults": { - "repeats": 250, - "nums_of_evaluations": [ - 20, - 40, - 60, - 80, - 100, - 120, - 140, - 160, - 180, - 200, - 220, - 240, - 260, - 280, - 300, - 320, - 340, - 360, - 380, - 400, - 420, - 440, - 460, - 480, - 500, - 520, - 540, - 560, - 580, - 600, - 620, - 640, - 660, - 680, - 700, - 720, - 740, - 760, - 780, - 800, - 820, - 840, - 860, - 880, - 900, - 920, - 940, - 960, - 980, - 1000 - ], - "record_data": [ - "time", - "GFLOP/s" - ] - }, - "strategies": [ - { - "name": "random_sample_henk", - "strategy": "random_sample", - "display_name": "Random sampling", - "repeats": 100, - "options": { - "max_fevals": 2000, - "fraction": 0.25 - } - }, - { - "name": "bo_henk", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "Bayesian Optimization", - "repeats": 50, - "options": { - "max_fevals": 1000 - } - }, - { - "name": "ga_henk", - "strategy": "genetic_algorithm", - "display_name": "GA", - "repeats": 50, - "options": { - "max_fevals": 2000 - } - }, - { - "name": "mls_henk", - "strategy": "mls", - "display_name": "Multi-start Local Search", - "options": { - "max_fevals": 2000 - } - }, - { - "name": "greedy_mls_henk", - "strategy": "greedy_mls", - "display_name": "Greedy MLS", - "options": { - "max_fevals": 2000 - } - } - ] -} diff --git a/experiment_files/test_hyperparams.json b/experiment_files/test_hyperparams.json deleted file mode 100755 index 3ba95ce..0000000 --- a/experiment_files/test_hyperparams.json +++ /dev/null @@ -1,112 +0,0 @@ -{ - "name": "Compare training iterations", - "kernels": [ - "pnpoly", - "convolution", - "GEMM" - ], - "GPUs": [ - "RTX_2070_SUPER" - ], - "strategy_defaults": { - "repeats": 35, - "nums_of_evaluations": [ - 20, - 40, - 60, - 80, - 100, - 120, - 140, - 160, - 180, - 200, - 220 - ], - "record_data": [ - "time", - "GFLOP/s" - ] - }, - "strategies": [ - { - "name": "random_sample", - "strategy": "random_sample", - "display_name": "Random sampling", - "options": { - "max_fevals": 220 - } - }, - { - "name": "bayes_opt_ei", - "strategy": "bayes_opt", - "display_name": "BO EI", - "options": { - "max_fevals": 220, - "method": "ei" - } - }, - { - "name": "bayes_opt_poi-gpytorch", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO PoI GPyTorch index", - "options": { - "max_fevals": 220, - "method": "poi", - "initialsamplemethod": "index" - } - }, - { - "name": "bayes_opt_ei-gpytorch", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO EI GPyTorch index", - "options": { - "max_fevals": 220, - "method": "ei", - "initialsamplemethod": "index" - } - }, - { - "name": "bayes_opt_poi-lhs-gpytorch", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO PoI GPyTorch LHS", - "ignore_cache": true, - "options": { - "max_fevals": 220, - "method": "poi", - "initialsamplemethod": "lhs" - } - }, - { - "name": "bayes_opt_ei-lhs-gpytorch", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO EI GPyTorch LHS", - "ignore_cache": true, - "options": { - "max_fevals": 220, - "method": "ei", - "initialsamplemethod": "lhs" - } - }, - { - "name": "bayes_opt_poi-random-gpytorch", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO PoI GPyTorch random", - "options": { - "max_fevals": 220, - "method": "poi", - "initialsamplemethod": "random" - } - }, - { - "name": "bayes_opt_ei-random-gpytorch", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO EI GPyTorch random", - "options": { - "max_fevals": 220, - "method": "ei", - "initialsamplemethod": "random" - } - } - ] -} diff --git a/experiment_files/test_hyperparams_ei.json b/experiment_files/test_hyperparams_ei.json deleted file mode 100755 index eefa1c9..0000000 --- a/experiment_files/test_hyperparams_ei.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "name": "Compare training iterations", - "kernels": [ - "pnpoly", - "convolution", - "GEMM" - ], - "GPUs": [ - "RTX_2070_SUPER" - ], - "strategy_defaults": { - "repeats": 35, - "nums_of_evaluations": [ - 20, - 40, - 60, - 80, - 100, - 120, - 140, - 160, - 180, - 200, - 220 - ], - "record_data": [ - "time", - "GFLOP/s" - ] - }, - "strategies": [ - { - "name": "random_sample", - "strategy": "random_sample", - "display_name": "Random sampling", - "options": { - "max_fevals": 220 - } - }, - { - "name": "bayes_opt_ei", - "strategy": "bayes_opt", - "display_name": "BO EI", - "options": { - "max_fevals": 220, - "method": "ei" - } - }, - { - "name": "bayes_opt_gpytorch_hyperparamtuned_23", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO EI GPyTorch hyperparamtuned", - "options": { - "max_fevals": 220, - "initialsamplemethod": "lhs", - "initialsamplerandomoffsetfactor": 0.01, - "method": "ei", - "methodparams": { - "explorationfactor": 0.05 - }, - "covariancekernel": "matern_scalekernel", - "covariancelengthscale": 0.5, - "optimizer": "LBFGS", - "initial_training_iter": 25, - "training_iter": 3 - } - }, - { - "name": "bayes_opt_ei-lhs-gpytorch", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO EI GPyTorch LHS", - "hide": true, - "options": { - "max_fevals": 220, - "method": "ei", - "initialsamplemethod": "lhs" - } - }, - { - "name": "bayes_opt_gpytorch_hyperparamtuned_21", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO EI GPyTorch hyperparamtuned new", - "options": { - "max_fevals": 220, - "initialsamplemethod": "lhs", - "initialsamplerandomoffsetfactor": 0.1, - "method": "ei", - "methodparams": { - "explorationfactor": 0.1 - }, - "covariancekernel": "matern_scalekernel", - "covariancelengthscale": 0.5, - "optimizer": "LBFGS", - "initial_training_iter": 5, - "training_iter": 1 - } - }, - { - "name": "bayes_opt_gpytorch_hyperparamtuned_24", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO EI GPyTorch hyperparamtuned newer", - "options": { - "max_fevals": 220, - "initialsamplemethod": "index", - "initialsamplerandomoffsetfactor": 0.3, - "method": "ei", - "methodparams": { - "explorationfactor": 0.01 - }, - "covariancekernel": "matern_scalekernel", - "covariancelengthscale": 1.5, - "optimizer": "LBFGS", - "initial_training_iter": 5, - "training_iter": 0 - } - }, - { - "name": "bayes_opt_gpytorch_hyperparamtuned_temp", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO EI GPyTorch hyperparamtuned fixednoise", - "options": { - "max_fevals": 220, - "likelihood": "FixedNoise" - } - }, - { - "name": "bayes_opt_gpytorch_hyperparamtuned_temp3", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO EI GPyTorch hyperparamtuned fixednoise new", - "ignore_cache": true, - "repeats": 100, - "options": { - "max_fevals": 220, - "likelihood": "FixedNoise", - "initialsamplerandomoffsetfactor": 0.01, - "initial_training_iter": 50, - "training_after_iter": 10 - } - }, - { - "name": "bayes_opt_gpytorch_hyperparamtuned_25", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO EI GPyTorch hyperparamtuned newest", - "repeats": 100, - "options": { - "max_fevals": 220, - "initialsamplemethod": "index", - "initialsamplerandomoffsetfactor": 0.1, - "method": "ei", - "methodparams": { - "explorationfactor": "CV" - }, - "covariancekernel": "matern_scalekernel", - "covariancelengthscale": 0.5, - "likelihood": "FixedNoise", - "optimizer": "LBFGS", - "initial_training_iter": 50, - "training_after_iter": 10 - } - } - ] -} diff --git a/experiment_files/test_hyperparams_new.json b/experiment_files/test_hyperparams_new.json deleted file mode 100755 index b7a8bda..0000000 --- a/experiment_files/test_hyperparams_new.json +++ /dev/null @@ -1,321 +0,0 @@ -{ - "name": "Compare training iterations", - "kernels": [ - "pnpoly", - "convolution", - "GEMM" - ], - "GPUs": [ - "RTX_2070_SUPER" - ], - "strategy_defaults": { - "repeats": 250, - "nums_of_evaluations": [ - 20, - 40, - 60, - 80, - 100, - 120, - 140, - 160, - 180, - 200, - 220 - ], - "record_data": [ - "time", - "GFLOP/s" - ] - }, - "strategies": [ - { - "name": "random_sample", - "strategy": "random_sample", - "display_name": "Random sampling", - "repeats": 1000, - "options": { - "max_fevals": 220 - } - }, - { - "name": "bayes_opt_new_old_ei", - "strategy": "bayes_opt", - "display_name": "BO EI old", - "options": { - "max_fevals": 220, - "method": "ei" - } - }, - { - "name": "bayes_opt_gpytorch_new_hyperparamtuned_1", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO EI GPyTorch hyperparamtuned 1", - "hide": true, - "options": { - "max_fevals": 220, - "initialsamplemethod": "index", - "initialsamplerandomoffsetfactor": 0.2, - "method": "ei", - "methodparams": { - "explorationfactor": "CV" - }, - "covariancekernel": "matern_scalekernel", - "covariancelengthscale": 0.5, - "likelihood": "FixedNoise", - "optimizer": "LBFGS", - "initial_training_iter": 50, - "training_after_iter": 10 - } - }, - { - "name": "bayes_opt_gpytorch_new_hyperparamtuned_5", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO EI GPyTorch hyperparamtuned 5", - "hide": true, - "options": { - "max_fevals": 220, - "initialsamplemethod": "lhs", - "initialsamplerandomoffsetfactor": 0.01, - "method": "ei", - "methodparams": { - "explorationfactor": "CV" - }, - "covariancekernel": "matern_scalekernel", - "covariancelengthscale": 0.5, - "likelihood": "FixedNoise", - "optimizer": "LBFGS", - "initial_training_iter": 5, - "training_after_iter": 3 - } - }, - { - "name": "bayes_opt_gpytorch_new_hyperparamtuned_9", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO EI GPyTorch hyperparamtuned 9", - "hide": true, - "options": { - "max_fevals": 220, - "initialsamplemethod": "lhs", - "initialsamplerandomoffsetfactor": 0.01, - "method": "ei", - "methodparams": { - "explorationfactor": 0.1 - }, - "covariancekernel": "matern_scalekernel", - "covariancelengthscale": 0.5, - "likelihood": "Gaussian", - "optimizer": "LBFGS", - "initial_training_iter": 10, - "training_after_iter": 0 - } - }, - { - "name": "bayes_opt_gpytorch_new_hyperparamtuned_11", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO EI GPyTorch hyperparamtuned 11", - "hide": true, - "options": { - "max_fevals": 220, - "initialsamplemethod": "lhs", - "initialsamplerandomoffsetfactor": 0.1, - "method": "ei", - "methodparams": { - "explorationfactor": 0.1 - }, - "covariancekernel": "matern", - "covariancelengthscale": 1.5, - "likelihood": "FixedNoise", - "optimizer": "LBFGS", - "initial_training_iter": 50, - "training_after_iter": 3 - } - }, - { - "name": "bayes_opt_gpytorch_new_hyperparamtuned_14", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO EI GPyTorch hyperparamtuned 14", - "hide": true, - "options": { - "max_fevals": 220, - "initialsamplemethod": "index", - "initialsamplerandomoffsetfactor": 0.05, - "method": "ei", - "methodparams": { - "explorationfactor": 0.1 - }, - "covariancekernel": "matern_scalekernel", - "covariancelengthscale": 1.5, - "likelihood": "Gaussian", - "optimizer": "LBFGS", - "initial_training_iter": 50, - "training_after_iter": 3 - } - }, - { - "name": "bayes_opt_gpytorch_new_hyperparamtuned_15", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO EI GPyTorch hyperparamtuned 15", - "hide": true, - "options": { - "max_fevals": 220, - "initialsamplemethod": "lhs", - "initialsamplerandomoffsetfactor": 0.1, - "method": "ei", - "methodparams": { - "explorationfactor": 0.05 - }, - "covariancekernel": "matern_scalekernel", - "covariancelengthscale": 1.5, - "likelihood": "Gaussian", - "optimizer": "LBFGS", - "initial_training_iter": 50, - "training_after_iter": 10 - } - }, - { - "name": "bayes_opt_gpytorch_new_hyperparamtuned_16", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO EI GPyTorch hyperparamtuned 16", - "hide": true, - "options": { - "max_fevals": 220, - "initialsamplemethod": "lhs", - "initialsamplerandomoffsetfactor": 0.1, - "method": "ei", - "methodparams": { - "explorationfactor": 0.1 - }, - "covariancekernel": "matern_scalekernel", - "covariancelengthscale": 0.5, - "likelihood": "Gaussian", - "optimizer": "LBFGS", - "initial_training_iter": 50, - "training_after_iter": 3 - } - }, - { - "name": "bayes_opt_gpytorch_new_hyperparamtuned_17", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO EI GPyTorch hyperparamtuned 17", - "hide": true, - "options": { - "max_fevals": 220, - "initialsamplemethod": "lhs", - "initialsamplerandomoffsetfactor": 0.1, - "method": "ei", - "methodparams": { - "explorationfactor": 0.1 - }, - "covariancekernel": "matern_scalekernel", - "covariancelengthscale": 1.5, - "likelihood": "Gaussian", - "optimizer": "LBFGS", - "initial_training_iter": 5, - "training_after_iter": 1 - } - }, - { - "name": "bayes_opt_gpytorch_new_hyperparamtuned_18", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO EI GPyTorch hyperparamtuned 18", - "hide": true, - "options": { - "max_fevals": 220, - "initialsamplemethod": "lhs", - "initialsamplerandomoffsetfactor": 0.05, - "method": "ei", - "methodparams": { - "explorationfactor": 0.1 - }, - "covariancekernel": "matern_scalekernel", - "covariancelengthscale": 1.5, - "likelihood": "Gaussian", - "optimizer": "LBFGS", - "initial_training_iter": 5, - "training_after_iter": 0 - } - }, - { - "name": "bayes_opt_gpytorch_new_hyperparamtuned_19", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO EI GPyTorch hyperparamtuned 19", - "options": { - "max_fevals": 220, - "initialsamplemethod": "lhs", - "initialsamplerandomoffsetfactor": 0.01, - "method": "ei", - "methodparams": { - "explorationfactor": 0.05 - }, - "covariancekernel": "matern_scalekernel", - "covariancelengthscale": 1.5, - "likelihood": "Gaussian", - "optimizer": "LBFGS", - "initial_training_iter": 25, - "training_after_iter": 10 - } - }, - { - "name": "bayes_opt_gpytorch_new_hyperparamtuned_20", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO EI GPyTorch hyperparamtuned 20", - "options": { - "max_fevals": 220, - "initialsamplemethod": "lhs", - "initialsamplerandomoffsetfactor": 0.01, - "method": "ei", - "methodparams": { - "explorationfactor": 0.05 - }, - "covariancekernel": "matern_scalekernel", - "covariancelengthscale": 1.5, - "likelihood": "Gaussian", - "optimizer": "LBFGS", - "initial_training_iter": 25, - "training_after_iter": 10 - } - }, - { - "name": "bayes_opt_gpytorch_new_hyperparamtuned_21", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO EI GPyTorch hyperparamtuned 21", - "hide": true, - "options": { - "max_fevals": 220, - "initialsamplemethod": "lhs", - "initialsamplerandomoffsetfactor": 0.05, - "method": "ei", - "methodparams": { - "explorationfactor": 0.1 - }, - "covariancekernel": "matern_scalekernel", - "covariancelengthscale": 0.5, - "likelihood": "FixedNoise", - "optimizer": "LBFGS", - "initial_training_iter": 5, - "training_after_iter": 10 - } - }, - { - "name": "bayes_opt_gpytorch_new_hyperparamtuned_22", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO EI GPyTorch hyperparamtuned 22", - "options": { - "max_fevals": 220, - "initialsamplemethod": "minmax", - "initialsamplerandomoffsetfactor": 0.01, - "method": "ei", - "methodparams": { - "explorationfactor": 0.05 - }, - "covariancekernel": "matern_scalekernel", - "covariancelengthscale": 1.5, - "likelihood": "Gaussian", - "optimizer": "LBFGS", - "initial_training_iter": 25, - "training_after_iter": 10 - } - } - ] -} diff --git a/experiment_files/test_hyperparams_synthetic.json b/experiment_files/test_hyperparams_synthetic.json deleted file mode 100755 index 91eed7f..0000000 --- a/experiment_files/test_hyperparams_synthetic.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "name": "Compare training iterations", - "kernels": [ - "multimodal_sinewave", - "Rosenbrock", - "Mishrasbird", - "Gomez-Levy" - ], - "GPUs": [ - "generator" - ], - "strategy_defaults": { - "repeats": 35, - "nums_of_evaluations": [ - 20, - 40, - 60, - 80, - 100, - 120, - 140, - 160, - 180, - 200, - 220 - ], - "record_data": [ - "time", - "GFLOP/s" - ] - }, - "strategies": [ - { - "name": "random_sample", - "strategy": "random_sample", - "display_name": "Random sampling", - "options": { - "max_fevals": 220 - } - }, - { - "name": "bayes_opt_ei", - "strategy": "bayes_opt", - "display_name": "BO EI", - "options": { - "max_fevals": 220, - "method": "ei" - } - }, - { - "name": "bayes_opt_poi-gpytorch", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO PoI GPyTorch", - "options": { - "max_fevals": 220, - "method": "poi", - "training_iter": 50 - } - }, - { - "name": "bayes_opt_ei-gpytorch", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO PoI GPyTorch", - "options": { - "max_fevals": 220, - "method": "ei", - "training_iter": 50 - } - } - ] -} diff --git a/experiment_files/test_methodology.json b/experiment_files/test_methodology.json deleted file mode 100755 index c830b0c..0000000 --- a/experiment_files/test_methodology.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "name": "Test new methodology", - "kernels": [ - "pnpoly", - "convolution", - "GEMM" - ], - "GPUs": [ - "RTX_2070_SUPER", - "A100" - ], - "resolution": 1e4, - "curve_segment_factor": 0.05, - "cutoff_quantile": 0.99, - "relative_to_baseline": true, - "strategy_defaults": { - "repeats": 1000, - "nums_of_evaluations": [ - 20, - 40, - 60, - 80, - 100, - 120, - 140, - 160, - 180, - 200, - 220, - 240, - 260, - 280, - 300, - 320, - 340, - 360, - 380, - 400, - 420, - 440, - 460, - 480, - 500 - ], - "record_data": [ - "time", - "GFLOP/s" - ] - }, - "strategies": [ - { - "name": "random_sample", - "strategy": "random_sample", - "display_name": "Random sampling", - "is_baseline": true, - "repeats": 100, - "options": { - "max_fevals": 500 - } - }, - { - "name": "greedy_mls", - "strategy": "greedy_mls", - "display_name": "Greedy MLS", - "options": { - "max_fevals": 500 - } - }, - { - "name": "greedy_ils", - "strategy": "greedy_ils", - "display_name": "Greedy ILS", - "options": { - "max_fevals": 500 - } - }, - { - "name": "new_bayes_opt_ei-lhs", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "New BO EI LHS", - "repeats": 100, - "options": { - "max_fevals": 500, - "method": "ei", - "initialsamplemethod": "lhs" - } - } - ] -} diff --git a/experiment_files/test_methodology_quick.json b/experiment_files/test_methodology_quick.json deleted file mode 100644 index 8c60c36..0000000 --- a/experiment_files/test_methodology_quick.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "version": "0.1.0", - "name": "Test new methodology", - "folder_id": "test_new_methodology", - "kernels": [ - "pnpoly" - ], - "GPUs": [ - "RTX_2070_SUPER" - ], - "minimization": true, - "resolution": 1e3, - "curve_segment_factor": 0.05, - "cutoff_percentile": 0.95, - "cutoff_percentile_start": 0.01, - "cutoff_type": "fevals", - "plot": { - "plot_relative_to_baseline": true, - "plot_fevals": true, - "plot_time": true, - "plot_aggregated": true, - "confidence_level": 0.95 - }, - "strategy_defaults": { - "repeats": 100, - "minimum_number_of_evaluations": 20, - "stochastic": true, - "record_data": [ - "time", - "GFLOP/s" - ] - }, - "strategies": [ - { - "name": "mls", - "strategy": "mls", - "display_name": "MLS" - }, - { - "name": "greedy_mls", - "strategy": "greedy_mls", - "display_name": "Greedy MLS" - }, - { - "name": "greedy_ils", - "strategy": "greedy_ils", - "display_name": "Greedy ILS" - } - ] -} diff --git a/experiment_files/test_methodology_quicker.json b/experiment_files/test_methodology_quicker.json deleted file mode 100755 index 64d6a79..0000000 --- a/experiment_files/test_methodology_quicker.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "name": "Test new methodology", - "kernels": [ - "pnpoly" - ], - "GPUs": [ - "RTX_2070_SUPER" - ], - "resolution": 1e3, - "curve_segment_factor": 0.05, - "cutoff_quantile": 0.98, - "relative_to_baseline": true, - "strategy_defaults": { - "repeats": 100, - "minimum_number_of_evaluations": 20, - "record_data": [ - "time", - "GFLOP/s" - ] - }, - "strategies": [ - { - "name": "random_sample", - "strategy": "random_sample", - "display_name": "Random sampling", - "repeats": 200 - } - ] -} diff --git a/experiment_files/test_methodology_random.json b/experiment_files/test_methodology_random.json deleted file mode 100644 index 2aa4acb..0000000 --- a/experiment_files/test_methodology_random.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "name": "Test new methodology", - "kernels": [ - "pnpoly", - "convolution", - "GEMM" - ], - "GPUs": [ - "RTX_2070_SUPER" - ], - "resolution": 1e4, - "curve_segment_factor": 0.05, - "cutoff_quantile": 0.99, - "relative_to_baseline": false, - "strategy_defaults": { - "repeats": 1000, - "nums_of_evaluations": [ - 20, - 40, - 60, - 80, - 100, - 120, - 140, - 160, - 180, - 200, - 220, - 240, - 260, - 280, - 300, - 320, - 340, - 360, - 380, - 400 - ], - "record_data": [ - "time", - "GFLOP/s" - ] - }, - "strategies": [ - { - "name": "random_sample", - "strategy": "random_sample", - "display_name": "Random sampling", - "is_baseline": true, - "repeats": 100, - "options": { - "max_fevals": 800 - } - } - ] -} diff --git a/experiment_files/test_output_file_writer.json b/experiment_files/test_output_file_writer.json deleted file mode 100644 index 79ccad5..0000000 --- a/experiment_files/test_output_file_writer.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "version": "0.1.2", - "name": "Test output file writer", - "folder_id": "test_output_file_writer", - "kernels": [ - "convolution" - ], - "kernels_path": "../cached_data_used/kernels", - "bruteforced_caches_path": "../cached_data_used/cachefiles", - "visualization_caches_path": "../cached_data_used/visualizations", - "GPUs": [ - "RTX_2080_Ti" - ], - "minimization": true, - "resolution": 1e3, - "cutoff_percentile": 0.99, - "cutoff_percentile_start": 0.7, - "objective_time_keys": [ - "compilation", - "benchmark", - "framework", - "search_algorithm", - "validation" - ], - "objective_performance_keys": [ - "time" - ], - "cutoff_type": "fevals", - "plot": { - "plot_x_value_types": [ - "aggregated" - ], - "plot_y_value_types": [ - "normalized", - "baseline" - ], - "confidence_level": 0.95, - "compare_baselines": true, - "compare_split_times": true - }, - "strategy_defaults": { - "repeats": 100, - "minimum_number_of_evaluations": 20, - "stochastic": true, - "record_data": [ - "time", - "GFLOP/s" - ] - }, - "strategies": [ - { - "name": "random_sample_100_iter", - "strategy": "random_sample", - "display_name": "Random sampling 100 iters", - "repeats": 5 - }, - { - "name": "genetic_algorithm", - "strategy": "genetic_algorithm", - "display_name": "Genetic Algorithm" - } - ] -} \ No newline at end of file diff --git a/experiment_files/test_profile.json b/experiment_files/test_profile.json deleted file mode 100755 index 3e6cd74..0000000 --- a/experiment_files/test_profile.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "name": "Compare training iterations", - "kernels": [ - "convolution" - ], - "GPUs": [ - "RTX_2070_SUPER" - ], - "strategy_defaults": { - "repeats": 3, - "nums_of_evaluations": [ - 20, - 40, - 60, - 80, - 100, - 120, - 140, - 160, - 180, - 200, - 220 - ], - "record_data": [ - "time", - "GFLOP/s" - ] - }, - "strategies": [ - { - "name": "bayes_opt_ei-gpytorch", - "strategy": "bayes_opt_GPyTorch_lean", - "display_name": "BO EI GPyTorch", - "ignore_cache": true, - "options": { - "max_fevals": 220, - "method": "ei", - "training_iter": 50 - } - } - ] -} diff --git a/pyproject.toml b/pyproject.toml index ca1df80..0fa0281 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ requires = ["flit_core >=3.8.0,<4"] [project] # https://packaging.python.org/en/latest/specifications/declaring-project-metadata/#declaring-project-metadata name = "autotuning_methodology" -version = "0.2.0" +version = "0.1.0.b1" authors = [ {name = "Floris-Jan Willemsen", email = "fjwillemsen97@gmail.com"}, ] @@ -44,6 +44,7 @@ docs = [ ] test = [ "ruff >= 0.0.263", + "pep440 >= 0.1.2", "pytest >= 7.3.1", "pytest-cov >= 4.0.0", "nox >= 2023.4.22",