From 22b063c0eb1eaae6a94866a5f5c9d6ac84c9a2e8 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 26 Feb 2024 22:12:40 +0530 Subject: [PATCH 001/169] Support batch-size in llama2 run --- language/llama2-70b/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/language/llama2-70b/main.py b/language/llama2-70b/main.py index bf1def806..468ad067e 100644 --- a/language/llama2-70b/main.py +++ b/language/llama2-70b/main.py @@ -23,6 +23,7 @@ def get_args(): parser.add_argument("--mlperf-conf", type=str, default="mlperf.conf", help="mlperf rules config") parser.add_argument("--user-conf", type=str, default="user.conf", help="user config for user LoadGen settings such as target QPS") parser.add_argument("--total-sample-count", type=int, default=24576, help="Number of samples to use in benchmark.") # TODO: This interpretation of 'total-sample-count' is a little misleading. Fix it + parser.add_argument("--batch-size", type=int, default=1, help="Model batch-size to use in benchmark.") parser.add_argument("--output-log-dir", type=str, default="output-logs", help="Where logs are saved") parser.add_argument("--enable-log-trace", action="store_true", help="Enable log tracing. This file can become quite large") parser.add_argument("--num-workers", type=int, default=1, help="Number of workers to process queries") @@ -69,6 +70,7 @@ def main(): sut = sut_cls( model_path=args.model_path, dtype=args.dtype, + batch_size=args.batch_size, dataset_path=args.dataset_path, total_sample_count=args.total_sample_count, device=args.device, From 5fe4dfd829fdca6591f2ffc070463d1d215593c1 Mon Sep 17 00:00:00 2001 From: Nathan Wasson Date: Wed, 21 Feb 2024 12:13:25 -0600 Subject: [PATCH 002/169] Add Rclone-Cloudflare download instructions to README.md --- text_to_image/README.md | 51 ++++++++++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/text_to_image/README.md b/text_to_image/README.md index b8573624c..243067459 100644 --- a/text_to_image/README.md +++ b/text_to_image/README.md @@ -4,18 +4,11 @@ This is the reference implementation for MLPerf Inference text to image ## Supported Models -| model | accuracy | dataset | model link | model source | precision | notes | -| ---- | ---- | ---- | ---- | ---- | ---- | ---- | -| StableDiffusion | - | Coco2014 | [fp32](https://cloud.mlcommons.org/index.php/s/DjnCSGyNBkWA4Ro) and [f16](https://cloud.mlcommons.org/index.php/s/LCdW5RM6wgGWbxC) | [Hugging Face](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) | fp32 | NCHW | +| model | accuracy | dataset | model source | precision | notes | +| ---- | ---- | ---- | ---- | ---- | ---- | +| StableDiffusion | - | Coco2014 | [Hugging Face](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) | fp32 | NCHW | -The following MLCommons CM commands can be used to programmatically download the model checkpoints. -``` -pip install cmind -cm pull repo mlcommons@ck -cm run script --tags=get,ml-model,sdxl,_fp16,_rclone -j -cm run script --tags=get,ml-model,sdxl,_fp32,_rclone -j -``` ## Dataset | Data | Description | @@ -55,7 +48,43 @@ CFLAGS="-std=c++14" python setup.py install ### Download model -We host two checkpoints ([fp32](https://cloud.mlcommons.org/index.php/s/DjnCSGyNBkWA4Ro) and [f16](https://cloud.mlcommons.org/index.php/s/LCdW5RM6wgGWbxC)) that are a snapshot of the [Hugging Face](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) pipeline at the time of the release of the benchmark. Download them and move them to your model path. +We host two checkpoints (fp32 and fp16) that are a snapshot of the [Hugging Face](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) pipeline at the time of the release of the benchmark. Download them and move them to your model path. + +#### CM method + +The following MLCommons CM commands can be used to programmatically download the model checkpoints. + +``` +pip install cmind +cm pull repo mlcommons@ck +cm run script --tags=get,ml-model,sdxl,_fp16,_rclone -j +cm run script --tags=get,ml-model,sdxl,_fp32,_rclone -j +``` +#### Manual method + +The above command automatically runs a set of Rclone commands to download the data from a Cloudflare R2 bucket. However, if you'd like to run the Rclone commands manually, you can do so as follows: + +To run Rclone on Windows, you can download the executable [here](https://rclone.org/install/#windows). +To install Rclone on Linux/macOS/BSD systems, run: +``` +sudo -v ; curl https://rclone.org/install.sh | sudo bash +``` +Once Rclone is installed, run the following command to authenticate with the bucket: +``` +rclone config create mlc-inference s3 provider=Cloudflare access_key_id=f65ba5eef400db161ea49967de89f47b secret_access_key=fbea333914c292b854f14d3fe232bad6c5407bf0ab1bebf78833c2b359bdfd2b endpoint=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com +``` +You can then navigate in the terminal to your desired download directory and run the following commands to download the dataset and checkpoints: + +**`fp32`** +``` +rclone copy mlc-inference:mlcommons-inference-wg-public/stable_diffusion_fp32 ./stable_diffusion_fp32 -P +``` +**`fp16`** +``` +rclone copy mlc-inference:mlcommons-inference-wg-public/stable_diffusion_fp16 ./stable_diffusion_fp16 -P +``` + +#### Move to model path ```bash mkdir $MODEL_PATH From 8a00168e53c2185dbd524f90731251ac74754349 Mon Sep 17 00:00:00 2001 From: Nathan Wasson Date: Wed, 21 Feb 2024 12:33:02 -0600 Subject: [PATCH 003/169] Add Rclone-Cloudflare download instructiosn to README.md --- recommendation/dlrm_v2/pytorch/README.md | 45 +++++++++++++----------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/recommendation/dlrm_v2/pytorch/README.md b/recommendation/dlrm_v2/pytorch/README.md index 64d2af249..6de850b81 100755 --- a/recommendation/dlrm_v2/pytorch/README.md +++ b/recommendation/dlrm_v2/pytorch/README.md @@ -67,18 +67,15 @@ cd $HOME/mlcommons/inference/loadgen CFLAGS="-std=c++14" python setup.py develop --user ``` + ### Downloading model weights -File name | framework | Size in bytes (`du *`) | MD5 hash (`md5sum *`) --|-|-|- +framework | Size in bytes (`du *`) | MD5 hash (`md5sum *`) +-|-|- N/A | pytorch | <2GB | - -[weight_sharded](https://cloud.mlcommons.org/index.php/s/XzfSeLgW8FYfR3S/download) | pytorch | 97.31GB | - + pytorch | 97.31GB | - -You can download the weights by running: -``` -wget https://cloud.mlcommons.org/index.php/s/XzfSeLgW8FYfR3S/download -O weights.zip -unzip weights.zip -``` +#### CM method The following MLCommons CM commands can be used to programmatically download the model checkpoint. @@ -88,24 +85,32 @@ cm pull repo mlcommons@ck cm run script --tags=get,ml-model,dlrm,_pytorch,_weight_sharded,_rclone -j ``` -(optional) To speed up future downloads, we recommend you save the weights in a bucket (E.g GCP, AWS). For example, after saving the checkpoint in a GCP bucket, you can download the weights faster by running: +#### Manual method + +The above command automatically runs a set of Rclone commands to download the data from a Cloudflare R2 bucket. However, if you'd like to run the Rclone commands manually, you can do so as follows: + +To run Rclone on Windows, you can download the executable [here](https://rclone.org/install/#windows). +To install Rclone on Linux/macOS/BSD systems, run: ``` -export BUCKET_NAME= -cd $HOME/mlcommons/inference/recommendation/dlrm_v2/pytorch/model/ -gsutil -m cp -r "gs://$BUCKET_NAME/model_weights/*" . +sudo -v ; curl https://rclone.org/install.sh | sudo bash ``` +Once Rclone is installed, run the following command to authenticate with the bucket: +``` +rclone config create mlc-inference s3 provider=Cloudflare access_key_id=f65ba5eef400db161ea49967de89f47b secret_access_key=fbea333914c292b854f14d3fe232bad6c5407bf0ab1bebf78833c2b359bdfd2b endpoint=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com +``` +You can then navigate in the terminal to your desired download directory and run the following commands to download the model weights: -### Downloading dataset -| Original dataset | download link | -| ---- | ---- | -| Criteo Terabyte (day 23) | https://labs.criteo.com/2013/12/download-terabyte-click-logs/ | +``` +rclone copy mlc-inference:mlcommons-inference-wg-public/model_weights ./model_weights -P +``` +#### (optional) -1. The Criteo fake dataset can be created in place of the real datasets in order to facilitate debugging and testing. We provide a fake (random) data generator that can be used to quickly generate data samples in a format compatible with the original dataset. Please use the following script in `./tools` to quickly create random samples for the corresponding models, which will be placed into `./fake_criteo` directory +To speed up future downloads, we recommend you save the weights in a bucket (E.g GCP, AWS). For example, after saving the checkpoint in a GCP bucket, you can download the weights faster by running: ``` -./make_fake_criteo.sh -mv ./fake_criteo .. && cd .. -export DATA_DIR=./fake_criteo +export BUCKET_NAME= +cd $HOME/mlcommons/inference/recommendation/dlrm_v2/pytorch/model/ +gsutil -m cp -r "gs://$BUCKET_NAME/model_weights/*" . ``` From 1b69968f30d6a742d5408f87b5faa065962c42a2 Mon Sep 17 00:00:00 2001 From: Nathan Wasson Date: Wed, 21 Feb 2024 12:33:40 -0600 Subject: [PATCH 004/169] Minor wording edit to README.md --- text_to_image/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/text_to_image/README.md b/text_to_image/README.md index 243067459..e353bfac0 100644 --- a/text_to_image/README.md +++ b/text_to_image/README.md @@ -73,7 +73,7 @@ Once Rclone is installed, run the following command to authenticate with the buc ``` rclone config create mlc-inference s3 provider=Cloudflare access_key_id=f65ba5eef400db161ea49967de89f47b secret_access_key=fbea333914c292b854f14d3fe232bad6c5407bf0ab1bebf78833c2b359bdfd2b endpoint=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com ``` -You can then navigate in the terminal to your desired download directory and run the following commands to download the dataset and checkpoints: +You can then navigate in the terminal to your desired download directory and run the following commands to download the checkpoints: **`fp32`** ``` From d30a0ca3b8f6f6040c9f6ec4bdfbd168a149536f Mon Sep 17 00:00:00 2001 From: Nathan Wasson Date: Wed, 21 Feb 2024 12:47:23 -0600 Subject: [PATCH 005/169] Add Rclone-Cloudflare download instructions to README.md --- language/gpt-j/README.md | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/language/gpt-j/README.md b/language/gpt-j/README.md index 58af37ba3..cbb5b3e3c 100644 --- a/language/gpt-j/README.md +++ b/language/gpt-j/README.md @@ -68,11 +68,9 @@ pip install datasets python prepare-calibration.py --calibration-list-file calibration-list.txt --output-dir ``` ### Download GPT-J model -Please download the fine-tuned GPT-J checkpoint from [here](https://cloud.mlcommons.org/index.php/s/QAZ2oM94MkFtbQx) and extract it as model/. The download_gptj.py only downloads the default huggingface model which is not fine-tuned on CNN-Daily mail dataset. +Please download the fine-tuned GPT-J checkpoint using the instructions below. The download_gptj.py only downloads the default huggingface model which is not fine-tuned on CNN-Daily mail dataset. -``` -wget https://cloud.mlcommons.org/index.php/s/QAZ2oM94MkFtbQx/download --output-document checkpoint.zip -``` +#### CM method The following MLCommons CM commands can be used to programmatically download the model checkpoint. @@ -82,6 +80,26 @@ cm pull repo mlcommons@ck cm run script --tags=get,ml-model,gptj,_pytorch,_rclone -j ``` +#### Manual method + +The above command automatically runs a set of Rclone commands to download the data from a Cloudflare R2 bucket. However, if you'd like to run the Rclone commands manually, you can do so as follows: + +To run Rclone on Windows, you can download the executable [here](https://rclone.org/install/#windows). +To install Rclone on Linux/macOS/BSD systems, run: +``` +sudo -v ; curl https://rclone.org/install.sh | sudo bash +``` +Once Rclone is installed, run the following command to authenticate with the bucket: +``` +rclone config create mlc-inference s3 provider=Cloudflare access_key_id=f65ba5eef400db161ea49967de89f47b secret_access_key=fbea333914c292b854f14d3fe232bad6c5407bf0ab1bebf78833c2b359bdfd2b endpoint=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com +``` +You can then navigate in the terminal to your desired download directory and run the following commands to download the model checkpoint: + +``` +rclone copy mlc-inference:mlcommons-inference-wg-public/gpt-j ./model -P +``` + + ### Running the Benchmark Replace the model and dataset path arguments with your corresponding paths. For evaluating the ROUGE score after the run, include --accuracy as shown below. For user specific target qps, please include user.conf. ``` From 48f8bbb8e014c4c7376cb827dda25f3d44d9e6b9 Mon Sep 17 00:00:00 2001 From: Nathan Wasson Date: Wed, 21 Feb 2024 13:03:58 -0600 Subject: [PATCH 006/169] Add Rclone-GDrive download instructions to README.md --- language/llama2-70b/README.md | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/language/llama2-70b/README.md b/language/llama2-70b/README.md index 979488e91..8c6a0c27c 100644 --- a/language/llama2-70b/README.md +++ b/language/llama2-70b/README.md @@ -63,32 +63,8 @@ Inside the container, set up the environment with `bash build.sh`. This will ins CPU-only setup, as well as any GPU versions for applicable libraries like PyTorch. -## Get Model -+ For now, MLCommons is not hosting the checkpoint, so you must first go to [llama2-request-link](https://ai.meta.com/resources/models-and-libraries/llama-downloads/) and make a request, sign in to huggingface (if you don't have account, you'd need to create one). **Please note your authentication credentials** as you may be required to provide them when cloninng below -+ Requires Git Large Files Storage -``` -export CHECKPOINT_PATH=${PWD}/Llama-2-70b-chat-hf -git lfs install -git clone https://huggingface.co/meta-llama/Llama-2-70b-chat-hf ${CHECKPOINT_PATH} - -``` - -## Get Dataset - -``` -# First get the `open-orca` parquet from huggingface -export OPENORCA_DATASET=${PWD}/open-orca -git clone https://huggingface.co/datasets/Open-Orca/OpenOrca ${OPENORCA_DATASET} - -export OPENORCA_PARQUET=${OPENORCA_DATASET}/1M-GPT4-Augmented.parquet -EXPORT_DIR=${PWD}/processed-openorca -export DATASET_PATH=${PWD}/processed-data.pkl - -# Process the dataset according the Taskforce's agreed criteria -python3 processorca.py --dataset_pq_path=${OPENORCA_PARQUET} --model_dir=${CHECKPOINT_PATH} --seqlen_limit=1024 --export_dir=${EXPORT_DIR} --num_total_samples=24576 - -mv ${EXPORT_DIR}/open_orca_gpt4_tokenized_llama.sampled_24576.pkl ${DATASET_PATH} -``` +## Get Model and Dataset +MLCommons hosts the model and preprocessed dataset for download. You must first agree to the [confidentiality notice](https://docs.google.com/forms/d/e/1FAIpQLSc_8VIvRmXM3I8KQaYnKf7gy27Z63BBoI_I1u02f4lw6rBp3g/viewform), then follow the link to a directory containing Rclone download instructions. ## Run Performance Benchmarks From 0e7044939f08388daa98983d9d6bf2db51ec923e Mon Sep 17 00:00:00 2001 From: Nathan Wasson Date: Wed, 21 Feb 2024 16:23:00 -0600 Subject: [PATCH 007/169] Add new and old instructions to README.md --- language/llama2-70b/README.md | 56 +++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/language/llama2-70b/README.md b/language/llama2-70b/README.md index 8c6a0c27c..f648b1b5d 100644 --- a/language/llama2-70b/README.md +++ b/language/llama2-70b/README.md @@ -63,8 +63,60 @@ Inside the container, set up the environment with `bash build.sh`. This will ins CPU-only setup, as well as any GPU versions for applicable libraries like PyTorch. -## Get Model and Dataset -MLCommons hosts the model and preprocessed dataset for download. You must first agree to the [confidentiality notice](https://docs.google.com/forms/d/e/1FAIpQLSc_8VIvRmXM3I8KQaYnKf7gy27Z63BBoI_I1u02f4lw6rBp3g/viewform), then follow the link to a directory containing Rclone download instructions. +## Get Model +### MLCommons Members Download +MLCommons hosts the model and preprocessed dataset for download exclusively by MLCommons Members. You must first agree to the [confidentiality notice](https://docs.google.com/forms/d/e/1FAIpQLSc_8VIvRmXM3I8KQaYnKf7gy27Z63BBoI_I1u02f4lw6rBp3g/viewform), then follow the link to a directory containing Rclone download instructions. + + +### External Download ++ First go to [llama2-request-link](https://ai.meta.com/resources/models-and-libraries/llama-downloads/) and make a request, sign in to HuggingFace (if you don't have account, you'll need to create one). **Please note your authentication credentials** as you may be required to provide them when cloninng below. ++ Requires Git Large Files Storage +``` +export CHECKPOINT_PATH=${PWD}/Llama-2-70b-chat-hf +git lfs install +git clone https://huggingface.co/meta-llama/Llama-2-70b-chat-hf ${CHECKPOINT_PATH} + +``` + +## Get Dataset + +### Preprocessed + +You can use Rclone to download the preprocessed dataset from a Cloudflare R2 bucket. + +To run Rclone on Windows, you can download the executable [here](https://rclone.org/install/#windows). +To install Rclone on Linux/macOS/BSD systems, run: +``` +sudo -v ; curl https://rclone.org/install.sh | sudo bash +``` +Once Rclone is installed, run the following command to authenticate with the bucket: +``` +rclone config create mlc-inference s3 provider=Cloudflare access_key_id=f65ba5eef400db161ea49967de89f47b secret_access_key=fbea333914c292b854f14d3fe232bad6c5407bf0ab1bebf78833c2b359bdfd2b endpoint=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com +``` +You can then navigate in the terminal to your desired download directory and run the following commands to download the model checkpoint: + +``` +rclone copy mlc-inference:mlcommons-inference-wg-public/open_orca ./open_orca -P +``` + +### Unprocessed + +You can also download and process the dataset yourself as follows: + +``` +# First get the `open-orca` parquet from huggingface +export OPENORCA_DATASET=${PWD}/open-orca +git clone https://huggingface.co/datasets/Open-Orca/OpenOrca ${OPENORCA_DATASET} + +export OPENORCA_PARQUET=${OPENORCA_DATASET}/1M-GPT4-Augmented.parquet +EXPORT_DIR=${PWD}/processed-openorca +export DATASET_PATH=${PWD}/processed-data.pkl + +# Process the dataset according the Taskforce's agreed criteria +python3 processorca.py --dataset_pq_path=${OPENORCA_PARQUET} --model_dir=${CHECKPOINT_PATH} --seqlen_limit=1024 --export_dir=${EXPORT_DIR} --num_total_samples=24576 + +mv ${EXPORT_DIR}/open_orca_gpt4_tokenized_llama.sampled_24576.pkl ${DATASET_PATH} +``` ## Run Performance Benchmarks From ef482c38ad4d83a593323c61bfb7fc0a0155a92f Mon Sep 17 00:00:00 2001 From: Nathan Wasson Date: Wed, 21 Feb 2024 17:09:29 -0600 Subject: [PATCH 008/169] Tweak language in README.md --- language/llama2-70b/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/language/llama2-70b/README.md b/language/llama2-70b/README.md index f648b1b5d..3fdf71ce2 100644 --- a/language/llama2-70b/README.md +++ b/language/llama2-70b/README.md @@ -93,7 +93,7 @@ Once Rclone is installed, run the following command to authenticate with the buc ``` rclone config create mlc-inference s3 provider=Cloudflare access_key_id=f65ba5eef400db161ea49967de89f47b secret_access_key=fbea333914c292b854f14d3fe232bad6c5407bf0ab1bebf78833c2b359bdfd2b endpoint=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com ``` -You can then navigate in the terminal to your desired download directory and run the following commands to download the model checkpoint: +You can then navigate in the terminal to your desired download directory and run the following command to download the dataset: ``` rclone copy mlc-inference:mlcommons-inference-wg-public/open_orca ./open_orca -P From faa013400b86898a4adfc67155ab903a5c13167c Mon Sep 17 00:00:00 2001 From: Nathan Wasson Date: Wed, 21 Feb 2024 17:10:08 -0600 Subject: [PATCH 009/169] Language tweak in README.md --- language/gpt-j/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/language/gpt-j/README.md b/language/gpt-j/README.md index cbb5b3e3c..061067027 100644 --- a/language/gpt-j/README.md +++ b/language/gpt-j/README.md @@ -93,7 +93,7 @@ Once Rclone is installed, run the following command to authenticate with the buc ``` rclone config create mlc-inference s3 provider=Cloudflare access_key_id=f65ba5eef400db161ea49967de89f47b secret_access_key=fbea333914c292b854f14d3fe232bad6c5407bf0ab1bebf78833c2b359bdfd2b endpoint=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com ``` -You can then navigate in the terminal to your desired download directory and run the following commands to download the model checkpoint: +You can then navigate in the terminal to your desired download directory and run the following command to download the model checkpoint: ``` rclone copy mlc-inference:mlcommons-inference-wg-public/gpt-j ./model -P From c7945ac7f1016daca0faae96ef7065473cc670e4 Mon Sep 17 00:00:00 2001 From: Nathan Wasson Date: Wed, 21 Feb 2024 17:11:15 -0600 Subject: [PATCH 010/169] Minor language tweak in README.md --- recommendation/dlrm_v2/pytorch/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recommendation/dlrm_v2/pytorch/README.md b/recommendation/dlrm_v2/pytorch/README.md index 6de850b81..12937c0c8 100755 --- a/recommendation/dlrm_v2/pytorch/README.md +++ b/recommendation/dlrm_v2/pytorch/README.md @@ -98,7 +98,7 @@ Once Rclone is installed, run the following command to authenticate with the buc ``` rclone config create mlc-inference s3 provider=Cloudflare access_key_id=f65ba5eef400db161ea49967de89f47b secret_access_key=fbea333914c292b854f14d3fe232bad6c5407bf0ab1bebf78833c2b359bdfd2b endpoint=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com ``` -You can then navigate in the terminal to your desired download directory and run the following commands to download the model weights: +You can then navigate in the terminal to your desired download directory and run the following command to download the model weights: ``` rclone copy mlc-inference:mlcommons-inference-wg-public/model_weights ./model_weights -P From 949ff6b31ae29b397a0146305d5b42c4f6328b6e Mon Sep 17 00:00:00 2001 From: Nathan Wasson Date: Fri, 23 Feb 2024 13:55:22 -0600 Subject: [PATCH 011/169] Fix typo in README.md --- language/llama2-70b/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/language/llama2-70b/README.md b/language/llama2-70b/README.md index 3fdf71ce2..b539b3e3c 100644 --- a/language/llama2-70b/README.md +++ b/language/llama2-70b/README.md @@ -69,7 +69,7 @@ MLCommons hosts the model and preprocessed dataset for download exclusively by M ### External Download -+ First go to [llama2-request-link](https://ai.meta.com/resources/models-and-libraries/llama-downloads/) and make a request, sign in to HuggingFace (if you don't have account, you'll need to create one). **Please note your authentication credentials** as you may be required to provide them when cloninng below. ++ First go to [llama2-request-link](https://ai.meta.com/resources/models-and-libraries/llama-downloads/) and make a request, sign in to HuggingFace (if you don't have account, you'll need to create one). **Please note your authentication credentials** as you may be required to provide them when cloning below. + Requires Git Large Files Storage ``` export CHECKPOINT_PATH=${PWD}/Llama-2-70b-chat-hf From 22d7072dc9985b60203a03950b89f590fb192a3f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 14 Mar 2024 22:47:19 +0000 Subject: [PATCH 012/169] Count error when logging errors: submission_checker.py --- tools/submission/submission_checker.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py index 0bae5b0d4..c79963096 100755 --- a/tools/submission/submission_checker.py +++ b/tools/submission/submission_checker.py @@ -2169,6 +2169,7 @@ def check_performance_dir( required_min_duration, min_duration, ) + is_valid = False inferred = False if scenario_fixed != scenario: @@ -2945,12 +2946,16 @@ def log_result( "process_accuracy.py", acc_path, ) + errors += 1 + continue else: diff = files_diff(list_files(acc_path), REQUIRED_ACC_FILES) if diff: log.error( "%s has file list mismatch (%s)", acc_path, diff ) + errors += 1 + continue accuracy_is_valid, acc = check_accuracy_dir( config, mlperf_model, From 6f2f14e3391c28d3c9750d820ccc9f33120b63fe Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 28 Feb 2024 12:45:24 +0000 Subject: [PATCH 013/169] Fixes #1648, restrict loadgen uncommitted error message to within the loadgen directory --- loadgen/version_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loadgen/version_generator.py b/loadgen/version_generator.py index a34657a64..4de930a63 100644 --- a/loadgen/version_generator.py +++ b/loadgen/version_generator.py @@ -43,7 +43,7 @@ def func_def(name, string): def generate_loadgen_version_definitions_git(ofile, git_command): git_rev = os.popen(git_command + "rev-parse --short=10 HEAD").read() git_commit_date = os.popen(git_command + "log --format=\"%cI\" -n 1").read() - git_status = os.popen(git_command + "status -s -uno").read() + git_status = os.popen(git_command + "status -s -uno .").read() git_log = subprocess.Popen( git_command + "log --pretty=oneline -n 16 --no-decorate", stdout=subprocess.PIPE, shell=True, encoding='ascii', errors="ignore" ).stdout.read() ofile.write(func_def("GitRevision", "\"" + git_rev[0:-1] + "\"")) From 33612490f1c4243600ffbc7ce14808fc5bb9ac98 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 2 May 2024 06:18:14 +0100 Subject: [PATCH 014/169] Update test-rnnt.yml (#1688) Stopping the github action for rnnt --- .github/workflows/test-rnnt.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-rnnt.yml b/.github/workflows/test-rnnt.yml index bbaf5bf52..a76573900 100755 --- a/.github/workflows/test-rnnt.yml +++ b/.github/workflows/test-rnnt.yml @@ -5,7 +5,7 @@ name: Test for MLPerf inference rnnt submission generation using CM script autom on: pull_request: - branches: [ "master", "dev" ] + branches: [ "master-retired", "dev-retired" ] paths: - speech_recognition/rnnt/** - tools/submission/** From b747899f3a5b6142d0d38a720a816334d417d495 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 2 May 2024 10:53:41 +0100 Subject: [PATCH 015/169] Added docs init Added github action for website publish Update benchmark documentation Update publish.yaml Update publish.yaml Update benchmark documentation Improved the submission documentation Fix taskname Removed unused images --- .github/workflows/publish.yaml | 33 ++++++ docs/README.md | 9 ++ .../image_classification/resnet50.md | 68 ++++++++++++ docs/benchmarks/index.md | 28 +++++ docs/benchmarks/language/bert.md | 73 ++++++++++++ docs/benchmarks/language/gpt-j.md | 57 ++++++++++ docs/benchmarks/language/llama2-70b.md | 52 +++++++++ docs/benchmarks/medical_imaging/3d-unet.md | 60 ++++++++++ docs/benchmarks/object_detection/retinanet.md | 63 +++++++++++ docs/benchmarks/recommendation/dlrm-v2.md | 36 ++++++ docs/benchmarks/text_to_image/sdxl.md | 49 ++++++++ docs/changelog/changelog.md | 2 + docs/changelog/index.md | 2 + docs/demos/index.md | 2 + docs/img/logo_v2.svg | 6 + docs/index.md | 1 + docs/install/index.md | 105 ++++++++++++++++++ docs/requirements.txt | 4 + docs/submission/index.md | 88 +++++++++++++++ docs/submission/tools-readme.md | 1 + docs/usage/index.md | 1 + main.py | 103 +++++++++++++++++ mkdocs.yml | 66 +++++++++++ 23 files changed, 909 insertions(+) create mode 100644 .github/workflows/publish.yaml create mode 100644 docs/README.md create mode 100644 docs/benchmarks/image_classification/resnet50.md create mode 100644 docs/benchmarks/index.md create mode 100644 docs/benchmarks/language/bert.md create mode 100644 docs/benchmarks/language/gpt-j.md create mode 100644 docs/benchmarks/language/llama2-70b.md create mode 100644 docs/benchmarks/medical_imaging/3d-unet.md create mode 100644 docs/benchmarks/object_detection/retinanet.md create mode 100644 docs/benchmarks/recommendation/dlrm-v2.md create mode 100644 docs/benchmarks/text_to_image/sdxl.md create mode 100644 docs/changelog/changelog.md create mode 100644 docs/changelog/index.md create mode 100644 docs/demos/index.md create mode 100644 docs/img/logo_v2.svg create mode 120000 docs/index.md create mode 100644 docs/install/index.md create mode 100644 docs/requirements.txt create mode 100644 docs/submission/index.md create mode 120000 docs/submission/tools-readme.md create mode 100644 docs/usage/index.md create mode 100644 main.py create mode 100644 mkdocs.yml diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 000000000..214ac3611 --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,33 @@ +# This is a basic workflow to help you get started with Actions + +name: Publish site + + +on: + release: + types: [published] + push: + branches: + - master + - docs + +jobs: + + publish: + name: Publish the site + runs-on: ubuntu-latest + + steps: + - name: Checkout repository normally + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Install Mkdocs + run: pip install -r docs/requirements.txt + + - name: Run Mkdocs deploy + run: mkdocs gh-deploy --force diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 000000000..477c13a7b --- /dev/null +++ b/docs/README.md @@ -0,0 +1,9 @@ +# Documentation Website for MLPerf Inference using the unified CM interface + +## Commands to get the website running locally +``` +git clone https://github.com/GATEOverflow/cm4mlperf-inference +cd cm4mlperf-inference +pip install -r requirements.txt +mkdocs serve +``` diff --git a/docs/benchmarks/image_classification/resnet50.md b/docs/benchmarks/image_classification/resnet50.md new file mode 100644 index 000000000..1a77db65a --- /dev/null +++ b/docs/benchmarks/image_classification/resnet50.md @@ -0,0 +1,68 @@ +# Image Classification using ResNet50 + +## Dataset + +The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. + +=== "Validation" + ResNet50 validation run uses the Imagenet 2012 validation dataset consisting of 50,000 images. + + ### Get Validation Dataset + ``` + cm run script --tags=get,dataset,imagenet,validation -j + ``` +=== "Calibration" + ResNet50 calibration dataset consist of 500 images selected from the Imagenet 2012 validation dataset. There are 2 alternative options for the calibration dataset. + + ### Get Calibration Dataset Using Option 1 + ``` + cm run script --tags=get,dataset,imagenet,calibration,_mlperf.option1 -j + ``` + ### Get Calibration Dataset Using Option 2 + ``` + cm run script --tags=get,dataset,imagenet,calibration,_mlperf.option2 -j + ``` + +## Model +The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. + +Get the Official MLPerf ResNet50 Model + +=== "Tensorflow" + + ### Tensorflow + ``` + cm run script --tags=get,ml-model,resnet50,_tensorflow -j + ``` +=== "Onnx" + + ### Onnx + ``` + cm run script --tags=get,ml-model,resnet50,_onnx -j + ``` + +## Benchmark Implementations +=== "MLCommons-Python" + ### MLPerf Reference Implementation in Python + +{{ mlperf_inference_implementation_readme (4, "resnet50", "reference") }} + +=== "Nvidia" + ### Nvidia MLPerf Implementation + +{{ mlperf_inference_implementation_readme (4, "resnet50", "nvidia") }} + +=== "Intel" + ### Intel MLPerf Implementation + +{{ mlperf_inference_implementation_readme (4, "resnet50", "intel") }} + +=== "Qualcomm" + ### Qualcomm AI100 MLPerf Implementation + +{{ mlperf_inference_implementation_readme (4, "resnet50", "qualcomm") }} + +=== "MLCommon-C++" + ### MLPerf Modular Implementation in C++ + +{{ mlperf_inference_implementation_readme (4, "resnet50", "cpp") }} diff --git a/docs/benchmarks/index.md b/docs/benchmarks/index.md new file mode 100644 index 000000000..7f528638b --- /dev/null +++ b/docs/benchmarks/index.md @@ -0,0 +1,28 @@ +# MLPerf Inference Benchmarks + +Please visit the individual benchmark links to see the run commands using the unified CM interface. + +1. [Image Classification](benchmarks/image_classification/resnet50.md) using ResNet50 model and Imagenet-2012 dataset + +2. [Text to Image](benchmarks/text_to_image/sdxl.md) using Stable Diffusion model and Coco2014 dataset + +3. [Object Detection](object_detection/retinanet.md) using Retinanet model and OpenImages dataset + +4. [Image Segmentation](medical_imaging/3d-unet.md) using 3d-unet model and KiTS19 dataset + +5. [Question Answering](language/bert.md) using Bert-Large model and Squad v1.1 dataset + +6. [Text Summarization](language/gpt-j.md) using GPT-J model and CNN Daily Mail dataset + +7. [Text Summarization](language/llama2-70b.md) using LLAMA2-70b model and OpenORCA dataset + +8. [Recommendation](recommendation/dlrm-v2.md) using DLRMv2 model and Criteo multihot dataset + +All the eight benchmarks can participate in the datacenter category. +All the eight benchmarks except DLRMv2 and LLAMA2 and can participate in the edge category. + +`bert`, `llama2-70b`, `dlrm_v2` and `3d-unet` has a high accuracy (99.9%) variant, where the benchmark run must achieve a higher accuracy of at least `99.9%` of the FP32 reference model +in comparison with the `99%` default accuracy requirement. + +The `dlrm_v2` benchmark has a high-accuracy variant only. If this accuracy is not met, the submission result can be submitted only to the open division. + diff --git a/docs/benchmarks/language/bert.md b/docs/benchmarks/language/bert.md new file mode 100644 index 000000000..e2aa0995d --- /dev/null +++ b/docs/benchmarks/language/bert.md @@ -0,0 +1,73 @@ +# Question Answering using Bert-Large + +## Dataset + +The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. + +=== "Validation" + BERT validation run uses the SQuAD v1.1 dataset. + + ### Get Validation Dataset + ``` + cm run script --tags=get,dataset,squad,validation -j + ``` + +## Model +The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. + +Get the Official MLPerf Bert-Large Model + +=== "Pytorch" + + ### Pytorch + ``` + cm run script --tags=get,ml-model,bert-large,_pytorch -j + ``` +=== "Onnx" + + ### Onnx + ``` + cm run script --tags=get,ml-model,bert-large,_onnx -j + ``` +=== "Tensorflow" + + ### Tensorflow + ``` + cm run script --tags=get,ml-model,bert-large,_tensorflow -j + ``` + +## Benchmark Implementations +=== "MLCommons-Python" + ### MLPerf Reference Implementation in Python + + BERT-99 +{{ mlperf_inference_implementation_readme (4, "bert-99", "reference") }} + + BERT-99.9 +{{ mlperf_inference_implementation_readme (4, "bert-99.9", "reference") }} + +=== "Nvidia" + ### Nvidia MLPerf Implementation + + BERT-99 +{{ mlperf_inference_implementation_readme (4, "bert-99", "nvidia") }} + + BERT-99.9 +{{ mlperf_inference_implementation_readme (4, "bert-99.9", "nvidia") }} + +=== "Intel" + ### Intel MLPerf Implementation + BERT-99 +{{ mlperf_inference_implementation_readme (4, "bert-99", "intel") }} + + BERT-99.9 +{{ mlperf_inference_implementation_readme (4, "bert-99.9", "intel") }} + +=== "Qualcomm" + ### Qualcomm AI100 MLPerf Implementation + + BERT-99 +{{ mlperf_inference_implementation_readme (4, "bert-99", "qualcomm") }} + + BERT-99.9 +{{ mlperf_inference_implementation_readme (4, "bert-99.9", "qualcomm") }} diff --git a/docs/benchmarks/language/gpt-j.md b/docs/benchmarks/language/gpt-j.md new file mode 100644 index 000000000..d1c351214 --- /dev/null +++ b/docs/benchmarks/language/gpt-j.md @@ -0,0 +1,57 @@ +# Text Summarization using GPT-J + +## Dataset + +The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. + +=== "Validation" + GPT-J validation run uses the CNNDM dataset. + + ### Get Validation Dataset + ``` + cm run script --tags=get,dataset,cnndm,validation -j + ``` + +## Model +The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. + +Get the Official MLPerf GPT-J Model + +=== "Pytorch" + + ### Pytorch + ``` + cm run script --tags=get,ml-model,gptj,_pytorch -j + ``` + +## Benchmark Implementations +=== "MLCommons-Python" + ### MLPerf Reference Implementation in Python + + GPT-J-99 +{{ mlperf_inference_implementation_readme (4, "gptj-99", "reference") }} + + GPTJ-99.9 +{{ mlperf_inference_implementation_readme (4, "gptj-99.9", "reference") }} + +=== "Nvidia" + ### Nvidia MLPerf Implementation + + GPTJ-99 +{{ mlperf_inference_implementation_readme (4, "gptj-99", "nvidia") }} + + GPTJ-99.9 +{{ mlperf_inference_implementation_readme (4, "gptj-99.9", "nvidia") }} + +=== "Intel" + ### Intel MLPerf Implementation + GPTJ-99 +{{ mlperf_inference_implementation_readme (4, "gptj-99", "intel") }} + + +=== "Qualcomm" + ### Qualcomm AI100 MLPerf Implementation + + GPTJ-99 +{{ mlperf_inference_implementation_readme (4, "gptj-99", "qualcomm") }} + diff --git a/docs/benchmarks/language/llama2-70b.md b/docs/benchmarks/language/llama2-70b.md new file mode 100644 index 000000000..7f8052aef --- /dev/null +++ b/docs/benchmarks/language/llama2-70b.md @@ -0,0 +1,52 @@ +# Text Summarization using LLAMA2-70b + +## Dataset + +The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. + +=== "Validation" + LLAMA2-70b validation run uses the Open ORCA dataset. + + ### Get Validation Dataset + ``` + cm run script --tags=get,dataset,openorca,validation -j + ``` + +## Model +The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. + +Get the Official MLPerf LLAMA2-70b Model + +=== "Pytorch" + + ### Pytorch + ``` + cm run script --tags=get,ml-model,llama2-70b,_pytorch -j + ``` + +## Benchmark Implementations +=== "MLCommons-Python" + ### MLPerf Reference Implementation in Python + + LLAMA2-70b-99 +{{ mlperf_inference_implementation_readme (4, "llama2-70b-99", "reference") }} + + LLAMA2-70b-99.9 +{{ mlperf_inference_implementation_readme (4, "llama2-70b-99.9", "reference") }} + +=== "Nvidia" + ### Nvidia MLPerf Implementation + + LLAMA2-70b-99 +{{ mlperf_inference_implementation_readme (4, "llama2-70b-99", "nvidia") }} + + LLAMA2-70b-99.9 +{{ mlperf_inference_implementation_readme (4, "llama2-70b-99.9", "nvidia") }} + + +=== "Qualcomm" + ### Qualcomm AI100 MLPerf Implementation + + LLAMA2-70b-99 +{{ mlperf_inference_implementation_readme (4, "llama2-70b-99", "qualcomm") }} + diff --git a/docs/benchmarks/medical_imaging/3d-unet.md b/docs/benchmarks/medical_imaging/3d-unet.md new file mode 100644 index 000000000..bd3ccae40 --- /dev/null +++ b/docs/benchmarks/medical_imaging/3d-unet.md @@ -0,0 +1,60 @@ +# Medical Imaging using 3d-unet (KiTS 2019 kidney tumor segmentation task) + +## Dataset + +The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. + +=== "Validation" + 3d-unet validation run uses the KiTS19 dataset performing [KiTS 2019](https://kits19.grand-challenge.org/) kidney tumor segmentation task + + ### Get Validation Dataset + ``` + cm run script --tags=get,dataset,kits19,validation -j + ``` + +## Model +The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. + +Get the Official MLPerf 3d-unet Model + +=== "Pytorch" + + ### Pytorch + ``` + cm run script --tags=get,ml-model,3d-unet,_pytorch -j + ``` +=== "Onnx" + + ### Onnx + ``` + cm run script --tags=get,ml-model,3d-unet,_onnx -j + ``` +=== "Tensorflow" + + ### Tensorflow + ``` + cm run script --tags=get,ml-model,3d-unet,_tensorflow -j + ``` + +## Benchmark Implementations +=== "MLCommons-Python" + ### MLPerf Reference Implementation in Python + + 3d-unet-99.9 +{{ mlperf_inference_implementation_readme (4, "3d-unet-99.9", "reference") }} + +=== "Nvidia" + ### Nvidia MLPerf Implementation + 3d-unet-99 +{{ mlperf_inference_implementation_readme (4, "3d-unet-99", "nvidia") }} + + 3d-unet-99.9 +{{ mlperf_inference_implementation_readme (4, "3d-unet-99.9", "nvidia") }} + +=== "Intel" + ### Intel MLPerf Implementation + 3d-unet-99 +{{ mlperf_inference_implementation_readme (4, "3d-unet-99", "intel") }} + + 3d-unet-99.9 +{{ mlperf_inference_implementation_readme (4, "3d-unet-99.9", "intel") }} diff --git a/docs/benchmarks/object_detection/retinanet.md b/docs/benchmarks/object_detection/retinanet.md new file mode 100644 index 000000000..f500f616d --- /dev/null +++ b/docs/benchmarks/object_detection/retinanet.md @@ -0,0 +1,63 @@ +# Object Detection using Retinanet + +## Dataset + +The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. + +=== "Validation" + Retinanet validation run uses the OpenImages v6 MLPerf validation dataset resized to 800x800 and consisting of 24,576 images. + + ### Get Validation Dataset + ``` + cm run script --tags=get,dataset,openimages,_validation -j + ``` +=== "Calibration" + Retinanet calibration dataset consist of 500 images selected from the OpenImages v6 dataset. + + ``` + cm run script --tags=get,dataset,openimages,_calibration -j + ``` + +## Model +The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. + +Get the Official MLPerf Retinanet Model + +=== "Pytorch" + + ### Pytorch + ``` + cm run script --tags=get,ml-model,retinanet,_pytorch -j + ``` +=== "Onnx" + + ### Onnx + ``` + cm run script --tags=get,ml-model,retinanet,_onnx -j + ``` + +## Benchmark Implementations +=== "MLCommons-Python" + ### MLPerf Reference Implementation in Python + +{{ mlperf_inference_implementation_readme (4, "retinanet", "reference") }} + +=== "Nvidia" + ### Nvidia MLPerf Implementation + +{{ mlperf_inference_implementation_readme (4, "retinanet", "nvidia") }} + +=== "Intel" + ### Intel MLPerf Implementation + +{{ mlperf_inference_implementation_readme (4, "retinanet", "intel") }} + +=== "Qualcomm" + ### Qualcomm AI100 MLPerf Implementation + +{{ mlperf_inference_implementation_readme (4, "retinanet", "qualcomm") }} + +=== "MLCommon-C++" + ### MLPerf Modular Implementation in C++ + +{{ mlperf_inference_implementation_readme (4, "retinanet", "cpp") }} diff --git a/docs/benchmarks/recommendation/dlrm-v2.md b/docs/benchmarks/recommendation/dlrm-v2.md new file mode 100644 index 000000000..1294b008b --- /dev/null +++ b/docs/benchmarks/recommendation/dlrm-v2.md @@ -0,0 +1,36 @@ +# Recommendation using DLRM v2 + +## Dataset + +The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. + +=== "Validation" + DLRM validation run uses the Criteo dataset (Day 23). + + ### Get Validation Dataset + ``` + cm run script --tags=get,dataset,criteo,validation -j + ``` +## Model +The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. + +Get the Official MLPerf DLRM v2 Model + +=== "Pytorch" + + ### Pytorch + ``` + cm run script --tags=get,ml-model,dlrm_v2,_pytorch -j + ``` + +## Benchmark Implementations +=== "MLCommons-Python" + ### MLPerf Reference Implementation in Python + +{{ mlperf_inference_implementation_readme (4, "dlrm_v2-99.9", "reference") }} + +=== "Nvidia" + ### Nvidia MLPerf Implementation + +{{ mlperf_inference_implementation_readme (4, "dlrm_v2-99.9", "nvidia") }} + diff --git a/docs/benchmarks/text_to_image/sdxl.md b/docs/benchmarks/text_to_image/sdxl.md new file mode 100644 index 000000000..2e9c95c66 --- /dev/null +++ b/docs/benchmarks/text_to_image/sdxl.md @@ -0,0 +1,49 @@ +# Text to Image using Stable Diffusion + +## Dataset + +The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. + +=== "Validation" + Stable Diffusion validation run uses the Coco 2014 dataset. + + ### Get Validation Dataset + ``` + cm run script --tags=get,dataset,coco2014,_validation -j + ``` + +## Model +The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. + +Get the Official MLPerf Stable Diffusion Model + +=== "Pytorch" + + ### Pytorch + ``` + cm run script --tags=get,ml-model,sdxl,_pytorch -j + ``` + +## Benchmark Implementations +=== "MLCommons-Python" + ### MLPerf Reference Implementation in Python + +{{ mlperf_inference_implementation_readme (4, "sdxl", "reference") }} + +=== "Nvidia" + ### Nvidia MLPerf Implementation + +{{ mlperf_inference_implementation_readme (4, "sdxl", "nvidia") }} + +=== "Intel" + ### Intel MLPerf Implementation + GPTJ-99 +{{ mlperf_inference_implementation_readme (4, "sdxl", "intel") }} + + +=== "Qualcomm" + ### Qualcomm AI100 MLPerf Implementation + + GPTJ-99 +{{ mlperf_inference_implementation_readme (4, "sdxl", "qualcomm") }} + diff --git a/docs/changelog/changelog.md b/docs/changelog/changelog.md new file mode 100644 index 000000000..8b9abcc4b --- /dev/null +++ b/docs/changelog/changelog.md @@ -0,0 +1,2 @@ +# Release Notes + diff --git a/docs/changelog/index.md b/docs/changelog/index.md new file mode 100644 index 000000000..f68abc5b1 --- /dev/null +++ b/docs/changelog/index.md @@ -0,0 +1,2 @@ +# What's New, What's Coming + diff --git a/docs/demos/index.md b/docs/demos/index.md new file mode 100644 index 000000000..1c23a5f60 --- /dev/null +++ b/docs/demos/index.md @@ -0,0 +1,2 @@ +# Demos + diff --git a/docs/img/logo_v2.svg b/docs/img/logo_v2.svg new file mode 100644 index 000000000..fb655c627 --- /dev/null +++ b/docs/img/logo_v2.svg @@ -0,0 +1,6 @@ + + + MLCommons + + + diff --git a/docs/index.md b/docs/index.md new file mode 120000 index 000000000..32d46ee88 --- /dev/null +++ b/docs/index.md @@ -0,0 +1 @@ +../README.md \ No newline at end of file diff --git a/docs/install/index.md b/docs/install/index.md new file mode 100644 index 000000000..d90c59eef --- /dev/null +++ b/docs/install/index.md @@ -0,0 +1,105 @@ +# Installation +We use MLCommons CM Automation framework to run MLPerf inference benchmarks. + +## CM Install + +We have successfully tested CM on + +* Ubuntu 18.x, 20.x, 22.x , 23.x, +* RedHat 8, RedHat 9, CentOS 8 +* macOS +* Wndows 10, Windows 11 + +=== "Ubuntu" + ### Ubuntu, Debian + + + ```bash + sudo apt update && sudo apt upgrade + sudo apt install python3 python3-pip python3-venv git wget curl + ``` + + **Note that you must set up virtual env on Ubuntu 23+ before using any Python project:** + ```bash + python3 -m venv cm + source cm/bin/activate + ``` + + You can now install CM via PIP: + + ```bash + python3 -m pip install cmind + ``` + + You might need to do the following command to update the `PATH` to include the BIN paths from pip installs + + ```bash + source $HOME/.profile + ``` + + You can check that CM is available by checking the `cm` command + + +=== "Red Hat" + ### Red Hat + + ```bash + sudo dnf update + sudo dnf install python3 python-pip git wget curl + python3 -m pip install cmind --user + ``` + +=== "macOS" + ### macOS + + *Note that CM currently does not work with Python installed from the Apple Store. + Please install Python via brew as described below.* + + If `brew` package manager is not installed, please install it as follows (see details [here](https://brew.sh/)): + ```bash + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + ``` + + Don't forget to add brew to PATH environment as described in the end of the installation output. + + Then install python, pip, git and wget: + + ```bash + brew install python3 git wget curl + python3 -m pip install cmind + ``` + +=== "Windows" + + ### Windows + * Configure Windows 10+ to [support long paths](https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=registry#enable-long-paths-in-windows-10-version-1607-and-later) from command line as admin: + + ```bash + reg add "HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem" /v LongPathsEnabled /t REG_DWORD /d 1 /f + ``` + + * Download and install Git from [git-for-windows.github.io](https://git-for-windows.github.io). + * Configure Git to accept long file names: `git config --system core.longpaths true` + * Download and install Python 3+ from [www.python.org/downloads/windows](https://www.python.org/downloads/windows). + * Don't forget to select option to add Python binaries to PATH environment! + * Configure Windows to accept long fie names during Python installation! + + * Install CM via PIP: + + ```bash + python -m pip install cmind + ``` + + *Note that we [have reports](https://github.com/mlcommons/ck/issues/844) + that CM does not work when Python was first installed from the Microsoft Store. + If CM fails to run, you can find a fix [here](https://stackoverflow.com/questions/57485491/python-python3-executes-in-command-prompt-but-does-not-run-correctly)*. + +Please visit the [official CM installation page](https://github.com/mlcommons/ck/blob/master/docs/installation.md) for more details + +## Download the CM MLOps Repository + +```bash + cm pull repo mlcommons@cm4mlops --branch=mlperf-inference +``` + +Now, you are ready to use the `cm` commands to run MLPerf inference as given in the [benchmarks](../benchmarks) page diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 000000000..39fab4e1f --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,4 @@ +mkdocs-material +swagger-markdown +mkdocs-macros-plugin +ruamel.yaml diff --git a/docs/submission/index.md b/docs/submission/index.md new file mode 100644 index 000000000..b5ff53033 --- /dev/null +++ b/docs/submission/index.md @@ -0,0 +1,88 @@ +If you follow the `cm run` commands under the individual model pages in the [benchmarks](../benchmarks) directory, all the valid results will get aggregated to the `cm cache` folder. Once all the results across all the modelsare ready you can use the following command to generate a valid submission tree compliant with the [MLPerf requirements](https://github.com/mlcommons/policies/blob/master/submission_rules.adoc#inference-1). + +## Generate actual submission tree + +=== "Closed Edge" + ### Closed Edge Submission + ```bash + cm run script -tags=generate,inference,submission \ + --clean \ + --preprocess_submission=yes \ + --run-checker \ + --submitter=MLCommons \ + --tar=yes \ + --env.CM_TAR_OUTFILE=submission.tar.gz \ + --division=closed \ + --category=edge \ + --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ + --quiet + ``` + +=== "Closed Datacenter" + ### Closed Datacenter Submission + ```bash + cm run script -tags=generate,inference,submission \ + --clean \ + --preprocess_submission=yes \ + --run-checker \ + --submitter=MLCommons \ + --tar=yes \ + --env.CM_TAR_OUTFILE=submission.tar.gz \ + --division=closed \ + --category=datacenter \ + --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ + --quiet + ``` +=== "Open Edge" + ### Open Edge Submission + ```bash + cm run script -tags=generate,inference,submission \ + --clean \ + --preprocess_submission=yes \ + --run-checker \ + --submitter=MLCommons \ + --tar=yes \ + --env.CM_TAR_OUTFILE=submission.tar.gz \ + --division=open \ + --category=edge \ + --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ + --quiet + ``` +=== "Open Datacenter" + ### Closed Datacenter Submission + ```bash + cm run script -tags=generate,inference,submission \ + --clean \ + --preprocess_submission=yes \ + --run-checker \ + --submitter=MLCommons \ + --tar=yes \ + --env.CM_TAR_OUTFILE=submission.tar.gz \ + --division=open \ + --category=datacenter \ + --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ + --quiet + ``` + +* Use `--hw_name="My system name"` to give a meaningful system name. Examples can be seen [here](https://github.com/mlcommons/inference_results_v3.0/tree/main/open/cTuning/systems) + +* Use `--submitter=` if your organization is an official MLCommons member and would like to submit under your organization + +* Use `--hw_notes_extra` option to add additional notes like `--hw_notes_extra="Result taken by NAME" ` + +The above command should generate "submission.tar.gz" if there are no submission checker issues and you can upload it to the [MLCommons Submission UI](https://submissions-ui.mlcommons.org/submission). + +## Aggregate Results in GitHub + +If you are collecting results across multiple systems you can generate different submissions and aggregate all of them to a GitHub repository (can be private) and use it to generate a single tar ball which can be uploaded to the [MLCommons Submission UI](https://submissions-ui.mlcommons.org/submission). + +Run the following command after **replacing `--repo_url` with your GitHub repository URL**. + +```bash + cm run script --tags=push,github,mlperf,inference,submission \ + --repo_url=https://github.com/GATEOverflow/mlperf_inference_submissions_v4.1 \ + --commit_message="Results on added by " \ + --quiet +``` + +At the end, you can download the github repo and upload to the [MLCommons Submission UI](https://submissions-ui.mlcommons.org/submission). diff --git a/docs/submission/tools-readme.md b/docs/submission/tools-readme.md new file mode 120000 index 000000000..d6f026eab --- /dev/null +++ b/docs/submission/tools-readme.md @@ -0,0 +1 @@ +../../tools/submission/README.md \ No newline at end of file diff --git a/docs/usage/index.md b/docs/usage/index.md new file mode 100644 index 000000000..2e92a6e00 --- /dev/null +++ b/docs/usage/index.md @@ -0,0 +1 @@ +# Using CM for MLPerf Inference diff --git a/main.py b/main.py new file mode 100644 index 000000000..d8facefce --- /dev/null +++ b/main.py @@ -0,0 +1,103 @@ +def define_env(env): + + @env.macro + def mlperf_inference_implementation_readme(spaces, model, implementation): + pre_space = "" + for i in range(1,spaces): + pre_space = pre_space + " " + f_pre_space = pre_space + pre_space += " " + + content="" + if implementation == "reference": + devices = [ "CPU", "CUDA", "ROCm" ] + if model.lower() == "resnet50": + frameworks = [ "Onnxruntime", "Tensorflow", "Deepsparse" ] + elif model.lower() == "retinanet": + frameworks = [ "Onnxruntime", "Pytorch" ] + elif "bert" in model.lower(): + frameworks = [ "Onnxruntime", "Pytorch", "Tensorflow" ] + else: + frameworks = [ "Pytorch" ] + elif implementation == "nvidia": + devices = [ "CUDA" ] + frameworks = [ "TensorRT" ] + elif implementation == "intel": + devices = [ "CPU" ] + frameworks = [ "Pytorch" ] + elif implementation == "qualcomm": + devices = [ "QAIC" ] + frameworks = [ "Glow" ] + elif implementation == "cpp": + devices = [ "CPU", "CUDA" ] + frameworks = [ "Onnxruntime" ] + + if model.lower() == "bert-99.9": + categories = [ "Datacenter" ] + elif "dlrm" in model.lower() or "llama2" in model.lower(): + categories = [ "Datacenter" ] + else: + categories = [ "Edge", "Datacenter" ] + + for category in categories: + if category == "Edge": + scenarios = [ "Offline", "SingleStream" ] + if model.lower() in [ "resnet50", "retinanet" ]: + scenarios.append("Multistream") + elif category == "Datacenter": + scenarios = [ "Offline", "Server" ] + + content += f"{pre_space}=== \"{category.lower()}\"\n\n" + + cur_space = pre_space + " " + scenarios_string = ", ".join(scenarios) + + content += f"{cur_space}#### {category} category \n\n{cur_space} In the {category.lower()} category, {model} has {scenarios_string} scenarios and all the scenarios are mandatory for a closed division submission.\n\n" + + + for framework in frameworks: + cur_space1 = cur_space + " " + content += f"{cur_space}=== \"{framework}\"\n" + content += f"{cur_space1}##### {framework} framework\n\n" + + for device in devices: + if framework.lower() == "deepsparse": + if device.lower() != "cpu": + continue + cur_space2 = cur_space1 + " " + content += f"{cur_space1}=== \"{device}\"\n" + content += f"{cur_space2}###### {device} device\n\n" + + for scenario in scenarios: + cur_space3 = cur_space2 + " " + content += f"{cur_space2}=== \"{scenario}\"\n{cur_space3}####### {scenario}\n" + run_cmd = mlperf_inference_run_command(spaces+16, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), "valid") + content += run_cmd + content += f"{cur_space2}=== \"All Scenarios\"\n{cur_space3}####### All Scenarios\n" + run_cmd = mlperf_inference_run_command(spaces+16, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), "valid") + content += run_cmd + + return content + + + @env.macro + def mlperf_inference_run_command(spaces, model, implementation, framework, category, scenario, device="cpu", execution_mode="test", test_query_count="20"): + pre_space = "" + for i in range(1,spaces): + pre_space = pre_space + " " + f_pre_space = pre_space + pre_space += " " + + if scenario == "All Scenarios": + scenario_variation_tag = ",_all-scenarios" + scenario_option = "" + else: + scenario_variation_tag = "" + scenario_option = f"\\\n {pre_space} --scenario={scenario}" + + cmd_suffix = f" \\\n {pre_space} --docker" + #cmd_suffix = f"" + if execution_mode == "test": + cmd_suffix += f" \\\n {pre_space} --test_query_count={test_query_count}" + + return f"\n{f_pre_space} ```bash\n{f_pre_space} cm run script --tags=run-mlperf,inference{scenario_variation_tag} \\\n {pre_space} --model={model} \\\n {pre_space} --implementation={implementation} \\\n {pre_space} --framework={framework} \\\n {pre_space} --category={category} {scenario_option} \\\n {pre_space} --execution-mode={execution_mode} \\\n {pre_space} --device={device} {cmd_suffix}\n{f_pre_space} ```\n" diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 000000000..0d0f64152 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,66 @@ +site_name: MLPerf Inference Documentation +repo_url: https://github.com/mlcommons/inference +theme: + name: material + logo: img/logo_v2.svg + favicon: img/logo_v2.svg + palette: + primary: deep purple + accent: green + features: + - content.tabs.link + - content.code.copy + - navigation.expand + - navigation.sections + - navigation.indexes + - navigation.instant + - navigation.tabs + - navigation.tabs.sticky + - navigation.top + - toc.follow +nav: + - Inference: + - index.md + - Install: + - install/index.md + - Quick Start: install/quick-start.md + - Benchmarks: + - benchmarks/index.md + - Image Classification: + - ResNet50: benchmarks/image_classification/resnet50.md + - Text to Image: + - Stable Diffusion: benchmarks/text_to_image/sdxl.md + - Object Detection: + - RetinaNet: benchmarks/object_detection/retinanet.md + - Medical Imaging: + - 3d-unet: benchmarks/medical_imaging/3d-unet.md + - Language Processing: + - Bert-Large: benchmarks/language/bert.md + - GPT-J: benchmarks/language/gpt-j.md + - LLAMA2-70B: benchmarks/language/llama2-70b.md + - Recommendation: + - DLRM-v2: benchmarks/recommendation/dlrm-v2.md + - Submission: + - Submission Generation: submission/index.md + - Release Notes: + - What's New: changelog/index.md + - Changelog: changelog/changelog.md + +markdown_extensions: + - pymdownx.tasklist: + custom_checkbox: true + - pymdownx.details + - admonition + - attr_list + - def_list + - footnotes + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + - pymdownx.tabbed: + alternate_style: true +plugins: + - search + - macros From 6fe12df90e7582f1297f72599b9734279e5840bd Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 3 May 2024 13:15:07 +0100 Subject: [PATCH 016/169] Fix benchmark URLs --- docs/benchmarks/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/benchmarks/index.md b/docs/benchmarks/index.md index 7f528638b..c4a970c81 100644 --- a/docs/benchmarks/index.md +++ b/docs/benchmarks/index.md @@ -2,9 +2,9 @@ Please visit the individual benchmark links to see the run commands using the unified CM interface. -1. [Image Classification](benchmarks/image_classification/resnet50.md) using ResNet50 model and Imagenet-2012 dataset +1. [Image Classification](image_classification/resnet50.md) using ResNet50 model and Imagenet-2012 dataset -2. [Text to Image](benchmarks/text_to_image/sdxl.md) using Stable Diffusion model and Coco2014 dataset +2. [Text to Image](text_to_image/sdxl.md) using Stable Diffusion model and Coco2014 dataset 3. [Object Detection](object_detection/retinanet.md) using Retinanet model and OpenImages dataset From 949f8f7a6bf7e4f3d21c58d99d17d2d648092fe3 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 8 May 2024 15:33:50 +0100 Subject: [PATCH 017/169] Fix links --- docs/install/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/install/index.md b/docs/install/index.md index d90c59eef..ec76718d9 100644 --- a/docs/install/index.md +++ b/docs/install/index.md @@ -99,7 +99,7 @@ Please visit the [official CM installation page](https://github.com/mlcommons/ck ## Download the CM MLOps Repository ```bash - cm pull repo mlcommons@cm4mlops --branch=mlperf-inference + cm pull repo gateoverflow@cm4mlops ``` -Now, you are ready to use the `cm` commands to run MLPerf inference as given in the [benchmarks](../benchmarks) page +Now, you are ready to use the `cm` commands to run MLPerf inference as given in the [benchmarks](../benchmarks/index.md) page From 120aced716384afd7c70d7cab3c0d19682460413 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 8 May 2024 15:34:47 +0100 Subject: [PATCH 018/169] Add _full variation to run commands --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index d8facefce..80643f4d3 100644 --- a/main.py +++ b/main.py @@ -100,4 +100,4 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ if execution_mode == "test": cmd_suffix += f" \\\n {pre_space} --test_query_count={test_query_count}" - return f"\n{f_pre_space} ```bash\n{f_pre_space} cm run script --tags=run-mlperf,inference{scenario_variation_tag} \\\n {pre_space} --model={model} \\\n {pre_space} --implementation={implementation} \\\n {pre_space} --framework={framework} \\\n {pre_space} --category={category} {scenario_option} \\\n {pre_space} --execution-mode={execution_mode} \\\n {pre_space} --device={device} {cmd_suffix}\n{f_pre_space} ```\n" + return f"\n{f_pre_space} ```bash\n{f_pre_space} cm run script --tags=run-mlperf,inference,_full{scenario_variation_tag} \\\n {pre_space} --model={model} \\\n {pre_space} --implementation={implementation} \\\n {pre_space} --framework={framework} \\\n {pre_space} --category={category} {scenario_option} \\\n {pre_space} --execution-mode={execution_mode} \\\n {pre_space} --device={device} {cmd_suffix}\n{f_pre_space} ```\n" From dccbe1e1d1c8c287bff0b2fdf7db184f5f9a5e7d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 13 May 2024 18:54:22 +0100 Subject: [PATCH 019/169] Added script flow diagram --- docs/img/pages (80).png | Bin 0 -> 242952 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/img/pages (80).png diff --git a/docs/img/pages (80).png b/docs/img/pages (80).png new file mode 100644 index 0000000000000000000000000000000000000000..0ca65735a705ff69d234b02ad9e116e6caa61d0f GIT binary patch literal 242952 zcmeFZ1yCGYxGp*|h@c_3Lju8ry97;w>)`HAu;64va7jXNhXjWSF2g_)G`RaffZ!Gw zTwl)&d!KV|-8!dU)xA}(>g}##XHU;`_v*j@^~?8ttHV{4AwijA2W2=pR6KHf?tPKUT-2O+ZdPU4|tsEX7F<|l!l z@N}OJ6&FaXrmM)}qf!RDG`FJV%;- zL@C~QmF&~+bm_W0>)L$5`^SRv$xUpA1tiak|^?z~AY0s|) zcx}q}HM5?6T-@k2@F%bNB~6q>xS=hya~`T4aVb}H#%?D8)H5N$rw`$+%1w|vj!<<5vF0J{}QRY zSr#mUx^a8XW!nr``1@I(wyg9?;n1~S(A{(Bd-XdpyBT%zyFtHw)N9{ zU4gH&(^c>_?rT1GS0UJc;D*z16CR9_9XcSubL#@zFBm`R_OQtv?OS7^lg}_c-ucZR zIP;|4U-@?O#}K$&sPYfWKZM78B3KXR6aP@$#H0Hm@d@7`INBK>_c2MhmRf}ldnl_(@L5Xodo{KvVO|&ir6Cjwd_guae>B&mIi0hefOKAZ`WnuQqj561^w1s zx>Th>H4=z{x2x7}L=7N=7)R;aTu654VBD%F30BBk6O*22eMvM56qek1B(5aGA@xzD ztqR7=U$slHv@q@79efKh87{vw@oMws=95i4kB4Vea;;4BQO+@S#`+K0altWm?FOyb z^Lfq>-ncXh&EDL38~e@aN8&H?Kk#GW^c`}ozKOrw3 z7vc4W8@6gMc$nVVeYA_=l6L%7`iV_k-Df;C3>%I$fi}l?)@A+`1BxR5&vr5{hYFBHAPQ0IBxu21vG_S&^^7`r0C+<(epLARr>p}#;A_~66 zZ>o<4cU5-<@1+E12Zu`MNS8}jwkn3(Feel}ch8blK@@frfC~wDg7aQ24joseS7A)? zPCcfSvKsW-7x2cvusQ`9JboF^ldzxIlTeb-NW)}KgN2O|`{CPDlQheyXJW6IUbuXw z`*QfA`W5!eUit2q)34-S5x=T^i7OxQ$)ro5v$cCOaqW{AUF65ngoBU0A4TbW6g{+J z*sYj45?MY8vmL#lN*GT#N|<8}P-rspS9-4c{UxnTw%UBrj}khmK`ly&o-iwFOIv#*!FxhGn@m zD-97%M6!{kgb;V_jWWIxhaAdWgHk^^%rVZ<@KM1M?z;Q1E?>=|qEpS>hR1+EWNIir$7WKs@we z4({XYd_QUiJtA`3!WYwiH&$?eC77t5?sa9W)2mC|!uMf3lRUeDSrn8h72l#RN7!NA zK_vGfE_~QBzq`-c3cPaBmn(9IDs#M*pgyVI-et}GN&RV^R=t?(um?e%JG5n+eH(M8 zW2^|Gs5`xOL*xw=rbxz{3K4bDfP)`zyu3AiM34CP`wq4d*(dhZ{#?&fgG*~0JGY{Vgzue6nlcPA zSMa=M?qV+Du@#zPK9$&4h*DJW>al0#uVFrPPjl6GnDK3tJKsJ25*!#@Bdu?0y-@$y zR?Szhn6@)$@0Z)4YA*^nC~Hxv5G;cfHG+pIb`P z%9}KsVRO5B8P5%EW~b9{2tyHsd$lXdo3E!v)o8S6o|3>F>Y<02Be><_SerVq{MY>} zKf-?K$zaGV$waoDFX)MrU25+x4$T(8Zav9+B6nzV_+*QG(y8)Fx=`9iX|46f(Je}g zo{Aq8wV3==jXI9{hCX4=PZ!^R{ZX1ztXBHkXvtJlVbrKl@qy;}TG-iUz zrc|?lt?1$n&*5s2P<}&1d?XO(H#g29cet8Ear;{lj zx;`eDBjtrsBc#JjoA=krL{?sxB$@$)7Sm+Rh{86JN9U8YtC^ z_&ho7lJ0NM=m6h>%Sl$F8;p1nIBxMie`0_NOqOBolijU%rkCtL_66~!*O*E2FMW?#Hr<5QC# zE#t9V)NCTNVj)y4zI2zL&kN6oPv@%rod{Cyb3{$Y9*HBo4&VGGzPlG&B~HYM$c)QO zRAy4-RU}u;%ydbsKQo0#{!C=zAhvvP`RTZFIexjgGGOv@l;i#*#U`Rg!&B&SbWiG{ zQ}oX=^?g&E ztD*)_IMJBBP*Vhfycs~Cz+e#Q6xbEG3Ie%51A*3!K_HH&p^VG2LD&%3!YE!N%vg0S|O+1%?F2K$oyr9ZDwG@52rCJHN2VdWD9F|VvPAaMt61icb&UNbr@Fk z-Ntbx=02O>QzIWw<5_Y)B-&?v@;jUO7BLH%_lGuP;kO~dwUF7iortvQ(Us#*8t#4S znwkBQVMU8YE9X9ag8ogXqB~otjq(OCKJ@nD+nGE6-ycArU&J}l|9p~=lqUb@mw~}y ze8>_Lz1_oixc&G4gC5dN-TmjoW9cgVe?Ghjv!?&&!+X3z*?&IJ(EfWNz!?9ZGu``K zcm6D@fQ$T0@$(Jwi-S!6(?5_a4Ntv)p4_Srfto(jw{5H&()&% z`3f%i)x7vPysijVi`u}R+(n0oQ1Q#<72JT+%Z{zgCJ2=6pHuBakFGWbuZ}LRP9axI zS7$p{r$!6$xki`k9CFR)zxp5nN8i&M^uQ-$1<-#S5d>;p9kyPay_zDQI5Tpu{ST*% zp%8riyK@uWc=!E{OSU7RlBYAIY&A7 zKRJEMzfKR>>fdRMbeMl5AZptD>kR)o1IiJQ|M}M${&j}`YtA63>v;=#^1y9w@c_i~ z`T5w`;o+Fp?#JR{<%|*_ts5f0%}TwlMcE=%7N@4#+oyEl4yXdbJNjPXKb6A2B`eZc zpnr>Sq!s>k2Ba1Kbq16b{&j|bo#Fpvnm~5VD2uL&UH!pjR61`FAAut&I#3pWI2tx# zJL?EI-|G%I?`Y6NU3#B^Kn<1aPzU}Cs36Vo-$6t%V{qh80)w>mrYC^E$&+npdU|$t ztT+`}N)icmW>Nvv0CX_Gf&>Hx++k!!5R#VO-+4$O>@ro%*^QN&CJ|u$+FbkF1&6o-}Ya3w%4!L&wnn z($`DytP5R-!MARL0{uO!wlRPk1byNa65`?#YJgpIh&wuUXt;2$l0a&OL|G(ihgoiZ zn}>NJ4!2VIQ4r0s9@=o=h|ZW_I1+z9C50E*$TOj~{{xK$Md>w8BtckHQ&U@8!^dA+ z%g^uW;o<4&v2c)_Je+T3DJv%@D=W9*h(sp8eV|8#L!o4X?h8;e3%%c48C8e-J!cXsODSW$_9dkPCre(#=~ zR2EN9rzAT)Ud+zk%cRhw5xYV=$j><(My06e>eCZYTh2Jdl@L7~mz1w?g?Gl{lh)zU8L~ykPAYGy_i%d4 z*z*Ux4`Hm8vXZ;+2%ausdiq~Lakod{2}R+25kTz! z<0SRm)pfkAx0e^!s7*y0Jw5f^=`p@-&jswSUS2HEe>nz6MjH85*O5pGhuS!JygxIJ z2(3uD(i0bcN?09GK`3AfmANRL!0FALEMps(sZkQ>c_s@zR2{<=KR@XgKOd++c<1K3 zKj;*2HYgW*65N6-wQe*0WJ-|#sEydw)ypL8uCKWrC!dj(S*&@jMB~Scp5xlc$XZ7> zb}e=KB0;{9MnilkYncGa2H`Ywj#95DeL25be+^6f;8urMut-;^ST6Q1_aV48%>`wt z6NjXu)(_Z}venJn!*^fEe1SC@>(m?xdu&dWn{}}3A8%GZAYLG^?C0vm|1&W)`Ceb^ zY9_#&%HE%U2{DMENv8pVqeR!K;FPtbEkL;F?qz71pEvzXQP6GUbJ5aC=) zt0$&hR#e0c>jSX#n`W$A`DO+A z8kYAE$QklN?KfFtrk7`Vr_W@nIK7J|xOO3XId9r{=@;XKReAX+GcYSp=Y4?;lLc)w zLiV>=RH||GPIxRoh(eS9Dt$vzBJf9*6@xg}QtEK!@R0hd4hA!VEYTdKnZ9SjC^uDE zZ;Nzl_S?-FWfD8O#e(#cq~?p}WAgcIAw`P`jqF?xT@>-*@z^2F z2eZ!o81Zh~Q>L(}ne}gj>eUhvr`9U(<4&rWC0~5>j3v7&s+Ecc0*rh2$*LC~V1NK* z13%ztipSza!h|8IOWF97ATxpgDG8fRM$x9Rsj*K0O{FzSp$=B5jC>dFJEF+@6kbvS zXQMUSaJ#w3rjtSnOPE!3L<8YnhO~$NBe>lUs9S40oOSI_W~p9;Xl*GElM?LBHjKWy z8Z-2s5}A}RF~t75saw0@bCi#Fu@G5bzj7VOo&Mu_-4%tC}Yj&s$js!z$M*Ije$nu9%=&}voFzz>aF`EChw z5%Z=39`Wkm9NQ>7s_5c;KDPPfq$RWI%H=G7XT$LU2sFx+;mC@Ci=&=Ot2~lJLJGuJ zS=p4i==tb^XM?jl7pu8*mxsBrd$YY|HVSJbLCTy}!)1It%B^Fp%(#@66D2xW@zGJ| zmq+yUZJJP_-7T+_*l)+P1^ilyaB_8i-U`cgM;hoc$d$w z?Lu$m6rrUzG%f0&RbpT$kW1k6n2rhmK9k|zvkfXyO$}T7?OdU9e#E1lrAB7Qts#RB zFN#oApAv9?ouzkMiT9Bz#oKa^^vlDAjIWtH+=w?+J?mS@>0D|%E39cKdIOX-F~|{c zkeF-a@4PbJeD+HT?oj9p-*5zStp;rf2?C@1$7pdPhY@Tl+$r{Z5y{htHCV+?y9BtZ zbo6H>4o8UOQGTsmfO`T7R#K+;>&F34?YvGVD=(P(rw(9eqfrcuR+bSgy6y|pk$Uld zU(O!|!yXG4+QA*x9Mg*l6Ps4`rZkrdhP|uvu_uL;y}I2Np0XKz;G7}?&ucX=A!DU@ zK$GCZ!FpISW8cxiL3H%N!Ez~O#{??ZYa3**xWym}ZX`&`7u7>MYhFQ;;AHMwxzn{f zvoJs59r$YGl=}|xQG?fDyZyzKcd<|2H-vG9p(wQ|E>L&wV<2n#0pf^ap*Cmv{CwY( zjEuCjj6NSvo3-6V59Oyi_7+Pgi1>Jt7aKEyuaX1X`ueuF`;-{R$_MM*)(oVAMipL> zRLm!B2D^;<`Un{|`=1>iY&7T%K3F&}g%Unm+sfcG?P}xvu(~uwUSn6?XE%F$vW*Vw z(~%QBv#KASiTCOL+PP0S3%JYx`ua=PVK8tzb$R&%AN%0bZcw+F1f|W~yZwZi3W&U1 z(WD9Qwhi)fw(4?GT)vGh1m_!+3wt0-qW!ushDVrz5kWV%)bu%+btm*&a$R#gWU!1z zg|l9N9e7E~sjngOf>f_5g;{R$LeDa(6f9#$6|;nu=SaNr zi!D%3in4?pMuw)DX*YDey;^uCeQ+x$y?|nPUDs1Lc|Fjj%FVH;2@quVYiTVUHRN&L zSs(Hd-MTo7rGhsbH7ewrk>XPOUmUcAw)xov=XWb#ksV=qefX}TPgzPPB^4BOVy6Lq zR|7mzTU{vmTf1cL(pzQ(_fcbUhFb&|awx49$)*<|RJ0TJhI!@im_lseVN^G=CU zD9HlCsPXu4nwTR;(j=H^@d#bV>6J=XSM49~Ld$L*gm8D&X7@ByfrUD}R%{&y*@5Bg z^YsT=?Dt>CdU9k3JW$zno1+W-+BOkTd0LdlTow|lT7n7*p1-yART)VEc=YanmOot)lA>mX}ROjcf7HW3)yf*2J=Dof~ zT?(Qnpj$4w6Q6>dtEjm2M$L{n@kX2CagGV?+zr3`qH(%xD_0D2zY=)#wKaxKto&bW z4op6@gk3iG1eY?k;&*kZERa2lXB8gL0L=X85T|WOP*;dH<{MAG|8|+!$sZBigu&?* zGw%MNwXm)8_0jF^Vz7Jd5^?YFhi3na!>x_+QDp)nR@`)to_ z+HW2;E}Gk{4d#CNLh@Lk=e7e2OMgabN$U=0*r@n0>;|l;=XOU+OHfcti+gPquZ@$IT#6R{1fwbI@r)YR$e z6L+h{rrNoini-f(%U)mERjaFDyW9Q*>Z)?1?WjA{RH?=X1yX5<`Qyiv;E0bt0?wU4 zglc&@^(>K*_{*0dY?9XI=sB}rwEknTSr8_eB7MyRu3gPp#tNT!z3PCjjh7XT$yV%J zGjPNSU*FE(%kZP?o3Pez7zx-V?hkr|5YbLtO{5YzUQWftr4kW%{rh`RgWi5*!BVm) zbB>j^f#06Fd-e9%4Uoi@8Zac#sxaq!eSH?XkThH^9XUT7kPzENiuph$-Fg5$`CiwVY*7@3?pnY zR{dZKc+il!p2X3{)=y5TOOcj(uv8bFRucc3gMwQsws^IzHF&@p3jGden5fRj(l@@Hx3)>dgLNlyWEbi@pq zudJ-*ct0HgAugoCH2GLGkM%UaGQ|e7Gil$4e|LOFMh>Hr1+{n)31*q85$bZu2V# zEbeib^vp`#1e8_n%Fs#JJ>>!PNIP@TUlpng3%Ln->k_fSuCS@($aVmk$qt zFvRx7e|R`9ZimcQSI6tp*W$~`0ZD7bd>P3bE#Z80{gkvW5Lzx?ZK1kEvKqW+<;ZVn zVr9v|I9Kh#BpuOx+hxJ~6LB+0h%qlZX0AIe$VD+0+EK7`{OddaoXDH64X3|8#&nG4CbW$pwbl5% zbhw8Wob07zXY!jd(oy$91lUV!@L2h$FeQR4wt{ zJD(#WBGZWCq_W!C!~J}&mdweuJ7xo%!Ux*P%Ai#bVvY=9&t219E*F_%e%5R7WPiw_ ztQ3Ee6~|Gw-^gk8b5HOujo@fV^w&+_ixK7Fpi#~cHaEhsPh`ohDu^Uas@>Vx&s z?nHrp6lz;lYR8#_u2Bn_1088i;L%A@PvaCGhK za%yA}-?1YGI@+3cll^EfhfYIG+$$LV{d)AA4zm%>eWIFJZ1B7#?Do<-;#A1amI0&@ zM)bY6tqqA(;9f<0HEak#@}5|Ti60vCswZp^;*Nb(I*M=!@o31f%a~X!PtRo}r8tkh za&dkp{Lb;9^i}A}_Y!{rYTR~IuDky~0H*6z;$BO{Mm@bUk8`||Nt`2?$-G>mMb3hF zkf^K$W^_f1O(_OPMP{;}Xh=uSf=uzPgu#r(4or*~YA33{qM0A{-r`HQe(Br&=g+TS zf1FPe+?23vm$55lvRZGO53qZ|UNus|UKJ0ft-j0w|2(A1SBj3!6w;*lF+5z;gd2db z8nW*0xKZfVz+Hy2iGdZdyO!id|#(Wjuu6(!m@jmkvA?Xjg$UT|m?YnLbHXJEI zpo{I*#OBlPK@Rac#KxSTbN+UXhGLZ}GP4B`Na^g(O?usW###69>O1AFZ@t~E5=!Ux zR8+#}G^V$i^78sJ(o>2oyITX6_;DA%41etVsn8 zzrCI*JEg^9E_!`M)%5Qx{A2j2P;A5d2$G4&Onj}dQ-aB-vK~oA=OYHHZP`FSUpTnx z_AcBW7QJ`1ey~yNWYp}q!}D&f>i4q>^(YHW0nO#Wb;=U(J+IM&BKwyChgE}-N(#Pb zTdmUidSFy-_jCJ0*ip3*wuSE=8a2_iXr*N-yR%z)(l>9sLAcOBaP)W-TyTGl!a#Ru z_V6~$bVc{!^x@$a)YECsakg*LSRXnjG`**FWPf`nh7twODk(c>iuDJi1t3KFU8q;e0Na<#i!N;=BEmO)ssM26i^OyYQ(z;#Hjs1_C)$UeS zo}N~WW%-kKcKhXMqH5Pi--O;x&2MDK*bcnhM?1!D$dvJj1dztJLGY|RZ(|kD7NC{67^&}3kqy)6RJhR-FL#Q<84#LI!rS>hTn2iru4O`};UU8md3re63n zKO+TY1)O}&VgqJo%Zk%M?nos%jl~6pgKsmMmI4~za9zjV?DnvGd3d&1_yggG(!VOa zk_)6s>@=M0kTl+3*Ru2~d&liaj^*VPBPSkq16v-oyS$b%K2&1oryHTTh=l730r~|w zD7hG9zy&r5`5d*yRDP$qCa*X@(S@MO%9Xy}zA<=vGY9jl4AY?zXntg8+v-v(|UUDwK!R}UNAaO1JM$7EwM61>UBSf0&Kixt2U1Qhi zmb<*{tTQm^R~1k4t0v$B2LPxw&YW-W@2?FH54+bQxzAsn_}5y0>zBb~=(JpZ0uSR` z)|4%gZ)Jz**L!Sr#V>nWSt?~_Bqv|fxBu17-)=}#IC~t<)K5+>d!qAqGMY;Do+iz1 zw^nxj_1ewN+}w?aBkvw?Q5E9rYYJqcBRjH?A&=hNz<-*SBIM&?YwLfx+YPFo%^KcMs+&hL%01JHTUi2U^u2Am~y)q?Ki+^ts^V?#p#wDJ+a zW}&!*^cX?eG)}sBVs_ud$#)NW=LnzoSeL2*a|!f5@8}S*)m`crFtsV>V@o{nSX)!pGXp-=w7)&Hs zF*BuW=znPMCCI@;N?^rLmdMUa);li`oQOsdkjsx6#rU6mPdB>S4ia{+{bHLFjbdCu zAh&^`pzddj}%}-B%LY&MX4R1pC-y`FEI{j!UDOR|gCHM~zc>J+PD=u7)Kpxw7G^XeRdSV=AB zj){qhii#2QJw6!6rM8rcrcH~VPe`Z9$1h0hk+YUi5X3{u!eN&HUcw(8FiBxvRj@CR79S@h%)xxP9h!^fCc(Y*x9 za$njPY$K6O_Z94AaVi@sFzFwWA*VEGKoKpqIPII6K~`c*qw0;WD`Ta?o-vfR*4poysND_4cCSUAK^NJ40Vg6h zJzZ<`j;1Z*OE{@QKLr5L{~4FBoLVfRq!1OWtk4QALZ_jDqU$6P$ovRwAMwnzI1`A9 z#2if*hi#D!ymgwJ?YS7lOMSArMq_YMZ z(xR*cjHPXP#c4Jz69Oefj!fzAffz6`U*{uz)Pkj~^nXsI5KOkRTxO7(T8vtMG~f>i zK!9UG7BXxJ=&5pCkII<|wdbp=ot>+z|8K+}!opCfu<&jd`l;if)HEt5$}wzsnODob z)3r{sznww}7~9)*FMP8I7VP0OGqbZZJx#z?Ufx92jEjwpgM;0R@_fBz?p$i?G95zE z*`A)YVQ%h&Ls04A{NU6oR4JW|THL*;Osr7{2J@yMKLgi(_%JxARFb>#p!Bt2@U|#N z!;+G4&H0ZXaOi^vyOrB)<4omp3$rxaq^8Zk*)sboae;f@+M7jfhVb5ksebCq<5Xa= z*V-9%ZNO>Mb`9`@N6DGZUatOP7>A~TjX=ZFE{vuN-aK)l;lntjV;-?alH?Tr1sowH zx#?|lhW73UKjTJ6!^3xX!^0abOdqha&OmFwer-J8oZNdo`_(&x#5DH3I-v0w>7JbO zK6!0d43o+)Zz_)|Pjs58^S@}1))Hrg#+p8$$V(qA!o+PlS^gg6QOXoL=!>%)t}Wru zx{OS|A)%O~J1lHJTs+dURo5R>uZZf&!;YkjvNm-MHK5r=HX^0PLIzKoyP0pFR zL-o)>C9w&LlWbCa=hXPLXVn|!FsBbZyqzhW?H>WSme44Nd z>mHAs+Sz3{a^zXCLh#a!@kr7#-42P+94?kSX8lXwOOHOGCqyOpNqMZkQNc$5`>ped41hvYQ-fl97A<*GP5^tpxunR|< zokR$i-x1=UH1df3luRQ~bB?Z})00jUm+aYr$a*(n&5h6s2&^7!D@D;e<4ApFpg&k-gsjD9i|lr^ayR)% zR04(N&B+pyg4tMVzXMGc_d!zPpzooiZc${>Mjql6hLEf)u`B@3y5{^-z!x*ow4nv$4e%* z?NQ2`tS?Q(4(Ds0t`9WxpzLN%R)S<;cY5SJY3p!ZDMJ~%WEPPVuw4a}NBGBvnP@mu zke*!Z$jMImTreY6@RnFgfNDhL32LNd=Al)HgnDnF-7W?T1BP1qNq!j9sbtXN!#M+` zHnuH2RVB43!UZ}kKqxmey91AXsHD`yvOf^+#NnuU7~i5Gdasn(^|MSJ?dg|-B29)U zZ3bhvBqomj>q=}0_N(MO)kihqBk_VhffhkD zdLG#c)ifuZL+P4hsXdotk>#ZT7J8(Q#_hsQekm#65D^s>6ckNAay)490a(FINJ*=+ z)5=W`3ugT7&CP8px4))Lkudg7gqo(HjSXK?e7ag?ic=_84oAdnmM;mXFurdM<4^y- z@67HpSWh`vNp_dkTfPqiJ^sqk3b;%k>ap}He(T=a{`w#!4^Rad7_AcZMOvl{aLm}` z+oN0So{w#`omzzBc+sx3>q%oUWYjr+hb%rMzR(d71X+1?KMK{D`cn7$B`{3YRJM@O z4eSZ`vL8CFn317FX5F>LuN2ESFWCs7!OSxIw0#{xG8ptFmC&<%@vZI4KP`ayT5xyI zzDq`1tlRhY`JdWQ`hoY8y`~v!O!WPV>@|g8-kZ=zx@t~$6xDt;T93^lex!Sxb}2uf zoi)o#lr07yw1lJa76XQA0W+ zhhWCWGZNVgcxv9CgD_5A>UQ2-1*{L3K^ACPbQbFsQ2?c;>L8FOa9>?SWai#zYfua| zHHmyZZ)Db9?qbq}QO71bwaEw}zLdRV)prSkete--uuXuJ6$8e5zRo9XA6Pr%e(dVx;VNYBFcTz16!H~g4Wsn~pIv$KhCjBr=EDsZVtqY4ulDod z@ar1-*zX!;+E~9jJ(#YA7>&am{=?Ne;;Fx~mTeAAdlx#YkaCU~G>+km+wKI`dPl1m zrUl_+vKn%|yK8w8rj`sEnEx_dod4C6Mu0=;d%8z$WG?5sEQe$kG2nEXLg+Pb-bN?S zE7t9PYH&ZH^UBLt&dkVQr}}7wqS}`J00K^K84VVUSTxGHK8vC9d!%r8S3Q6I^su)# zk%OOqV-{npwz3jBt*oAvkeon#s>qQ}yj^wA<+1cqTiv2V(=5!p8hW;_31(F(iOB63d zO`qe|@>D;*)7GfC0bssWQS{ltO$iDb%fja)EpCZ-$(mTEj*oYr=SJ^3Egq5VM`r@% z!{;*cTwE6qPqXeDIPK{k9O2^2*P?xaDzX5wNAqnH`4;ORiYEauX}|zAtFkPmq^JW# zczDq5Urmjv7S^_GX|-19?@_zp%MbnsT%gVf;n9NpBu~P~3DZ{6MPyNQtG+Z*pJj!aNkAITO9lpKis%@)F9D$Ds`5WZ zjIz~}{#O1bdBNltUc6IOqTn;+VIXv-QUqk)s%Mtg4L3!P-+WL>US z7d3q-r=-T;!hQ1%UL`=}D=KPml=A!t04}cUNRogOu0_xQV}`V|bEoa)D0r_hzNYDP~UC z8d@VdI-PN;VfkJxq#c+dWc^Aq2FnD2rWWfj+9ttHB2Rp_S8PzA__{5v#L7M*{SXOP zLm3v!3H*c3yrv*7@_|CjtPhqJAuEXqRN}L- ze3k*RBw8+Txva)wFo#i-clDcOuK6EIPYFt*E~PU;`!$!1!|cPg)MCDwrP#ik6W_j- zn{C0la&zhFb91*YdKwP_pTEW-m;j zmv5XHMT^!y+lz{lM{rN&THl#f95-52>@G0N96%a9a~bQ=L9NJ%Tq6Dyp>%e<`}XZ( zM`R)rt9LLmm{;n+6V5IZ7YCf~*E7@&+T!8IEj<5@UPM!DuZeZsCgFrtk7IN?=WUaMj?aG2oJ#NYMkRpcgG)iQ~s2ZSfh!g0>s4pfRjA7MQ5Jje)y2 zPF~N*(B!;rw(M%!YI&xZ-`?-F;l^3voaj*nUoUh!1n7xI*jUPVCdr4^MgjRP-0z@h zTMJnK(ieFN9gY!hZ}jBnH)@)h5t(4Ll6q@l`blRgac2gfe0*u*K^viH_XHBNnEMFV zh5;B^78FH>j4)udbV-{>u)L;}9ia?&Rye0>$r)8Pumv?uAhD2y2doK3AN|#R?ZE-R z6ANkFkD1le@nD{YysfM9*CasSs7Hh@z~FV=?Hv)m=MI!s@oY?Drj>6=PU8?cIxBsV z{?Os3H@h?Sk;I7EXqfZYi@#wLYF2imWn3p9L5Umc+U~e>mwhZxTznJ8W<#C8tUyTsToY7 zaYq2<-G)!sb_ZBW(=J93fE$7|F=Or2U7}kKXG2^0#L;Wf7UByQNSlcw}ZgmMv>9I0EO zhe)G!;m{@)+Nv*eS3)`hO-J6^@g_XlLTj*gm!A(5yqXXZfQKiv)?%UJexlRMmDT|t z!_Q1V6c|2l^LxqpH&V&@7Y{b9V&)L3p3nNGp`olLN~bfEE;u|Opj}2jM;2ThpB&*6 zqH>{=44-ldSsrbTFqS*Fug;I&N+#ZqQjTQ)7@FzdQ0sHxru%W~;GHSV4_TKfWfH^R z9YT# zJ!j9mA7;@0fdD++cSjS*$Q1xTQrrxz(+Lu8zJz0X0&=raX>u6=6%CYb`>KPLy@ZX` zfoFgP%@E=h^Iz8hRyH}4h^Jz@;EKN>rOiMiHARzL=vNW&;qv8QbJP3qf4J$jg3C@^ zm){*a1Luxa`KN{8VsVyK87xlko#k)RX{G2zWcauaF*~izU5C?N;I}QUPk_Y*iMB6c za$GuV6t{GT$`Cnv+f}#>Q?5&0)pp>r-|z04!E&~Z`JRytD|b}0mdRPKGe2%5?wfS9g4es-%e4jJDgYLu;sFb7-k_K5{W+3h zdw)zC6v?TdNSV=tlb4X$$$@c~JX$`HELsU1+%`WhDUjFW6Z#<|N}|M)gP8>DZqv6D8n2w-m4UT@{DMU-Jl& zJI+G8D@dIEk$SZ%5@F8jZhfzZC52uiMhG#-3`)&ML{1s2GWMwG*LIJQM!+bB=O-)Q zS7`a{Y-;>cWmCWw4^-M172XQFhwh@&3?Q4-Z01+N43eQ<2ROovdhc9NRCd1CSqBjn zsuzkJi;2k);1u#c-lCo(cV}6Qw_obxIbsEJ5AJEbZLW3hHO>tT4fJzu#d`7o>MPVb z2T@8qkXtd@1J@m80(7pWrCOlT!C@_BWnf(i=>1J(Sp@nOI5x(|H#WBJct?U`co>Kd zpn*(O>I%GNi>k5z%Qydnas@aVxgG2MZwN+oeT)79z$Ra_HJjh~8SQuj%qV#E z+VcO}8fl3CZsj=2`_b0=_r(t=JN)}f7^E5gb%y`Z8UBu0&UonV8m%E24LEB)?=A*k z?QvKnq@lRNCK#}6aSLEN=Wwu)+ix~*9v%S!9v<|Pl7nfK>;R1?1C6Qag9n~F?5rd3 zh(~&F&~GppfgK`;^X>VyEU96rwe|S86<99~tR+|V)EnYLy=}m@8~H2r+9u!;vJ>5C zGV8#=h@k|C-9i-z&`*Ov4e97W=~jN1_dep?U2j>@Zq&+iS{i$>G(XM#h-_{woA zKxYY(aKb}LR|2U9`>V;#;EFu}j0*z6o+G<-fZuLK5yRHan(R@!@v60e_#e3&v~^8O zi$2MTzEE%enrWwr4C4|IRQm=B%mCiN;D7gACXCxwa}2qD9&{OSxXoA4q7~R+h5j0R71IJ)y71THw4N;<7L2NJejI;ep`w|Tgq3+{WRAMzqfuc8C3KU z$&KCuDV-zWJ-NW6JG{0}6P{vPZni7jY%(>>@=#3=iLrvR&{_o0{s+erIEmSIbjTEP zw*(rN0VEeW*{-!35>aGvs!kQ#<;d?q>1Y)NpeD|VUK@YD*-jRR+%18)zMmcG0vD6{ zQ$UB((6QG`=S#}Vut9|^oj?OND6|JO0ANFMavdGHYrt?$UbV7AzgY00et>iRgK;9F zG6VTS4>O)@rDh%Uch@?Yf$wLXv{3}avrs{zrUA^8=hL3t61KH?AK;$m*@;^H7l19yLtlCVAU zbTpLc;d~o+tKf7S36Dzo;7*Nt1I_#Us45Wj%T7rBC8w>FYum@MUp&$+t*|84YGbniJhwlUGWHIKz-wgS{4LU;1MB%gCY$lF!#JsUgXIJ zw$L^WbQ&yL9#p->wf=m}PXYOjM8iMd_*Sssf^MM-N)!3BE&A(xPEN zjix&_>N5Ul9Sun1Vf5$RLbpLa4%q(O<+MD91+@(W9#Y!}+Gj-HmoaWmp&L(9U%(M= zz%k_Nw1xWe(CO+d;A*=0VjlOZkaamAw4}w?)FO zXQVPW8%yYp^XkiX=uO6(vw|6K=BfCX5xUg;nMl|r(}eeaZw8#Yk}{8=G^D-345mSf zgQYwgUfcHP4o_V@JbZjSr1?CU&o^`C?jZD%lV)wnvr`JV?N!&bxmDLe%x@>LIK~R8 zYIy~Rqj!D&oOSyMuY;44*R7Rcmdvkw+dqSpH@_bN3(=`9R`r}QlaKl5pN5E#kKoO2 zG5y0pRfL^_o0a)+b6k1DE)~z!Zqm%)MmLGh`#Z}L7x!i|!_CFTcZ0E2Gd;cOnbtVF zgCJhf0&A`n`J@ZFai;utSx!^h0d3JNj={&!T{#DBm<2RrX*8CxgvJuU@&r0TErl#9 zuYgeiuM%Sv%;Hf!8lZ1|JSdPTux>ERl#M zfMaD-6%vxXJl3} zBm!fyidzfGIy#0~Q)OkVZH&q?j#u^N<9nf-2yBH})xW(T7!V5(mtp^{7989@^OAXU z-HYNMBXyFcPv|6TWV&J?5#XgZzu1mSX88H|+=b+1MY6-mf|+;BZCXpG9B3R!7jzt zn3}ch5J&~a#YhOIam;6xKSuc~ns=4N_cq9+M< z9-VF?CUE7qw8V#n$vF2Y0wx+3mb;9drBm%H3JW&ct{~@`)dWl%6Z>=wV-LypC|-GO z@3zp9Rwa3^&G2;k1zpXB(OBH-r&x544KKJpkpAm>U7m{l>;MW2q)V!V)kKw=U77uS}6op&Cra@DblM29PaQYXN%V44Bo#vAOL zguXF2Ki&A>()P<#+B45V z8q>J2{IV|VePd&3npp-O+? zZ1OS3M>CgXATfk&CRnT;q_*Oumd0947*n7|WiBo(VO@APCopN}jzyoiUGW5Zb$iv$ z?ilzCiB`0z7z&hshwFe&vXP|oE*Z{H8B3&DDgg(4*q`J4U0y#r>%w9$bi)q0Ps4|D zX2_-Di;sxF<0CXSGyR%uZf?AG z6|emAA3}J7!sU6vMB7rHw8jP`W^-=o+qX!8fBuPL3UZm#w~0Is(9`ZVckjAGQp7_t z-Zt{^NK$633-~I=Y=uaiR2CFZ@)}tp3ksrfRqG11N(&obLfytz=U%;{HB9=WEx0H_ zPj90m@wMvL zMR&QhsLc&WH`WWU7<)>zKAjg=lUyDHJ$~^rP1Yytk6Aatnh8~vzPh8a(9TSm#g-7c zwJXNmJWmVi%v8?;*H7Zq@T*CtsCV@8=a27yHY26+PMM-Lxjl41ytJkC1>qak`r`-a zNp&%?FDxlyJ35szVZ<8GxqCOe*x`fWcu3R-%N%S9W@z?R!7l1iGpW6I)FwP8FD5Bm zzVwXXST|*vhDVcD!NYYj6+xv{wOUV|L2JJCcI^`1O*w%_m^4?f&kZ_QwtHFl`r6z3 z`ogL?_o$71gYFb^V@#s;fz3bym2peMBX1b_IZ2&TC2Cbm;mEHUqy2O0V{mSEQ*XA_ ziZA{Fw94)HU+QU32mkXJ=QF12wITEPTmd!K57qmvum#`m+t-O}BoH$ndvcpCaSUr| zJZdY5K1sX7U&faEUSR)=k5lS@Ne|noYbG9%vZ`yV_p5K~@m0fqX|SM~9UH&0^fJsf z$zt*vQ0*kuuE21gxUD<35JE3-=qEe8i`54|O%y2y%zEM$8fN|1D6=Qf0S3&luyg6# zhq1AL_F1=Hzn*ZpkerPF>-rvdy3*lq!U&?zXmrg+g(1q;bF|QJM4Zbn$MP=~cn!Tf%7^70$nGd%q#+aXLXz3biDmik2R32Mnu4(xpxG|a!TPse~r)?~L1<8u9 zIx8~Dy5skqAjzxz;|kw*V0zK8g)F(A;Ej zui3y(q_5g1?EC(;>Ec0;1X0(dg9o|gBrm=tzoF+I6=oHSVf zHR$l6gSC`r)Uz5KqyS~Z7Rc_TaQSD`)D35>2h)LzAVRWujN3k}KyMbprI#yPb(@y~Ikq zmHf*!gClY30%K#z;&vO+&c}W$+y6pCKkpg`aJwL_;ujxGEw)?k{6p42_l!?(rL1-n zn&f58s%{H?k&{xe+vP7hd<)?+;ON@_LoVVDVxz+S$8eo?Y~PBAZd$&A2B&LL&l4q@ zH<4_fGc@zg8#a`4bR8YW=6LtVVAoc)Kr!F3sTpgKLMi8&YhSF3`x=KKp)41M>%zds zbwG3ZfRAIbVTD7{?b8o!e(g0R{e7{H{$CF0_xaZbYsu*31i!7N9;I0nCJmV_Dou(M zmxiQ->{?HmkEd=@85{9bzx5x0ukT0392WQF#FcTPc7(-o$78QFT9_2(J1!OrIx4qx zr9%^ZQitZ8bJru{PpC~qQr^H!mQbhQkc=h8gthx*xSb0^!*9K)!)sl# z+Ud`4q(K*_cD-8rstC$PufR(|f*#0e3WRZOPRYBzCpu&6TZaJnufrmq1cdbX` z1){ktCo=5cJa+0LX1C|RzLPnWV zuBi`@R$sp=Gu9T}!L^RBh@%&tA7emX^sjTF2uubLc{^?=z%1(Q(50B_ETag?U_>8Y zB~oa*4_cR$DENzLwZh&Xarup?*mJHEkYxKZz5bu@&JV0^H#=l)!c`;(#_fP;g06IS zqG-m44COggV&K34Z38_f;G5}VH&@%+=V-Zu*3p5OqMvyi6R)JQD<=U5D-b%bEz-@c zX0>i$ph=AJGf1q48%CC5~0?xwH0x$T40PN{v38{BBh!Qe# zkQiHPcW9KGEqSg3B9q=;BRCETREG0k?V6yX3bt~mbA>7Ig#r7j`oA807E+E^WsyW> zEsVD%E8~>5$YEXjmxBbEvTrXWbY3y&b2l%;53ky7VQhG~B_4e6b?>75IJR=cxDeF} z3ME=v5}8tNb(d+5F6@fvX9^boXhu)5+VA_v#l^=5a(#wAd=SvwyKhp7sfm2x9H9yo z94R0--%Zbd&A``x%~)$l`)Dd5>JD<5^%|N@_A(%7WFD!ig@z=;n?p>&2ApD)r*Ye5z7-Fuiv&l$=>rg;8R3XQu}J$=^4 zm6MY$P~YVB#ZWNhOJ%-TCArEt-J%-_+Oc9a8#AL#x&F&S)<2*T4wH03=q99z7Nz0k zP-5*McI}D?3uA1yg@vs(2*@JOJTp?)cZ4E<3qvOQFHB5LO*h}n6@%hoVpG#%{+#IRfIOOw<4n>58>fhW{d}QhYo^WTq3XzR z-~!*-bP}?YM5{~M5lF<^bI@7&b=dtxzxB>=7@(ATV_EOz%>NnIQih$xF=kClLxtYg z^$z(=ZmyLM*-@KKnZA3NuHYpNF!1vEspWx9>GKm7JhfJ8B2rdZ^I>b{_!l+}g|+gf zl_fSsQqG#k=&9pE>rTz}+o%syI>quSP#RR_&mG@Q|IzP-k=4Oht9Eh`iGMs*H@f%` zz4p>gyu{{Jf3>*UbZK`tdiy@MIP)P!t-##M82LdwVrz2^+OLFAckyTTWI(azk&9&g zF?`3sASTw(ps{pE(?8!GfqeN-gLnUoP&_hx`wJAFg$tr44Su&EcA4N;yb^R1FViE# zQyoKJwz7wiB_m{p*hhb3ru=HYhYaO4n=veI+i z?Qn#%F&Yv=prJ1E^L~u_MPeSySEs3|kuQ;v!I#IbU=bo=M)y2!&g%k#xA8Vl5A!2j zOre~!ZuVCDufxAJoZf)S1?+O&bKugL{7zsk6Q{9&TThp1g(^pdNrtjo$N7a zcZhbF?%rHFU6OTax9g3+>>wLEA+zJgoif)!WC&9)?Xp0e4#f#PJwBHfyfTWj6HZ(A z;(rz$5JmYC?x5J@kT$a4p^i*3Xx+U#=_VcPmzGTSt`8$jjx z%jCYwbilvgs@qrip?dc|r@_A5;u5a<_3ZEWJ9s^=7;8pMMP&HWu4!JP^Bx$?_0e7# z`iq=AD9o%{Jm1O6+(FdXYa(kLM&FeanAM#?)IuU0pF^Joy5e_yO6Cv$MY9p#^G<;R zM0MeUWB=5YUq7>UrLx1*5U6OxG$9wLR!lqLzmqXx`i?3qmyK4KcgCWJ$ghK#(x#`U zXfSI9oRl>9Lqwyy-VnX@LkI1TJtt5zx|6&bY+#mRY~h7qS&f1SSMhi_H9NM#_FF?s z_@*@cV2EIW**lU1) z_!rCpT8IDPQp^9T(5#)^IdKDa?8Is|Kk-#>-Ul@{<~B%)&)n%7hkpE@o-*Lx)!NFA z2JW~b@}-ka0nUazrKiO6#F%$QUzER3jyR|AAyI+ zbjx2eYwUrlP_)l2=f1x9`SXvjAe$Ii1UNx>hnZjYf4kws|9e-*e>p*cGj1z0boyWgQ}YAI^~Me(3v*Hb z6W#xRs(t;h+%vzb^-w``x95z5!@#?Rk*jHQBUd*!;7Wjcf3mE&r2BB~&!6q>KYxZC zYaoDMC@D!xD+NrnzkmPgN+P#+`0@e>pOhF1yxz+W_LCGq6H!u{tQe;G<_!wxZ%3c# z07Y-!-0JZc4|*93P=4hWj`3^$eT`Vdux8W>rWu`Aj;&Q|6){OJ^YL%Gs zh8eA?4Y>BuYZ&GCEuCf>qxVZ`a9^ysp|*0EkC#K%PnP$5r~=5a*xB8`Z)eA>29&J5 zz|{VUNf>QI&CCpTcUO4oB=d4!_Aj@IfAIir9-AGwgRXJQIM-@)Iu#FNyjt4ccXrOt zcXmR&p-SGJeeZ3!ZMn@VtKrT>I%*}#A3NL|&uJY_U!DdN{=_|scyu4EZ#D&;=Yi>^0mXKJJ#q8;_vKBLaktj;~$noIw5hWwk_P2j} zt<^@y#@X4&#|2_a-@0tAe^gc5T9)a{tShUL#b6PwVUgD3r_5h!5)L2cReYKInPXx3 z=l8L(PeI~GC=Di3yLGXbmcLeGoa@Qx#FiF=DKPkH6{4E?H{cu3vm-w()uTVqyEpjr z*ctUog~hA_V_q`@ARp1H=TLk#XrWyg|S;(@h82&V*!ipYKEr9VeVkZ zAwM^_g%%l_Rw0tNPy+0iT$2h3gB4dO>&W6)@vGGDgYL2s`4e!|nO2m%eOnV#`$~b0 zGqPa|$rsMniT0cOk%9J$+$!x@ok-IV(5S9C`xzk@ehc&HIGpWTpBhRN81&kJ)GBOrGTu8PxiP{T#|%I1npr*rK@6CF9sctgUMy?~)%xvKWJzHoC`re(N==aCJ9 z)|1T7M#rc69lQM(#$?pBS3IW>emT4@pf{{1x4)FAd1OThwc@*CU^qM2Bt6?2YEAI6 zZ1(P;KsmKvly!k6=Sa~Ukp)&lQfwigjXOLAzV-H5=L}6;9uC*7x)hM@SCq?hwUUkU zrn@D~1|#vRxhFJoWc(B_;TvXSI2w8<6(WpC%s(PK9<1>PGLwNMJv(&}D)GC1$t!Yxqg zW{pX^jop*R>0Zk@iKi%fciJd`Gv%-0_f{eAgc~obV!wYl#jxbs6b!h;@4VaXGa^Rm z-mb*wLDRfS0|S1mZ`}&rzNt8q59b^;#?zyxxvnHUK&0giiEu#+ruFN(zmR`4O-gAv z)_6#oU(g#$c_+cTrw~v!abCt|52ZOJ?{|u+sRor%E{)QjK7cK7a=)aCPgI~nZxUz# zE7_Fav-MTDd6feEyvm}|k4(ysD(2dF_3&=A%6G8h*NAC& zioeESV|g~zANg;&@s#u3X%9xFj{Eli0Q`sw2DC>$ZmyHGwD? zFRF4oNx@b!Ng)dqPKeSTZ+?^Xwr&HhQ;H_lE$~p@g*H~e!l|uc);u?tpFcNOJ`Q~r zOK;6TSf+9aoLSynYi(T^N0wB=nssG}i^FTSs$BDi?~NNbZr;36Cya7`A|fg(CMIfD z<+BE8P+N-VC=(Mjz#0MNz;^;Sjx8wm8T$SSWPy&@?>Y?L|KGOo^UN3kzG!?dqp9TK z;PSIKzr3M0p8uPya!r}6GP>x-9&T&vj0|h*m2!scN>S=zxafSFT-Z=C1#9ieB<70x zz_~w`8(FAgar#2^zC%CGAS5?SY-o**0f1zna1PsbKwk<>HUcp5Po8ub6w@BR0|=g* z>*$!9OLa`0AKrd>o=65@NdPO~Vc>g{8KwDd=QvQU933$j$J})#fi8~x&sTz=RQw4( zJ}%+Ix0ugqQ2h!Mqm+VAwWCD~10dJCqxWhzrM0!2nzXe$3KUX9M>kBF%L51i@*8!Q zO17kyoTAe-_w6I_=K4a*!!ogh9+ph^>8^*;j!DWS-^wn5T@eclVPOjk>ivKLKp~a= z`)3DXI)!q8ogTKBF7WWxj=y7k1-^%umlXUqSAV)hizdVS0K5mrj=n_Bc+*&V$JqGJ z9pmik7I-7^Df)~A`#gX?z1?I-8C=`blwI4T?>J@c`b2HI9xM+(?Vo+|@ke-%J4WS_ zy_TmYCMbniK2GLUK5&kxNX6SiR3Tu%Mj^m-MJwwor4WOmvT2yYM19Jj{K$^43B_EvsI;Kt3$qX*gaEV zp8z0EX3`qnP<(v!w}#;4dhcidVHKIVkY5PJqFxvbd_ue-)iKXBE*{JZib@OhGF2Pp zHFJ}R>FFNf-`5tPWH{9efCPuU1gaxs&AOteMjT!rlL5R})6+Ui00M@NgA|_KgF3p& zf3r@~jeEJn=nLv!zxNopgl9h$)B$MCFHv*Y@R1#dX8pvzrD8gS!knKiP!Z@X5b6Qm!|6B<3GPYc)CVfw-o*WTbTL{K>%6d&1EkzbxsWl1P6?@Lg#4d)iFC`^%l~QxyoskK7x!%NtP=7D&sw4b=6`(bAB+=*@V7c5CZRowio#uK{u= zt|niW5O*EBZf!LNtF(puR(Flg!|Iysx?R)L)J-ej3JFbSlJ>_0=wfhlhuc zfJ-w#cL{_)HFt=@iyAnOau;vG4y2=J-3N!QI=b@G8t|u+aY>4us1skdElh4&r{xJr zL61cRrw4cLg1A~ON7Qar5lr3#r1*1M(vM=D)BF}EzJ2Qey$u))jpplk`G#CUu>x(W z3if!a-atEMF(V+a?;QA#sWYM9sN4s=%&Y&OXV0OJ{#TbFa~1xx{O13n7`L4r)=H!M zl)ctQ`8MwYT3c`1tQiDQW6V$yRyws*`?=D7IG!UbUH}DamzcpV7RThpdb$_p%5m&7VuBdqR$`HkQB9yf7p@I3ai7x-|U^N@OAJj9| zSN!<#7<&~M2-N4Z>;Ty++wby1iGm+rIf2zD%9E32`Yp_BuE>An_1TAYJ7ICOu$!AMplt;>LV;E-Of{)tzv^|GzX{=y18oeI z3TCxH#W2*EXDvO)Ms={S-lAi8IZRAxYcWi0IMODG3pKgc(Oa^*{ZxNwWfA#-Zz;>|95e zvf?{$cNN@(3MrIOF`~S9Ny6>Nvp!(z+l84<-o!3lUjg(3jGv#oyPv{iAIJ%C@06dt zu2Zvcb2$TSfoh6gZ{Mwb`w%6Fua6R>G=R|(C*)lDsqP2C_6Yzq zm-}U{_y~RYT@m1JVZ^pp7aaR+yDmY_<7YXnodsZO+oY-cgD)Sr$aE#2Cd|`Mtl4pc z-z*kF(Sfs_Hjh8c{NR6liM(eId3NB7asXV>S*sHQe?7N;av)Kji;KD_a~9a6OUxqm zv)f&v<(YL*n_q&g>*i|6v9$(>*94+)2W#^5sg3#;oJ?Dr1!Y?1saFXN&6Za3Eh%1_ ziYw^?w-0x4YH8^T(`{7!hilv0Wsu*X`LzJlB%t#I|`Simewb4-_|7XUcSF*v!vK?uy#A`gX^3Ex3ZE#C3)+sk~)&B z+uk9l3Z*!lcy#vV?Ftp`CWmJw zys_5Eu1xb|Ycl*bor6nTDvpG0Obxd`$aV z(2*ktxs|vH7O|y;O&s~9Mvq=3CMG2%I&S0U8n!WQ6l>~afH@-F@0Fs}U3fsj{(4r? zA@-0h0)Y?_K`6D`YyLnmCc?PF;%Ea`0|#km6i7LF%L1MD;(=8?XIeR|GFD21gg%5B zwA{mmzDrC-n<{HENzh+q`!xRLxWB%B@W>+>Z@lJfEdeYYr6lP|WMxG|>FVF85}%0CNz8{}m? zN3T^roO9mH3#r*((AJ+#PUjp&GH7L;j3$6uggUST^AJsc8A=iClyeax#n!Vh!^kuQN>IkMm`93L=pS2PL$(wgP^i!iG-y@PHx8{T zcHaAPZi#H;R>-xu z^%ZA_{?6+Dn}WxnDrF^ZA72TfeC{!C=2NeH#RYOaj|YNt{WK|WQ%Q>{g78Y~zxedznP+UK0Lt=Bj#Rxyj8 z!F39iC94xEBo-4!mRzPZ?|USdwK@B8^$^qb{jqVA*+yDzCUP?YcIX40 zNoI@a*Y4_s&ED8<;ruCgIIn6Md!;jy@16*tygD~_Dni&wAVzqY>UElqsy#R|E>ywr z^FPoh*zX~svs6|UYG3yyid5}0Z&7(0AnMBw*KV8vLw)JyGZ$=e*p@G1=?xWDHWSf? zPoC&`?W@ym;P z8jY?=FjqbwbfPeHyWX~Y>oQPNPLP>+TC_Q=W*}lFb6ZC^=1l1F)7-@pvSP!>HFh)r zRkNcl7%dtu%cVKb%*+Yt#YW~7`vWB$_qq<8Phy0z*JN2;Wfdo?OS~lmHoQLB&Z!~u zFD@iZQ;kM`GW|v}ri7MJ0FJemYRCLD;}gED5_gVghq;Lji?r<1peeL9m=}cgV39o` z;+4yNwYBynR={6;Rz`x$bxNEo--~-zH)fyg; z0*U2|e>eNtUkOqlH?q5dzs>e{ed>PWwa)tz1>-u~6{CiQ9-sIA8OC2ovDYFdd6wd7 zbzu?`lyUrAEvo~r;pj%V=5Qj%?AM;XhiC)me9+cyc?OCC`rjkYwejdWQxP5g1yXN` z4xHmgdpK!Zdg-cetX1yV7L-mlb+TT*%72zRwe)&o$!^={=&J=k*`-9!HyKp#44wv7 z^Q5n&9Q3wtOOOXJH2%z;kTBoJn%~tC;iBhVD`+B`gEYUB*BF!E&U!$4?so2>)6s;} zs4x!^U&*h>b9l^8uTg>{S#pI_B#EI0B+>MH7BRHW5;Rx{)xKa;;ChN0FW*b^^pLleW?&mPozv~dl!q4kk@`a?g zEdPF35!f_$|j*AT}iAVJ4x(`iV7(n z6ZF-Y{^1+K!dOYp3|yIj&~QX%>1^->-x_t+bu>g4v(Z|#{jS^~WT1OP7jIA;+KOf+ z@NjV}c;rZEA|hL!lS094vG4>r<0B%N0PbfYtAA&%;yviRBJ`_25nPf_yo3$4Sd z``2WfU|&e&a*=Z+=~`kqRF&&5oYHmto*ONCvdNO>O7VzG^scX*A3evfZ+M(7{Dszd zELU3+507Z|v&yaIbZzIFK~TZayK%8~q^E}xI-ZOAl=#CJlXpUlThkkPrO z%S+SO!}6C~9G!`J{Z>%QbJC0a+|VV&u+k^}lTh|lE>B@T*`@;pW`5Zem`0C!%$oLn z;L$~Rs=b0%+tIZx<`8?m?;4zu>EHVMc|b+ELDuj6al+mMO_uh;C3Z>5YZks6Q9AR4 z2G)Y8$OaG1`|+-D?w^^5?Nw%;ebwmSqlIp)8>jo}zAG;NkUbG};xaO(EYM*Uyo z_eAEA&WEm=H3}Qn2^-C-V%FA?fDSz>5$fN(qxtc2;GN4@g0|HCw>wqZ^ihU0bp2_2&7(DSlFZo0sltA9f^2U;(hNDB%5;`65)Hn@R4}N)ZUm0yiRQsb)cg%y^wxrI(ezvEPaV^vavqn9T`)q4) z#15qYiooPa1C#d6qC=MMfr-hrmr62ts(VqxrQl#yP1PN1#&BQCf8 zoLN^Kbz=K9qqsP$TK_icYig+!-yie~`JW8~x)8hU9iOUVQ=Jf`fN0Cl$5T1FW2K2A z<>OGg(YAQB7nQvg*IM9eAN%Qt^-OTK3#2T9-D~4OyKP}^XIg-pnWtr5-knjzd#`R| zR+r{kM~~%?k0}%nZo>pj_v}I1HW*g+?wzn9|Cp@oDvj^R;Uw4xQy7{#JNU!p-?w|8 zK7%@ucx2T5&EP5ZY{?r<2EW;Aczn<1!FSr^0^ov@|7O{YZqX1_-SkDv&po%B0^g#74%MnHB&Y3HIb9WdeseahZGkXJ=y>1{Z39Auf!(`VRAeB^puoP@4CS7tc$Q_iyO|Q^?=t<~NyfyQU$rG|5F~EV z)b-wyJyNkq5ncRdE<(8#j7$j-0-PIs@{>FC#{5*3N zQUS_V!VBsSc@uVQ*5s_)O-&g$qf6DDL=DN6GSq|wzlD~0`L_LMFNhpGG+lMreCo0Z zqd1D>r9gglCFn4Z-0=r}9s?2$O0<{U|2>P)vRBm z#WO2vVctedw>kLPS24a@{-xx5YT06^jPc;wT9fuL0Q`?E6VGGq3>Y99Cr*U1eowj*IcAiFK4r4y7 z0+i8sTcYrv;Tn&VGzrl7E*o6;lavLpq>ttK>8v$jG#}O)>jiS&etfiHZrC@&u1@=v z7iMN*`hW0fAgAaX%{@50#4Cy7%f!zm_0@3vEilDleP4~2CLrEoI1}FZX1a_4@@4Bx z?kAPbY>{yFuzdOfE$Dz%9AM->LOZW!`BAVq))ziVIqla=yoRLKE2xq!OK*t}MtRk~ zp4g?C!Rx-oX4La za`z5ike)eRvpW1$r)HIZLpP2tPAS{vdA1T5<;C5Q*O1wQirojg)38F$FWDL~I;W`T zb~Ii&fspMZbw9sGC>*)VZ>O$_<)dz^XXaNHqeZkd>k>g5=*IVmU^-Z<*kC{8yJKOF zJH^0x(KKGSjXa0EywO(R!a1so_Lhy+BG1g+B${S*vXATNyM@2NR36AqF!Vk2lALbH zDB1N3PEgDVqaGS#Mqjb5{~i6tmU7kxSrXDGkzIj}rQaA1J;9b?i#v3TXsTXe&s&h% zcVlfyxpm(wjq*f!{%ki*wbEeU_)IUuq8kJ0MWpIPqWwFe*)0c$e4BWO&HP5GY^1rq z^6CfBX83L2jw@hoC1M^iDkb(QQ64-IfKPZ%9&{hwx09t)aU8~_8LlP&_+coYGML@P z$n2Dyp2T-nSKYj7oggQ5={{Y*!*jzet5{1U$}mf8eu}c}F&J4*7u{@ZdoWMQEU@%U zRG&jd7(|bnR4=qHF3JOt63Qk9-0tKw@JV#^D}@<0*FYQUxCODS+@Rz7T{&EF*3nh_ zXa~o6j|An>;WVkyh7D*m<)GU>kR*N0q|vuCNWarfYrxmj8nA@t>xp3Nmr&o><`1wo zIU~X+M?RWJtb1n_e_mbf?JYHv(aSV28Z8bh%=eLMB(+`Xw!#f!SrY+k-$K7CUL3O# zU)mbxqdMn2h&g386P%f+3_Miexl+rq8y;(#w?SSWl@Mp!uoVwhv~1V&8aee39~?&V z8w2`s*Be5O&(NQ-)meXK_*&rjNFduN_v;?7sdaH_pZdlsrQVtl_d?d~?BmCw0mlwR zAz!{3ZHp-`URolZNKY?*gywC#_Ptr>M)bNKflt}rc;zTI+BgJh{!)Fo>eiW|6&Pr? z)T$v8s#)=Ve$qO6%s(+pyr=E&eR9A;cEjQHuoI}AlEg$n2lY4|M!V(udl-`M&(WV# zBkQ#aertF^)m6VJ^|2;hv#*5G&Q{j( z-Z8(7h2wBGa`xfYPH`gPxS2AM;07=}HeO|en471DRTwUswWyt&M|Y$GXkb}(t>P78 z61lrLRLJn3os8G2+?deJb5Oj3GSAz&ZZmXHL@6&_zV|2T1WSNZn9&*34$bU%V9;;Av!d^{V=p8456R};V7cK0;2>?HY$3%i%p}w z%sqe-Q%}66C08HQYG(Q;xi^&pwQ~hJF-Z}x1~c+e=}$B_`_~ZFD?`J>zO!}Y{HNS4 z8T2^PHk$;xI)Sgr*tNvslVhT-d4;lilXkW=TZoZ{_wKT(0((LAy2RAk`pbUs^vV)C zCvlHrlV-OP<%II!{jD6~;|Koj?kZZcNM^!LH+hC&;6NIZ^|_fDG4rTi_?z#D7sSry z@&$T1qm3C*%-CQ$8bt=B$VTO|T`^1R#dIm_( zBOc68j>DiNLyB9)+-VZ4$sjvXmhlwj$%ynqL8N= z)>U&)qHg*_Y;9MgvSfiF(Dd2pGo7~TiR3TAcGc0|V7pgX&Z;gnGEl(V>}2kOFCjlQ zqf_+kW6KKopGP@9WMfR$6d$r6YFWEhWkw^bvc)peD}S;LJBT@HS5*xja?EOK>)k01 zcqlkGo9If_da>8+WCN;&d??F)YqRzIuGV>{cJx%oyXQ64#nxbK+(7PM2I;?B1mrcA zh8X;l%Y&=*nA^ZIQ%heZ11)Qgvgs8clk|m~p&RznFQa46EmA}eUQ_Z?| zc4nSxc@HW!7}RVG_vtB0##Qx5G3K5Gl$FouG;4SxYsO~nnCp{q_Aeiomz@I&tE$L_`QGZ*x(dA39VnWs>salb=gKB8 zN$v&x-)5mQO{y_Ef02VRgEgS}g;{QI%AZ?iL!Ry@2b7l^Jrg7BDfW;qm}}e;ovDvj z9)2=vZCJuThSKD9dzKit`2utL$`w-OB}x6`uY>iDL?Bo~Q;XM;v zKm)}ivbLvM(Y1slYwJ|kv4$Y)?1nu}ZEJ8zN~mVO)+;s1bl{cLF}?8d#s~}CY=u*0 zd5KA-o-6W;60cjojo+efn(pj=+U8{#tFY-w8xoQ4ZyA`hs^dIrt-U;jC)x`tx9U5& z{A^I>#x$cUQ+>Ih#pd$h=96Rs_g7ds_K*X6; zBbU(MOE#tRUl{5S3b84?4Do+1R#fz9r%n({>Eyh+ahwMuaz~x}t%QJWbQfEy9FEO@ zt}Y`vU{0O5%%)waE)_7Wz{!M7D8pdXO+Eij{pCIA#MZO9xHM8ZT^ghljSCrSjwBPdJ?d0_CU=`b7 zzpeD-MHMFJXg%KgOLYXLVxre%dAnxl@*kAOl(pHOrF)=UV56a|$#sMo`QCL-T}Ezl zxuyqaSg>;xG--8nvDeSnPfa~OP*^yfM~cdo*5Ardg3%Nv-Ez{!k&*)acf~7y_EZ(0 z<2Q;Sl`le`m~FFn&vt9-R(C^R@?cs2$Viv|c6ok(sH-xbKAaOKtT#?{n?_dNzL{K& zV=VrV9fE<)4wZLtg$Q3@cYplw9I|x`7K{N|I0Y+NR0YR9X>%jRJ#m9>OXuvQU$544 zIae*!bB$U%+h581QPVU5%0{4Y0GW;D5b6!Ma}ZS5JoX&>%BkaQYGxwOMZq-pWZt~7 zjkznnT_&wziY@K&>IOqy9Aqzz5+p|X2ox>HB2cKmjO0Rs7&=9>c7EFF#$zyMOd&+G zK)ZEqubu8IQFDuJc3f$bfl-LzJ0zAymy7W;u|vih+mp!n7Ywi0*P>&cA`^pbj5r&T z5`!GRove-^+K5E<6e}d>e&i5nt9eL?Y>-VmVYy(q?KNbR#YwQRm&MYegND9=qF1@V zL-)n;ZmGP}h~L61s>_nP6!UiP^FT*t-YDA;f8k7UZ3wzI!)%KX+>o=1FSJHCtY&#h z6?)cS`TXNvMEOmjgaQ1u+)&vNOj%{l8Q0RU{>{`2E)|9LjRF5t(iuK1;qTE4E!DQA zt4-qDp#X`O-={&N5xDS9;l04a1KXkoMuvQgXGQJ_dE>>zgiPG%D7$P@cPmreU$ z;kD9p2*jC8EQ3~8SJxy7cf0VdO$rPT8Y+Sde`xkCq^ezLDjqJ4_AmhlFp093ewSw4 z%a;k8>w4z;I_Hhczzy~$3+kM}3+nJH^bYv_Jjwfop{Qy%j10pS5;SvLRx1jasG$v} z=~U?qwR7|%#A>hZn3$N@8eYe6A2243>HY4&9xmL{ zugK3-0pH1qWgb^O{@C-re6=1~ie*WJQgX z4YlHoa?K9(%#fK`m2c^cu8LsUNB1P*>!j=bc)=f`+ESVtapYqiHJk|(vpK{>Ym{Gq#E$tcRFT3uJr37z z!8)|xm~WZ+&A1MFtFElb$lDq74@xTOEx`Z?{#yeKAO~xVNx!~12&(xgDY6N;QN{l* zST%4f`1j9bRQwCC>0*c9V~yJN9KnY6Y5Vq#fK!k)>WTpfQO)R;`7O0$^!;Dd6Kj}< zzf$DjLoNX_i9bCuU?pMF=c(1*4Lo*FYPEK5p;(K^UlY$I|*V`BhsN(}$! zRTx||H<4RPcjgSM*J6*rq0V({dkx!XAYcR!YXm?ud?ZC%8?Fc|v}+JOWUm zlf>jS{LZqr`{YkVAG)MbQSQ(S9REOfwYrwoeoPK>E1Piz&+TXQ zYX9^UrK6+JdZ%BFiMYhBLx5<2O-!A+=iUb(i4`?#w3X`3DNnqZG7+Z31P2COKmPyh zLrZ)Et^bbvYfJ}oA^wvl(H(*SVCC9K0od6gQb2Ll7Gn$7-#=;s!@7GEq0ovYB_3|n zV)0%69>f@D_^p12yCWgf6B)3A3o?C!+LgGm8LMOMkvK!zLS9+X)kbb;Xhx*-ukd;{*`1#MP}DNxA!M= zb4~xq$w>?Q=bz_KS@#WJe77H!8N6#B`R3_iVNI^Jl=$L=#&2A=V_D-99x0j#@ExVr z=?xD?P2Iy#9Jl|Td$-y>aw9_`K=WnXN?csqKJ##eB=4b>M8iwXGBx|OK9{622@hEu z>N`8v7PPfVRV8nK_Sv`KmYQP0z(l{xZ30KQId~LZQmRP-lReG>ZvPj1?;X}u_Wp}z zW*l{fQA9+f2o5NWfFPpu=15UQ5u{fIAxKAh4bE8Th=_nxL8KEQ^r{F1=|~M#X`#me zA(WJ}vUmKQ@AsU0?!Euq=efV{GkKoL7?QpBTJL(-yWaYFmxrQHPVIC@pN!{|I^R2` za^7vlIL~v%c%uiZv3tic1xiR6tyv;0H84SbQ$+;u#>qcnN}bNi90vL*0+H+v8nd=W z@c;vhMg?!>j*b8vjH5oYtgskpJWCF)t~xrdu6>T+h(LsNjcifs(_I3B_dXG5clJJI z4u5u%ezTwP)ypyuq|tD9>f56)lY!Qo^6DsD2eSN}k%x* zKyL)*rdh1C#=P;S=~5oMogOr0;W&8eW8qz|Y-Z-cocUL|9U-!tFfZ~barbbVHs3Ur zM%P!4FbIh#eZ$Z|5xs2I?D)hPa6@T9JDrNPU1$L)|Mlh4SB8&&+qcg#??z{GBrc9g zDf|5}r97V{20;|5u1)Mau_449Zm%Nz121eQk}%8|e|29Y)qK?{&t1rEl727gnK)N`{ z(@TIURTXiS=fM8my#xED$-haG`-BFdhw{O*IF(TJC=4!%lKiQ3VTSP?LQfc;!-KG> zH-dVd^h$y6<7v}@$1`wBpk?Gv4MnH6X05TYKy)vjUD(d(zqSOtnkEnfJs3f&auG^!_$}Boee&0l)$N z+0Y%AnRiS+mV7$_rPPGgN;YKdAA~-0u+nFxlldUmQ4G477ppa90DV2nT#t&=b~3Gz zty&(ujMYV~?}0w#z?<7(v3cRuYR8^S5TRqQVS8a{iIbnB9;APS_{#_R-~GC|=z;G} zSMe!cC`a^~DDSOo^r-AP?QN;-#I24o_5ePopTkKaooe4VYoH zrvgu)dX8<3=>(GxjOiS&fBBX_ppUVIGt!DJ%m5tX>gVkE`0EMbjD4*d)6*e^aVpN--vYQ2q1YK2&hjGyb}?7$MO>S&xvYspa?2t*xc#xK7mg zZ44EG0+*5P*|l=v>m)UBia@+U-ED2HN5W`4+h0*|N7yXvpNHfG(=x<6LfQlVO0XSP$VmW%uF?a zHo1ISJgqU>S)J0W@Bl=Q?3jFlEDGVU;7_0n2ik#Mjf4* zrr9^>MWf{6f@!c8$K%MGnwhUi<52&}J1-^-5o|=C=fdurgM?BiB zFPH=Z?v6Kln*$JZHCkH_pY&fTx$;z-`zJ*BQ9t77=u`gUBGRVf_8lFDFD3Y*tnp?I z7)HoVNW5kIY~D!Ip?gyOcZLOzm;>knFRx3={wf}J_ZK^p1_#NLW)@v^O7`b|odm5b zSHSoVIV=4kW@X@%$sMMeUEVSk{wbQcJUjdKV_Y23QQJ>!R~~WkaCfuTUXpI86J zDfP(u3cjTUhihq}_*u_J#9+W+9OSn<<)M9GYqYpMcy$ShXH&_D3aPnyw+-Kk+1K zw%Pz#RXlz^96F79yJ1FoU0i%p!SwFl{6R^iofe4iBwl%mj0P%&-}Fx*-tj7T53J{U z#gP-&jW-!>LqfcT7)NHv>E`)-No>jP#nO=!`HH`f$-<1fstFyNzl7yQMMPjQ5fOtw zVH01TcH4)j5T|iQP+}_2DuN01+Czu()tl08g$?Rn`=a~%K~zY|0z4wb&_G0q>Y1lc zwL1AbA1JZj{?-gMR)$_NngsFD(fC8W1u75jf`sVDAenGy2Ndob9YT}=cw>uNfB$K# zlf=C)CB_9AmdwK<zR5~gU)c-TrI^SvcX&rLM$UNM;(?a9EHac^ z(5;Zx7RucPYQq>W(^zQMrk(J1NW{mouw1{+!lI6bEt(8zh?e$}FuTwB-RpEQ2Rq#G z3kWHKSJ`3s&Tl9f9wWm^KcsE;0tsHN*)z~uz8sjMw+T|Iete*(eM)^K#>4+brI8vq{lu76)b)iqQIBj6Jp%cp^Q@qLhDC zN2hVUebY10S4GZm``hg7(3MBVccsVk_}KN2-o2c?A{6~c_mzEb%Ww8ZBKf#SD_!mF zJ$}*M$VQQ7c5G5{Mre{>z7_Qd5<2fMf|6CV1fK5We+1SnHyikdB+4gTVDnYV9c zg+f_zA}w1|#U?&YG}Ig?D$9A=-<$1yU3B1aRu(&ZRu*yZs=lKIa$kaacGTjO$zaPX zjHYhGx^KEsO-^Z&6R(k?S)odBRIONMs{(rO>(g>`TQK9R51BL8Uj9Vi?otmZ504p% z;iKYuqWR9T9-h9k8y5EQsjyIf&$V#YJ)6;6ib!cL6Jt=pZ0x(3t+$w+Fg-)>Dw6Jb zl+zVv6SHxPM#1Ij;=T#2wK3XY&a`a zvcBRM8Yv`{ZYwo+Rj2g+msB*0#FsnFm5VmsSXpsIcXb=e7vPMAt*chjU-npZ@VQFk z&h8o$g89aOHc55kY?ehZq!9~B48rjU1#v{@RgJAclzJkPO?{f$0x7xn@HaiElOo$~x zd7GR4*?MWnh&QK#qwt;G91%ypw#DMp6ds&TlS=)Pr|1?Y%JJz>T2cf~PJ3=xyfiZL z(~CdHXO@sHzs@KfAv|^Rz8%zU;(gq-?!fd_Oo{&1Zz=Mm>4==09A2c3m@K;ET zkWPaSr>(K~N(D-5g^6j2LFvuh)56<^G|tB>Yz8VuU~zlH8w?p9$Rz{)xOH$}_jnpZ@Ghet%jFq+=4V$qmJC3VRFP?0Vf$3xuKhj9CZ_5QA$ zl=oRHs*?2e&mwFkOrx8Du*l)i*+3)Vi-a3mD)i=z4T<0BrOF+AGyE?!nl z?UdPl5wp`b0KVFWhliQUmSU;YuyUQ^?lrB5jr^pM{%L7+rn}wcHH~BxI31?(_Gvi{eGIJH zvkQ=j_k0!~UpZB*xvdxUO84}SW=Yxl*GRL|3w`O-!pXZ+U%q6;OrGm$PkY*CvJ`KA zZ!S#E`b^Z#26_Averj5}DdDQ|+U#Z0Z^`-S)5YpVabpe!Y2WPBxk*a}5)Io+1*Fs~ zFN0_Yd1uJ+O{Md0oG+FE$Gz1x^e0Te4_e<}dq(J}`te(5-SRH}o+r1YpqUTt`!(sPv)!uM z2kk`7sagVoVY9_UXMOx;uibr_AYo~4>C)V%-p0W`Bck+H&J5Se6+P@QUZ9T`&wW!* z_-o~2yH?D7wFe0(0h@X|?w)Ns)yR_BSGKtK!wEhn1`wo-A}c~m;%sk^XWrh0ZqwsE&PYd4>7 zYtYBB`{LX+B>}now8+`7OGYx%(iim1y9W>YN#BV&XCCn?Z6t8*ucc9fc&LXlRlC)q zH#9^nO^z-^=nA8AzwTcv%&dq6zJUaaY`zGze-@ zNrOv-f}|up0HK81mlW+?kEwwcw|FxrR(jsg$Jt`5*lD6V4Xm)|_r1}j{=1*uF%n5> z{i>CfR7|$N{5z7|<063`v4Xj|2+o8A88W3fn2$=W^^sAoTv{3}$uhay-qiH@^F(R# z{2QmUZk)4yH+?QUait(%-O0M$;Y(xTv_w&x9)J1-&O2ejw*aU@yu0h1`0?6bwhhac zQ{#oPivFJdy9|&_;5%uHZJj8Re0X6}MWS9`q<&Km(=7l7IPercPe+!P1`7%bcK92s zlR$w@^n6dm(xTtD54H8y!cNyGhARWe6Qwbfq?>X^Dly)#o^Ti`PD%~(<5FIn`+4mz z#nDCaw)DScgzF-FjG$BKbKY%<2Hz!pO2c5~->w7&(VpY0?)z!e5{G%WBN|5<=|n7< z5R(*LPyp&xIvUsVBUCF=*KUCk$zz$`awp3)${z(o$IhaT-wtYLi;6?p8NWHYQ>o)c z+OQ{;<(kvpyk-BB%jYuRst>1m^#jsUyr>@^E__TiCxaavF@;?0ZW|QGa}8v>#-Fg{ zk?jOCN@0wZU@mDA$lDBdSl$ZG;`mC;X>DhFUS#bqrG&ZrD6p1iNA0BJ1_|1J&$G6! z<3!v!J>N2fTf>;eDY3Een^l-;EAx^=o=%Du%Tj7bxTfMd8l|ubZm_~P6YzjW_hML1 zrjMD*O3_$k>(i<71}$3NGp2zwSZHsgC;N*Q`3T%JFwl$>y-LO4ffSylZW4J)`Ah(d} z^?a{V)2tlnms&b179w8V(&a^>#rM;$-7F5yF#)FXU|lQPgVU4S=dy?Ad@2V!?~L)= z94kLtfBCoPq^HXT_$IzHMQp6m@gNZPMr24&iTE=J`QC0;hiB%>=E%FM1%umln@HOq zRBOZO<)X#^A4i zo{ou&d&;VF(QOtlg*`&Tf}aAp^@$5~!hN0%8?V=4r?Eww!&cR;@# zhEn9$z0}l%!?3V)GeAX({)F!5kmn2-2#Wpll zY-MHEvC)>62Hc^x+)3bZ>!;tPvU^pt$lp#|PNN4%%#H4Khm%{2EkN_?Vhg6B9~IoP z-EcC=Ce37`qk~gIT!J%*A@RPf!KJ0u>DL~baS>iaL{>D^!MAGUsAkhWtWttQ?c zFG{Efh!JU;pyqZmRpQJq#lgsSDRO~Frnt{#WV_1kcN`yUkCl;A z)S@23lFuteBSp_{FCQiscBjVEg}v&V>C<{=`7F%#zJ2U&>t`&i7tA?7z+3eskDigg zcBg$xk{@j+DQUN1_w|(+CGUk>B&=HCt)RzUY&_mE(KodtqrID)sv$XHcAhP7dH$_$ z&2FlYUZUddrr@nV)^4ZsPYeI)Xh8%raUkKMh%^b2I#JefGh(<;Qz{2aCiPmSk`i!x6X=H+ z9uDry5RZNeV{L?WXuziHS@4m@Q=dPQQL*dTf~BiF_sS{kjXGj5I4Ml^xkSf7it+C~ ze$h4(@oE0~`lGv|fYN!p`^oC+7Q`Bchr;K2z;cI0Z<3I3y6?1$%3Yf@tBv88hsMpw z;Xd7|Uh|!wKPO3bS)r=e)+}mkY9lNe%Hen_B(IMiO=_Z1V^kVYD3Xai+YgcVooTZfb2;7@y>FG;0Lkz-?01VNL}XU)SK)ZOZmc@Y+jc< zyk}rTShrtb;^$PM9*ZMVJJYGS6oYvB_J&dRZ5eLV6NwnkiP1iyTh|@ zSmg7gZQs4`RuX+%PBb%+A~`ul&m3g)yI0xa)GQas^k{Ae1>TvW zEJz)F*mi2&Nw7iAU8q3Qf;efv$>m1Eg#TL9UIh7sCqt3Z1-wRedS6BsMERx$F-f|l zMHgJ-U?{1Ir4+bb4(8mf-=5C99X}Z`XY_{6F|_tn@boOM-OnSu=54AQvZayCT}iuj zKh3j*Zo@dM2DBLbrfL?(TDO5Z&Ch>c#{aXWBC5`?)0To!*^{8p_0Yt6D)M8+^kc%q zpWiqlasv@QQ$&c&Y>T%ZTiXsQJe?UgpZ+v4=0szwzx94({iG3_s($(cj&i77;p-CK z$}bClaNgwFo10M`w(I+%-CLGxpKNz?^|_7Tj{Rnsp3dn&4Z9`kbJ;Lw;^+9uhb2!| z264N~b~y^zlWd}S$}?^r>*m<1oL6y&E0T2Fub8Wv`VyLi4k zP&+RF;=>Ew&S}m{9W8Q-u$@x+RXyz&9W%BcunP;#YI7!$u`JDKwmY_Qpk2h&qG3jh zYm?;#wF-2t^;#*PEr#P>G+Pe0mpN2$dy&fDbDWU5UHwcYr@+%|TFg0!ySO%E*YEJE zC+8zY!pEX<7IzeANd(mU7OpmSE+f4~MeOj5;xM;ZEBCY_vIozwNe>?$+Etbq4)-x6 zq2!tsUQ-mt<|ltQaSxt0x1}rbPEEbqQBiSHFPa*|Y^|N-9j_%v$sFgL^38&RxVbsb zJyUy&75K_@${K4$I*5;V2|2%D#K4`iE&5w6YJ;~wTGWPHs%JH0@v(0~-{8o|s7nHU zj&HZ!l$2A7k7YIMVr#{J3|+^n-C=8ddEgMPEzceMT*bMw_zmT6hC`v9N>jjgMk z$6vlYeq1EGhSYwF`?W!FaGI!@6>*|mvGyABO!94NICf8}0b`(*IvtcYq3q?iGu<*b zx9vZE*1ZVx3&C(bp-_Xe;=!r^x!C7&p-QoD_QmsSkH3BeL7b<1yWWJgQq77}=Q^y;1RTSWEfvkStXN1j zzeYhS=O4qL?$0n4_*HbEH!uM7riYeFc`jm`^Q4e3Uv{T_3SLM%#IlYGnGo3!BXmal zZ}xrofWr-2)?&`A%}cjN2}W(0ps(3P_4662DENufr^3UENE2@^rN-;)QN!^`DyGGX z?bptIooJT}!&b4Fx}S+x`^y{l^Bgf!)WfWhDOH>f6Ctu`eh=2egoT8cIjjv!t2cPL zD^~*ymnThdxsm~5?l&;(ZjMx!r1&U7Gopep5Dzb;%KHX1fZ|zHwmhIQdZp9Yl;PHm zj~6v6wyb|}red{w{~z%edJ{bdDXfb!GUX-aJLcJ1!1-{(DsT8n+FARSf*!+Wju}Ndl%##v zq3KgKpz>^u?KDHyN4vtIZt!pL3PHn3H}Ho?|DS&l86Bze^RSt5iK+Hlc7fthAe#X8 zPw1Du0||Cpb#-v)uy2`hhJzTT!H0)|oViMDk0(#o zJ~*TE?nK;4G66*)QuknAI{}I~($<;U4oA>9G|Ol5&}rBEx;wqmmq^)x<1@HjjQW=o zz$YgM1;~Ng1Y6Kw#I1j|VOSIv7IP?5v@C=^k2nZvk{MUd!KQgV$O-_jFRaYi9rOwG zIZ%hRAo$a1W-bc!>7Ngb-d_J<)VXH{C8wO?kZ~EW^P6dHZ&x4`79L|HqGVt|Sx6&o z`2hbhh*#h=r!VAU^J;%mdwgGCSXf^lGrtd^iZA_tmnHIFDI@1rhKi;r^Fu|MM!?Qt zX1>W^^&wwj1y&K`i6Q=@B{K?BxRR^|1Hgf|bRY(XIrG%-|K|_PI2~zVY;0m;tk#F{ z<;!IK9Lm2(i9GJnqclYib|sJ-tv0oOuy>}fOVG)5T1R8x77m#nv}Hol45sg~s;uA- zBcd)s*^ak185V4Aj_V`JnMFhC0F!8N5ZrxD*8wb(wjKLmpq=FDyL#DM7rb z(7Z;U8Y|;!*%-WG2TOyHAQ?%8M9{TDOZLe{vib$YaZkjUY65=fg zCv`h);Zc*rWIW0eQe{9$<{sQ&hw&>=iS&O~rT+Qmf1wUjw?tfQhpT52B+$i{4ZMw( zHD#p_7WJ~RtiNrNS$|u>^-nb-d)}*BO}&5L*7ogNTU&Rkg@uBTG_tFto0*us-42(* zSk!{du2s`O+dxY1Ya(%SGA(U#lG4MAN2IOE}gjgf%*V;Qt_iH4QA z-@9jTfA8KhmhWb|rLk<#?)sdlk+@Z{>v?VldiXGTq(pCVq{K8`I3Q~JHz(WFlKdS2 zSuu}a_IktMH_zz*%};ehWs6yrWXAFEi&PJ<(r%~JoSe-*hugQ=*>B$-%sH=dWM|Zo zg#}%IyoW1qoH&EQU}H32`b;-pmq?Xm`(RbH9BC4n+`+XGAto`DqVyA+{nnPZ4|&T@ z_(Ch!-XPl0Fql*z`x6d5ygWis#P9BwI})9u6n-)zk{ruQ^LF=T7Ah&kzGHYsjV z^(f(+ot-JNCDrG`u8_noJja^mu(lrm%(Nl4h6e{_rpzs(F1fnw&4eBnbcno4fS7b zLSIsw=uD@PeBlOLMA8SpKO7pSE-7iq12fZLG@=YyS#lXsD3>(@q$N+~{Q%k=2 z7etIzSRqtp$yMf(6sXg1-52=n%eQYI{*I?@ZZ1k8X(M8vG-P2k0f9Ux^E(O1qhHGb z-rjKWrRftnZ-s=`D$aQY$+Q%UE2Lkjedz092`bz{DnS$!&k1?cA~X<7H1O~I@b{G~ z@mU}J^BOLD6uRACr0lfzutj!d2qML#-_-{Cd6CxVG!#8%{~q)hl*LQ-Vfr**)tCB} zUT56xjwsBer_bq#B_cPb{qy+7s7pqoqK&%-9)_QPx&70IT7l$8fJhqZbLZZR9hnYB zgbPsthhbGSt)=ghK$LO}%{C@!q!*MtKd*h{Hx=sQmoGXf(-oq$>1^^K+O4pKh3Yujv5r{HNKsT`?;Le0cb zVWE>Dr>TD}ZRZ7>>=4T3LV9;9hXqa18JDjvQaXRN>hRzp;0wFkZ@U7?aR9G)xr;8S9> z+=UB@iWl@Y0lW*C?adOV5AQvHjXNkmXpK^g3H)*NUdeT79Bh}2ZA?{C*Y48w$qCsxQ)T@vmP4&ULj&mn}abIx0>-CcY zAl*ALaUgLb+~2{$+S(z1VFqS5C^;K(N-kh)leCJ9!^gs>uKA|_+1Bmt*H>m{2Kha9 zHTd+ z_kFh@2tDy8Twn$myn=^DLK_*OsLPo~TCs=uc=;O4BAHEP<55bPt!?~1Yb#5m)gItP zU2%!~_h;rRjvc~2893YNq6o9ZPX{zKS$RYCi$_SF(O2&j^lV2K+@>kZ`MCJ)++6dM^@K@zRlnK6?=K9aYn=kmubgw+#Omr9ENEm#p0ct+(W%zM{rA@&y&lx}rGVW%^2%8UrPQ-JKL$X=`rNugAVYDa z%H45I#yt1OlbFCje~dpOSaoQC}KoQixnkP?~5;Gdu4?qy!WwThU2zIy90MmLQlDQT@D2j&*VN38AsEfkxX?M^Cdx#qi6&kTC9~!{wMv zT9qX_eKEeTY~>_xKJ`nqB*(h23a*hZtP(bb)tblaq`pc=4;Rc@d8nn3{3p7KP79rq z7C$|aqKG-M(doljZ$$Wd7gU!RRJ&k%aV&v6u{)VS4kGpy4N54Bz?I8RmYp{uqh2Sx z>`u2b6+vn53zamG{Hub}gA$%l@R;9Rbka&h2HztgJA#$|{FE4yI3X?kjKy&TE-cZy#eo3*A5_+UY^fqvsI;9B&;A`PmAesn00v0bj>{i_wjHYrFm)h79qKNbdiiK{^CU^1_9S7cU%bY%Od^)H)Gsttgay zZhop)&Tj9&%&3Y*lpTfXd<)aKS4-bfp|26d;R9(`5RPi87X9NS|8L}%_{X98 z&tidn{olut_8-l#Ea@X5Kqo|8Q@gDHA3f23o7{i-Mx|#)%gf6hhbK{}AS#udOkYY5 zd@$20oIdL}n|)nWG(mijJc!yGo1$%10{CC>->NIB_OlJ2Z$`oL!vp&v5RF%;V8%%m zQUoj3s@KDFiSHlw_5E}Jq_SL6FLHx4A1Ip-WIDIO=ncKk;S?>PX#j88n}zZ>*_D+! zIh9QB7&C6}1Q&!OOpW&iD)@MNt&W>)n(DbvH+Oc1u|x0+{{l+$F7cMEva+2*~9p@w%2qXZ^C7jJe)cH@Spmr5bYehI`#>rSmSI4LF`apKK0*s)b znj9LHXmcCD#(Y+cm3jqtj%PYS5YA@=8b>k!8q38i!aezrapDadV<8nsZ~q3o9l|+p z0TFCk0^1V}z|bK&`G1GE2`_Vp?1+qo@&T*PH)X#-;2&k8@QGhmik%;{NA+IHiQlf<&QjGYsK}EyzmNN>b%Qi)3veb7cfh(?NKqgmT|bdve0# z%>Na_&F5AK;U@4*>2(-*=D+5Zm3^E|!!3zshhvOEm1$FFWVO8QDfz*(L^NzC_N;3h zw5%85R3F4Pb*0rBdFHLO3N!+yy92SUp%$fnLpw4lR#K&OL5e{=#4#q9P%Pg>l}LJe zSV}(EVj|y&!L$8#Nm!@5Ger@XG8~>*Iui8`vY-c;rac8bfHxn8x}_2ni3KUw}kAAUVr9Hh15rll@m&KwISm>_#$__*0hFqe7pK z=uwgI=Kz*TZlk+h3lgds6N&#VQoX*>akc5a^4%A$D2uH} zj=BgWuXUZ2DLqtwm0Qf|g#4S!)my1fzj@yyIqmbt_O*Xliq4Fh_~U$Q#BpZ{jo1LE z>G;gDeWvQAw%>#zZ(Q_Tusg-g^?b!TV#~bgahRu>; zOJ6^-c_;=4x_#RwrlsN235$6(*${D51x=nYWSl+EFu>sLBQ5Jj0aw<}6n#$^z%#55 zH!Ax*@Y@050n6eVFZc=@sF6A1t=O#R)5Rbjx;AQ{<2chlJj}0bZe?Xjo8eDLAWYT+*olcy zsd;L_wo9GH^-Kd&B`BZ;uiS8GX+2Yoo2wh?IV!7!m|;Op%OFx)?3Jg%#E2_T1#TJF zJw3=49ho94dEs-eVatVnqF0$nL-eWnm6fKZ6(=#>lZ1h_-+zqO{oo!*NoeHdR|%%z zRaAoOmT1s{gOJAl#lS0hPv`dzT+EJAy<1{9q!wAB9=RT;bDUFU=k#Ckeq);7h@DAc zVN%HH@=L4G*@oa&hfb<(S0$w$rQ!bDUZUY<+msC@C8J`>R;5w#i@Y{j1>oYxU>M!J z!K>F2!u<}Z@T5_;w|6#hwuKvN@7=AZeHxP!b87~nAH9!*u!|ASPJtAoMs9uy%v|<$ zDX9eUa}xALosJHJO6X)l{2>6ztFjUrEuB`@sAAdSgqrVk4YqN2fbMfO86sWp*6yB_ zf1Y_6acv@iyfztiCcnT1(#sIvCYZx9YH+Zr3iPv^U!Ti2gR~7qpbWH)GFC~X`fY0~ ztDV&>@0q0~d^)<=aKamUz!z6|pcW3_*@Pj`0Uf6xd~QU|!0ztf2bPx&Dk1d&v26zN zsOr;^az}i(MQyvQ>t*Ipp?+3~-rqsntnF_!>iQUAl$6EqEc7cBqWj(-zV~QA00=-ygA5UbEZ6Sx_)n z`JE?n@H=>(l2cy$L>{ z)CSjdbq8;MU0F$zVrNHoH=hiG*)W>1K39{N>{5ES@PdG~LGWH!*c8>p+FGz->)Y2| zfA?EihL0YG3tF>{GFjzd=&?%S=;@}%L?R34tCQF;oyc#uEXM>+5A<)0d+?n5)JiK= zl9nV&&@V``eT|QeWf#`aSaL;C@qehj6Ost#)I3IgSGL)wdHs5F(FyK=Xpc(V3ESnp zH$mtzS&0Pku1@osiwd>x&MPGQymct3)EJP6K4Uo(shJ@%${+MRcy8E9<=9lCUn%w3 zA(mEZFx=i_?jt%Xo>ry*))hdyjIB?rx=E*SV>tpMwy=(o`tBr`z_oOGU6Nx44?Jxq z>uovA9P<-6U4m29OX9v`#{MCzKSx;TW_mZD`o7DVskPqJ1@Z*_vwMWX$K_kC05zd2Fd0*<}_P&3I9j5 z((Y?-I&Rbzx2>=Ci=b!nYW=o+w34OZacN~*hgEcp%+_z-crTvUxplHOo8J4@!O>om7Xizw5aFP>LkrtJ7k%Ap!? z26AJL=I!i8_X7><*jFRdZ~0wfiD+T7#l9?hY*M@W2Uzcr7eS3nko>aLr=PO0|8t6@90dLYeE>*qE7V+#THZ-Kur>6xN~b4h!qp30^oGy76G4S0dPb z;nt({(hsQ%4|!xj_BD@;weEmyjc=5Xb+YVM-6@L{T!NLd|K?mm$5YRO_)#w&qx&g& zRpE>t;y-3RcDHO7d&~QK^-6W|Nu|sKpXGeq2Xt}bYk8x$Y_6Xl5;xHf35qa6#V2JRB1@fh>DdVI)(FAJK`tT9Q8B@`J(*M3haY?7eTp z)+wx+C2)uB_q3wN=LBdE2$R62T_1oOcA{8g_9E@CHirm=vZw2rZ4_z)ZdR=*H=GNy zmVRGZPSv!UTfLh#IcXSuhQK@b_;dRvQkwVWp*6L=4x^k1H)G=9^C&%a6{n)79h=vL zEh_t-*WI$lCH_2Znk5+#;NWTt+b85JX`LT`!|t^JK!sX~mBdBrt4=5(+Y-h}oN@X0V1 zYJ5$MZpxby?yFUiX^a_r$r+l*VStu)xCXl2#WiWn37#}#K^+@D9Wmpep;ov((vz~& z<;s0>Wto6l%oa|~a#B${R;c?Sukb=_q5ZWi5!re2wLObUwvu+1OBz94cIG7}IA8l! zA2TkuNQYRznY&3xmh4*GTbg?PI^3|*qyd+eH|~q;b4f`Z2^d22B`exm0XFZp@J-O4 znjS0x%R}S4_yUq9N07;Pd|<()2x!gFYA#p^z{Ute2wEu&p1yNSSeUm`f-@e7>k>q z;BRz$%@q_sOarYf=L(Em3?I?Mmk&w{Qj8h|W`-Piy(g>6VBv}L@ecqo=~ZWqJ;xSV zL%Pn#ZrNV}BV^P&71S_HX4O3tE7Z1IP`&Q;{hDfLc@W{~4uZ;`th$3|=WIps^T&_R z&zG`wUZ9Kc+Y?G}zUfp>!|;pUprb-PQkf_{DV)zz#jDam7| z2p7lj0^%jZiXW|;y*DZ%ZBaAkWy<4m-CjI*8|vQOF~43!NDuum7iGleR_8uLZftbV z(oH|1bc^?0glMXWT(vk$Iw*H*cecovUc7+i;Y*cDzv zQrs)*S>4ByKOQnLC@Qd${x-1t&AjLW3$Nxx=}=jY>-kA3o`A>_yzjZgr=E08 zVkV@;)%7bcU9wKjk^0wD?A?7fhm`cjK)>LIrHZn22SIYjP=Y@er1%{w|! z6q}sk1(9M(0>|%-hC?pD8CX=h7aboCb@uM8iJY8FRR{>!XyCzn-e1g$dHHhM+;#PK z`#`(%R+2xV9vJBL)KMjTtZ^wOl=gQ?`i7l}l7zsO@9=4#h}6X9zvYG$zNr65E8`qaatWlY}; zT@uJM4_GtJQ_N35&G>EcT}*)B7RZkzfJ7sWf!C4eZ39onlDjm<<-r)(v5IiRt2kkh zx)CRwEHl(%!HZmlm6LY*7;{UUgynnL)n9)oywI0BB<4+7)uHsoq-}=THtDRofWcY% z>iHG~HW%sU*SmD2kdG}@#^><@oWC7>L4g@daC}Tzh0u>Dved+`1Tv4s!20r2PRN++ z*cjHs%4Zk2l)hoo5B)}2K|Vc9o#uRcSlzj)%8uJh&0WHEHF%LJr6YcZ;_o$o?%3aP zFS_t5CouGU=AVP&E|X4dKZvo1YyE!FT-lz3h|YC44yISSqKGZ4W40AeLHpkEO`I2R zsZIks6<{Y@x+phaO0Qp^Sqk5FGG8TfOAZaC5WhPYRTTZYghAyy*PR*@TZi~d zhDqT*6L&CutwUw``O8~dtH2$@IqxXp=ty2GDstYk0=5P*MkgFmVIy0})0!Q++45t1 z$-NCb@)6yp(1$}$F3;3qgSZW(lG*&%zE23!A61S1xQ!Im)~2lau(LDlB4l-sh+1=w zFK*h!IIT9G*kxD~K<+_JQvMpSeOcmyt$8v5w*D;wWrs)v9dx0-o^N#nq8(?}CRkPK z0(`u;KC&W_f=^Cdzp7c@qH%6&oVo8Fj)i_x>(Eq6p_C zRvx&GKADY5mGL&0hb_Lqy)AtDR0DN$F)XY$))F65BhOlQK1H>9|^oMVXPI~bmAFT_#;JsWA9CpJf z>I!CCUf}PpP%-S$&%pcY-Kxv@>3uyo!$DL{sk%%2$@O;mfV)H9g&Md>TR(+>v)(g$ zp#@4JY&3&$@B@^`=~Q-vYIj> zHRJO=$an+fZ)mcT66w*kOfuF%(REOxQ&wKrlM8N(mN*yvYLgUVGw`uscUu};r603U z4;uk4v$q8ss-JBP7NvOn47MqaBA@Byd< z4~C2Rsk!~4JMl5Dll6Oem4xNWV@}Ro#0zBCon_&ZsY>E4T)65D0zDYt{aqqZ!`z3} zRL|F*_Dsvpj!`x>U3voROR>y{h3323Iy9ObsTKjdjkd$sZJ^@*F=zYFO`}F?h>igl z?boJv#d6H-I%u>6%hAA=uJwm)Tj2=rt*RIG7i{w841@i}`ptO{f%XUh#pmL@C%ky8 zS0O(^Mg*@iWvSpPE2nF}Wnb=1J!L+h*|q{SpPZgofZ6NL*tfTe-gQ2&V(T9twVt+J z8}E%KyOJZX&k=|kfdA$su-J{JEVBY%CY9dke7~*2)ihqGGog; zeR4CEn_KJmh9M`0;Gc>6@rrJJq4v3t-M54`4I4&nH(vZQImmuB>XuQG@x5gmWO-(V zm6)aZeD`XadtRr~q_L#oiO0xUK+?_7rvRAUsGSiez#%zs@0|$XcBgTg*1d(Yag$6Z`9}cAtiBpf*>T6 zsRRB11VDN4W&FEx+g^pee^tXhQ*E&O52Wi07Uh4_3LUv6T{r|Q2L|OixXi=mL}@lk zyV>({OjKkYv9;|gQQE+w#P+ZFD0v^<@?zNquGg*k0wLwUPptd6$V6Tzy7rrx2QYZY zeq;C^l9%Brc&5yhBK^bU>4=~pK|$f0v)C4a=G%kQ6$#??bXxXxqL*mG9hV^E9BmbB zNen|{o0%zcGLskRG^*oW9Up(n-`!Q-;H>*mf>7Hf>tp*@T1$~bn$uMujE@UgKMxNU z76bEwa#%~77;RjQgcwf`>bivJ&2Gh_NCiYP#(=V9?TD!M&*`?GT^gphNFq`5jCwC96jk}yk zn0|GC?&y*afwt2#MKWn||LOwigr{sFO&ejc3d>uN$*SA(?$YJV(6$t1B}w;P&pHpv z_Ln;x*UzWRZ6r^n$g#8RNfT~o8bvItX$0PI6K*z;S3xa)rUXuM06lL=e+sm*x1#TM z$!s)?0fYy(_tPzTX8k%AqVe8#`DD$`QT zXKaPf9=j}EJW}Tn$|G4D7H0S4whi6S#?GwcK6cu_jpaf4{J5>HsL8y6TcpQu-DIenS=pPa(P=xck5$K3 zT{SMASNO|zDyE>!_Y+}Fk)hF_x-^#W0{)Na74y@L39VK;e`ZO-xP>;n+98at{O^xu+nns##8`hn>d4wDR9v!2-;Zr$RE71l7;3gR^H8D zssOWpvpnB5HaR`~L@f?;X|D`mT$zbh+$eL^{F}6r@X6il90)@$}@NAwaNmQ+b-HdF_Jf&-Rsd} zn8moRE)k)Qu9>hS3rCX~7rGXtp8^91u*n%1&_SFEfsi2oHd^Ol@8;%m){Tcq9+F7% znU+}@8xZXZFl+FOXDe?g^;tPCx;S3vG!z@Wz9d*vErS;;0Xj&AyAQQmS!fi-UIN)}!g)m&Ek z*-qK*pHNG|NWLQvuth}l*`LVk@VoeJ$A*|$0M31ssOuz#Y9wL;e1gb7zQEbsU?XBsJU73Rv~hcdli&A3Sn${t)Lt8 zbWodWY-Ql|Xc{l{ZuqS>P=0RDukd`{OzR7`Lf7NAUTch_3glNKy($49%?g0T<=t(2 zMR3P3tr&V)n)6-$kK;pK0Xn+fc$$>~{qkRtqcVRGME>+_V}s9S?e~v2G<q5@hzx z9V;&fAw@aU`p$3P)VPF&#~T7xCdTXZt3drmF!BHwC~!chjS3eop*g<^L5epb|FMe> zMCsrN`-UwB!)!4JiUI}@nlYFW;TA}j6+%3O3<=2J>LHd`?#*Q{3?(M+t>;Zo?`$&t z(K2j0aZQB=G^R^Yqp5^Yku&-%`}r+m9sHOYEpebtPgBu!BNI?)aX+6BiroqxfRYuo zS=|zZK1TcHzis00e_kr{zidSNIa4M3T1bR&La}L@%78O>@6z@S;4)FuW=gtajeFle zkx9TZ-sxL?Jm5mZmVnL{Mzo|TZ4>>!CYJX#G5BCwE;VTs;EzWW>tZ<-{MYAn2B?zb zZ*$1j0u6+Rp-dRTgftF1VN*dTCADzy)%WeR%p-I+r5n)cr2Q89e7_eX?YE;iC}DML z2m!b}0Ypgt4+Q%JQGXipXg~7_2YMUCozVyNVH+?rXb0zzfJ3&u63NCus?2&(tSRGBck% zNtH=Vbej;GSp5WuqMY*iwX{}5J;9X?YwMQarK6c^cPlwczba=y=>I5jLOa;d0RIQ( z>Ho_%_rGog_RzmM6fl~8hW6IKnv7EC;a_Txwj2K2ZifAY4anHk_eIX+I^X$%JOpUJ9vlQZfZ2L^*ukdPrMw!TL2Pq*T_hk z9HHy??^5e8U+R2TM$iF1+X`y*O_UXYjA{%~Y~%u0Y78kS(a=n{`ZhU%vIV|vVQR1A zq&(^OOYkyR^s*=xQ&=lZ9o6{4A}%7L+#d93cmFuz!d+Wst)|r-bB>^C@l)&u{L2@R zW1t7TV^GKwP++5t9#oWX^+lOCfSAn2{HJF?wbReIFeqHU>bJf6^Cz{0|K}SP@5Ho} z*$*EEZMiv$8xQB^+1uyk85+1wS{OfczAqUg86(oQQsXf-HQVLlt%xgUXtB_e%hs2W zFf8flket6o-GLdLqZ58RqTpo)PanM7(OfwCZPgk@OE1!SCWikN6N9)aA#1{JA00X} zJY1k_ZQZQT!gG3?LMks`Y2+eB=a2ma2>ds1VC|_XwT~V+3S)jhVbFtv@oW z(q2wJE!H_+7XTu>T?RJM!F66ve%ph{xtEJ1`23T6Sys}+iPE8|QG0v88W`zaas*Vp zL15p*;70#}Db5!+!uVS@ZQ!Md$w(xNSPO_lBIMoiU1{^lo3mE&q^pcVN! zxUZG;Aczp|hKx_{_3M^)f?61&jRaMWhm~#Cm4^}j^KJ2P&LXII107;w*Tv;FUYjE-`s>tSAwBlYH+oJGGX8#@=;gIRA7W$B(dkv|i4zkYt}4~roNk|F6doTbRr_5U_|e^NrY$}>dA0XTXXIM&(#Y;E+6mWO zSB~%Fv0x2C_@HS=Y+(Op{EVBIUMM#Eih zzl&PT8rX<(uvWg7(yJqEWI}e)x zb?d!nHLVqWoEQ5G_=MHR^QS+we>KSM5IHxIqD4EV^L#X5ozj;k?ta+!_R*QS-Y+lL zZoY)ytC#esc=X`M&W6DNEV?;^r_0Et!<8mVa!N_jTld|~q;}*5{G~}O<@S{ti!^}# zEjZ-(pVZ|QGO{D!Rrybij@P2=0~ZVy{AUaDWb3@|C-*q&p`0z)@yDWBY`2@sXrt?g z9Dp7=o%j#c7S{blC+v+5zer{W*Sm*46(2mjL!#%%C+4T}Jxy(%l_`0%__P;}t_Gk0 zSX#8UwIGU?W6IEhd+jlQMZw&U<}* zak#f_s!c#QS^Pi@oIEz7f_?ci_T0i4TQYS&^z4TZz{S*c)CRBym4zLAm1W-6GJ2;GSniW9sd-+*S|@`cB_Kt@SS4->-nKG2+sdIC2Zxo`3%G<;#+i zm$e7-TFkvS=681oiu5;WAks7?DDqMJ7B#}Mu^svBh3D%dY}fcluEy0{j1vBla4J0h zIoFGKflh)@WA_m$RiVcuA_B2AdiZcGM=QNHEzGji4Edr;3@QWIa(ka8*#PO!;pkBf zt?n*Ot(Mo*FnytkS=t1h1ruaT(YGHR$(n$d1^BoD^sfFU%~DCb`M<~GyGPmCA(Qm0 z5*-1`^?y%|5l3&-BO7o2`O;F|yuzsU9G<6{`#~MHFVI8u+wM^s0ybB``ayePU@3=@bN0ByK~%kjTaXnwl#AXMK(̒+j0|_;k422Y!33#b zAs4n|l-CqRfc6t|=PPx-La|V8E8~wy9;Y9DT84&-8I{4Pu^$OE_%}nnpXk&f+*wgb-NqM@^MQbzsh_gGq z`-&WTsFgN}KXvj!lo1q)fpEHXCr>{>i$5u+%02j@up)7q3bzpo+?`Hza4I*J_jY(H4BOX1duf&9{U;=)9t6RYi!9I*) z8uIK(M8};XQBhyVBYn4|6yshwy zID<+7ygMvD%3+*p9c;riQji+A+&$Mjl6!l@uO=K-rtQJB>(K7Wt)><|j$&SOL zE-5Ll;Qh|>sk*nfFHcQvDlNRlYd#CBuirFQEUvC@pLmKHDx|S|9=gA-@&MLpXl52= z-hTXwrlv)$=RFH)DNXL(Q_+{k{PDT<4r~^pfB~#Mpzgy0Yh2z8F_jPg7?@n0ru+GFX!+N8sx9yqz(+1 zdux7ZlDT9}v;!~^!2rFRhHxvYE}utOEG10Nq?bXWY1;JL37A{waWl=kX^M(cwh-vL zwzJX)L1_msHCxZP6g^4-VkpHS8_IM@Y{ zS-$HjbU}e8*LqSA`~J#^k-e<&!x#10QJ3aNM(op(Nc0SiDE9ZcaSoTUec1J!>a%Ar z>FG~lo%-ja{GY?_MZ1)hHm|B*RdZ>i$^ zaf+vnlOJ|FExG2gtsV9%ZK~zGJO)&I!9VIE(|%s}b@pOcptK3}m(WJY2!ek5Z|hn9 zAJN4Bp9fCNLH}H;la(F<;32c==?Fwx8lt3D1uU`rM!&8Wk|uJRR-6cRqh!*;!Y0@d zBH(1+q1iH?#YrdFSy;p7?PcT0aeP%QB0%C(5|vH(cm<+JWe2)>@E{W zoNnGkHF(cy68TtFb{>%Qzy1wHfYAZEu%x!|LJCPHSsJB`%W_edR#`v@r5_ z;I;Y-1gk|&4sm~H@#3)bo9@Xpw1_|zxdLt9;|N{mt#iH@??6;$=qhd0-WS-g+0=f3@2YW$E3k)6}C-ZBA&?4g+xa+PFjn2JD__goy;wnM2hN+g9 zFN>Ci9o&}1=~B)Y2H8(Wq+fv!vMS^6!qFq++=%6!9gE-^lqja;qE!iaLoI-#vwmE8S*=$hq3>0;vYRH`ufU2 z%b6-yVbRH2#@>8i#m|Ik!H>;r?;Ut5?n!@TX}{NEzB_1Y-&Mb1P>&J-#S=ZHX@`4c zVs0*3B0C$``s;|al)walmQ!`=)vJELzWuZrXgQYFYW|m!gpi0uBc|Hu8aMrW-lxs8 zUGUg+K>S#2YQmKh0wyCE__PpL-1p75DxmTl681&KaP5&BDhdc+zQSKk{e}a`gJ69v ztgU%?tgTBW1M3;$rjN)O&{o};mQ>6TRzb#$6cighuPr_Pruj_OzTa5S5H+isR%3*} zjE~2KoqDo5W=kh|(?zAq_eFHNXjGKCdQ{Z$<3lJ>A-(Z?-&^tY!`%HW-OmYmt~xj| z_CR1!%04Tk+R_y{pcARb46>?hrD?Sf_R5i%r{?l?t2pU?Dop_9&Raa``XO>KgS8Bn zzt#FJB~L9Qm=MUq$Wgu$8-uu}AUjWg9}HrRCO3NQvhG>yjYarmsam6>v{TQ%wDX{@ zhgvM>wZL_U0VG{pq=R)6$5a}yEkBmd4}!f1hw?0=;a9AMWI^_DUP{^4Z{%C-tu@rzY5sc4kw@$Q|))^z`UutK&^3=CO{d>A;wy$s&&89 zsRI))3t`+{{&sZhH~7xNAs^`b=yDj%i>UZE>IRa~WN#vf(kxFjc;pdd@jypFHkuN9 zKC%JkuDU@z9<{o9F2&{mwB_#&8sk#P8yloe+M)X3MLSd%gn4tah^j;~wD$*`!2HhH zSzG6629f9|xcjf{Zy)M%S87r%m(cfjwYod+Y7Jo&wnDv$ z^MimxMxi`A?8H|D-u!!o{s$8i$k&!kiv0m-A~f^z)w}wf{IvuYFw#&cJcc3$Q5VWV z4VtP830W9F55Mi1&XE<>0W6RP;L##i3#CDvjC!mSa{dApFloSfr5mdWSura4n@bA| zq>^JC;GWvMso5i7-|iFSZhdWQTN<6zSFzrsU@&A-kt?;NVnI8JEanzOeZ;#^_Dvg= zFMGOOpxKGAR~$Qkh@iN$xdiI}PhyNd@H?_sluh@Jpc*XzI%&a|FCo0v!`@}7E6vy5 zcWb!x#}6Qhmqr@ub`jZ!t@cVmHA_VR%m~@D+tgb7sqdnPDrJ?H^KVK!`(wjtvN!&;qONo>H?UF&#h@J|%dB9RwLIYzJ(uD&d9wh{JX&B*4eW8&b8r{v zXcQ9jr$&l`6iE`jQh*Cb1K|PsAS9E-TF){tqDPzFe%rVH179~{|g?Z0Iru_V# zprM^zW#vopPf@BH4}2mqX*2~WUeaAZ5tCs-OiQAS zg^vJtY*sT`Q7nvgvy0u0Ly>}INAF2X;es!5s;t?N&Z|}e&FeqEh^xNC6MUA((J6*| z#o3N0iJy(+9MC!yC~Z^&VLRoC`g(gHgV!n>RW6^ zSUtXIf>?T)Bpr1dv#57>-BeGiUGulI@*ixtgE|lSiG%f=`7Kr5y{nITh*u>T_FPmn zM5lvf3UBN8=`M*bzpbY;8xw_n6^wZWC|!`2!};K)mQ$*q-79)l25SK&13e{0ml+!3 zkj+K`XWfTvTHY)wQ|9L8=jZ02Gp|#cTl&YR`t(D#CY<~0`E(0o1FVmcz_y}19yRx51k<+MScy5*`NrJ4JYn8FmC?Fc z5XB+8-X`dtXY(ac>{Ju^%%)~?Z4$fgNXR)4$GmS-CBGH|wx=lBmzHv~@VfG!&hTEeL=%G>pnV}2_lUQ>$^&A^fx84aJ68w?>GswBr1(ua z&1^(Bka>30ctGg;0ZmPt=6)}QjS4FhZZ~?|9kQWo*f2Eus`jc`UG(N z#3Yj_{)A4EPfWtm{^$3@xtP1w~^#!WR?T`OJCvzLgv!M(F{OIwCG zqBHX1c{l&t!tl;WM91-gW;C;V!p^~L_p~9bRoR3ZTwPmdF78ht+C;t;@9 zF7~pE`EmH@R-W8YK3Lu!VjqCfnH9tStasV*_mK| zS)s_cnJQ|}QRVxq{3M&U)R|ThUj04$=84q-*Qv6 z7CXCSv=h$?rpv+{zN36;f9PaoW&8HQk0Rx^X4Wv9Xi|iKCrYN=}pC1>2JG=}r^}pgO6)oy+1 zm({OpIL?HzO}&%09{U;zo>tcckk9VeHG3(s%e}BkjO;S4^m5|4z0?>v6=@wE*2q3u z)U3de?z(-;1R4-hXMh3ryqc%d;3Yb`i_<)N^V{}GZY+FcZ8)FsUfPqhWg_M41~sm- zq+KTltc8E9fLph`Yh~S1XY?(@`CrxZFV#DqM#8{zlgn{!s+c zYN#tClQrF9tZ-3I42`S*_=G=ar-8DQ>V~2CyNtc}^SO%<*Y{h!@p7ZxK)ii4msNLT zT2S1zlDijDiqkL-^jgY(@-QzV18h8-I2#Zx53_fkh#yqQ zc1r$gjt(j8&CkhlQfpQS@DJEreAua@Y$GwP1BxwCClmKtBO>B4W zo-*bs}mY#!+G<4!p%rU7u?l~ocEyh+OqHe4x14oOr8gca( zD%~=L0;Z*l+b*4FtUDJ*y9{zGJEdt_hTZ4zYa0vVACt1>z9KeWhb|agW#lrIc)-Oo z3ngMy?&faQQfv?N;L9sPmo6tKF&7R?R0-t=eNmDAIC4KGf!s5bjpxe}zsE9a`h@|u zbP3_OAADeUh|;#!A9~%34ip~7a!yfgr%u$K3GWfkA<$Q$I~cG?+5}&oYFNF~@9Dhw zxWhfHYN(rEtNb6inXj>3kb+NPN<&g^X$M5y#^AGKP8(T2J6p(rj=Dm!%{{LhZuK^O zL`dS7g_G?xap`eiX__8MBJJT6R`X?Fr#vkFM?`)CANMzyp{AZ=he$lr9JTSU*@mlEv;F+oi@(mz+$g zh}_U+{oqJJZr&BlXy4j0KozGLgr{)3Ppp-^NxB?0?YdAfSqT!A{kvh`9Nuaoh04%{ zmu)En5VTP*C~zR0f8MkzA<=C-Qjjkza$Nh_OCmlZTpkgA%h)U#cBp-nH+TjB4nUMb z77DuynmTxPwhcb*B&{X+Dk4(m3((2tBNz?kx?sq^nP?25?2<{ao6h}+DK-QGAj|z_ zv7^1c2t;o$OS+7yM414A{fYhke>__4D}`@-Lo4gkPG8 z2={F29^BeaLUu5h*3|b3m6RMjY-dl;amj+IrZl<)J>fgGMUa`e6ucUJAK$NMIXScN zHx^O8VQK1zohq+%n*{k8;*oyospP9fR_n)Rqg;W%Cjz(vwFf>g*g6w^z`R4Yns)MP z-wBRX(;E&pHp#Yd1Y*72zS{ADAW>f{O7S$#>~jp z*7ig`zNo3=Vgu)XG;ck;my6M~H+19PfB3Uaw>F>{CX<#F;f;G;l7m}89@^VGYzM|Hc zuYVb`3|Z?OvPA7T)S44~&)5qbh?(9@HX#pH-o+*#5N33+4rgvrqB=ZWd zU2B`~|JpoQcsb$d@1X7Jk0gP=fD=1eR;hir92RsaOaEryS2DZibK2rpFrS=8~y z$k~H{-(F)DWpZQPNJ^C{V>ORDr{KT!h{y30Hi;qoj(l1IYr!|>AuT;@kmAU;l|rY`JtwKs)BI0lhqO+Bs#g{9UT|D>{h_G7 z3iKg{_UNpo0eWMdrCi_L*oEu)ao*rbRzVy>1drntm;h8gE&k3TH(>f1%uIA+nbs;> zv0UxZI`A_{f68{6TzUuvT7HMSu__&<)ibt6KuC;)#~S8TFXzT_{o`H4N9@5ZW2Nvr4Jg2#S;Q85}H(S&z5;jM`$mv8omJlz%$>ES++2rOyc_Y`jF{qlHrt7H(kEyw&aBiK| zj_^}={hVLBE*z+%GK(X1qXO5a;3wVO)Vweu8a;!z^tT92Qy~D_h=lu?)SUG~3@Q>_*DY|0GL8-SNj2}qxnH2QDJ>VHLN%;2ti|VIzffP-nj- z7NFG#(qfMKat}hFh+r?D-<18f_wwZAP*q4s`RKbFO;478M1*(F_w#Yf>Nho7Ma2i*5jQrLMwwCiM03XWdoPK|prp^nAm|%O zo9;7h8aY3Oi-y?58s5Thqj`;PBE=cr;F^2OEY(;_s@rt*iHpCM#>?}FL2+^N`(8CD z<{eP6+QtKySZ?l(&C@Y#z`32VZxA> z0doz*ZrP&zSm>Jm-4L&;xo!Nf-K4?DqKoXVPMzG9rf*~#-aA?&!fnrr(j}&OyX53o zzu6qGe$k!A&ZX#6qLXgKJ7+vrIMX_~^)Qi8UZ^buq;ms={60AO*Pdg1!Rvco?7EV`to{s?V87ctaz42) zh3KqTw!Ls5Bh_VXP`Rhv_cLarL5Cf7iBr)B+FBimRP)$8onBRSlX59wbfDG{;d6i4 zQj{*13}a)ws58kOkDINPJh)8Ym}pa^ZzmPw1?zEmeo2%If0GE%zv48H?<%t>LNSD3 zHSy=|CSwzcA7}gMva|EFVq>s!a1$lzyGx7+!SYvD-rRg-{$2c`us&lWVJR^KvpQ+T zkR_9e>fUJElWWRhne{8G>mU63-ibZ13=~G^%23~9JTG9g8sw$>sryA z%{sl}*T|k;8+}^kYD3Y|eFgF`{bmMOg=+G#L71)}B|ENDyp~B&@y`45vtCF8RXwLf zaZw3z8B-#Eg#lp&x#B}uyEwuzTjXm} zd*wn3*)^MqQZz4;y&(^?1%B4xZ7t;ri{u^e&ip6LiVxbG8u9!^I@W1GdL-*`hs{}( z=7-_xC0JIO#I%0MPe2&KB~o*v7jUc!XzEMZmhatau7*`)QRxN?_lwniFDCGua74Z~ z4SSN>5bB(q@YdL_YNJg#<6F9Kk&6qshQ7=_?HK`hK%tI0c`(v3+>c=Z?c+mEXVL8i zwT$d4_tsvxvY1!#`e#=cT9UP-kiEm&&Ia%5e3v^hmNQo?rB;`Ps2RG9hQJjn5tvJ{ zq?Q#)>@24b+3GBh#rRk6%oP=mf@#sJEdB87S6_CvpF7++4B1KP-huPaQyrMaw|>d~ zoVLW|AjMpztch+rr*!3jeHrXscA?OzU$b2*`yzoK3JK_9E_Uk5+cL(Lc^e0dmHZyL zUG@QTK2sam77~3Jn@R4BF4COLhMHz7XhU^!yrwkMmYj#3rzC37P(uN`U>6`&Qk67= zYItVlrlV5k*RGYwth?GAmw_d!N?X*RTgO?pd0dSGy=(=oTq~!0`EZB=7mtefsFmqN z($4DBv%PLO0kz&)ceNHNZ4P6Hmbz%*b-#6q<;yZjTUS=jZrac`-IFWX-I@Zn)+f9Z z1+oqz)esI^{coc9D*SIm5{z*P&Py1lmR>Xt1pD(}2Py7&Ls(>FWtcnp#EBCpPoC%x zmO*ruHJEpoyBe6CcSqZxCRof(w`Vn){R2W)-%L!5klMS!?2h#QyBHxCUfwg@;;&`M z9JNW{V zeh>#<9E=R$Stf@Q1||;@jM0BI)hD!!$1yvFORFg=*Z*z8X&U_dQ@~2l@9P$)B}{Rp zfk(0fZ8FDK#DVp#!wCZ9U`ok&e}~T3%eJb~c2#*4Qi-|h(BRwx3D>=|1Bwc2wy&wy zFS==<#hMlwtdlRh0wT7&U3!+a8*X;bk#xaE@Y!H+RiR?SLD0F})1$(j=KRu0^Ep~J zJ2k~~BVMQ7|EkFoZ&`ggR|bm(#W>iVyo1u5(V44z^9!h6@0>agp6>yO^mIXsZIGP?F>^0>$L zB}eJ^TRRJGnC+c~WfNx>DnkQ&+TvS}r2~#SnZi~!6y}h+b_1h`^0E9{cl~qfOJ)6qB2t5PE%3QL z-#m@@MDMtzmqs7NV-!Z|aUZV-p38R7h%=BKB_+YDt@Dm@s?I=$rDnq?A)>_Z@l}DROz3_3SqHl@cJjhNag6h~jjK(QvlsEY^ zP6KKhajmWDiM)JHlZrk7tTa|bd!T-G+B`^~Mzgm7AS>qFqv)ew_c0)}dv>15`~nd| zvMN=r>m_9UXFBHQ27~3ci1+&ErE*130A^2ORtJ>`A6BaZDv+n2Q;WGIU<BIZcupoq%3kR!~E5;V|Nh8PVg;vfODh>j>)d~P75 zSXI08Qn47+?p4i5>lg8+@42MP;_!B#MR2lWK%kMu{W%-Y_bXR?w|;-WUvCgpd^Tc| z1mHu)5);7Fn*EP}tD(s7qhE(C*xCG-(9VYvlX~vCfJlHW0LN2fnWH-ak+LG4MkN65l5g;wxqL#X%Ip$3#|b%^ z?#lqqq)P*{^kA(@Yp8x4Mj;HyKbjEq8!!V~-uvtTuKbS{Ek$#4@_HZD)m|QJ4WL*; zq_O+=9USiejd#asTW=pKF1WL_;sdjW-*=*&!W00XXL|n^*vO~R?QfcuhX#t))(lY) z>MM@>J&Qg|`3k6XO~nMlEQ*mc2;?e@sX-72nC2C{PlIpXZrK#r60Eg;*Bv2DYG{nr zJdiWBYC`(*&{4d;4`PpAQB}cGc^kwPMgR5RKeE*Zww;*KA+<9ya3- zAvg`Nt53zB(P2`xL1@Ot%cFv@PbOd!@oXQJs7LuKq5}kg658qk0b++kWQ%T!v>{D$IPUWpd(3oZ_aQude4tqFyRp*j zJo}cI!cnj_K3X=fxIzdrUz$n1a$H@_5S6>K@*d@Un@_Wi& z#9%zZp!{zs=w0yoUr}U(cC-JsH?n~Y7GURfhwK7g!ybWaZ^LqPWp5~i(arDo{gAy+ z&-P1#IPg(pLSWW&$$g6C8JWFxEg3fv(aCxcbb>;F+Mv1s_&}())qYWL>c{`ZN2bmK z-M>)d)LHn~W}wc(zu64`>kL*}(O{_l5S(E!WQPiY6cn_r9xe|d3K`YZ!eR?u)lC3rhKq}n38LZ$*%E)7%9}{Pdi! zt+C6jji)}`KWxE_o67m6qc?(KbJ4Z&`&Zis5!20Gf*8{+2RcDzk{a$dP#~B3-8uRF z>piK>^tR@(_7V9A@@D{Oe3!;QqLbAY>KQ9wJekdF+n-kqQL~k%Xu6P2sRdx2X9uJf zU!=UHey69CW^B=ASOYZ3o2Q|$!RVB?|2zj1G^)8rz}OeUn|Ya5iddzj)iy~}zxir^ zj?6R1on%x=a&GP})IVn=&ctWZgNB!> zX)`ddoe)a}{5=!vee}5MX^Hu}2IEY$>v`=z;6f9=tF*^sz-8MN_Q|O1>*V~}?|9i`j?2(;rK3yBT1@sC77 z&-to|^|-2S)M+MdKmcmhB{Q{i-Vd$3k86t}=Q)WXb>3eJGd#skA@ynBg zC=45#bHCX~k0!JULhk~GHkFsRJsW~Z+L{e1f!e_+k@g=^T9_#*ra9NC(Uflxh#lc zZimjpt4IpPyl#hNXb427dFQAqkRqY@|70DcS1z`IsvYXON}AlGY&Px%h3u7>lgS&3 zWYpfy7cR;|BV|gFGPy^J-MdtmL!Duz*p~3-h4g_r1`I z)H;bEYD|?D8R-9SPvXeRrReP(SifjMyGG~sT=pKhQE>-Y>3g94nf#?XVLG~XTHRFH zLIXLkLIc4%M(RhEJQ6@JA9D*P>P;p7lQ;bz(XQ!V)tSKZ|C_orI0J7G)}gyKw7NQ8 z*WSLc5X*@~weScE(sLFUpE$8NA_eeG>u=IeoQxk=ovX;n5ycppv|T%6*!Jles{Zls z?;k&Qr5!wITNS4VZTFqk)rNrfc3exEgpwD_Q6N4;J84-F#pHK^d*s&EGSgDOn>R_s zlP5PefUi0vb7kxE@n_F$tHRF$N*@wg+|g-XCoRL=*NRIYJX{HyFL238UlSR*<(8Dz z;O*iq-L)5P*)#=MN{#ncn}!A7=^XN#?U)Bd)_KS13Kc}l)q|d*Jm$Wk-fK7t4c!Y^&aiH`{nrr7!iE$heMyY?ms*LG+3} zUG#J6VW}f$EK&vi(wROuXuZ|Uy!V64B9PD&0U)XFpZ}z4F3-BHrE8`t=tU+l*oxOw zX4h{%C#S~k&Jh#xRB8PPG$YWIc6160$-9b8Vja=Cd5-A(_thJRhVTQ{rTsZMpunj~ zq{<|BSJGx9K93+}at2vOXXe#(<{PeDq3d1#49o26lo-9v-QD%|-9Ta-i}h5kky*W; zhkGeu?#_VZHT$OHp#qoB+%U$Q$oZ`zt27bCmb(hUb&Y|5DAJy2eez}5Cnt;S+_YA86<(7jYwer`u{g3`+vHr%}Nh z(}{#zQ`31oxgOs?I-$;8IjtLOyD2#x?jar^lBQ@`1pa&1WoU?Dv2*9Loy>S_tD-z@ zyOYC7SL{Oz#m?NQQ^|L?u?Y;cd4H{+1y8_^QYVmBBi>;|b=Z9pW5wW=Rvf|N62&r& z3JD_bphC{Qi2{wi?zTzT^YV4Si$JA48)2ZHYK<1;k8k=P% z#4D2#h~#91NF7xHrqb+?)c&EWpn%}JO=p!m#OC3QM~hUJRR2&5^*?y9&f81iy{6kG z3hYRzh3$psgO1*DoB69o-bG+BrD?Uh1U;7FS5%1N<~TMO(ztSp2Bk5_Zw%HZB1CnNFYi zx~jV>xUeCu72Tx^h1cKG3KN3&GB#;;Y1scw)lt7f=TGCV>Gea>p}8vdP(k~CKQp=> z64l7b$fB|m=YD*_kJ1A%GoW#nW|RVV#shjg#r%FP%8f+CV9NHpo$af$j8~sNF3 z2toP`9MTo&$sbl)ShKM0v}(P0(MB6Gc#s~_KeGa~-nglS2!=d2SA>d*q0PG>)BBkF zwn6L(3D@=lAM#^XRwir8L6f$sVb1Y}x_VFvs54c@)rKopnoEAGP4F=NsS}H1S1>&r6>A z>sKEuYs_@!AO~!&ZpskFO2RTTgGeu4;3v>;43)_%{p_)UA=A<%IWLmuN<5>1!Qh~Q z0saIq1}7Tie9B0#Hyg9i&z^NFDqdlkZJnF@{dK|9)W7Ed$BQXB2$Hyv&fx~9L`9l0$ zh^6tJZjw8JX-QA4C;y&GCF*{^Ton9X#JT_p;VkEpSC`1S^f7Rd9cWbT} zIq6O<`vs3~+Jx!2RHmr}=eee@ql&Rvow$a4H@~S^Z9D6-QcHq!FJ+6((0F&SoKQ>8 zDe3VR9EoGJo&R`#Pcog4@_oYjUH+|$FFGS4@W0LGB|)_aDaFZ=U$2HI0U#m>bUE8;O-=gE!Y=3S>|E^L8R6VMb#>vUHLS;3`titZ zF&74xE~1i#no5&EjNYzf?xQl@MfO+}(7f;KS4#8z9$=F0L?7+KdYP5QVU9er;{qZ# zdUpOvS%uqfG@*WxEU3Eh}Rx?l$rn!mY3>t<-bbeytI0FJ*i8}#p*Fl@m$D_vMMYHuFb#JpM=)g z`1y>Um(LX4p-cX$X0ZMM={>K9H28_L^5?~1J?xw+n%;LsFt{~%;g#j>Wn1#E`KF{{ z9L+>8nCf20MOkr`zlp&GH9npms}VG8OW}qK&i!l!rh#7>CfKM#AV!lA(J?H}ek5NJ zm1US6(w25%@StZ@y;7h-w7l3oscY8d*Y|e#iC3(7l+stlpH7s|XA8r$!#~_0q(tG4 zX>9O37<;KKmFtNWR8uUH-4s^+EM$1yfU)GM!_8e{;wHP*#7aDTQ(^Sr#?X!RR^UuF zcz;(4EH2q(del7+7Zfbyv+I6i5cENyP7O{1=QMxwSs6PJi=LmcPw#-f!D1I*@lkBJ z-o&Mn*2pxwVs~W3Ex5cgfxOeo2a0g>r8~Rg&*!po%bsSqMGye2J{8`htGQ_Dp~8B@ z=xajP!gNuD3q7jn`oYEaL6zT2A&9ZzoV8cEy|ermtnp5e&H1j@YSp&oXN~>v6=Of2v+i)o z05up2adJ}m>-Qm0N`^et=qml{+=>Bw#ieXYUpk@JZ`^nN=DNqk)erHcfh!{V)ox;E z{xZDYFo8K{^=Hha_NqUIuoB;Fiwg7Aif}qV`ob=eSfYq`w;U>zTQc}HAy`CMYcf99 zFg`Nh{_w9fz`&t3ZRnY@I&*QpND85iOZx6?s2}0mwWS+OI9l$^h=oV8!5u&Lw;)%Y zZ5@%Ateb;aEB(&#?Y1;vOVHg`zOHxo*v!LvA>WGkV~5)3pXHe*+nj{Sw93&>X&`8# zbGhRhc4GRjhO}wK`|2yp?)rm!6$HQh!?lK$FKpw4ry`?{u3S0hekiB%gsZ&`kJQGj zW3&KF?kl@tDr0wY(B@eFX*0XG!bO`C<@gR!&Qsw55xyI~7fMnLRG6|Z!dpIu2ic7F zur~Otf5G0XL1AQNN3Lk=i8}D#K zodz*uZEG#P5y3HRZz6N*yWEL8KF!ZKwfJ0gYd<$msa-+Nzs@9SE}cdDPeF700@(J|0^=yOZ- z0D4Z^d{6dW348+kOxwk#(I#m-+fS>?Zhj5AOz(wq{AElm^G&dr!Lsq-pGy;7Q=rpP zWBLk(9JbNq-`^Bm@GD(PvciM#a`8f$;&GC}LXC4&vux(Mv_o*CAJgIEnxid^o~ovS zt4-*euP21_Or9Z67wAu>!PS&R2MdQv;^Rw85u9raW9!DxUdZGo6&G&RxNRrAy`sn+ zjjE;?0nccJ=E3Pk=D&Jv99a_3Cy2||oJ$IFtL86ZeR+-<$;rdR%gZBnLS6K7X?5*H zMj6Ya&92en*<+=o0C zz1Kd!M3IH4(#&D|x#o=z#~j)-PVQE~i1+G8NogY zy|{R5v}-R+#TJUIx_ueJqU6T`REw&=)=2YU#HGd2O(Gv@ltW6GQ#ZMpueoWK> z8PW)&$=C5yMd}soeP5iWD5Q+cwfLO2vB}5xX+mv2pi^N}5eOWOT2QPMU3`7--4pQV zl!*1!`gSt^OkG`FRaM<^Y;eq+h`2C2CUcoZAMvHW$a2<@>D9T%c$}H6L{)PQK6$tf ze=Zho^$K-`=v7;?QKLjoUu|qjum*}eX)_UTE3*QVv#iAzGvE~PYB#1De+X}AHU_gc z?>G}p&9~V`hbcfFV_YDhI`km)T5H&bEpJpreEKw%;pf2wgKg5?$?4$LGYL9T-%<|l zRa^M7g8WTXoKRmap;l3Qdq1f7yxkg~%)xO!g@H z!HC2))Ay5@o~$olG~#?iLbIx~q_uVoEmqW1OU(4yP$Z}_g63;3e>u9a!5LZ;SManH zHkxF;J8NPc!eP`ji#(Mh%4H&4J!^EK%+8KD%Up!5(UrR@bmAVav%y#p)Qq%i=}zrP z7H2&anUYc>Wadm7J3pU()EVt;j*HxkkUJzbDRkl^;@J@cf4-Tsq`Na0PAo-y&?kPl z=@}bh+vrEIIa@d->{z&7{CIwAf51qyQFknS(0_6~y20=K_TVw2`m$ED%e=sRo}j_xeW&6ITQ=el@zkx07J1l<7Sm(jNSznTzW<5c z>V_LTbDAV~M5$~VXST&vC z`o{B_X)M?xJXo0BV%&yzMpP{l$Wi#Edxl`p&q6f8VXE1BmN_>Y!xiCH{`b1lb*c); zl3GY%W!9eL2y<(~+Cti!??NY-FI1b|BTLg@11IYh70`w%pYyT&T?;*6pKaEw;&=9v z9?QA1C)V4)>$X9>urV`wtQ9G{rM5B_936kD%}7sG-12u%8Kd`a{;Vfi5;x0{=j==j zBjRLpw(q611g^FUpTNWv%nIr>tPVu7EYzD3NxYuB%?DEDu+d{Ml4q$7)BcZYXH|)eYq&INU%8#lUBx z<7>Y@CJGszS^)3gX}BCP@n@XexWL=}jX2F7|CwTfJauS!>pfYz__0;p;i&M>@+=L} z{6Z*<5m+`u;0K7X2JtxCevw>&VAMh~VsfvVZ!Nn9CM(j%Q~tiUobi7V-}49Q;w z?74t2*DHvO4&4Z6x8~;H;Ns$l#1PY!E8|Q+Ps{o+9b@Ay8J$>;(mlnVf2zj}o8Q!T z%WrYC(cJZ6MLi(TD=I3Uee`hT5-rAZ6?HP9_L=TeaBeX?6ISM9b2z9)YV{5C*gZ<|0{US>}Q631NI*DY@JTQEwCET>)BI=ooi-<_B zuC7kH5>Fd@B6irl2RSus3?`ZNQ!dWI2g9S86|o^w9FSLlHGD5@wQkI!b;r08N}}=^6i+753-O7Lgc9-g{oxlC63- zc_W;@fBVLy^g1<0sNKK8^Mg=garA`K&@_bc=5*yrW&Q5Ik`gLRD>&*(`f)D+)Km~or`@k_!PyJ|b zfPmIMep^k=3j^j>O@87C+cZwn-Nv)A_sN_(u_4$~hvsSz7ql34b6^|FG9TN{Ywl zrJv>=X$p7O#$XdS-R$;;6-3H!MsS7v4aEiT;~Ob`>Mi;-ty2T!66mdku(*5I@(P?V z`;6)j_em_LD0)(`6rAl<#t&%Z$>!1tR2OW7?DgIe$7}S83kr?janNiZqJ84A#wE z3$z-j6d0;RZpzvH$I|&(!_N08e{jUaIF(D@Z+8PD@0Tu19xxZwsi3ghj>f*W74^XT zQ_$!X8Fu!zMKFgh(BQR?wyLbp`72dKC2(B)cE<1b-#L8kk?C;Z*TmA)Qr-*Il?XMd zq?R)YdH562xCH#z5^E&VSgiHCqKG|Yc>n=Wyx97D!0x8anHQoOs5dcB?-oCH7TzT% zxc@Ph{kFQKaNa{Q4TXsvUO?=P1%Fa+G7nWR1k$)rRf?Y+04#UkKedd!7M?rD@Zb>D zx6WM9vkNFYC?s9qkk5OejXrI9Wz7P&R(}4R=@VKBt&NIlf9FgrUoVtojk96rc(g3L zWn>F^0ePGz*BlI|T7`E*@WPeDTps<-0Y7>c3tj=&Ye4{d?au+ru!%K*zCce^yun#V zth@f8o1LAjtDWkGlUI}$p0xypRZ`1gl3|bU*=rxEM|^gqa`@R1;8)0cO}5vQ(yto$ z!rM-=d{lFmT5VBVXTQXzv{)=) zl_rAxO7U?)Q$onkF9KMV-qBh7`jSh7o*@H{A05!4xkZ?M%fp)c{jy$jyD6w~zO^#sI*iIM8+ z69X8L|9C|Yo3k?7j*g+z%p1hL2boJc6R?~1`Cq!bn}YFrw!l?trEh@y)N&AQZw?-k z2zYr^_m5*r)a{oA8_ykrwpRz2pV+@E6tFsLe99m$Kr?0L(F$NY@TBDC+`pfbQ(zE{ zw2IH-HkcRcKRyfjY!Soyh-#+=|A~K@hH=|72<1eUsnENh>bG9QWq8TL7I(>^eYN$x z&(`h3{Ed*gXtNV9vk*!9Q&Q@E-Q0_p-J)-w z4Nyi9t&ojm^`e{ua)c`5$VOQ@-s90Q@$Dr*VV3NCeiK>cT)%Ty|1W<$4)^StAbNQC z6WqA56LT_Z<$*i^O8zyGmr7Gx)}JrDMAZ6`pfQq?ss5nXLXutGq5vgsk~oAP>(Mf# zVb|&DnAiQ^Msk)lo7LT7UaJ7_-reGJ9p<9lx8UA4t49JbS4X)VKu(EXXYP83 zJo9{F0=_O+|A)=ik}!*8z4jFPsb~BL6N-(xgi)epg!FmkDAl)S*@cJN0_o8A0(@6? zJfN~tI!8=UaD*c%a;l5V%jQ$%8c(p|@hF5@Q3>h9%QtE{(pgCb91q%C9xU4^F7PH3 zx7&4!Ws03%?N~3&aggqwDSlgXT|>PEqoF=b7VZg05G~R;N*-rr7(Ii6A~5t2ZuWql z%=(!IU)6Ze;}(HDJ?0lBI?-^|o~IAYUuGHC^qnXY)U~%XrDUN2_$xR!o*BvsI z@%({*y1U=I=Uz-sEPY@lnDkiqqI}v*_N!3>W5a<=602}VR*aybY4cl&QOTW#j)h{3 z!TWP+Y(isoc|{2_k4!=>s&=D1vs39N{^x?FR=hg?)!ITmucL3j^8!*Ml#TCDxqvQq z%rmMvaF`i}_(nR|u9l$@ROE@x>z?O9Pp+)NjhQ7LqX!*t#fTNvsgK7*DS_bqrGT$7 zYituGQI!6?*y6_xS1t{AH^B#M_yNA-!tJgTPdqhf=#crPG0?t}^yCJvK7O8vm?MEG zP-+M3M+jcW`wo#l@b4Ax`XkQp{1b$>axOm%;oB^5_@q4s%(+aa*140qf|(?uxZFLSQ!tRjU+Z zH4Bjb#uD*L-ss@SR_@j4TlOQ?c99i$qizPKzFjUt?^6EW+h zpT%&`3;zT7N%pIF-(4kGqt!Bzyiop#~@7GXJu<8!f(W#Z&>tGl=&X z$f3zd_WN^3tl7=gxK0EMT8WIjSPTvox)|vCa0$YHxioBL{1RK1F|Sj3l8HE*ia1Kt z_?Z_CrIMZ52M<*iIMhujA|q3f&;j%2V z8dV)>XYTeW_n6@1w@(q|sLT-^ks%`%XM=5Qf`TAgZnK4XT-iD(y-#t4Q%a~_BO|(~ zb(zrh6}WwMcAqL2q|f)Jd!(l&O&%x+r^&auqt`)hFn4)Q>%FT;=;T<`$w1Nb!Zu5y z9ya3V|IAF2NH#=usK1?E!~>PTZ|~-xdX|fcp4j75)IfZpqglm&{<46i9-mV5t&$tz zRuGeIgU|?KXk{HEM{0M%dYiedsxx^%>=%BxQ&q1h6qA_N?{63D<>c+{@9!FD@c-tiRZgzFzgICJQ_-_u4tM3<=z1Z4lm=J1Tb{C9X`JUiTiv-YlDObh znpk>AG*9kT8wZdX(4BHWla}Zj8~*{Pbg1~Tq`Yoe4KBbOIPetMmcnS{8(%m zx@*FwUXO5uZu^YVm%}s&OlST9W8uK9f9jeqy~4a6EBVdJww1p`41l=wJ1U+B&%ne0nF zch75wTh2>nsL{k&y8kskNj3*}xTYgE@h~R$B?2V9e70ucN zzTcfVa^KcDd7?`^`kC!#c>v)MS1UDT*dn&u>h$a_f5)$;SJoCFR%b{5`YcVyv*@6?xNDl%d%y8ow^*bw*2rN6La5e*em za7mEot(vLMzCX1xpi+x8lZgUECin*i(}wD(PV*E+0ZdEpDcC_(ri`u*Z_qOyU{{5X zUyAfnJlZe+9ktGHMT%%;U|TP%0#>GPjnFPk>k|h5Ro6Q|U+Gd_j;Ov%tK;yrw|8^1 z&z@hS)#$0YiPQsk8yulRHqrFW5{D6Z?@uSFK6yw~-5#^|fE9e<<0A;5(%Z{!Eh^@=g9BXEU5V?dG*kfg{22XfZ==;% zTcD7TvNanzT=0Nap4oki{w0-%kY$;gYICO*it5Gky@IL^7)Kb1Ru91V?GU3{k@o99 z<@L}o{C7+_+Hn6jUTCD%9b6JyRM^4{-?_ihaEwGArRAq${xG5p|jEs$1EU?0S@TYDl-2o)9&vA&$3WzclYiN zAqcRVb;#HQwTH0FJW2|eA8!0AGS}3sc9$CK#RKOYTwe4axi>HjXI1Id__X)%KP>n5 zuQvy{wYs|M*kmk3ys46ZyWvoSx4aQ{X2Q(e=Ng4(;Hgh(DA>atz(qp z17gl)&>a*m!jE6ND~K# z6062wN@v+(yt&kNoiymYPQolF;SUg^*_t*dXZk#vILOVeg;%a4HmPvO>HJ)qltF82 zv%sCzp+46Hj4Kn6kxCHZQZ!x*0Y&&Q*aYPo4yx@3X-YLjOM@OUTNBV>)K%1?BW@5) z?NoVXO#Ch-KHr^8sjT#9`W9kowL4!vG2ys5nW!AweGjmEyx$;EcfSczcS*X3rIebU z>8DAcsJa;;7KnEKOq1onx44gv71C>x6xY%}{mR{t%Iai=(gvS+Dpw;9Rb(W+qjDc~ z{W>~f3@m}|T}y-My?w+tN0%oe9ircwTWOksiS7Gox~24UAai9*A}FJr6nJq)`0}LM zmUQB5dU}XOg|~>}@lTlXJs7&6NR$Rs3abX7te%=u-j%I?>Sx>#*esGYwcJT-s+=~W zqXUM9M6*cd^CP#jTNId$+bB-^xXme1X!gKZSzE)B7%H}=vdHZgHzYz^1bSnORIiwe znB5_8k^Mm?o4o|OaXMc($jn^D!K}xmsoPydBKWc3znX_lVqc0@OXjLB3I-~)d+P7I zD6;9l-}?Uf^Z5AZ>n_%Q0=NCAyI=CBgDD|&|v5>T6~3bX)3gp@I&7P zG>-AeJv>Xw1q@gnip7a`6bn=cv1N6RxzN@6WAV*%pXn>u1EZv)6@yT8x;r~?F$!;5 zrDX5Bhuql!@pQo6Yte$Au)~pRc>*7ksypHZfVOk^G4pJ4Z+j1r$K{fdGrD$(?Fo4P z+CB9n)k#Sn1b568ji3*nj#7b2vw>A`-EvIzO}Y$i0YjJbu+RxM(K1(0q$H>=-kRy{ z&CT#F_Dc^eIBc{xTGb@$FMG=H@u_>w7eJPrHrcQYO~rGnMxR?&IkO6^<=JdH`}`kG zcJ?1@58Rx1_b$o$qF>2~*mw#e?qRAJ?)NP7IY^`xF>UvhQ$7F^gMz7S(HBYHQ-0Rt z)`3o4=}A_zc1S>yNFp@KlX4MHUN8KDtl$Fa(Ny}}9DK1>XVIjQe^p4}`_S-Uk+t*o z202Ru-gxIcG&qtpsvlU`?F(fmeGJ{^?T}FoHi9lP#<{+K#;g3mBE7tdVx{kATq!U~ zH2*?t|LA3CbjFGB9(A?2QCht~P?oF+4aF5rryfj@9-9M7w8M@JC0ZKB2d<;IxVA7c zF;X4virXweq}_$RQP7CxRseLrM7LsVlqYd(wFqSlR@OxhtVDP?6Avfv#d?~_J+1b) z?XMX85M*)&Cv^Z$TNqexpF+EKtdX6r4zNDK*MR=95Jlt30 zh_5x!BCBDJ2V%ZHiPZhzZo=-?YxK-+# zI)4d|l!LNhX2t1hNm8~w_nSol@d3c$3H7po@=**ckeW@{gCXP8D3r`{|ElE9w~VZ{&NGpy+!67N+_D{762{i z7++xO;eeB#Kfk`ap%E)NHo3LRB~xrIzMIPI9Ao-3mM1t^-uDl%!@c5rOEV^q>21~@ z4fHD|gk>k)yjExD=dnn1k}N7D*wa(sFp%gzD4W<=0zJZW21`Jo1-Sr$7I0Z9XaOvx zK#qFsys#9|L`{EviYw*y1vpI_{H;9Qsp%$K(I=d6*~G#KP;c@`!dta^Z*TNuVd7-Q zCTFp=wSsC~q~j$emohdh#q5heMhG?RJi2=`F+A5!*t+TIL5T_5wl*ZD>GU2#HCBHsUk_Brp`)S)l^7zUMI=b{H*~A)aQ@+ZyGh6OV2{$_mM-LdOm(wIG4T(et}tT?%YX0pqM4Ztvb zfJ~a3Dpxsy#2Bo)IOx(Uy!4kZY)2aV^r;apc)kTAc%G&t8?+E~B$OPTpKra*-xTMF zr%!=beb!huMS|@2Q)l{R&C6CffpE7!5c)1#hfW8pOJF>T7u_>}I@SZG(!PE38=J91 z@}FWqQ+p7r4UFf6#-*7U{xfJ90;tL9kP(Wx_TfgPEOOek9@uOG~N-|?} z694W`9`El*M}tx5v}zoVl~xF1S6cFG1t^#o61wWiq{yTp_F2Sy1Vk zfR94a<{J99XmmAg$}iF9rJfTm%+|`j$<)23l9fA4_$Pt7^Js4!o*^g89J&gUs*rDIFO$``yYWRMl!D>g{oxmvNc~k` z1~Fv9RD?6#ZAAZQ{P2L-^7;CXG_y)fK@_vS$nG{xP!?%Ye&#d(sz9`Y!x}k4Q~mem z`G?=sRN?9H;u2Er)Hr2cH%s_la99c87&Y1KiQA%tu4IjiIb|HIokU+0w*+l7z2}bH zfjpFM;gPZ3IqK-zS|4ergP^!$g>##c{99j@S0EFiZ+}VrIs3i(gR)KZ_+!umh{|r5 z@g=3cKE;>udZ=}&uk`uKk1qlO;Y@xx4h-LN);4XX;(hSan#6DUNF>WaH5Oh%zvoLjssO$BUEH$X=!AnuL*bifVJo@3JBUjl&|IR zVNEey)clJ++(+bXkmmu(UXSqHjtkD3 z?)%N@&dZ`{4vhfX>*CO017`>uD0ui^Rb>q(9EnUcEE~E8I6kib15<)w_%{eI9W@8v znO35%dGJ9|X;s};=x*1nGf|9gURvG7p|SiMtPjiC0bB_s)YTGdAboE_RG*}6d9XJO z(PQZA|63+X=$4!Ic>myBv|;!+Q|lj09~z2PUf3DkfVC9;DID1&=r$dC&2o(z21>uMD?m*N48n(LIE> zTr_S3sG@@oqW#TDWZqi_U;p?Ku!*{fLX14%r5}gcBbR$ zu?F2Egt`7j_pZGqSPz$H{$<}^{`VPZ)2%3yRPz3yd>pu@M8ofCU0>HtuMtBz)ok^7;c7 z=zIlqtOOQlEIPn_Lb=*W z9wYY(%E^PnQJfykz{3Z*-e6&Z)1@LK}(+xPF!d==h-z;X}r-XIs{J?`~&<;)rp$zUJl1k91-)Moo>L z6Med5eD3XHb}%dfY+DSHX&7v`Le@ugzojwP9V8zuWO}7W&$n+Fpqowqs9P@Kl@iDb zvBN3SN+Dj+C`(|ljhA_Nz;4#rVACI2aS$+y0F%QYvfED9x%dZ<55GJ|4sEpP`k64; zcCzUyIkrf50#Y?qzY#RafDuclPr+ZOnB`)A&l(|brxhNh-Gw(hee}OeWER+np2811 z#0Eas=b|-efIqxL$8lgkqlOX8UZY#(YDaHI>(KyG1fxPox4QtHm9^gv`S=BNDU}j> zR1IUWugR?c@vot{K;zB^Z4WImL^^mf>DEU;xtKr#QsrH1t9PH$uop)Ln9yC(@(7K^0H`XY@!2P6tu=nwzasr2ItnV1C-ib1NRv)pbQY8&vsiBs=DTx4Jc zEi%@Y>z#gg29U6YX`|_6!@7&;izrMxN{>jx>s1v__;3SlP8=jv4onHQiq@+Su2&r) z9YTiz3m`S|zss#*weEN`vpS!-Uq9mGzl1|e2LBF4MYB^VPH0bvR>24v_o;P{_2x(|aX3)O@WiW^kb1FP$roqciuiKLbMgt^n^mr60i zwhH}C&tk~n=%Zck-|c(!)>>F#AEUH$#Dm(W+_>Y5a|45EMglQ39Q+`Sf^z=OG@6lA zDWN$NZ^eA){nv0j@_+pmvh?!ZJKARzR-*vN2hD)6z?Z-0(-sGG2me%?(B8xUS%o?b zAEEou-waM7QJkE^)gF$HL`ufC9T83&6FUD2!1Z!yO)#zQ=t*;VIy)J&R}6`XH)tZ^ z$$>U^TCxk)O$Xvs*iEf~PGP{+rWS$V;4Y<+LBHv4NIqLq+IN0vXy|W#w3A`RW@cOC z(b?Gv3Ba%z@29c2=ykAzOaJS_0|Oz;ATP9q0kRVyO%@!mIo3EaaWEEY|84KEzwuP9 z-#J$Y>x>(YLQ_2&2DY}cH0I{G7HYu_1kh?n&s`TI`vZ2EdrI%LdJ%1@KH-ax^hm_86$+(X-cna{xF#7^(BL8hhOK$X9^WYGs49*!Q7_CR7 zw^9b@DK^6mT$dSLh!N$GgA0`ugmvA=4c;K;QCt0a!~}GZf)*Z{~)$ zvA7O!mt#@;xcBrXnIy@?l`8TG2-w;R2r%00q~Y9*(q>&=Wu3#>my%gWk z{$)(N*!9Z?yZwcg_paX9e{$a5^wLnN6ovnTU{!{KsYB-UIo^||#AnQy0R8!^(nLYE ztCy#RUf7ENxNe%_n0)W^o=4KVPImMza7%(#QPz4#staCZP)VWl}4DooK~V5cAxSy<3r z?fbdgw0sV$tRrG~SFPh>6GKCSj!5F2t6Xtq_HGM5zEoD`(CRE#m}r~T+o^;|>Awz2 zrOU0R5I|?QH*(L$R8DT_&fLt*yLVIRT%5uo_Z6mAW{4Qw4EH)Lbegs&2blPj{JXtD z>o6>Am!4x?zo=q##lbya_6Z!Zh%<1jExO5Q#hwitanOV+m4kOm^n z{(0RQeD|&HA56;B?OzY-scjYgu#!B&^=(QQBTz@5`qn-ZX65# z_5H_}159sdT%p|_^t~AtR%O{(SK+1 zydkGAY?to)tVDKbivh)QB41pLB;FS`eeTK?6b0X|#W+Cu{tJ98sZU@4~jwf(>jBfn;!X?AFr!Lw(g(;PmE#)c~6}}4HEw#0^#l%VhPPpl# z%+3m4_wP@(C+E=cx}R!0R4zhPbw?JnSjD9 zZnA{<`pXwU_N5RsKO`Qa@9_O{T4M9QahzES6YusH<)Csmjkw)$c8$2jA@cUJyzi^f ziJ=qHR#Y<413AIL5>Fc20jwWm_{BLE8}leJz$mM$+z6K$2q5?WDqTm!G;EB2{W?9Z z8!9Kqd4}k(Kxuc45Uy$gGFG`=8Km78Eo`isb)_*GA#YGVXf{@gzNoEXIHIU{aX=lJ z`HWx@)j=Yb_7@p#jT^}O|C&-(_Wv-vljkEO&AS;mH>;zI-RkHauC5p{ST*Kj6$7&` zWVbha181lSy^m{jCv2r5!j$zuL0L{Nbm~)|(9NaQIld78oxu7wt`a_S;lT#uO1G5^(-HEw zs{MHW7$a`)Hub%>$Ow~YDL!SQSG)0F(L*kNi4t5tqwHg{-TlPeRFzRJ+v-&nq!akL!4&E^Iqut%eFpBL?~u03vKG>#|u+uol<$Si$! z=z*)|T@0zZ=;LaOTM#qP7U$xW58dsM>B{hs$zdze!|bwZ^0=I?k85r^-o`~S+?E=T z!&S1ML8H-{No?%V)!`=t3$_OAT)&X<{aahi{m~h?Uj+58O=!;Btv`0x!rzy04SA)#0I&*811b_`3ggYXWL#M8sqtcUGW{bn}3FaFYUAj$A%$Y7{_@>}ub} zgE?ZCBq36jx+Kx;`b(m!+V$Mm2%snG|NWzQXux-seR~Y@G-Z^OBqWr0-&HhZyC0Rh z?`G(_Q;$kXc5?oWe7|zPoh@JFhQ}u0^?H+2B=O0Vs=xQnNiJ~b(oePO&x+W;s3@deBd=g8?>`+|qlo>mkjx@yHxput!P)a3U*^)5qaHc{<} zz=@M3xUJNKf(!JeXt};p;uK|SB;Cw5Cw4sp+hii`JDL_|>k7wRvYzv@;hIy(WFK)5 zwe7EMZ}QmQ#VB7Cr;_9BtA3)UI@}is z_8BC#o4+JCnUplGk71D$gLZS(cLdLT9~xBJ_zr4Uo-i#vM&T~_x|*aluM??>0naX9 z`)OKTdNRcY%DR8_OM&_YU@CXHReV$Z$LWI z7Q=7X7cCaqr3l2WzZ*^&?OiUIK(G03E^Tj5v}qTW=GX~->;S{=Io{l?8cgCUEvS3) z9{&e$4+gm6ar2b>A{xF+TczhPa-sy1J@S&fp|hyj>=Mh-YA&sq;Hzt`d2F~%IYR2t zZ{_;^m**}~icZd5y8bau_mm~j_JzA7*Fb=fN(mhadBOJMbwp+3N^iV?c;hP*A>q## zm@h;I>I-T8;ylK)wb{kQ<6OsTUgi39^NqV2Tr5(>G-#2lG*@`916nNds|nucqCp?$ zbj%J?UTVlDki2E3TrfJJw$j>68D=%_*#PsCNMwG}2G=n%H`~+|4jhaiBpf=K$dvAT zLSH*~m;jm3t%^iDk(2=wLD}nx=n`%)I2AKefPMkiEyO}g@bTQWrjAD=gjD4N(he}~ z{@?=yk>>D?2X1aeDe6jYLc(~RHFIdBqnQDgqiwpudGNHn*JL$H(d+!cJsu;$wx@YE zHsn{;rOxQ3`+OX{A-kJDek?Y?3#a7H>vj$7w!?J<4Ynpx9=NEOo~llPul8H15ueof zxJqj;-)nX>C)+jK3-cs0Q|?a@#PRy)1BYt!_E>E1oXXI7O6iNFd43i2%8==sPi@Pj zkh3fjNqO&BJrQ++#S13P0Yb{9t_GP+md$>086kE4ZYPxmuUBiA#JNXQrd>~HjgN;& zR`P`h*v#Rcr^T;3VU6L4bup9@aT6bz#n`Xs4JN)s;HIZOf9~y7&;I7022rcV@*o9x z-;~W8e)mC{R5ji2-_rrVuNBNoNEu&+Ub8se1Cu*Z8maSH1$V*7GsX$hyEHNJoA>|m zyJt)1A4h|!l*Z#;kzeQ4?Fzm{Mb2PUkT2NBV}h@6*&yrB^&kyd-9)dbk$SRhZ8hSI z>rXBeBNq!trZtIuF^Wk1I^q=Jt=g$&@6fGz#M@APqqdEpEE~s4mO{uO_NKLc!@A?; z3XIf>nj$LgZ*Vd9e;tU7(bLiD&KEX9MaezTu6`_LE`7dkp-9DiW*+erB`_E(xXjlab{T8oXoi>X;`O^ob$x~WoC8@Bzz5L}%Kq$h|w)*^A>XsMj zY>h=urO$!Gp;QX6oCIo<&HyVdwc3t7Pv+7 z?WZX$*nP?j`!W{j61Y`VJskXOnxpk;`rNUqmN`tBY6jb*&xqZHr3o~8QMb0($~IKi zdm4BP-gz5W2TMXXeyq-ER|EYv*jvT(kO&kkP-Wi}{_lSq@v$oc-mEA;WnonwEvd0! zi9`PVj$pQ{?k76TzhfSZ{%B!z?~{&IZcwVr`XV3Ie|<47|SEx@WD2`)vmG4Ver*Z8yj0OiI=FT~N{L$Bhgg9lZyOF0KYW=xWp24HEU7msNia4E= zd@0b&*m##_4yOKinCI!elB>+hvlzUf&Xq#T*a8kR1TBpB}I}+uMspYFD!dVoPmN8|YG7lj7|Kf>rIE z9JAR;)l`XDd0yFHZPvW9-M60A7JQEH7J2Fo4aY9S+j$Q;GVewu%b1@F8=#JUWntx} zZeS)Rly@aQ$F4S-%oi~~7`4EHLswafd@+dY6XZd>HKBVDd++@#d}FK>ot zFboA&ZQ3_^pkU$i$y7bmL7fS7ZkYszOGat?R}11YD~Ii2d~!010=CmUmb=^AzLW#g zK21b$Phq>Zv{~}!X}~bSME&!yj-#0>FFh96`2GY%r7$UhkS!>{vsCL`?Xzd=W9Ft2 zuWggpg;^d*8Y%9)(^9Reo@E1W(4ya!D_zP|kKK;tE(}S#8iFFHrD@^ipQj6p0Yuqn z!}sJyYP13quR_qq_iN3O3_q)Y!Xx0$DuJ1A9tbG5rvCaeH`vy&w1`*UC4*O1Z(BAq9!A91XM_Zj3RGD0o?cRU>a?@W^?GC4uxF=`WEFR0La!*M z$)tI2ewzS64%oYSMVGaD)B{a66<507Wl*=f-{nAT=`cMyDtt-h&CyHf=K3cM`=`}J zL_)U={IU=?ZgeTiNE#}=el7Sp-hQoq5hG)+hG>7i&%)BERir#z{apW0`J4aydin^H zI8*5;urRw@(SHwGbVdA!EfgAkA{=eYcfPc@lR{*!u(-&}i^=|sEXvk8zDAG8i)VuY zCroIFf6YubG~>2t3_!3;DsoI~Y7u9Jk*hXZzvy9C^^9@1rcH{jgA1XQ_>8^DEpt{u zf2Xv@*Y+uA34QGAQL06ID(M^POo-;R7(Ia1+fI#Go75&XUA15Pw8Vm$`nGm;RvV1s z$3Gd^glWBjf}hoqg<0(SSbd;W=FJ7;axRWda$HKZh?+lA>*Rj3e+A4#W%lK)#PjEM zQeVHm5Yjo(w!EAq?iU(a2G5Rs0@q)Eg3@=W@aJD0ieiLh$l&#N??|CkrxAzB1dWuP zH7GsthQ?$ATlwHSSFL7Qk&}}q7Hh{EZYbwB)4L5aDr9&iYs0C1%qWp7t8PY11x=uL z+1FakrkjNY6E}5$+jgc5ej9JL*BPtBt>yRX5T~(7j<|S8r@4o=E^5`7M*??{lMVH7 zDp<`oxWE>Ii=W)0C0M5KEf~A~k+x4yE}7p8dQrh$f+oblL8m~G&(awi$7`z;Kky7Hf%~j6R}} z2hBf0k2tt$rX1OmWYWNgZ=_5q6B_GlO*$7T_BN1kkqU}F@=a=+rn)l7n#!nY_4<2n zb*^jTU>5bCS{)t8)Vh$JFH9xYCB9;f= zQ<5TSyl+ic7RXMqping?ZjgMjl)!4jia6AP95#qT!zXs623jVSKFw1L)_*?te8iPcw7W*Fu6K_ZX{O+kb)HXdBH7^d9dml|C!!# zn=xMIjEzTo0b%i@y+%e4Vj6`KFRryZe%k~{;#8N3QPRb+iCJuRHqsdUktfl{i9&3YiQ601w8P1fl@IahkRQ?fZ5T0zcHuW_1G@SM47b+ z$dO#yOM5q=Bg5X1-xlWn^@xXB-nnshiKFDnOi)3}tj@s^Kt>~G1eRxojLi*%e>=xK ze}N?;GCS5$e=|ytd2bX!H$y38du67hQu^`f(*0)2Ne6vZpW1t!NieN@%_(K&=H=61 zv(wz==r!jMxH2ss;<@5Dm&@%|6S&&A-o#NH)var&BFdK+RawFOJ7sr0{4X;Bcf8ZB zpo^vF7~t1t;!60u6`PB|z&(YrTe9Lx1lMOh{tZpS-duYyP?T4$Xu5V4)HjL}Cle(z z&2Sg)Nn7FE?XFlARlaa^#ZJJN9KUsi_-_2_3aOG2?Za?U8$%ds7!!gU;DOJRf>~pSpux9+}`Z9pvQVOKGrH$~HoZmLl!>9f_Yywr? z1tA4=YYqFsAYS@P^OpWR?VVd-)6Cgw7*r^b_bRyKavRsZ99KtiQZjSfR~;U61!e~h z+&2gFy|K(df_bwq!`PT#onud0gPwlj^1;65R|Dj`s`BzB#Dw+a$x7kh6tV6;xI6$-W@<0%?tRp%NcJ)pJkvo!li)L@}+ z+2r_+OV$L|b!2swDd5r9MO7|h-!@~j{rZFmLo7C-yB+~WjCi9LaZkLImd2;R7fg~^Ix$Q-oN2kE;V(#vOx2c!u$Yy_qa*5UR7 zxAki;m7KR$P@;Pjd*WTONW_C%s(&a0Pve}tkE>BpN0W!4cbm>qy?$t^hLK$<{)*OnLCoOpsLN6j0s%k!4l+OPw*Z#fACX)XRn`S>J zKj74?f9Z)4!Pu?%VCD;gcbxaWviDv(irpEQ{1(E3e&60x)lqpVaCL~(TX=*k!)zaS z`I+yolI8835y94ZP=fm)Azr}8)wV7g;;B;SK4x*Z zHemVJ4>I{n_{a{5xhNu1DaGp8e#@uNi>)lp>H>YdJSz^mseVM_^KD01uRYIe7?P84 z@%?QgqRK8;EjGh`Jcd~@((G=L=J3`{X***Dd$Gog^1m;ytPr+4R=krrvvCV+475_X z!d_6L)4i088pgY)SPGV+?ABXdze=seLmTDjkabyGg4uDz5`PqX0{e2h!TINdW?7Q; zJ9g`_Lm}hw?mBsvMGZ=4C}M1FtgPH|ygXd;f&DU?;!%yZUZwc_{K&}sd@lgGccr-m zUIhctTe;dn+OeNY>9W}V6qjB*%dU;(jPxGe!9zg^49{?+ zjx*D04q!J44EzQ95Nz#(v>8SJ<^QYX=r1uqm!oWD9XjmQv@!0~1wrv|Eosgzpkxta zgcJV?(iq-j-vSgMO!Jx3@gfi(n>Vu!4)7{jn71kMoaSH0IDg zR|#O;*>jPCOUIyy|Hh4?&>~@OXnRv()Cj=#I(U+yNR#fWf2^i3;%e|6aBRF`TnhT- z-zB~O8%7_xg@5Wzp;7pErOf{?*PGhWGhJ_$fm1h=>qm;EVfkZFiYROrEfGq~0=-qH z6~pL716@7kkQGPzbfo8dz}+@BKZSNi`tk4HAdc>bfG2(brdPVU3JO*Ri4JqtsP#rtFbPg@se@wu_ig`k*Sk4bZ29F&y|1>mww9K5 z<*pBnnYteimtMf@2!KGUh4rwS_jm{r$zJy~iOnglNUiK4{9petb ziTd?}hyic`G){qC12U-fq#_R`FY*E9z)71vw;5?VE((48PeCnx6#lVD|4at@OaMRs znGF9-hX1F@@VD>P4c{e(1}%3`H~4@6J)hzhCe)RoxQt>H^bfCrY~>=Jx@9dxmxp_+ zocx?-vuP*)Kda)5;Xy)A{_i&K|I`anRyhMDC14*D#muWj#qc$y72Z>wv$JXPs?2*eU}?JNr66GPF14NNTq@yqIc-Jl$y#02h+7)`0#3DFMcIPx0OqvS z2_my}g@9h~D|HJ<2h_$7ldh0W96RXi8=#fB!1fQ~$^i`CrJg4-9#+&9IX$+vaO1{E zHHJ<^0seM==-W#J0~QtogES1cfbppX(U5hU!D6wCR`pe()amKnoq(=LDWG_;ximlD z+q*iasTtr2cmv$Hx#ayHxE3x;T>^~Fmt_3te20cyrx525+0k9=qqQ?LUQ@%tY4f=( zZDY6Ovy7l>vGy*L%J6=DFvG4kw?r=VL6rusazgG=7@HiQl;g@Mtq)Q#g#+} zinD}~U%gAoYG34F%uj4E5eSwOWnK<9z(b?Rjul$t~z7T_Ipe=V-tVAhOZ zGP{HA@5LR;ine`L#P7@@hVy^cp>g(xpHc`P&yRHxxmUBJA(_?aB+q38&)AbcCfQBfpxBSqpqO zU64=An7%q5qu+RW*TmBW>2@oK4R;GE&)(Cbb3GXwl~_?6?yvATZWS#8O_Q(!dM-$> z)HO_;bhQMfBthPq^J#z^&bJ-e1n>sh4XL|NMBx6QdlBYxYyW8rv$~LAr~CrH(TEeg zaw`=N94blXIYu}Qe$^D6eYF0E3yfZ8bd-pI`gZYRWEkkC_MZM4-+%(7$BkQVm1pP5 zAAe5|qFkAxrTFkTqFs?gx&FlHV7zu8k=DI3ol1||d4%|miWa{cA^^S|9y|&)W9)sS z`Brq2jIgLNm#`7NL1QbD@#R`C2cOl>G?1SS!L}@F!L*TpNq|@3;rorNZEZ5{qT+tL zR)ANb#;zX?EStBZYvGI!^yXl$V~n(+iU)kfF?gpZ>vER@-1q+EJg%C+%Na;n$T8P$ z)Ja$Ho&Wi5qj|aNc`eA#)dH1H2t}H+D_ESEm>ZPgf5Gg`{WKC~?&IaWYCKZSbkvRD ze&PXig#vo^L`5Uh%&M>G!X1UHOZ1RU3g5DOau0@X=9bf0_F(7$m9MpOm7;-uI6cU6 zpT*mX1A2Z>^6>w~-g`$i)wTPgvAk9k5fG%RAP9&ENUthIrAhBW=~6{{jfyB873m$M zm%Q{Eu+V#l00|1x14ze&z?m!I{m#92kF(Er@3`aqvv&r=p@yuiHP>9veCG4~N_o}@ zi7i5uB`cnUe**dxTYrN6-)=^pVvyz?^VnLn%m`aUOz-5-<=9+-zJ2rEKVc|UW_hC8 ztff22%w)->D=(;(% zFU6wibU#~hT7#OJ*=izm3rdQ|?oHC_a_0p84qK==eipW9RG>tT!? z9fc8!V0DfduQ+$ke%x*SSFxE|{&5n2XY(4F!ovO7vgT*(d+~s=FlXnpXPuqjQAR|t zN;9cfTaF8gUOQ@NG%YBdG`wu?`3Juh_K$|erpV!^FFY+5cvYI231%S&jYB&QH`7sC z%D=XT^0oyS38C@iyP6!QK(k?2YAPF@o6}JTwPazvwTCCK9QQ>Z>+&5#*04Z%D=KWS z*yl<4=PDm}b68m%0)Uh49U?(dac7mUh|KocJQNqlb8zOXYl%I*{~0;17$6GH;wY7P$PdO3AbDcD4`Ny>_Z5D8YKkq>$7uapl!?BRpvIqX? zO-(zKJ!eIMa7E1>2?c-yt%8Sp zcdRW^F(Ufx&1eJ%bl~2l!vm`e|A^I1;p1Q~s(5Xt6##s^fo8Oym|sM6XOBs>003?X zL_2!QiGlX3o8d**3riu975U1%%>rA@q{!~Ko$6AaO{rqWM+g|zkJe6bDS);?ug^bsE(rf3t;f+A! zX9ejHzcZ|E31gxisO`!60oJl|j2Z1u;8*y*xdD7frK5|>Wc~1*+~0hD70uVPvp6`i zvc%*3#5q1n`*`?(qR~o0obAXZI;62}m}}O+NAX|0#&YT-!_t4u?F%K8&OsNpK>y2n z`Y=d;xxzroOKSwzSczd5Ldp*>l;Mb~kF1Xk`Q2YVa1 zkK39`!kiK_F2aj>8f>JY;?QQ)CPCF7JZKzCe><5Z0a{7b)Dhp@`T~nKa%Ir5y6Yy>U~Doq&+#qCI6QR8R<^`$NxK$ z7x^>)2L+`6zigJs3mf@V|0}2n$fkEEC&h_*Z&%MHBlfH3$c~!v4kWH9td|i8_Zy>} zIg11uY2PVEK<)%KHO6G@itIg8y`SVJnw1rwaeNY{)XLnvC%L%z{=4SpcrqOk9Kwyf zFRW_6pvwgJwD8%B(p{NE{Qj{|%rZO)Q-ku%_is)KHUIssD}jec#IZT`J7JaY7I6Bm zum?N83L%UF>3*b&G)4u0qa10;nciLR034#d7kgOHb50Eeu~znbNm?`gqrjhU-6x5} z)k$AnQaca^SQx`E&6c!-vc^xrCRZe&-RJtRhXY;n6(HP9y)%&_%vhNe`2Eoz0DQm` zy4c@e6ol zUyF-#_@Hxv(Yx>uyA!q-7njG;EzGs|l1FV76o3YmIlG_ZIcZ^VE&J8~%YCGOzR;k@ z&u@P==x)(Vw>ksN^U0mn9+1V0i;Q^jJzM%~SCX(35xz3X2L_EIoScyuR_f?DV1hez z*E}YUYt%ndWXi3Pq=3o#u7?`~Z+gR8cRqzv9d+Smy4F$W1e(hRi(^pRV{w8=)TM|7 zr86&Gpw#@%lNalqacyn5>%Rigr^jhSes++PxCF?cg-;(|?_}m#y;pbztN=7EMUN{_ z?I%l!NcpZzkhkm2L&1RIlO*o9H&))%WIGb?zy>36eMnuwFHeH>;1ixq5skjz`2%}h zmPj}B<-UlJ5I?^VttQyr1NhqCW&?%CUh*133rr2iikBTTP2j^Em9@tT%Iw5l(5sDL zZRr7U#=WPmLMzw8UeViLgr#aT@mWMTXvhidg<)fV|)I3~Xv#VHD7M|MHgSa=D-t+?$v1Kv~BAb`iDuhQRPsim#0qtOk~k z!X6ka%U()TpbI(zd!WBPD)c%c+&GP!znLZASV;u%qF-zN^$TLkMX*3~^aQlz%PyyD zm}%LEDo&k$5czoI>)#x(FSATyFMj{{z@`F8*eBom77qS`SCnI9o28GC%i%u?{(b)C zSBU2NI3rN7dGj2)B?F$38@a^x1K4R4;pA_^p4~zny!e4)&aTFm=@>FE%U( zcwakKIs*K5y)Sgku7I;Z6te2QFMC$zb{I@n6BBi+=15xO9 zUoK}nVhu^K$?{I!Y998!+|qoWg^5v!u;JYOIyDhRMJV`Pa zPFlcn54Ss6hDT(%v$9lE_ATQ8uZ*0rjDs0NPXons+JuB!FIR`6t+hzb2o@>Nk_D2) z;1?MyZW~xhbMtv-21Z-n;R{X7Pyq6W7~ZJ@B927`6Ln~>2tu+H8=Lobnd&5IFt{F< zQ@tzzlBZwz2_AzEG|!KxN-j zBoDU$v~CWx8(baLl+7WHF>`CDXbijL(q$C7^h#F`LUUBu*W@NkOXkY_d zgy5{bAL?>*%q}iUc`GQ?FT+U@*7}i_jh6k{mjb~^_gsnXE7GJ|d53dXUP)oB&` z;RDJ$fgIO;c<+TpMp|_%s|o}02Cs)$?T9-4LRhLZsj};TePtrJe{-#L_YLO?EF>~h z%G+%fTgn-XZ6QAWzm*b$1V9}TS%_(1jYyOPP9|-U&>p7P+3JsOxX=Y8t0vea~=cQp=cNHRafD}9=&eTnT4kqSjoJkaOc&Mymgj1F53sC5_Vb_$L#nAQwY6YO?5 z+!_z72{O_L&l!=6Tq=Ab2_rSxl`QII-aZ1ciX6+^>BWP^CrlfZO6iuP;MXh4$Pshe zYQQ%9>qYNI@@{2jnOVa9JSE_lJK+W$&4N(g>OAEjc{Q-}G);FBwB21s+!Xzz3(^jb~TAL&Td6GVui=Kg5F@n za7`tum_;>F)C~u4#Pt#4eLSc~bxLK=ou?%WC!G?gC)tQ}I&5m?L^xJm(>>f+;=}p} z^NN7_lO0P@`wh}fBXaSWX&8R|KX!!iB_g0aiq}jmXwKDLdgz@vDE}mL7yeyT3@9rH z1h`{^M(mjBQO|!CRsuI@cG*ju%dT{t5d#>UabGcu>FsN`zFDZbo1x@qN^uX@n=|p; zBeGief_3v9&$&z`r1Z5vqD6|yupS8Wow%~T&*1*a&0ED@$IdJ=(y@+TEKa11X56OY zk{R~BsoPQ))?Y%bx<*{=@^`)R`PI9i_y~I_cUQ|P;u)UKEU(`XWy-0fWhrsGZ=F6b zWLJIrn%7OGJt4TWGu>dvhOzAGVif6qoV^#vEtEh_cN9?K;0wB2+!keie2sIohDz|T zMANl4OP&;MZ!>y~mPrK7_2(2%ZSh`AtXO;&fjT*J)4lN;($cBBdIpqEJ^i@!> zQi{R)Ngz$ae1*faOmQ~|Bg4UO(HU8Gzw;L?eDPin4ju^9cKcFBfABk)WNw=)zLDZPTTxFlQut3<+WE=~;q+oPY^--xmomxJJ zfOrl65AS~!!tegXF^rbXv`2*r4R5}n495M6E$pXUZrYAS{p!Uh(!agIdVhkxV%KZQ zp4lbjYi?EP262auZW7VZusXKd0Oyyu{HQ{^m}aJO4~Lpv6NC%a3RzaB(A$NHOXV8Z z@XXe}#BZ8VmWpi)q;lhj{QAA@^tF$36WVkbucphCVa0d5oKs55v!*7*4ApoDBebFikRX)N4Td&jFcUp9Z(xp;WaPV2G{zK#hk zs2NBo=JV}2>$uDJNtyK zR2XvF_#Q=vrCM4D=Ob(0ij<^_l^cXpW{;mtm*8dSkbn9VS$%%@DQX8Snx1P+TANYJ zGdnjZ(x<$Zs*ta@aj2JDzH@d~#WS&c^Ma?TlHW|!+L|uB&G20@Hjnv+o$utRgRU+h zQVb2*e7E6JetM$w{e4{ziNW`8 zdzQq=n|1INT{MOtY~?R+bEAW@mqtTs(fMU8YQw&JDelJ;_N%XYj$tCEG`n?jf?M)~ zr9TBplrhHVKEm}mWleg~=ql!?%RECdU)S=!#Dp)``P~8)LR>-rH5z15W1>79I%fvUGY+?#~}c#E5LOq^ysPai$a$f-LZ*pI|Gs2BH$Q z+e1~V zRFDbWTPDa@)L9vpAk@w6uTvK^YJ>T5tlo9!d$pL>pM9WmuyuP|K+%XI%Z|55peNM+ z=B&=g14`?Q-y9DVNiTkyCjm<5^{bE3KHkMXJ2`VdBJvP z+>=~5wToq|_?eg29fQs=>*%RSbHGpvy|qL1<4308C}S^LYg&SCXG?$0YD=Z(M6=P5 zll->uh+Sb7jRFl}6&kp~13DfS!CbcjdKh><=O4u7+AYyH-aKzS#2;$tC+<%#9TX8c18oE*&DS%4~LuEvXk?_Sx0GaoSbQ*{JTVCW}{K-U-jNX58k0HpW+ZMP-cdJ5)ELJa~b&1^%nRoKP|yogjZeR*$zBa zqiu-bp}5z;uyi{~@RJ>;Yq&*XRoftAt{8_o4R6s(aBVn?In1$}dfK+xUA4|pSX>i8 zKQml1qzDMiS0=y-bw&G#cm_f&YC@Z=Nv6Mwodxp)vl zdlGIY0VS&>F$v+^-Tel7jRL~?iho4Zk(h|O#*-aRRXj(S3t)(e{QgN;(GCl z%A0s*g&tKabk~l(_N?l#1GnbntzpA*?U*Uw4&5@FZPlgjEqA1zbhjbXYdhVzrzQAX zBlg`_zzpyuV!oHzULZXauXD0L7Oq|->b+rDDmo!94Og_Khb~;7IX%4~ACgoCv1aKi+ZJtn*O3i=E75o6?z8rZx6;k;s<2}J2vRcc{qEYb3NeH>32%jo3 z!X7sSE2>;UXW7}}*;>t4e1t!4&hc-tXMRTICU`*YC~e2$Wq9XM zC1lr5(JLpPi66T#{*+?iQ%a3fZ{tfPr}?Yjs9h;OHS)xyyv_1>aLWQ*RP^M%JWO=E zoy@ZnyC;6uzFeVy!(!=t;whhVqx?aTsM+?Esg@y^rgJ8@{{%8Rh>4faqyg@Td_lzb zYZzS&pKe|;I_2nAW1piIml06K`*|bxY6y=)i_^W*tD#d^iQ)U&A81f#{8N=vt2~i65ZrE z);wkbhbYvqVdCkC;27J(hh*TR>1ti+-@m&hntF029`CNyHXIOZS1%#No%m~gP6;sk z5Adi0aHC|Q!id6O<%{_)n^P71;LZ)&Klh65x=g-vS1WqRX^Nzwt1CGNcWC~!m2Z*$ z1}*E%+>wyPQam=@JXQL4)v2T;7)l!Lc*B>&mB+d4La={OQ@Yv>tiML=9p}03pFj5m zTZZ$Bn>ImTMgwS-saF%r`R5fsKFgc-etbX)%LRau)N>Xq8{@U^J1Z;aEiO?S4ZA<& zK%8?iyDbv%`oQAW{b8q%lj3O#JDif-f}ulQ+ht?)B?x>{qw722sVy0D{pH4#6aW0> zhL$pal%c2XbFZe|!E0~E#6Mjc-y@gw{FukVApBfEF$aw1*ICtDEbFjbjfl*w?xU12{j(ST9!|dBHljGO@g)o<0LWrAWR7mlSosD5X^6fh9(|qIpnHffrMdVRKj`J*r-&$LH{DcM84>ws_ zleunFb)(%U=f;CAm(iu(I|KNkbrIPL?t!zJf0xHpZ(ZNsT<3?TFjvK*K>1J2oYAo^ zSB<^IyP_YJHJPC(qxxCQ;H>N_AbX{LTIc-IEN7oae1-aE$ngkbGuYb*Y?C zu;<3|ioE=IFRTszSy$OguW;<=x9)Re?nhjEztb?5L6HP?y&^GZFc!tcLR*_~Mrcvj zkk+IitfwdsR9Ev=RFb5ngtS{;G)3#|(+h>O_mHZ}!nYn!QXc%`{Qmv0{}(YL^-#p6 z$fqm^Q<^=x-m!s3VKpYAVqTkGd4>>7(;Tok`#`|g)d#E)5+BB_d0{9A2=tsP z@=1nv!B5%d`N}yJO=T+`$wuleF>%Uk=4$&(vEK?(K3#U9w!yuD7jqYLOvACGwGcju zFbkVJ>>6S;=2@gD3YnZf)SoJ98nm0-9?938n{6LFTj1|kt}PdxQKXD@PS@qqEY8vs z9;Elx@?8?ED2n}?FYA=u4(+wT%S3xyWI|X%3r!*>#emr$DuT9xQK-t7u!0${On}!f zmM6Q;_T}G>Z4c!+4<=nW`+#oCPU?O`s9x~hQ?^G*CJB$Y1c`EigdVJ)xApvTWrBkc z<^o;KlylFx>Xe3#vg|}#Kqu6GUrPB_AoJ<+#dBMtlYU6Q9&_b0 zHlp|2oZtHc5L#0e3Dkz`6BGU?ot+t-uxV*+ZS$Gqg)7H~xT*Q9E1R7gbc}6w?6i>I zQWr*i5zb5xw}Zv&!IRW9g@4Tthb;p9i7eMbN+Qi9C-*n)O3|5H^@*3W=ba3&D@!ex ztW6sQ+9@mhPNPhdN7dL`0dfS2&uhY$W7FLvlYCi<*tc5O|}X>OoqF)tNZ&qtMv)re$`Phti4_z3#&K|o|i9Grumx$ZL4|o z^}^|m{)0L6#10n>l8<<)(VmhgpA`Ew?)~e>mg1#V>=o5#m(k*^P68?GT)w4H%C_9H zkgk!T+ZinTo!psjIu^)JL;tf$kJDmB#;rlZUxD&Ii=xrzEBG8;_+?haOjOm{mHMty zss0W=R?*+<9iAxt^cUbk#G7}v0u*K0X=Q%^)5_Xw%3Kv9NR$lB86yY&dy=}3_RK2 z^{9~ghJ;^{<>mCDA@$#`(8<<r>OWB2{7?eW6(A&l$`zj3+57o=&GWRPWonNhwDw-JL^c%&Cb@v&l8vmn)2&EN|DDocanxSpt)}>z53@3#EA>oN;YMmB(^?Ayw*mf3Q}>gX|Be z$zab&7fa(yw&kD-}jRsk+T&G+IlHz&mx1Iqze zVQVRhgz3PLtVWA!fuF$aswo8Y9qRPT$vHYA!5C7TKIU8;SHT^CLD#TyK>2%&J^h zz6_GA=74uonSoMTIaGN9qEZw|&l}~Dfl4mwQS7T?PDTbM6L%X76yjK}Fd70u4&i;h ztn9o;W!&k4EfFc=u6rKt!+J*VZxwl7j*c!enLPF3etCNerjo#GB7L;qkd4A(ip`7m zgnpba;<-bd8+%+mDZ#Ly(RCKdgON}5x~hD8;``5t7J646&A5zC7t6`SZ@2|oxteEW z?ArQ%{I}aWe+WUE*7kf1Z{Ymf1F;+>yK#pN7eGJ3w}5Qk#oj}mdw1U6O}vy|An-^Dz3l4j_2TZ#+o9Ti{UQWAPJFFyViL;d&of}ybK~>vIHk)@ z4X%3t%l+F0Y_=Xp-Ibn7r|TjELzq4Qt2qe0B7Q4+Nw*`=FLhV2y|ytSOfuRtsV0h2 zO#6CXj6_!_`loiX%?>x-R_5q3YHulw}*Cstji zM8hoRhgk(H(7rlMr$nDT^#Y zTZ{pO7Ci7eQ%b{EOh@@T`)^`PJfEPRiQjy-w3E{vj@|h}R5{p13q&TkWY@=}Jw2g` z*txBh*$`1RFGF>snq16n^;WgZC-*~i2rKv@fhj@m`gz9Xwi8TdVe$#nfW|4GlufV_ zXS7AyF&)p-;CRvZ@JCgrr^%mS{Vgu);xVI~>mUPSa`IEpa-04UIto%aeGHVs=`>AN zlMyXYe|er2+A>}}A=GZP3!p-|hJ%bD17kS$qb{1&3!U}#uwkzcHv^PRJvHhb1rkM_ zaHBy$)iKETGx~02sg;#wcXC({s&aeSZ<)5kk(P}2HgxaMq^EhivNXc-R(x{@Y#KRl z^)=B9ft6yPJ^W(6mkH6v(*pm-9jmYBF_*L}HLS-}Do>HFOrP$6Ns+ysqj6uv!uB?u zjI+(+=E5GlQZLSi*!R}5<$gPEE2-e!{*t!a3KNUp*zsD5k{3_w@5a%Od@zr%*^nAV z5D^#+Q94*^3TQ(2RtWercn)A)6%)&>)U0c%>(jL3ekS9}EYOo?y>(S*)Jxr#8CbC8 z`CDr)Cqk!6oZ=WFRnFCvpO@L1!iu+|Ofko_{RNAueqH_rVn3b`7~)6U*F0tKNQT_m zM4a5rxA+vqX*mwWfpRp19w`F7^5nq*jdymfqH0W>F7W0ZA&Ue?qIMh1=L^*wa~ru5VGv1Hu zpmgGM+!>E%_FKfDw2^u#S+_#(if`Wj)HkVUw3#2)mfMrklaeT2hhIecZ7t?&8w-pe zqjhJee3hA5VuU-CEiu06QHxS7#U)2fgv6(XT%Zj>#-4Rg@;kh{k<3(@D^=v7oH4f5 ztA=_Mclth*ALl!SD*ei&qh&1=a_w-bbF2{Xbr=2EDR^qqpE!8nehXQ3AEEf{h;=Y% zGbYe*zG*t9F{H|>qhOR3Mm;9#v}~S!Bu``Zqo>iukdPINTRqvS%CwPL<5d+DZ?$nwD)@`N>Ob1i(t;~)pZ3?P{ZHs=i6FR)CYoHlXk$Iu^c>ak z^`ta-@UQ;5eT@$z78aqI+twz6jZF+y%!_}{sj|>G+kQi^@9DOAmF}%#I9lJP;ucPM%Of^}Ep9o!!Q<0M8O*qXPxf|H`Fgu?(LKspW%t{@q;`T=qV~xg?{YFt|Yz_QaV~gr%_hbh)>{ z(JN3?yIWP`Qns45%ALn>ak_?$tYg+Xf6f|nnOy7$wwFdcMz(W#=LyIlX4?0B;0cC1 zS>j_Cj@k3i>sT_xba7!>579D9dNqEFv>isx;kkNw7z0ylfnkk;9&67v0JVQx<8Qx9|p4pm)IR)Td%$^L|i0o4tOf!zewZBU5|p1&E~Wo3va;=De5q18cGi zJM(ovb!R{HH6=u_tViiKx0rZlhPwyZSBdsg&pEj|y7nPvwZ8{rapfc3i2&Ro-g2os zF)p!Fg1W#P8l)GH9Vd_oAqrJ2b}_%2Brh&|MKA1;9nJ9@&9qC>Ho-(G#veS+8#ccD zwu&G1{Xv*gBOc6xe6L{2_ByQhG^ihNzn2Q@sg0#N7CwX>xO9w~ufgZXmkz#`)xM9X z9JOmhXAI#xQW+Ko8kXY8D?HSZj}AWSfx%+st49=bpkt*ILqP(xE&W-{O9R zYd(U7BLY?6O!R7$$4prl?h@`azVRQb_uk~5xgUv!7j5(^BX;tWLR>YB5SfwF`3`x) z3<*F!BXQN7mCspD1rb;pwHt?F$vd;%`Ox3n?q`Q?UgZOz*Bu1QXcwnrgNS!4S`g3^nAg7S1NJ@ zwth}^7S}WW0?nqo1QuKbn0D*kV}rKGQ>FiNl?K@cbga#GmS=#xmA-X`BRl5pZ27j={&n%MeR*S_W^i)9e?%nxTn36o`eUI zEQANOG0N%ZiX+NaK$qt1cnIi95zKb0JTR84!45mj88hhZN{6d`A3uBmE_s*R0F9{d zj-la=ir;!0{9oclT9;~NTXX5QbSTZB8c?aJw{_u77DK;@XwCh8vo+xcGEkw(SGA1O z#;>tyR`r%ov$us|Nj0&k5v33Kst72iB_n{IgWmfM{@QK{v6i;rW%i0V zoG_EGoydL5!EBGEGiY9J@9n{d5mbg+OVt`Wb|`hJVsxGd<3JijO^Tztn&)Phwz75G z8Y}>UD+2bd*b;gq^%8r)BQq@?=}CH5u5boNhlbXrW2J^#aq3+KbsH^YIudSgFUKT; z{p7Ou^IrS`fLf5sI)#%o&6i<|75`CkLZwW-j4D;ZN#N3b3eT8#)4zmOZKsdwEjdr! z5kllT^^g7Dz`1<+yEYL(w-gk*+i=2I8~**p{4B2wdzJ}Je^5Mk2SRO zD=sF3NV2yyoPd4_HXgHMrWEo7J$u5drXd1woFKZkHp&3G8C=*-!SU8k>TTVFyJbEZ z!QUaMaIx@I+-g06FxfzGfaA)y1Rln1$6WrUl6_J&`}$_|rgyAM#j=ZmMv9-*R62FB zl#A7)9RXi?%<_UygHnUZa{b{(xmLLU83NuABgJNmdXgyR#!KB6=HBKPk1oyjD96n# zTaKL8=3-rFT{&C$Dq*xHz5N;duNmAht$Nzjc#Sxv#^6BFTBQ^f%* z1@FG%fY_~>z*=2j8ZJ5!^LE-T0Oyuq!1NAJ!zvVo<}*zQlDbxr3AKM>qNmnI`_m)3 znn%XMglB`c8~e=Mhab*Iv9*C`jHwe;o9qgEha0j?rjDeJ1PKQVAAMXaN)>iWX>Opm zkqd~PL&UVCkp?blu^w!f{=U#*z}|>(Dq_}1AZtUc}Z7? zN2WJUl`>IPY2NtR_U#3`ad4jAVs8JkxPw?9C4 zZQX7~J=&ga?n>H{m}N58Uf#0U4)~N2=2{ThOC%4@ul)u+oNb|IK3z}bFx#K-N zu-UJPRK3QmUlFIZKdOp&EUJ?!hTTW8@3ry=gnjG4af}wD%szc(Wf6~U_&9BQi|6(qKRQPdRj>!v%axz&r zNH?Ui{uV;c^&b*i`Kg{t_IuTnJ}OEVm;!*5uf2`VJ%d_|G9={T45o0G%*Pxcp-|i7 zNqE9rei4r|xaNS~^9A#sze?Z;U(>0r5eVx>h7g*1mNRDbKU|Uu2j7+yA{f4VIci$M zvpj5U0~B}$lSLpYd5YJfbg>DBdb^fo{EI@(Jpge`YJ%yYK$*RR=b>3T8RaxZ}C>pfTp z=XX=v!R=@&p#q|GzVy4)*GvXEmeA3Vi}!Bio)fr=-{mstXKc-(Y9Mbl?Bf&b752?Q2{yX>m~mnrM{T3878YUXxo`=VMiib zC4jHL133+Wm0=7)Mmn3Hv|__~kl{9Dbn~Q((h*=daO&Y5%$Zb~g{P?o?kfdF*lNvX z+ve+?Ra87IPNz$yz2sl&;1ynGH5RHAYN{*#d0W-r2d266w#fCX%h&I^0)Ayx;$>QW9TEmcdr1F^<&zsl>~!Ak|KQN? ztf&AtioBoR&!NdVMP+pymJ7HlJNqSDT~@dDe<=1CfkF$0W9$ZaK3uecBlVA16zuYS znVF?r@SSo=O?J!8IhPS+rk7k<@h{cPcCC51WMc;u`ItT6kz>7z2EQp00EdUuVPHP;e}OX8lPyXHF(JE(AP*O z(QBck1Q?3E%rbt9gKN`k4<>pTjwtTj1^DL~`GBY@QD!S@KwUhuRyarJB)xt}kQVZI z!*E&If?HjCX>A&)#LGbXN>4g2P5RrRF9M_olcxF_s*_g(1{&OojFqH$dl&ej0373J zhT>e0j2BJJ)#?4#QxeTmKx-IK80OD89TAucQV-xK2s&g0H(2r+zRhX6hs5PZz~)fiMAs5uflqL>rWEYjG|hTnSoJN4%N%SoUt4mHdyc9b^%cy>~ATjrc41o z9caNkoVX(-;=Z*wVPuro3AYRH=;v2|qHJIv59GH%1?T#;qDPHvMT)S`j_>oq`U_TI z*hA}cqGY_hk@D4%fAH_{>w~9yYfp%c?UD-gfS}K)*=pe*N=AAnoRc8Y+F+mPTqmZ3 z?@L}AJVllafAB!dCiF;v{rKCr74{+CH%c{IAxaetMpH{><^#uQQ0D?3*ANH4WaJ8j zg_gvS2uELJ=-ic8NOyD}-YNS}rsO#DY!(n$Qb<0~`_|Mq0CJ7hh6v78`LZDz@Bch>{V z!Q4Te`zr9F{uimoSZV4v_WX?2*d^#u;=aQr9#3sa;sIW*US z`QG5LUKwf2W&77@@El%%A=Q3JPy#rOs1s=%Bne6ifq%ValNcW#8yjy^!#hMzCfC|% zKs*)jO8v+AbMD$$6`AX;QfaYqEz1*0oTA zSLES+nP(xNJH05CeP~c}fl3Ho9zC$+`)=!oQh56hcbik>elaC}jb^c>e=@|If3=>6@A#6SvWakAqc3?h~ zk>?6*@>+VBs~jbp_#r#&J=y93)~c-t<svOzj&&>g6>i_XxCZHohN<-wQ z%w!)Sf3u&RB%An;j;AbRu&u~`2jUOkepAC;~OK4I-eV9=ztSA6~=wwXs;;kTpKUADM$WJ_J|MD#?@cpG&tk ze8aHuEriM^6b}f4G^CG8kw5x{qse@Lf6Ybm;)j;Hg*;L&*iMcIF!7It82W}e6)n=J zJZ}EqQ4IpUBX?#VlB(?4D4~(|B~jaWk!aQA5zvJ!7qxUgUyuvl!G}LVs@6{5!jb2V`gM2jD z4T$mG%elXhLjD;Z2G%yi!~YV?gxMvZ{`YJ?$vRuTb#(CEhJ7R*e6)NC+;#V55QU)c z`tk$FZLduOvokDlbD$RRVnKyP1(IQZ((&{A!OA44?@AGuOw`mw$dT&F^TGn3y|wms zvVjj2DPD5?ACNxVL*OuK*2Z&5BEOaSDeO%Q&kIUiZVS*C{p<9}N6KdyroJ-z6UW-^ z8ufyfqWq(j6%uYt)G41*elw%8`1#^mJ5`h{Pe<-?bZkv%>eaN{ET>IZ8G@|)x&})o zFO8{Sn^tMXmbzp}xL!V|T4vRkF1D&ya3HfgcN#d=gEwn=no(;+^Ut8@SpTP;XDg(kg%}KUti86rQ<`Z9liaB{fxgHQ*!-j7KoQCO)m~HiU3van1;Qn5Ra_2^#LjyF<*RUjA}}F zPz|tgHQ;JCa0>+vtTuvz7L{k)sZto$JZVzs_{G~D=yE%f#i;O3v#_C`45yYar8O59 z2L%-u;~$#^&M`WIB1|tSk@aeH7+q@&G955TDhQ&t522sQPQlg_7qa;Oh;=tf^adcGSq%{c6JV6fgL~@w>JUG zx8}KgeNR%(j>swBZDh(2*ZuHV7h;H;Bb%Fxj4m@StseA6;0e6_Co!haf46>6ZCY8l z-gHoQEOn>q#68>Pt;q=WnPQv{I8_uWt9nGOw00?1;Np>%g#}9K*yLNaLtr96uVs?K z_4WKAxObrrPl?6NvzbR(jQ-TAF8A*?_}^^2LQCLZ1Y%f($e&Bn&5 zN)sZiV9pzz15V{LkO|u-Zf%u-r57f zkU-e52d$8rgp7}#58>-qHpT6W0iSi{Gn)efKp?r)PBLo+N4rBZk-Ati@>doCf1U-m z#8W%;$?>14MA}dRcjLF9cLH-M{`u1?`7+C`TG~%s+(Jd5T3uRW;6~b;flY@HG^MVb z_AVTuS5N>RujZd|6`J9vga8W!FrcHz3qH`S$UJ5N*{gai=4K}YZ6**GQM=pI(?EP) z8VektA7@|s9A5KsX$f%VkuB3>V{<9U!WnxR(a}j6QPDe_or3r8Q+<^LE#TZzD+%=M zHDQs;N)ZttK12ZX$!z+oe^?k!dX{KXO!xhJOZYj;2vS@Xkd_w5+bgm&%7$hL3u@+f zb>fiaRIidqVKA(vgurL)-xmlmV$_`zKO@K2;;a_1dSuG3?Z*LFlV6D2o z$RZBD$QosMA@6O#TkI-W-@XlJ1_r>kox_!aV=2UAQD^DkJ^;8qmBO_azlGGCJcc1W ziyA^HB6jGd{OY_zg-H=lu=#~#=|~1^p4<7V6AXWC<=sA0av+SB!c(&RT6M&kg$C^I zSo+v5{|XGOuvpc?NpEzuhFxlIaPC~MS2si~%*AP-<3;cR7b-B6pK<6wJi%mZm{C8Ga zRtfiQd)^V{1I77>8LUXw<<5En+$4LgkFxKWv-8G=v$J*;N7acF8CUYpWLzQmx*`r< zcleofv4n`K-7=zQMmsnVz#0O`qkJGw*~~S(x3Y677(tj&MrYA1gOl9xLxwR&2srEHO6| z6p*vpqSiOJuNyveLJa7JjiW<}0fLJbwq>6)>XD{XoZ*(o3s0PkU_vLI2@Xoc$q@kk zXgnbS{lzFTZaJKaZn6cMVrd(}S|#X|s_1AGVkV?MDTz3r0o3#Sx>dClFR+6JKmI;C zU(>Co2IuShLPl8yMOYcsB3hZ^2vIzAeDLc2Dpifp_8egvq&};Y!+tQP-ieC9LVOIi z1bOP{AM!r4%_;Tf-h6~-v=L5QAPD=G>nP264H+vUDk}{Uo27Zb!>(7Iuq1rF(VF+| zMxYx|ec9Y(bo?bRT#9%=9kHb-vn@V;Z0l8>2Qn_M5*r_fHt9rR6ziC4Y|UpDq6t6& z^8Hb0cYi~FLYePHtymUnoIi|ib^o8`mcObhij|fStQOZ8jm~YskMB6;OL1HrlVYaT zK9gQqN*MBS;evCr7&1O&S5uZ~`t&qW9{sSOS^IuTl>F30t;EUL;D%ISxwim{z zi^Nv-{R!&l4t+$F<;!ijHRMRGn_Q(KByCoT`0`k$kHyMPOZSm8|JUY}xBM%nzb<)} zURCOoJtl=k#4x34sCt$ujOY% z;i6YxZ_K0AFW3jhBhU!P;Z>zIdQ;(Dda1`Um(tZ#Q~VY(2eMQyqQ!}FFwW80;*#^avK?*`SPTb2p2*KGW%zsFuY zOkGyw-Nvpsa_3yGjl$M@&ct2|Cfe8|Ds0Bs;8xxQkp?u@VNdTDD!nJ9&S1L5C3>Ej zF(Hc=O&(p%Ddrk#C|R(5sYR`l^I&#!j3 z(X(0Xd=)YMChf>A<&_$@@!9El+CFtdRb5XzJ(oApJVMxb&jYoku;^GjcjGf}9Ai;`q&3W39W_h?OUfpN5!t9rDxDgbE z9P<5rp(IOkfpEWqb;>!9*)01(ESo|9N~^AGxZm1!hv{)YIlcdRq1sPrMmoE_c& zEal~G>%%G8OE(o^A)&BmgSKT<{8}dRbZ5Zp?c9>+=yeBfP_QS)%EY0X;;%qrAtp&ezTVaD4Ti z7scz;96FvLkDBQjEKbBuj8@Fxw_Wxj_OU*DX69;0sOq!13d%>KqRu13ldOZCOm6n-2fCY~yt*#&PA80(NG)W}GP-)up zscj`<`3HQk=1i>}?cY_g=-`0UFKl@9j$PSdewAMh3*QV)F~k;b@Bf6I+n4sW05E8f zOHR0@6PdKp7Z4e;Di*MhL{Y>}|M|v9N|3yB>rs=DIhP>=UJfv~hPJTpU^?bsKR@m4 z++#I?8MI}+nIj8(-5lxDJZ#_u z><15J7u&e(aCxWsqTrfLS2pRAf@gOu6X_p<2|?ty<+sV{>Tf&O%u}DTmriS-WcUi3 zp2;n?J*;?y>;14EwJt2_6R>*GHyBxr;iTRtgQRGSXIJ+ zGtjDA$_@~_LWb)H8^0pu@JR1gy~k<3?5?$R6Hu zc%H-Js9g;AG%L#}!uI}sXXpFbb7dul8mdzHTJfL9l(%i8(Y9^#!G;fXG&Lp4iioJp zpf3iLzP=XZG-Q6;%IbSva8#6c{X$-H)T60JBvxHRwG72xZXa=r2Bvi06IJ6rjCU2Q zLgpD%BlA!~SlR4`qQ||{jr`<_AcV)FZh@;mKs;f z;1kiblpq54Zz={TyVbL6KfmM}DsDb-bhNN=6eY><^;5=9HFzw~G`Jx^5dKXjC^=wT zuHcW3nlMlnAw0w3CMJaD)z4m-XYXwmM+7XjHpjg!xcf>-SYA1?ZP%gF;$tXdTYhJT zw_1jNllca)x(8eXRe!`=s;8plqHJ}EULxnMrBo@~aVMF><2SBG9ajOveB>lA_B0w? zir8O5(p9*mVsET+1cfdjO&#l0|5E`e!@?h}N$J0TbGI5>il4RZM>piNv!hGmeC6>qW>|=JNel~-vEc3JHhTl2c7M!j)ZRp*qtZJp-ptf1DtIk`50>M9Ye!xlo&B)( zxn=j&nRG(#Vph+AMD}@~t@WkJWVAv++q2`kyTXv6Yhh=LzIyLDswz7u6ka~VWX^&T@QiL$1z4o%UW`<5b-Fbfl{E<3aVI zUb<8)J>rdo{;rggTT#;#La6Kab0^${btbY(b95yZZqh!M+gXwgk;}P_4B?!W!QfR? z*V%)j4ys30Aq%zRcZP*JWt?YTb(Z(-TZDgU+!@4p*;b22o|I(tv5i3&^)lc{N=v(p z4vmD}(bh0g_C9Mb8FVnp8ptB#>!hu(kEf@NO2n~F3>#j%RU#H9TAW#>R*;lS)qUS~ zC&;}BI7e5L1FeICu7_=4oG4Wn^U@Hv^B)?kf;d!d?6ZYD!Pk#w!XO247tK)Pf#Sve zjcex>w#Guo8TCLi9%~P1I(d1A3b=krGvc6t>7;Pl&+DO|x`rvz!YC~v>ZyFG?rufu z;IfCtJ^WI*z-?FakooZ(BV=-VE^jjXX#NdMnpt>my}pp|A=J>&_{5Z9xWirfv=sZW zM{QX{%q>WPGJL)U(tXCkrt|X-K68 zGc;wVrFSnI@*YCdr^cG1rL~V6Vh6eoh4=u_RJpTMqs)x89#kG>$f4B4Gd6o3?y3>0 z=jOJYi@)?SN&C(v)vLC8?o6)0`De~C5@>#Dl9#wG_9!~fqRBXr7Q!7PyK9^;vCPge zdLaDt&XGC&jTpV0z609CGu-y9V9m{@AYpj5L9Tgq?CKtgr<*J%!5&Ldz3C_enP-WtnZwy?j)ba{WwURRE}r2bYv9vI9f zy)U!)xeRkQ(kCpgmAI=a^{ejlgCA7ry(xoKU@iWlvDV` z%a+sl4~EESu)GR_MN`H#RV{Y56p;+0ML`q(LnURD^+^YZ++0SjA4j&|!RVcZU-dLJ zU-dqJg@Z^v-ST?Tk>^nOnVj*NS6<<7nBfUrHp=oL)+*G9$}NzRZgzIjR&nljadvibaaKJ- zTAYw3V*Dyc^Fjy_(@*fBpPt3U0JL`lnL(HLDNR72zZ%XlnJ?|PFU2^Gz2Pb~ZE|)o z2}#K9-F@-*eez2!9e1L{LX~Ijo^v*lVi=Zs0Nqubp9IpPwE>U>>}?;c_JSw{p`S zeqQcsH*3cadmfJR6OpODx=~e?D?GFu*L@#skLcYFd&G~f?OJazsma>QxtGtYpS_^% z-^m&?jV~9I!h(m%6Pf`>jFYCDVnUA9Wp<=lFMdpgd!jLSLwjU!dU+s$EkZ#uL z9xjzsP3!QHgAX6it?`~vK+fHanXEF@dtr~coy&+W^j7TXliZUdxF!0Y%=f)+9bxb@ zU*RhuTn)2FAlHz@i0e@y2lq_fMU8QlAdX`$qb6o^e#5Cht`b}XZW!j)Y}n`ZZKA}) z#bI%Ukt-(pPKAu9oX~IkOic^Z8d9Gexd$|sn+ny|222&U$sorTU>1M8Ex_m`e!7)} z_t!T_IYscgch6`2W?@-hNc$uHQpIMI6n7*gEz-tze9|WNE+Mz=`O%PNcZUM7;!Bk~ zY`A{w)%$+H?Km3cnC$qe1{LY;scJwPWw_(aSpA+#$aWMX{Xq2`2s&<_cwY-+IWBWO zd>CU$h9{bhNEKLEjGr`gm7sN3px+RQb3NM9ee@d)dE2Ca+T?hRggmVn04=)COy%Oc z=h^M1tbV_US26yhn<*FYQLAk*hZ8FaW6>h2c0zGsB){*eHx87K`~I!sLV%7#JLJ(v zrYQpOr%}Bq$QAR49Oi3d9GhKTU|f<8VHqhD#}|Ik5}oaq~y zaNveTlmTO93$MB~4?C1LhfFM>Gw6Aw?ugsClf|C0hSc&Jv4{YI2TEw$K3D$*D{uaG zrV%0SX4!)L5iPwz50}o!Bzg-Zaq;o-idi{%+Jbs8Q-WeT!-8>QFEW7b; z57EeM+bq{_#`>JUeZz<68oG=$$ga*r-jE$Sn08XZHAHj*qiD_BEgbfNP|Z+b^Ni{n zrO2SSLpYaeFG2q3qM+=F+ehoZ6Itd+uHdg}$uOpKu;fgvx6;tFv-3Jo0!uk8&w(<* zdZ`E^R)sT+lJsQ1r$th9dIeQ=m$ed3X(y(ZioxL7siXQnAtmn{_;En-<>agxn4s z4sVKgOV0COooXC=*H{sN&&vK`YZvuEOIE?!VvpRf?ww2Q2nRpf0TeBx#BgxeHMoWp z>PN|p|7^i-&#+a0Uo$E}RB-vg&Gf&_17fTo;-M3z5 zf3xt|VD=juinvHdTxx(SB_Zs7Kl1tSW4y^AG6)bW+PT?POb+Ru9K1hHGe0wC1QRDz zt;A{PdDF7q1GrxI>QD?Xx=x z&rvi{2frN=ATL9HU*TRNWLlpduhr#U;Hd7qa<~}2JiNRNK%(SFFCVYQ7GH!bDiqHs zR*p6C5aUABZZ77hyQ*Hw-)@F!CEqu~vRRbvQ;A*q5Lc69yIOVQX{OD^Eo79uIGHo~ z{oswdEs8XWr{UIQ)o8dHQ99NpzZdvXPEvbvB==m4TKZT@GzonYw)B0)U5<~2>HlO= za%$tsaI#8m97kukK<=B>GbZT`tNC(5T41$u>*RWMbw0Ei1wdJs;2)N6Q6hH|HWsMp z)dNI->|B}08>q>sL%{da&;XI1#<#K>HyYw1D%RRMVQ4+IPr~|8cl3%xTt!U)sN>SL z1sERt=T$X9k2)gKbjBxID-U#O!AOGMBXyD78!P41;?mur8lS8j^y+f0GtpzcMbv?K z0Zu17OqTPUp)wycvw$=)rOts=KQlDsD3(CG9dJ?dXrZEyz{SMaIV~3u);wr?*Op^N zWU-O3K+*dKXq;fk?%c4qS!(N5aPdX%E9T66Pf}!x63?KcpHo3$l3ZufWn|)X^uaXG z@?Xiphl$9xxo3e`F6;F_kxp{t1mt*6WGIcy?l4dGO>d!Q19N&VH>WEyX18aWu2_RilBw4mAuKF zqY6bi4-l!4`DUyiA$8(xHINN=^o+?tZFP6L*vtFJB_1gkD%p?lM4HZ1O-0WM3RoU5 zTI$6VMeiiHIKTf|IjO2z6k~JTP{m$UcDJ@c!%0(peoCssp{T_3ZKygMKSqL2= zD;vp2(p+4Dt1m9k0E_5U{vk%XaNhrB1=8X-pHlt@;SEb>=fF9*mQ%of726N#f9lpY; zr^_VvR281_8AHS+`z7pzmMea8#rB*@Xp8X6^3Lc`X^6^|docUL1m z%s`0*1MO1nURAafwPpKW)h_nZgQ+=0JpS=cZf?kdt&C4R+a$hnw7hLR-azbXX@6gm z0)*;pw=Y!sIiH-V0@~S29mFMFV1FFVyocwT5!EvT;RxTfb}tt(6qxV*O*zd zdj+9yI(guy3_eXu(rnw(M%~%0h&lFl@zn(Ru|Y-HAZ`7v_6xTc z^1RuMXx5u|AOxgj5c6m6h&%(to^2>opW&h(XTZWY5mz?@lV^aCOGYXsTer{EmWncxU#)p{N|q|eDr;^)$x zaz95geiz6HU4bohUtQhZU0ofNhcLdb;k}quPK+<0ZcmzFA+e7CwrMV&7~0M(dYfQYg@JMiQ2GiTO1;@k9)rI9OFcDJgzy$MA@fef}^U4|hbqe!#|8?mrYwJO$?!E%;FuU+bMYzev+Bl=CM@c_H zR$2-)2aHxbICzny2#cL*^YdzvP0ldJ-*nuD>Tpk-&HY|e>|7=)@9sXg4f0zMc1jfhz5Hoy-iXOoItJ<4k z^kyt(b8#;$EDZDW8Y5T(4&QL5@%_=3zS$0XbLdNoaZs@{0D3lN4&Y^+_OG{1|6`>C zYMuU*?tw^SINaPE4xs)0X@nYY3^Oc-nUCb-JE0J;IcWR*d9pucjr{K2c)glhdlE0N zykA8D>F2k$k}E^&R3d3^b8D5{5{CfkgY^mUkCXqpwVeBO*VDam;=J-Qm%W$m$c7RA zk^Shk2ey;nzHFZ0Jnl;wtf^ICyf&rW&DMSd7$MS+h#L*lq%N(Vpwp7EtC>W3IK7+1 zDzVN3$cP}HgXIONp}+OH9R8&?GkIYawK4522wUWQhgXlo*MrHD2??)XCpZfJ;4J)X zcW->vX>L3+E>>Y{?R#TXRJ71iS+GBvR8v{GN-%}Pjg8@Lp^`s>?Z&63;w974T1lu1 zANPCTezc-iL6&ON!RhWriZn$cf+S1ou?H~y{r!z6(xx8T3~}YXNyfTb2Gr`K_As8n zlgE43h3@?|k5o(dqW>ISZFW|#U#)>svusMm{3q8oW_5K7uy8)xu)v}3-(y6gqr-F0 ztDam;i{#)?*fv8w%RHvf~eyR{)Z92^xVna?eC=he}iT)yGFUf_>J+kQ3mzSTe<4ds<`BPw?Mno&S~yGrPtz*g_^RwEl%8Q z-D0207IA8R4O=unhyb}C@Tg79Z9a8@-M%ICsH&WEK~$|A*`D32+2Q8WUJcu0Va_}v zAC8@Bw)?UmGN;JTmCA1kfA8WIwNP2fQ5I&J4MU4i3{`9@r}?8ytG z=eo+84!^v-XHX^A6m)a8DodJ*nCK9>$$s1UhL06M9nDY@7(BpUD@#A0;mu*}Z4K~I zIzF0afq?hGVr`s9%8)@fthdimOlAv%_Frl1-#z z)Bx0(&8#kAV|!_ETGYsjJZ5m8r;OTL(AO8RlG|4DevkgP&?g3I&#b9KAQoG9HWU^M z3WWcxbz}=vR&n}_UpbyuhLDZ5Z0q)p|Iwy*@OK#I{DsBOdnXoLs#W}o$>i{*)75u9 z97bzvii)@s!|qrLwKG~N*s=tfaiT-|?|&UDZr{I89>5iwdM;0W`qY``gLUGmbIh|% z<~`v|uLZ@q2x7OEmrdm2k!&rlU+Vj6U1>a8 z8$ngv6?P#fOuzQM!qRs-teN$qtm{t_YF1WIt09hq>p9_x#yj5lPPQjsao_SAcVdkV zT3-#hV+qDy%lw#{8Z1srvtET1s~+xQ5!3av1_Au_%m z4}MN{c6LhIs;U8D-xTUa%Fi3EsqXJLghrLJ{k6KD#9?$B)*hPv1(1!khyj0vrJZ zV1FQrLO?4D`8Usid+e~Bgm2b)zU#6SK~VUTOWQg2MA>R=|Hp6#KYVAlq84C7QZAgsIAtL%BMmxUXPc| zH+fT*$wiHg!NHA<7$<%+&d5ZUZ(3*>S!#DOzMcJ}EYrqu+sMsHZRg&}BM@+W@@5%) ze0|BbEkO}Mfe<&?h|J#kl)Yg50%J?Cy94H!FVD;*22@Aqg?O7OPw<;aU-6l^6|g-T zDRM~1kG7JX-H(WTZ5gG+Bqo%l0c~O7tHk7Z9{|?-CWs{^;{z?XkN>HLlR!HWNzC5N zK&*v!uBgNVIsM=r^y+U!Gat!ZAV+G8rJvxR$Th1rwnP+($*9Af&z>LfLY`3YTc0q5 z`>ao_T^aMxHY}2)z0e%_S&khY+L(6%-Lr^lkLk}(Egs^=OrgkS;%O*ssT2p$$(Lr9 zaH(JN^1gEFpQ4lbw34^#(P7DHXO@_yqa(UfZ2@1Y}K5N#}KgZ07$@QjUsJS@ddKl?-o5*Er{Y<{OG>b}1AK0AVY^ z+2)y{rLg6Fe|kJ}c3UEVt@K}{NTg-~-RW0Y+8ZCsVA@YxdRKaHQeJ==~%#ws?t3v|TNXGwBMOd)Ymh zm3s@+#DiWT1$}OP$gG=kWp@ZY9sow)I*wEyYqv`CUI{(NCogmU!SWPS;^X2H1TPJm zB)R%cPP&Y|PWIAZNU9r%zfwnx3xeZCg0LDl_D_Kd{iv3O%>jM+C)UOw!+!|8@<8cT z@Z;5W5}hyr7#qM-_^kazZ)K8WnK^G`B#J@sKxbCWcWw|%r-G3)R4lQkx(vDMJTAs3 zKST&v&SIv%*hcFfK@cJk)&Il&{)s39s{G3!X-sVc(ea<097KkHNXH<a;4U@AebqdXumwx0UTTA90r54CZk#i&@Cjy&hpJA_UiO}rgyTRwFqr?dkryV&WFueB7gpg8`BO^ zO2?(P7qWR&iUns`H#J*9@y0}!(J1H1Xyomv> zE{lt6MWA^BdY>tNtl!g*b>f4rAQ%19;Fg0Z2=D1VRN@3mZ|D{hQ<^*W321<#qMu-} znKenC@sKa}FnR7PjAdenDwmVYo6LWOf)h6&)E+jrpJMrl
    f8vg>m(o<=gR)bVV zAeZeu3n*`mV?H;BbLAvJoEt;L5tReEMT{H$M+^P`#9~&;2iQvo&}(1# znw$YvONW11S?z|sLoQTj!&-F13XOyYB&P=q=0Ue$OELKuO-nbX+t=}yiE`pQn}??L zmlK2;5>06qz)(<0Sn2|^AQ2LsfpCb#N~Sytep4Oaz6c1k%S&I-VaTk&zEIL8RDKpn zuwpbvv5tQd+7KcBhjN%KTvKQX|9#eV11o&ndqJM|$Ik9<>v9Xasw$(7c6wIU`(uHN z_gcK|v**pukgj0saDezVCC$cma?;ke*;z$o82zIKR8Y4mw*ZhX4-Y>~0jc9%ZEX<% zWn>&4B71ch;`}xfz3fts_SVnm^kpruTPB)aYJJuhnYcnRqcYI>DO3hP85+hKW;6NBKX@* zlE)Jh8NH{bro$s61j0zzWlIA6C^o#_?q*5OIt98Ln?5ttd^1QTK@EaRI(-5JbUkP0 z2D>v6zSGOoyN*Z&Jl(~vsbPk>EYX=`fVC#)X{KLZy=?K%=R?x_x$6`y4PPL=6V_E4VIS7hC+1D zFA&rlxJn5GsKeHAK~l1Gs6_+GH7m#d;LrX^(|yo=So+GYklEYAZ`#(@*52Nh^GS^z zP;S`NlvSu2sL8abOdPGgGfKP13L7Z8x};hMSNkL@R>`dabj6O{kUCQ`4Fi1z+M+|8 zQKd$yRdWW>sVgf0>9fC?At1q#$W#I)Dj-gO59%As2Pg3}qxfuk%#?ix$CodleO%7W zkD4`_rNFn6z(nWLXa4bNE2nt?mC^@pk;|g;X*ZvRSyObuG65Z({glA>s zhf+TfT?#Q4R0|0fICT*+1X995(3630J~n({|A{L_ac5802n$~lhgzcVFs;D9&!tp- z0(qT39{c-)#K+=oE%eVnmOWVVK81(1rf1KSeQdld<_zv+Wc5^3`A)XY*t6`iKhH6p zD}`Hfa``M&Mnx43YhK~+xqRu8JQ#b;y5ZSw*4AAHT|*22zfc!NX7e%wi_H^xbtY>+ zYlq0?Gttoq6y^qDHsl|<+EQ$*l_-NMTM}z8h5+3vnHYcE+y_oLI$EuYljtE7R#o%` zFwCr>VQp<{s+>2J}O;lShF*+CAA|1W_WtJ&$20KCZ;G2q+ z1tEOMQp+*L;EwP)S>)rFiD^3)tfG`tVK>r(d9uY`*gW3;EJ=U%+V3Aeo^-927PN;Q z6u2$o?K*sU+n!vc?8!p6$K1i;o>!gEb(+ym-x0VnX!eWzmyy}lpP+q~@{BlEnfGe4 zB1FbNN3$lgZBn4qK2afI$sjRjRj+0xZ}K-b`>1g781{$nt4w3|b(sS>@T$K87)$Nr&_Drx~{j0SrWoc8s2Pps8dhifBqS3M6=)m=*@KC4c)!$d(U0#mAt zvKRzuHJcwbt1}T%rKN_`*76@*ux0P`5xxoDnqoIrrO;rQ)|LU z6{G6r<`g%-%sD=pNk!zLs5~EBo+Dw5{oNtP+&mnK-yX}cpUaZ1IDG0E=Bsv{HNT85 zk4G~;QrsL=3#-Q!Mb@;%cUs0Q)<0^|PtW~;~88qfe zt;Kl`^$GROH%6oBs{rb}HlU1RDy49AK9bXw(j81JPQL5-O;S>kzTToyI&9@Xdz#Yi zMTk%IpGW7B^!%=VI0G}+i#+x`^e}GRR089(D zL^g-@&;9-xZ)>7ffxc7A`|RF1K!w1SbtD)k4~`*03m$9!U)fiK1F!Ll1xw8hE)^%w z4<05zb(|ASJ&mP~_U2-#<{Hw*8NP@oCaJAtwCph7sbE8d{DZH(hRten)kkB}iylTA zr4s2|O)*5^c>CTZ%k&q}DdqhI^|QPDdW6be4BQgfe6??(PKqO0#W7v`{)GU%06C#K z4vd`iyW+wfQB$j%p9_#3M2IgC;;7~fYwfTm9cgUF>q(8oE<%D8*zt7LF zbSnLX+}SPTA2k+EaXzJE$)kOHT2VyIWs}0S?Zt&39Ulnw8LiJt#-8EsoA%R0z75OE zifb?NPCQcn-bne4MSD1#F-CoI=TZ1Uw&OS zmJuY$V$4|uJ2iUOy}FCpIEl1f?p7i$`ok}Sl)vL5Q%}Ho$!Q-P6LOadYSg0Ik`YFi z(F)WX$`Vz13Ka_Vk$gZadXJJWOyTR%i_w$*9D`0Luie%EEwiXexhVCFzW#$2gphT4 zyIrG+e!mydA1;^gcjUhRyYzcg`c0wsQ9d-k__Oxq7L^vhQ<(b|<5y;LM~%!xBx187 z+RMF%6?_&+^jhjCSJ_(Lp+W`(Ib(<#@X`YFRq9TU36dpY28YLH+S{lNhbC^2sj1_d%?@$B1%#kX)ExauQ| zug0&{Y|g4FpR_;OE8l2Y=e%ekPo>aWiF3pK^pDr-3>N#0l%w(y^m&&81el1#jxYUKczpr(})Oha}zU+$2sn2;!D_~*`H%L z9IkZ6pDe>3Wk&Qp@eH0GX0#0_Q9U)En}_2jnby{aPR0dS<>%$5HU$+P>oGrfzAP1U z1K>FEq5Yj6pR1!@?0AgbN2Pp#KHU}ak{krxLATB_TvM9q}O*drnIi=sAuIOf2Ovr_>9|1 z(SCtvy~)CXzvqv=L7yt?-Rd#jHy8w`Ur$b47Cu!LJ(hQdYJO+_0$!xPeT>uMET5{V zB-!`FH~YA=TCQI&pCFyFAB+HsPW0a>I=kRqm*BO2TbmQjx69M{_~&%Y(u)>SY`N0X z3NROB-mLcT@)Q&E-9I5=&MMCdBLMtlmGC;DW!S<}MC|aF@H4{uZ{cT`i)}^n z*()%ck@Pjp(8Ir3v3G-|evBg3<3JH(SNhoGMN{KEKW+xY_z{X;?H);xyCREX43_|c ztI*PVCLvF&hZ9r~mSbxIXrKnbhJY&AD!mcYr1w7bu9>1~%(WW)y;=Q5<)Dumo?oi^ zRWSj}Qrgcie11hnDX%y09BtB&!1z-m14{f+{^~v1m%9uzg#L}xiG`%j`4>%3^mMYy z^)m3ofCIKZ7U>9PxlVP|)h%6O(y2KnP*FW>L;p+p6aAMO=Q|jUEVI8I#s(vh`wn=c}DQlsOB;OpsZbbma zxkM;ziT(TK-8ssfcd||JvyAn%-_JcsZ`vuqpU>Xgu$XihUE}TQ3Z^-X?S6XwL;74* z2u1m#aG%y@Sdb3gQhph+PUA02un`Jgh`hI;FEd)XRIdoe#{GzY+-3PHc*nD`<00o9 z{-dxxe|KSI9R!JR&O4fCW*+a%%>@I~J5)8IxT4*VRB?8W;1n&}5g`k{@o~lV#x-v} zW$tH2!yl)mh6a)ia0dN7GF(64>D`Nqid*ALj{+YbWf#GoiOYUxly~9ML6S#)Yz948D0S zEh%=l%9kdxB^e8~f6dI|;SJ_T~s4P4du#Z0X^txCTRVPi}G#%E)j zc1WLBot>Pr3#}}l7nHPYxvwiBXse|9Wjx&4Nq&U(O0Pu7Tg_v+@v-CdJ}4MZMr)E=hU5Pcz@M5#pv&g z#X_?s0)Of5Yr5ImryhE_A^7m~1Id?NX&WwR6m4Uod9oc*YnEX_#DxMiK(_)C3Jec^_SsNP@yb=}(O1>f^rANC0I&!UIc3)Q2$7IA6d{~T8+nLKZ z8;SIjEwN;nz;_3jj`p?;zmb_O8hUGzy871H#TfTXD0P)jt^%!?>Rlf}Ta2rZ=p9Wm zYG?nYQF9~S_WpFYR$S}`mM~@DsBWE93FZf46?ragJ}WkCEM`g<>QdkJ!MwA-S;wAz z6}*{|@iz}LRo17#;E$T|!Vc$pj)qRdn&sU6HUYTI0YmG!GlJc7lR9>>Om`|mKF zoVzFIhgYT1#V!c4yH1vffz6)gU3#9f>n`iwy81RY0b5J)@fh!pyZXi4HWTFu7f{A; zr}RG)YS2s4sMT1XA1NT21?#^nrk(T`)dJxxs%2~3NcZ!s+IY3Q=>truj3MF&hy!Xs zP}L{h(2W-p%884~wipofjXBtvf1$8p>v_A3%2Vv?&lQ)weSdx>Zq5Jk7+1hCTc+V% z;>_CNEL)vB>)-A!mQ?0P>hjbEU%pYS-`Aw^0c1uPbKUyMO`b-?Bjpp}5w(j=Q+D(? z2hKVvU7JWFqSO^28jNw#OV5I&q`0>yB+^z2Y;7$Qk7yk`5!HI3nPr0A7Sq0r4!HMi zbKU66sOICpN!YO$@)Pd{CSG>^4j2;GeqZ>6w&1TvPrb!}mxc+D>CSW>T%?h_ut#Q5 zi3Fh?t@-8OvTt*Geho?PZk8I{|79fCN@c!kECrO9zuogIQq{8_AF%eNuyBm-`SUB# zC-%}(E9=jKB$3r-#(%5E8fH9ehf$q#NM5gyl4kzm5p+|H@@bSHD1j$Y!p7c zEb6T9U!*;{sGC*46~OT2*PHci(An*k?tB5>3ufibz1e?p>s<&Hhb{2oDgh-aNverE zwiVQ*HQ6R6`r)a{3{LNT>jbWP+1+;p6B3Nm{_9FL-(MYWvv9yYzn5$rC;E+eaiNv1_;%4c8BqEpXkuEDkPUj#kT4%(Ha}?YW{A7Y44OJ5OZy5I44*ewySOJe4dUfZTo|kVNF1m?$ z(z}={%zj{qrRZ{IEDFmad8reyNZ#hXaOE{Y;U~T2bL8oM_N^e#qs;@AX1M68e61&_ zB*QZcUk+5qvq@7jj1&u14@j=iDK8OD;=P14;MEW-c7? zV|{7w-pTe=_o2bXXO)JHO$J3R!_wo2Ke#w4*NK_vn7J2X96w1ETtLj-?(N$@H{zCh zhW%9DjCs*K-}0Zv>>UC=14?s#IRoQlcYU$zfiexZ@-$)WC3>?(Rcleag->p+9F6*U zmzrT5HUSJohLtFvA>R5-x%k`LUE+1^&H@(-la~060jdotZ($~|b1UeJ>YAnJ${&w= zdsoB^4YmiT>X;zk&S~XU1g&8$)R1`6kXc zg4nFFHGHm5%W<1e%<4h4aqA&?)D48B5oj_oR<+eJ~;FQG6$DtPU5Sr;v`JH z#NcAp9<=qj)P1<_MZBBJfJ1XE35^z8fD}Xp#*aJ(<_ClpYFe208qSK7qMJfW8RGAC?+ktYI z_JvIAK2^UycCUr@<3Un&ZWI3EI)?K|1$gr`1CobL~=GA=kxvDPreV%^wfs_LQICzu1TRdP#wJ zSO|=qb>)A`>7bwNvOLU+WRQ{_nWoYgXoN}%s8ENDFTZV_s?V*W021EEtld(1;SuFL6UsOpu@hwX;jGF5)tC$-F=KvAaChnz0z-fkNw zzGc{R3XFWd?ORNFP~x&owniSqeWBLQHeEF8x-?)latBxNIH3aa)RD(BA+w4Q+bEFX`F}%ckTTuF%R22zDH-@@izDV zCDZQYxkxj{3$czsMP`Wy^Xq>@qBO7xL{Pprj@P%N+PX8_QH*g$Y@g1v3Qd~KFl*%c zV=xXymbJJ*MF0G{Mavl8%Q){vYWQuaDTi&x^*RniDA?PoaN-mWSbRU9%X&3!=SV9W zPm%`^u1WHimLMYuR65Y<7p3~vTUZhG=?RF6Q>-3|vlBz5T8dy~rq4Q<$vH7W_z(Cm zzY8%-N3x9c?(uN>095X%Hmt$X2&kCDjx|m4?8M>(7!ZhXu41OL=2z#LGMBMvP*`FM zu^scvKV`cR9sYk+Gya0fwt>hhFA|s&NT?YcM9_VaWz%JV7QLaCTNwahg-uBo$`9WO zMQ>!%X;}Jna$BOQ(PgiE=rnNV`5XV@5N6i>z@G$`4Ptz(td@Sn!F@p+E_`Nl7K|`l z>5T&;3>iLd7c@Ky{Erfjl@eVWT<<6OG3F?e{4%E*C4Vo(nGqFm7C8QkDR5SnP-5hy z`CIA8Fh>FgwYs*pFq(+=-&h7t%XD`fd?OU=J)I85J1Svm&DhOk`Vx!|V!~U{fAGx~ z>}Ipm@YW)XF?IU>V3ntb$7~;yHdVDNbC3?dG#VB_oxx=i!3PZMU9~uIj5P#xNBb5n z8VuYdRup3V##PCZ$#Nj207i~8g~wyt@oM6FH{J`e_O)8k``hD;;Vu`!_mDJzp6tIt z`a$>kpX#aqpQ%R)Vm2Z{PyRnOcJLqZ3&{Tudoy_OpOQFeC;mUc3{p}|JM!+&%~2w# zS0vI#v0JlgYV?f)mO!@SX4+0E4)hhQGjR(+cj(T8Za>uciuXM8|=T z_r8pcl{$omg67;0BelGWE|3>1Kw6hYhK0`$fVSI#fwn|hq$9D`vc9ga*8lL~w^HF@ zjzc}|v1$&CbT0@7dVeR>fP}$lwU>+M3c&jKKGpenXXJ^|D$nNFJkw^Z+lPnVG_}YR ziUG@K(emI@?zZIxP+{_m3>SG6g`t;kOQ#WQeaX|S;G-Fql<&8$(hUK%aKbY+TicRk zj=sYOZ9^&urJ|U|0Z-(2l@kIz-pf<(uB%0AY9(KmkRL_0NLW1WY#Z}}Ay;N5nhgsU z2=WTGC@+G$#lDu-L`6mE@`{R*l8V>IY+DX@;)8>0P(p&fh;2(t$6#&k#9IH!w)x@S z9jcZ$EvLtpmo5W~$@& zxS$=Hw-e>AT@#Y{-G>fMeSuvANID;Slj;Vbr;K>d3i^xE}2G2^bNPZZElu!!TUBSzV-S+O26(m?8Pi-o0dquy(ADWdAO%ugxLCk0)|hRm@-kcn2E?5!Dv*&u^CX`}qJi zi_H)4zGxZ!eN!d=g@ol(;mHw2>fFW(m2UgZu2OWnc!!P9xon?$2_GQj09X5w-L+B0D`@1*Uz^aHm?;6%UEYnT?s zq;7Q4M5ni>YjAg3sO8f{etRx4W7N|XwbR%2tXlR`@^1I~Z*5AesP~u80bCBg2cX~l z!M%I${FXnOk}J`RjT6nuelHU-vLXcev6#ZPr7{l2i%&6T8U#J56$Tc{O}^PZuZ8#> ze9+;6mZc=4fYiw#N!696H3R?`EjvRtwi3!)FjwYA?Y1@$G3ly2F-G2{(8q-3V9Q}4 zXyA%|sl`lUXq4c?E?D#WYBgLf<^@8d$!w%-L;2$c3*=~@uV|XazRx&ZdO8l5LvR(l zRpCNh`kdHX>k@BZ%@=&oNrqhmTHR}=#dK68Ka7)HT_yCZ!h^fS+HE@*w`3I2(_PZN zFF&-$i1g>1S04JKX~P0(J;P3@F0b+>!AnW9^8TA^OBw0;_A+x9iekpowdgfKY* z2`m!FzkVX~tDjH%l?Qi8gw2k7`XQM%0Fy%;Ufow9^GJfrqmJus_#guZ!;Ok8E#M;m zF;-#!VJU0Mc=dvf;^2is2$kaGB_<8^GBk`nwtUCwpW8n?imAl-E!oif+Mey3&*wD~ zEIHmQGV5^N@gaMfsG_kH zELJy9-sQ}Jt}RTo5(>-jixE**cj#=~b&2ARf;vh5B>QhodP*ae?o>*mEOjs^&s0sV z+_{zN;CirlzpPh8crSK`l*&8HZm>;%I8l>lh)8g6Q zXztuUH;VeIM0N+5$H@7n7y?lQKt{xcRlPoZm`b^=aj^ypj|8p^Gs{odI32;t12(6$ zoe@&tdfA^H9j%&t_wLq8$0ZFSC<^knh3sv} zL}b`b=V=*Mrijy+Hh0f;_L$U*N;7N4D{7bQ2qBFOo<0qeRmd&rB-mDMsxBJGktdXc z7n!HR^xr%Xc{$8o*!mfMMzp&tRYF80VLRoF=v77BEr!I@P%p2&oweCpJ`2ImY{)*3 zzwJkoESZ1W@~;}7{q9uDG*9w`P_0U^Vvq!xy}rzrovP|R0~dEJ6Y~TLwlu_T zIB=dAZSr>~^Mle^DY-bQc(p2=Kek-GQbE6wMn!Y&--TZc?G-5%d6MM%`%8mQ&v_PT&C8Y1IL_24#a#ZWD6(_lN?eqC5GP0+CyHA~ zFRX@D?c~Sz;_pTy$jdEuLRzfG&rfb9$)V?O@ELpOh@Kr+@v0b3z4-W{TT!wRq>(*R z1<}6o&DQcC>##8JB2%3cv2``XBF6N4W3dIy)EofvN|rJ5Q?&RHE4|)1izC%Qcc)%Fc;m?umN# zq)OmsOI}_KYfkp9S^c}0Nj*ufGXa5A?EHQ!Xf3sK19^99Weg06(>CjGy06usj>ULl zQmEoBrr{x_h~)fWbfJKjvUScyT3gciPm<|#GTdMmv%-xU4t}tD{`+Rm&hbK1;KkZ(9-#X&PhW~Z&lT@?m-u9S^G6m? zQ~38sJvi@O#-hqM=GyPPS1Kf7P#CvwR6G}#QFU=K08qs{dCG|)e6}48uee=a3X5?^ zsBpImY%&tN$X|GOh?Hi54Y>;LtsrROzE}h;DB!-(o?vp1RHWM9l+umL+-$z3#cud2 ztpxmEHIH><)^|>we!^$+Ef)X#-K8W)<;D=< zP!fmr-i?e1O-nxB_q3*UXGB-Yi_VM;pL#KsXp|!PpyK@M{p#h0-^{9Sp>YAg2Xk_kibOvaHtOU3B)x%dQBx55j)Ru+5Ua$DtMFp}6C#Kc zLc#j4l*gk-&7&EC6DwmNQ_;W}W%2M;YCoG3MqIqsZHesFW7Nb{#Y-#hPj48LT%fPk zM4;c77uRR=LC%Bx5)rQnxo!P1GvZ%sOZp4M?}(jB{`~(&i9kR6?-rz&$xus8N(C$@ zptR^YGm|h!G6znz@}x2S@~qkiW;wI9Qr&)8kAX?j>tLrZGkfeL#FFR|-o713Lte%G z99MtU)YLFaw6?l0fBZO7?c*U%GGBe6|BRNXy3d#6EAVWEVzehZEGC)ZBi^%?eZfBmonz_IX zw4XMY;67I$Qm9rr7q>k1*#U&$pi9%y`g4=>eGh{onH*}t8pR$eK804C`fh7xh8+(H z$>kk)1-LOfcm5%drJ?rrLPK;LW1Vc;h+S(d)W^-9%NI)_MU0)}Gyg{eGm(HIZl@@a z-5S$iQ{0^*X}5Eu2L;>4PSlAIm#G8qIn?J=+TRXJASfTjtlWljSr;F$AY*)26 zO;nrejTwY%b98=369h8AiXGV`uaf2g#hpu{?(fvlH_TO(8DVHcr_}qqLxyYBHJ>4m zg22UOj%R19rKKdbLuDgBrWPBQ&^4qaOL#asl5z+XA2{`jjO$)z9TeUG4jOe^kw(=S ze43P`pvxYPyppvHhBgI7F-myv9658Ki$Ihi@B-OnG~F@tbe?m*e`WhZ+YQP+`vt>5-7 zDmYm6;N*0seNNG=52)>_Q?J-z44Y`{VWi&{n!Ha(D;t1j7EGBB)7`YMjp$|4biXN4 z#B}odRa&I6p^-h9cN7v}QTCVeTq=xVo$%t^f21Pqz*j`8xgjuQO7XS%vUUH;KafYp zDl%6;DbR)-gFG_W8WDOE6>0p5gTIyO-SN^WpnkB?Jur}|7{x5@H2}t*4oWna6=9Df zG%NU-LL_de${L?;N`gR^Tz-EUN1eWSkEk>Is^N@;Fy@?1RIso@C-|C%`3VN%V>c;- z)OWf9rFHlMeQA^;NYLLY3{0Dxa>T12V9f22`uH@9{Gu1riWX)XTfyE!f zw#AjjATp%=k8PhPt$FjcK|7+WaYL073V?|q)toQfN}-KzBij={-2B?KVpNhN3#5ri z`A9c&_L>m8>`jj0S1r z7BK6Lli*nI19G&<0SqQM{xTKEvKipkVPV4vNxtaFW3CaBGG6!EI`RS&d1ue+79Ra1wxS?JHp4(rHB(+ zlCn zGBd|2k3s<|KQNIMnn7PzXu=ZMDicSZhUTy+`Gu5v)AH$`DhVn{IL1!j>>H=H)>XQN zdNJ2MDyFukY3FUb_PjN*yDu~7BBSK!?sKc-57$!2{$b_9jMbs|Hm;i`)GZowM zXtm;phSM7qj|nD)BiALEdwi4fe6)`9&Hv|#%Jf6>O3k5$nYKV~h&d>IA%sx6SwZzCDI}A=VeE z4|PutN&8%;-f;>#9esv-=E8>jzInAfakIOyfTu9o<4I?HvTC|)8Cm_vWI4{6A#OWG zVEC77?YiNJ97aH8hpy`R@bUCWTkQq zgFSdYUB~PF7sFit_d&nI!hR_cBQwgbraL`=*!V1|ws@=fi8s5ynK zKCi817#0*ERrcpn8(rf0$edG$uFEi&YjS6=IVN zM|D!t1_uz}Jj_63cljWYQwd~>P+-l=_LM!yGW63~?Ul$09-IIp<-1b!< zT=RMEH}r51_1OA6V0OOO%Bh<0J=j^Pg84B%x#}~#kA+KYS-v$<Rj0qwR5D-Or)6Yy;K)yO^?V7!S2S*9g4sAuXP()<1$wT@}) zdaKHx>M8R8sIWXT*|fdd)%mL~VYFAFoNvm^`AHIE{VS+Y^x9g*v5Ef(dS5(eJ^Ur` z0xeHLVM%rZOy7dFZ8_H0hPM*Fd?%=Tu9;76mu_mbxCLG{-uU+?yJX->-0PPL`Rr7}%-3yDtt`lPyxU6#(Ner-0{slE?A zx|u3i#MpNz);Unq)-oFDrnBwUi;&6bb}8weE6};-jjv}wi)zmWn=Lc)GbMj3uTN!A z3f%I!v`CO>dF4Ad_eAFg6Qoq-k%O$Qf4ghiZ_Ab!53yQr-O^fz*LZFQT9)PXwou)1 zy*k#T7BaT)6Tr__>~3%<9XfolT(&eUy8VO;RKTCm|j$*th^-rXd2rh$2CJN%c| zDA*;2+%4a)_3TmoiA=ZD?SCK}rNlrbMb9O^A;XH}?^3b!V=Evh;MZ9C#6h3?-Pydc zVNUlg*^(SRiC0hWzo$h#4lvuNdVw>RNg5q!8tGtgj*j$IThwEC^CJeub=}a{Ic|P^ zbQtMAhOl5wD>CO@wMPE*yl=3*<~L4hzc&~7XTe=MR9(#Q69*HLV^8DmYOjM$Hb;qK zTcYorrV(RdDE{+?d#uKa!3!HZ*tvZ#R$4mh1DvwbXBQHwP!f z)TQ63dPctiYT%5`Nl5IJ(cCoqaEu+tK(#PUhXhtd)pA0_P^@&pO)G0_GZbC-cRG5c z!;G2h;!QWi&KoZO5X6yV1%8_7+6Pq#ZwdOJw2s9x8e3jpN|Z#JQ$kWUIK!9K{_& zkFt%maadgUyeAjT@69m(pcMuyOE~f))?7*@l^Au7KTHf(aUUJp8&WzUwJsZhUdRCn zib^ZPITOS=(M6pbb`BYnPExudP8(5DfSV_^mm4XVwA?3qFupuc)%4=_wZILS24N!1 zuu50g5n~ISmD+gI4rP^e?S{a9^xvr~)kSj7<(Rr$6M^BC6R}R%w8MGPJ={ z5R0@EVnj4b9C-EtYnxJtW)8efK57tjIR+L z9k}M2o1dX>JyGp2Ha}nKaA*)(-CN|uU^S>a*RGYFrnTJXSw*|G8X+?7y%8a@o}FGe zJ;J4)c-;yW0LO7$Hj{4ep6z3A=vr#{bW38;P0P-FB=UBLUH^7EzER)#w@_tM_ShL2 z7$CJ2tF>$sTHc#sJc=1bi=8H3ANTZI@%wWp+c>zta*yX*T;4Nh?D(RsWKQG^Pspaz zt$}4eFL&_{-v*}~0guuxt;9X|lJWe)T53RJ&Cn_Hc)DGLL7DA}Iys*yHM9);u~=tR zCDruTScXth7ql?MNAxe~PHC#&F`T%wS8_&Z$oPWm$ll-?7256i!~5CoTO56GTtnqF z?B=`tL1H3k6^1^D(?rTP^h)+X5(6#)}Libs~+5KIDBQ zVr+N-A)~vj=n0eai%d=OX)7?5_`sE^I`MXoFn%rOL^_d<oxEG?v_hZ;Ma4Lae2ug+x_*ej#CK++WBZ#YQU_i zms6NVlUwj|$|zoxV&WvOpAW}}xgJPdx>v4N{j%Ua+XjM;q*VBjkBVr@p>1Ml@!`Uc z>0ANBlR5bnKHIB{ii zCcfQX`uVAyz{xs)JY67@l~154n5y#oaw+^UXa4-c*u0u2ai*(!9deOHAoY@8r3dma zM&ZieoPUkL&rf!#RbAL*cPYPI^$sT`WZ~R0FRErE9H99N*5;MV@gYOclrHw{y@VQe zglZu0>r@X9m?-uL8V3>ZSsQN1Td)oH=ZO_N8$KECt82n36|$&)C%FKxqVQ|^ne!Uv z@2An5(`smC$3ytv+*dV~4)lze%0g*w_N}Aj{+i3;MQUGMpOf!KpJEGK&8U-y`Hf23 zUtB0LJbWLiF#Pmn#UjF$VycoKjNE|3E8qlNBahtWF;LH>Es{|xrLeZP_{HS6Z?e+n z?R$d)n&0fxYa(a5)S~oPoXyt^^o;_#)%>CGWsz8^nA$k8r)OHeHkN^gw3Y$Ng~6 z8ZLjI=al61_DA;@FF2hps3Ym%PXjb0f_|DM6aK_>BuMXmj$DyWt$yLzv|%jc6POcZ z0G~v61nvfhJ)l+meB>6CaJOj6F$m%avemvqu?UBa1Ga}%Q&V2rFY+?a*(dTn7`PCT z?%*?FK*D$>idpo_MRZDRZCq@btnn-AoLLd)39&R03@p^5lTlaQ{hEZQsBHno z?MZ1G%}=bp%(%05frWbEueuf&=itbq*5aoJ4W;XK6NLr$Dp;k+5%@#D2{ZE36#^Uc zmt`CHAr0vJt8d%6o$v4Y%wol)Z>a$4$N0+*GHdxJ~VsbEFyRVMqCr ze5|&FGt_>rX~RLjjJU}^1m5p1{SAq@g?CRzMmv7(U4eP+(uG3bO;Q`WC64}O*S24o z6KD7B=3`LG&Wpr8IP~NF$xwLm*$WV5(0IH{^?OR<5?gPcc(jkzc25Pitsj z;+Q7xsM{~3f7UR6d3k}R|C%nsy}H<=Y0FrK5y#gO=s;W?o`rC;f*J$7L5eM_*|HIP zgK90bcGG1Dj^87@w*CD;|70?h0J^wQ^QyRZEKTpQgwt=<# z=LHleGxy@wGJEBL0D<$J;f;3e{!Avf7fqnnZo{p2&Yx<-=0Co5X7_j>|zAoI?6_%dO3CP3|agFPT4jJ zhCrAw{|2?Vi5qVK7e34k`V_;$^ON>##mA(TWV{zKl$z-E+N~Gq9{T;|K->?Q8jiz~ zV~DyV@OoxczLR9z5DnE<3-xPEgX{_Z%*K^vw=(CAk8jEdT8M`^>m}bDGf!1HVrpEL zbo)E>P~BzS_lb{@2x8kcGo8^Edy7P8CxKF#pTblXl0@8#uv=!s&o~p%|25;zJSD5= zz3MrpWYivdHW*g(-gBn?=aWONj};FSkG=ahf1dUBG1xd$j31#RC~hcd_Z8Fr>1*o! zK}pLV#$6_Qn|5L!2yfL33RN6g3&UKKl~~lrBwCZ;w&|V11TTB^*;STYG!HY`l(jQYCkKdL|mBr5mr#FTqLfCVz}g!}yZcoYi%+VoGr@DI8g3tM zI^Py!BRv&u*wXpdSLBFO)@Q$$PrTu|XY?>rv2{Bytx_`Do_kgSNtE;u$__T^Ukbdq zP){@Ih#S*WQ$K%BnXL(5OZZV%7$Ff?))Vr{^!O6RB}ocTqF=w`8-<;>S8bR&A6nwJ zGU~QARb-4B{)bDwF=_65G>Eq|us1Z*e{C+aZfosMTiM#2jw{o7w!d40)H#bOf8L)R zJUHZBM2);A>0&?0?1M|5@WyGLi`fvcZ$96?Bs_Wr$3by?IyChsRxkCFbwYV$k1_#n z?K6=>7AAVk?w5bn%`DJb_MB|uLg&m~2zAKvXbxUO0^JIkzye)%i~re?p1b1 zZb*{f**Y1JkAcgP`_N@^Af7u%ZRYfog!rqp!+MlkO_z9~2Y;J(Q1DN!-In1!<$~dc zh5A7yhj2;QY$v)*m0O593|i93BpEF7v87MhhkF>&>)1*oA@;!@zCQ8RV2`l=6De}` z%QLdl(C7^wrPc+XahIt8Zy%5KJf;(QA*h#RlM{B9BzDJG!80;zM=pCaoLV0V=1An2$#om6d8~6(( z4^Si2i#yht2q`#yky?7LJjJ@kyMw$Wv{B&SX@v6+g}H5%b8;<`dRmoct{aTr2g z<)ji1-}^OVB4~J0xH1T@JSI$+7GK$RMplBDgUu}0>$%~<$;)%n8M~4 z-)Uyfm4p%i(VItxS=i1*P zLf1~w4+;m6g8kf#+D${80L$`fF@~-)*FJ6?Bz278@w=yHQj*9lPxd~gDn;X4I&?dW} zj%Afj=BO9q;kgZC3U%a_2PsDhjjQd~s2wod!)?)j(sQR_#TT1;2}=(lxkR#W)_(wB z3}#P!aOV_EHE{PT73c|h5LBsqKKEgwkFXPmX**pSl%MB5HLv{gC)o&%xXA_6`FAWl zr{s+?!t!qe^`z;ads7#IGu4iRphwKcT}s#51SEBIGIS_Y#k_Y$NTT++GZmf6P|e^_TC-da?y{` z|0N@`FjxQAAFcZODyNB>>CP~(U$t&?-GzogA;G+DYXzesL~nlH(v}wkFn*)15X2XZC^Q>91L|H;}9A_S?>4YqP_UsFq4Ds*v z1C^|JC+ZDm6NyMRH|Yo7KbxC^ zs!r|`5DlW1wlSMl(fbU^K+5;)yUL{QY+Zp$Q{l zr@v~A;LW=dt`+b~%o0ubvn@xF^f~0Qv22@J5(G1$~fpb8ECk`yw8=4jvr%Zp_i9>S&wX`;Kma z&1JGo_%5QV%B|PCXZH2U$teHu8XxtxgYP< zw-r^D6?$BaI?KPrgezUFG2H5=8gzTeLP-iYn{BXTq)M1K3tgZx7@~th`c~8n z_&76C1RU*QH~`7gL|15x*~&O z(MvFcODjt{l>oTxGk&6uqU@z@*4_bzc8T-ceeFkbF) zNXZJs??&+fPV?GU)vOW@23y;7_jrqB2MWlp(31E4)MJ^|_D0#Q7Fur|Q=8P^R##ng zLF#YI4E|NIKj9fz6ZEOxo!Plhm(qc2e2a{&Mr^&3X?&`X_{yv@)dU_Y9+&G{&Ye1~ zB+2deJurNu*;y8A_?sB-TH6{UDoq?{-qlC%d)b&EpUl#DMIo$)NA?!8`eFrOs#{xu z*<)jHx2Z=0+=shN_9ldhy84Qn17~CyiP5a5DJijeC7M>Z{Kf141&3`5^g&QzLMGld zl7Xkp?0#H)vu&L*2K|$oo1<01A6s*9d3$}U(E&Y~^A%&b8M^s=@f+RehnF5+TalW| zC>*+*Bjs|}@8#m(R(hbh&`PgkEC6Pi-t3f*3L{g3aEFQ&#t%JWS*ERMSA}n5Y{zMr z=#lgBu2?Kx)kA9aNb~@c-u-ON6)zht5a9gQOU$w%=%=rEUXyE4v{S_hFk+JnRVJ7t zZ*NdEZ9hu}0g4$K4;zoztNz)f*o?ER{{0&|D>zfjX3J60PZ3nZ(7V^uIV^b++wPQ9 z&qHU|z0b94^FObRpcgJM_GM#o4HIH^bKdLE@1D$gFkv`0z!_gtPr1lcUBEFadVlk{ z4MKR6Zy={o(Wu-oF|yH3x%MKe_o1|tbO#JMwUY8+m~cZmQ3|N< zZjVdADo0VJJ3{-?mBqRem2BhMq=&jP@wUY-d;3n;D(x-ax4E6wX3b&yVp~@ZRkSiC zBX3JwAU_qf**9UhlG-eRTzE}#_XtFX{lj;Q_fRY%N-OVQ09}643s%*I=Gk^O!9LS1 zi&q*vCA8|elx}L8o2M|LV@7SUeyL_VEEzOwj}jNooR*jHY7s~fk@fM>w2vsEZaF@0 z`HP&6bYEDsaUon7@;HXFQ@kXof69*6Zi$k%DP84GTy?Czg9Fo{?X{WxP}l5?)q$Q@ z59aR0TJBko-t2Z+iX3g;GV-2fn$vKTW6tO~An&4YBMHkrBeKEm4TmO|92Rv#6+;~{ zwJz2x_O*uSWSWjB!S~rC$)VQ83+e@8l?Y#j+zrtW%l9ZI-a0H2hFK-rM}d(%2^j*l z4gAxZUnnFV>)Oi>3f7ugzHRBer#&cld?w3fO~Ogs-zu)LH*Wn-Nx{hM#_7%cl+qjy zvKjwVQRCv2pLv15_v%|m%WP)@_|)MVQ4xk&s=$0m&?ROG_nj9M5P9WLtP>Ota-3XxIx&nMwu(r7Z$;l+_cz~dT-+h|D&Oy* zc4Qv~aCID1bOsc9degptJ1D_G6=L&23* zJs9oM=W6_8-Zs5MZHIfpoKjy!Cw@plXM>a6gUjM}5;08q=eAnhlT#1Vwd|dMxP_R@ zmc4X>IPXwDx?>X`i7t6p?^6Q@?0?i_+~1bNgQG+o@S3MYORUVsGxzOI?`}@tdD?HA2&$$ihvBv83O$bK=v`{@0zbdLcRA; zNa(?OSPrL&Z=b6nWYo$bA5FxswufD?M7(f=eJgW6yZbW>IRV$3<@q6iO!T?0#*)dT zUR9x#AxY){{qp>F&5c+5w8Wx~Qhf(G?H}i)_zfiXpPM-k?&*_r zr(K~dpUO&K+3z|Vs_iOAwTvB*T|b&_vA(q2T}s+ySk2#S*f@tDds}s17xnhpu`gd1 zOvc4c96DNNp|7^;mXpDwseTce&CBB083tC)%9s(W(y_)vlU{hP&LSF{m+0lG$41hQ}HH;Wp{N)`At}dS-o083^hlD zX5o&v`9PMaZGFSxE##$6M@>rwp|bUEW-{NXVtXUAl1b^}*!J(@O#+UnKRyjYVr~M)hr^@;2sWaroe1p^FCA1Ks$7|By zRk2|q1#qyePS>W(rQi4;K3A$Zbr>rp1yWNRQ+8|DR8^M8MEFaH4y=T`E0iNr=sm`; zY~R=ZIVG3jgp4|2JDR)&`kb#v9}6wfO8naVzOd~*Eqic0QpdTPhy)%|xW91(0&EXM zxkK-=_u~cL)-T|`ER+X){os2krn@p#O?k0@h=s?Ca_pgBwkvPH-T)?Te7g>}KZvKN z-(OH~Yb~{7N!^;;grNmXyU(|x&4&e|f4xdr&PkMnh3a)c#~s~MHR9HzBbOU`uT_T> za^|PZ>S%O38dK76G!!DXZ^$!d%y?F~m6@NXWL>nNXCtLS0CEZee{RgEN>-ypLJW73sZcKTrFg7(9ivr*^%0D@_uC1M;T$x%wE*3 z<514eIRh=x-(p&udeACRdfaa&h{Zetwbq*K)Uc-DY+11msuhyEIL_+dE*@$CT^vn& zCmh?BD$&;JzPh|66t?uTkatvH2bPb){#@4EM{Znk8?aL`Wf$MysvqlY*G}Yc@=+|K zh~JbKFC~cF-B_795Z@|2gQL*x+O*y8@hu7hB8#OWB1PDO3H~`gge{V{jQ2BxLxFi} z0G^pNarr=A$;@OaLbDQ1A1jONizZywYvhlAhM|)^sDs=R@h+p=gfql2q49 z>AKYBQn{6$@i90mN1Ex!_@-1f>YTP^FO1Hs4?Oy#uYB7GG~|J80^hV&Fo%G%0ZL`% zp!x=*+ow{vm3!#ca?UjZr&Ry4fqK8PI?5}=)xu5Js(nc$BOzF^UEB9MHD(LmRdgEl zc3C`Nf|%}CtLdGFCy9W4fEj~u^cv8!AK_fBp3y=ebVlT;t&LHV&Sv7Irq1W% zkWS3Jb-=YE54gx7=LZ4O2 z91qt3D0GUqxfST$o%G?)r=Mv6cM_s~Hyg9yeLp+Hh6LDKawd~+nz5eNk^tMb|3{ua z2XG+ZX^!CYayZz@$yjFi{vaZ;d=8(OXp8)yCvtKg^^=(oZE^u?kLGp(&8b()ndh=} zS(M_*?dt140S?nfnh4-3y6uNy3}5+4i}Gtj|({S5RQ1gI)08+fE5I&#NIm5>0r2~SVMxIAwsgTyj&Ls@OXguKrvfjj)r1a+csmNBdY%8%KZA~)b^QRboNh>I!9cd1+a#`&v~OetYHx8wf3D^G z`V^s6zKaZ)4u-4p@&lR(lFXG7XO<@OPl@=&7ewk#uKa*w3Q$m*S~}KNE%ORH4rDr4 zg@`g7&ot!c|B^|as|9^@Q7R6jEKj5DLbj+yshh{g~C>r3S%GkOaMYttcIg zjOyJTocUo7)&}y>Xsw*9R{^_I-v&H|-&MCXpjh>y#%ptZ1_96wCvk6?vI#i86`u_tlb)J+=xe8^GCq?3s3pm z?l3W&Lc6?v)qL%kjID6PeyTJO%X)XIVgD_ylua%7;a#FRgWz`@SS|2KS&a#OI1DKfG4k> zONOZC!ve||uFSW=gobMChlKSz3sZh-&AmL~=}S4s0wfRDrh=FiE--yk09ljA+JnGb z`TCYsJqhXv87cm{eS=E{ z%kS0F0^cH?)h+tLI|78AjQqo2nQ5DL;={*sr;zB5_?xyR&72=)F0ae|@gVbnBnYED z8ldOk*soh5h%4x_x50v-j`aKMWWvn#crboZ&3_*lkuc(rFE>Jxv&nelDCIbGSQm&9 z-#r(ugisX(=B|mQNUm8Y0m zGa;NWS-PJS0|N~xQ)b(%m_|SBI62KG?S}D{u4o?|0OQef7QiUCNU%lWYjvfG^jd$`cm5c z=d~!}t-svRtRNF&h`;^+S0C#C>hXZ?{=ajYIO+bknsrAN7(o4%YBFgiStKFBX?&Sb zM?ezXwpR&zCV)}BIIz5&r#0BHKMev}(!Tcvh{uQR6f%=r(hJx7DN}L3kwnw@SCILO z%xI;9tLs?h;eJo*SYzWtC3Pa1G)2IIfCo_!Ll}i4_SY9LGKU*->gQ0QVWmArToUkU zK=RE&se^2J$vhJh{D^qMDSbJ5me~>{6Jr{_dIviCB=jQLE;#b79=bEBH>;)YDP$C}%IN-_@ z)-&I7b*{N_CXM)Uo=q|@C4PjOOz|S=$OaPej^HTQHDtAmOp4WnmmUe!4IZgD4%wQxHRV3M zCdB@48)%Q(Q1fpa@(bPANR(l&e?R$HEq2%0v6g#jINSd&@%kDnL=jO4Bv4jmih)X8 z1QWkn?)21w_^0$Ei42P92PwT`i>jbRNO zZqnKEi*07Bvaloz3t{3f&=1)vW%o0Y8b=`RKY&x8D<9XKsd$M|+^ZZNFL(Go_YNBO zl+>0aqQeW0gT~PaN*6GN6(d@09`@UjZfkQ?O7<_I*`gPKl5(%1wTnqbIY|I0a2n|o zetuWKasH|BMIq8zN0KIAhUI$s?uHcYqmuHeNUZbEN8PxSl>3=3Ms9eUN0VM)QBD}u zOl*P7QMpQgr^^*!rs}V7DoQ?z(EzRu1ma144u2yz6*|Vv8%t+YNq1Ty7ubXR2|b%> zzCI5GGykIzEk|d}1@oDy+{Sb)4QL)GwcLi8Jj}nvGDg8Cs0O{@+9=KAr3+c4{&>Pl z?vGd?X8;<3_meMFJZi)Yd0vBjV4Aj<#h;l3MTOERxxzNb*prbuRC>A z^t89)jL?xH(e5yj_QiTPqrv>Y`%vcRj`x1#QTIz?;`G15%@A9tJHBD{E- zY$Zlr>xh5ze^5b|a2>4sm-crILXjXczT-q#i4e(nlbH!Pbx#RfPK0d&eys7Z#pGa) za6s$Vw>wOZx`Z$GsaAoyIk~M>q*yGvEIYZ+$pp27gMlA2wCZ{d2N}edQDzhByBntm z^{oLkJYsX2$Y_OHvLqu(pDQ1@eE&W1MdW`{>G6NbF(OY6VspUyYj01~S(15rq)0qb zzA%#@;y~n}fk7?ShvMM`7f*8Q^GNSu!z{*Q!fOfR(bfhO#@gD*riG8|zv?rCF?#p89TQt$pgVgbn9pP}ZVu zp#^oC_=7k85~U~J@!fheCY~lU^XSsr=cjk5+Hh~aeHR^m#ME*S^HJ_nK$bRYPw8)X ztkpVoar=!-nU^M67T=uy`sjMpm5Gc=Sz}S%V(_Q{OGf5y0~oww$N*M|#q)r9@hG^L zv;>o;$c>w_Bx6nb#XsN-aSM8MT)9PFe@MUnPoGqp2MEXDD5eBZtF;zT#fKB_U0V9V z$J=tI#}8_NZY0n)jx_r1Z#j%rIyu#ShPd@n5zoMJeggYMnVyen_$KD;oH3r3)`&Xx`STqegf-@l4$4xhC}fe8XK5{<;SsH$ zo@eU3^$8pV^+=s)jQokun{I6cv~K*?%*L|jt#(;rG`MP+#RP=i|9p?8yW2WiHVDAiz0xcAW=xD@zYu_ zH*$yM;rFz(8u6P0+7?fiy{Esu+P3a#I9NxQ&CEz2KKB$8Ru6yCnn9rPi-2o>V2g{O zHhsax1FZ!ja6Y&9VnzaWULE5*gA3@5x3L2$_OS_2*1&gTyPCrjivq=gP}{e*y~k#N zX-J3vAev@e{YbN?E6xOO>5BUvZGvC_9^KLsMoUYJ2s*BCc`{F{EizSnu<*t={l%~+ z@~@_5IzVNYnGR;ZgGT&t8*>}W%$T~O|8#>-lC@}}#sS6gWGOvwt-X7=_35IxA-pq@uw-o_l_4|6C4Xgx_En0nQcz5t#;X8SDh z1$p#+U;ya{3YX3#mYAm8_l`9*_F1=Zt5^ZG(%#qh_4(~4Y+A@xu-^x#xKR;%{r!7= zje>@PSF@d1UuzfJP@rK&)Btx#_y^)Y(4wjfeChFeOXXd`X)%x%l5k(Q7|R0dX*!&I zzwY^Yb-$qOE*y<`WrrGvz5E5>&$kHoGyAw6?kO^J^Ag(pxOm$x2(ImT zA+^An=~N`!@PX=l&>)<-hQ2H;(t-KdN(|Vx`>V;_$b3OGBco;yf&7cV6s3XAD9_2f zYPY4POHLTuI z(i5ke0}y%%@kLOF|F{S%@gg@g)#1LoGyOS(@J3M0#iq*h<47{8ERW$!r?$89*uBjk|uuUR2^IuMy0(rDvCCMTH>ZC+KcJn4=6KJaR3 zD2REycF-N@E)Wm#JER+nEp-jkDJT+VOcWHycqCNA(pv?svvYbYxSW}ppi6+2IoY(#fAtV48VcHsfAQ*- zD-M9}86{s@*e@qbMcdPvdE3tQsH>(*H;%tl_`5_DK%?bt?Wycxu*|?0kPPHg?S#NU zK=!yE`U`QG@Y)R}sjzi_d33Xhd3AJ9H!7OfI#!zA-*@LPub=163*5^=6G4LbuHa=K zt4%#M1Ky$66+QEyu8&maktNIIZev+G#5Fw|Iwon#TlprP+x1o?e`~xQ9M1};W_DsQ zFMmg;)B3&U$5l$p4ro+?rjTt{Y{t#4l z!5?nMvC==PDlRTBFV-k$NbX^{){#+SLo025OyKHB#-EbTA>E-LY=bx)GoDW}b)r}7 zs#Io*{P{)nyG4DK2t5h6#gP)NoT<64WDu;~uoA8^3plCGYhAA#%jc``1JNiOYugd8 z2VJnAbhlju^`Md#8`EPPL&Pt`HN97tx|*&FaoZXXzz5#=BfeMWWz=4;U^8SbGvz_c zA1*BT?N==<=%>dkzW}X=D9Jt0uEY+BgA(`^FqO`#!h19zs7ZK%=fqvMo1$cFz-Ig5chgAFV~fgwumIRswfzuhfC1ljj@%DOQa3on`0JEaU}{9>J%(l6bm z7O5;Y*8Zku&t=can_LxgIjkKRlbV-nW||t0+(X1=WlH_Jtm)fNQCZXGj{)^Zc#xpA z?{=M_^}3r> z`@kQxzQHWvo0obGxe<@{^q4`nnleyJdQI0TNlpmJNNFQV|j5o7s-A9Mi0~63*05fm5;i?vGKiP6_p&b!!i* z1VF{F=LTJPtOWUH_9y4-ZqaLrn@TRqRNAA{ZqmQdN}J?LKFPf(!(c9~!U0=O;3eR< z6L_ok7naVJmA@%?U~^tG@cV88qt0wpmKU9w@L&FJnFj7=F?1DHuW_`>iCF5dr3K%e zg<OIW~ z9?ok&fmUZ_<*C)8MP-R4<12`VC#YL*S?+Fmm*8~TWUWs0hDX80g{0DWA2M3!+0=;3 z{g(Bps_hsK&0W#lsN3Y3!Roi|>*lQ-4%Ck6j6cbEr|?mJ&RSxeGrnnh+h()xN*Vp* zS)q-PxrD{C+QHC`uczJWm4nAc*E@7~PlaqST&pY{lXB;@`1E+maoex}zW8PIZn589 z|N6@S&Nx4oIYWc;_8s@7`;As70$Qpp@W{2KDks8z<3t1IKzPRMTY<0b$p?mPwAL(( zeww<=qKm76ld}^dk%v)!_3kU6=+d9MyHGKDeG^}V4rY3;mzf)3f|Ob;a~Mns$v-qeau@QFWuGJagK zC!YJE;2!c~fy0OW4ihG^E2epa1gG+L`x|-#0o^XnG(OXcysl(lDHT4shzl?{96tA^ zQsZ-BdaEcr`lR4~1G+Oh6-Z!+JEyUrF}3UB_9%y_8WXnZ+`@|E*1JTO=0)1ya(AbIWM?hoh=}-JKfEE=?|WL1 z_{*+94pRor4_0qbr#@ZOp7f_kbDkDW7>-FEFVx!s>v{$5s% z3tkWjNKo?}=vRuU0|x$`^*2`K%?1tBpO9q1t0H<^X_I3&G{7;0J@N1zj4v+M_q?qL z`*DEky@1(6a5l{w;^PEmx|iwVw{5*@@m4zdAgY5@NfYdA{gK}JT%ioK_c<1s?61|jS34YVknUfs+0Wy*BB(T}-y@T0BC7>x$sfM}E%SH0fv+DAB@|v{ zbQia^NSpU!<{y&RhrMHqrCuAurZB9I4Vh*keMy$AK*P~5&1|O|gxsi{6ZLt^&sw7} zNcOIesS5;a$+nlpDRuQ_r1d-J(70Kw$rknF4yKp~UZHx$W0PmA$q&bhj=d_UKmuIg zb4t|=cmIP2y+%|Elb$m6Lp@vUotD$vJ>Fh_e)f3R+D(J%0z0EQfkj*MMD1jyU&7(w zONGxF%H6Je6+WMO@VLt&V667=oS18$kmlkBY%<>fjAIHsEt-99;kwE>B-Y3~pM^t^ z1>Hk4r+F2uO`Y_D9rGWq*IL#uJ#ZK>`3&c9Ftt41umk-^gV7Sg)JyNrpUN$gi;^5} zS-NOfc$3=A3JIs_@xQ6A@#E2qzFCiU1ltdR8II;2?f$i9MM!&2?PuZh9gRb0jFFqP zD)1!{m%&x6ZCv)1h-GYIGULBak(9v;)86Q(k?`tl1&?p<$z~`Wi-5T6?w=m-{4()z zJxwjRu#Q06sL~?CI(J;{GDxOd3k64>H7RGAYh;EH~ZyH_LPXY84_&N043& z-(i$(BS&Z4uucrjd*Z$(h&rab-*v;xZGbEn&{i9>A3l(ZYHNQ&ca8>d6_s6*bjdLs zDd03S$u_0FFfib~+qktA#aU3D{IUG@JKAI_HduEHUz)}_lJJE4^6bf*k`$X&?g5e0E+Ar9kX`dGQ%C=+aP2?) zyR%>>g(zOYD>SWDkk39aZvn zm4D{^qIco%&60N#CKuPuEp-|Q{-`+7ET4J#4;*;C2Bu%Qt9$)ggn?x>5lhJaLVou& zzX^L)+hG6j<2$+ha~JzJR#0Gq6YY7Wx_)xWx{$AnoW~mb-Q}#~1=pGXmb+c7#37g< z=)5wTrFP;I)>ODIf)d_HNEq?HSc;0FTNISRNWg&sko}xv882&Iw~5Z&-z!bCR>0oM zFwAN^%zh@`xUc{QGc7E@jCAOz6$ldZ;=ab;Z-Vq;Hqj+ouXN(6;|S&ra_nN9NrFA^ zm~HjOfgd)}<=Jc(T|p++4YO;1b}n9EV|+S6Kn0pLW8Erg;$l_F$ZrQ)YIVgbCh!NS zzhd1sWjTa4;?%}=XC*}{$lgup;~SK1miJXhG{VuzLwIuUclr~t+xU$b$pSLBQq8j+nE zd{VX@jeg#E$I>a2k$f<`T=(kJD8c(Nz-iQNc#)l;_MPg?x@zzAiR~#rj@YShowyni zVk{qCd|=Z{m5%w{Ql_0pFNoq)Qv-PlH8tsu@*2>GzixHyySCq+@)@m zoFaBA7qlXi`Zwt~k->kY<1<@_-;krvPzLgvIm=7Ijt=iVwc@})vQIjmU*pP}PQQuN zS3E52HEjOX2&TRNYSieKiAteC5&& zf{DFvnM|Qx`5_(mxwN|NA~h|;&M=Fpz(0Y~$qN>W>(wgIUUXjN3e-eZtP&<o4vK zH``wcrXgBc9Mh7PA8BZCjb)PO8$L|npAd}A<;D}_0g+Y%*xZ1#dH!L$?KzZpGDA+k z7PMcF7)9Zep(Ibr*Ak_cwdO<%0R`jaYkEY%%*P3FX<2xKS$mU5OFy62S@F7Me3 zGY)S`n*z;4gZ}nMgz?T^-}_9m@E79kgrh}F;!SV57ko8| z1T|&iy6Y$Aq0(JBMNyf8gb%pceNFq@8@hDG!|XU@^FY<)XM z@?QLGI&Fj7Hq|Qd`qMC!(-qBGlyv>@(T~OSbUfB5BVEwDfB@c&3A9vqJ92?3$4GYW z3lCRpDdB9ik=s?9^PoWV0GC`>MuEJ#jsWP+R`U0$F_g0+*u=>Sq}Vvzee%n zj;2EE*Eu_+&BFqz=I2MEHV9*f6Z~?kJC`ZehvtUIrQOIo8WnOr-Tj5(9l(W|M~A!f zu=K*DufD-ay1#$$@M|H5u9@EB{`7RO!{w|jO#SdEWpTTzG;>BKyzheG^t;Nc-@csL zalh8KqjZ+)6Z^=2zxvBJ{TaKZ*IeAu_99l<Mnf?RW_ZgWyt_+c`V2+mC)LU#eb}KInx@L z9z@(0{lMJF91tfa_Q)CUp+fhGKILl^#r)CM;^N{!#%vgqY)ea3`CpFC7TuuBtVQ=A zoh5}uX|-~GL3yX@xo;zy`9%Om8MrXkuc`N`9)T&;FqHC?El{4 zu??^#IGH}R8lF1#1jF-fQt2TbS^D#cX)BvWCs6y0|9OrqFW(>R^7 zEX2~sBrWH3KrfHQVkx8<1#XRv?Cxf%&CUWyAg&$rCcnzPgTwh&39W5RY1Q6b`_gOA zf&#)@6p>_bf^P8p(_qrO`VIyzM`!j)^XjyaMw!D@Oqu9_gr&}*6CntE&QH+L;tT=k z{H2QyU1$HKal9J}pd@4B0P{tTt*q^b^G$9ishwoYm)udWn4t2L7PmwX1*52EgPOyB zhvq-imT@jz*-Q42)XL9g8PF(65#UwfXEafS_NeY#Hh&71|fBd7A^Jb%`#T!-`#jacz&jovh~O3I7^rAS8m)PtP5WpV6- zk$8#k&z>^2Wvz+Rh`coPE(g%6shOUH%H^O3gZvI5lw@~Vd+aNCG(=l1)wqz5=1qX`i#*4Ad&_K|JVE0`bD|%=cK2Ftw5G|nUieDo**uiP;PD3gdOI^` z&Sgo;Z9VKzf5xID+WLM|OGtU<71N_gDxGP}^mBB8uFkz1yn`xV zT%#=~hvbV4VLy@|>^x&{t25uWw@IpE!+u9F&9h0gNC$K~3e(TK)tm}c`6$qX+I)Nr zR4hcrvcU??E{_e#LT$#k!{OHBWrauXhkG+K*k$*fkdU>>@ojr>-S}-_`v~vJt!;U# z+MOP514{&lBKg=okXi5%9V|!+j^}U~yp47o;w+50@%AF_R;9M9(3_uBbpme^Em5wV zb6bS8sef}>ve}|7=OUVdldA<|$bm&uoi3)iki*esUOdv2V@CCHcBrUTd&h!m-LMt~ zy{U3m+GZN>%aF?{%}og7Iyx?1l$XDF5p|$lQ*>)AbvKtOH|d^wc{OD-O}Fh0S0dy= z6Ds>joMXtMab%dbSM3gxyhew2{*IJUVUz`MB@WZ0D6o9v=|$c)%)MnWOOvuYIAxac z;ThM>hKDGouCm%>Xl_dRO4Qb}c%*U}BB15cVnixA><2a>{IjHTjM8O=j61aC=uqWq z%ZW(3$LqwzLnNK&2TYh_1mA=6wz)@k7~zo(#mQyPoVo&0VhBF>x6O3|X6tEoRg!j` z?ImB}OXPg^6|*ropdp`3U-F{+mr$P%AJ@f`;<#x6QM`<-l$GqXFml*=gIM|8n7 z=s7f`(p#3QhGen_<08($mIPRCkF9_NOGvyhN}**U_yeE$k)8Fa>2~u9#qB6(x#BAk z-bJW5$04IlR55p*{zWM03<4!u}6*KG^_Q$f3LNl^pk^nmgPla=~5b#ZKh zMJNwmi{tOsp6g|w@*Jd4iuALN{x>BFo(P?2^;GbczwJeta$l}qI+{gw+o5!7eoNg` zE@@bq#r0U(__ld>UL5iO$s#W%M1$E}#cMCO+VI|%To0MFhwdF!SYx_GaCFIz$rP7D zR#*}y*BmuvRw{aUekng@c6h4=sVdI8|K1^se@>By8Mn7+Bo z4wkdfF#`SIN0Txyc2w^oAmVb_`cAOew#gU-}bNt8R=u9wuDFdOx+@+6(K%v9f|qI(&__-qr&P5 zT#mGlNwsdD)deCABl9@=hCr?da0H!@AaHdq>;lJ>G9QlPa07BFuDMx0nv!aqiY1rkMPq+2R#HT0>mktpa%$_M8CrUvBT&ijPlW-3oE$n zw%7h2KAf+Vz_&@<&o?sCulA~0C^*8w3539TZ~T8e@#!faL#!CSb=_+R`~)i?WdOV! z1AHeY>~Dzpyo}68KUo<-NC3#cB0&u$lCGYlaMB=$Yt{;wwiiRa=(z~IsCmET=}&ow2? zU;h0*|A>n}*wb&DKO2xp-*fhg*GFWP;iB%_Tdx2|lCaxLgpImJanbo4X-Fu8umG>) z#!P4D+GJxR0ty^>f%|;JE{lXm6cf8hG(A$$XQ;vN?d~oYoYYC$M1#U{z}XN5zT4-x z^RI(GR0%<&qSq(<&8lhnDXHuxMu|g=$GL!~C+;{ScrV|dv7QGbu!btOn@$kfa3422 zX8cwyjssGNH~zqFyxDSWgxec;SYU)-D(2Rz-OF7qg49m~`hkf(SDnv@3k@*zV{=vh zN>YcF-riJ-s{LYCi_l!3yvK zPa_a`&Eg%*%-wNcW7-V)K>gKR+s5oh@oN7m$X2n{2xxWW!U^|3wLnprOa0P<3`sen z2_jGd`;}25&7|Bw=G*2%AINIa(~0c;j+G7lO_h;S2D)*%uh$%Td9ey6z>j2NzE-;0 zn=^Y+{RHE05a6}Nx%ZU7G&4X^zIewzBaYNzV#2Q<7F76F26EGJPcf7%lS`1HP>f$4whT7?tMn!j3Jr6z?-v_;3%1k zMj4a8j!i!WdC2sj3edX4S&Tf$#F+k=dEqf=9L_rC`sdn;ryTAg6u&Ix<;!h3IZF+y zrYJ|g1Jd>p?$4hF-`Ux7q%HU}8}7n}o)_!Z0=mvJshN1qE?OJ*Bx}XQB4F59zPxU= zXi!2DO#`m9H1OII|N7@Fw!_+PgF5$xg{d0CoqzF0`lN%#hp zuq>X~F2*QrYst%y&*@M6w%5g+G69iSL)y~BW#HF&*+57|nf!=5 zUf!+<&ay@>WGc1%V4=PNUT_Gi^u6^oMBJMf1eM1Gv&;T(TC^#VqV%{MoS%VAu+rZ8 zYbyYS+MKtrC=Qi^=$~x_TVz-?#p`l}MwMj+le+#sP63Vg1K8EH^sP&2x@E+)8IljNYs}K<7x5+ zF2%o-KR2-TEasmh)zqXHzPFcSW%D1;c*qaME=}m;LAIFCW3-wej*zXo!ooA>T3Rr_ zZ0*@~$J3TC7t(Dak+KYVxlJTpVO>04p{+KkOW-NuP=FBl=uT7?S9{DWiH1vYY>(0d zCPN|(*&y|)5L7D@GzHR%4X~+YVNs6%78f)<&P53CA%{;vd_6cbR;pK?Z_shF_)Rc+ z$Y~37(ha3TkAVy*P^AQtaaFGHrmUzM;c%R3X~*l1V$0G(^!IT=`-DFcM-VhiG$$wM zpEI|0fL{7q`?g%JL<{HC;#rv4AOx0M8tWDxt zKM?C9UlX1*(xf>EUE|=Mt_(l2GPSWr1c86+n{&@Lo$VF5uG6{Cco?c?S_*$CcE_!_O2 z2&|-PZ$U-a;b520o^5@!&~rb8Zpi3yw|KRewT-?9z02wi8lClJwesZLVAgpnbvS=I z)croSY0bC;c1RovD~H}~f+ENGe>D1$(t{z*MGi+^xZ%zXx90iXF{_l6f*O~D-7u!s z(0MU1;Q7Oc%1VEKUJGXCna9P&^F2M?W$uG2JD5<5I`X0RTmcpp{-RLR<@OM|Cr^Z2 znldI_+;HYHk+ie1S70jZW%{0vJ{L``8vO11_zYyL%8p=60oBrmD&>_CNKEe3dEs(; zYV~nT#ek{qi(Wx30%0)uN$Z<&;AkAPzmx+Ip1s$?st-9rPN`iAVm`i0D<1c5J}JLv zfkQ+iygcnU_OLbXva&mCoG_)hw_m@?N1K~7-pRL=Op;UJ3JPAG7ULVvfAHpMwCYmX|^<`7B6oE7G5xmbHiFxBN2Occ4RB2~H&LxBOpd3mg~dt6#``yJnyHJ>{F6lmChMkAnzN0T&_s|y>fYVLg!&ipo#z8&nx9mJ>G$7SXRbQneaDk?u^Noi^d1D*1cP< zmflIW=H6M%06o#XbO);fo$$Edt~RSsq^e<34XyQqb`Izc%RFZq*E6rt%`{OTPajmQ=%E#m`cN$^0E~Fs4(~cC&UA8+RF{ggo!P3wqo~2|$XL%v!5HR-;az6$r zi$_S(m?Gm*^+Tm->cN&C*RGYxV4mxuE-Ov`Cav@$-@d`NGW8mFZ2hk(9UEXW-|Cg# zBWdXK;1I86y^T2jDQ!cq*??tgUR`e2;=_OsFA2aDQZQp**r{M#f#ocKQjCI>9!a- z`K_+*y-uWojo@F90JF}QT6X)9l~8ctf{-ZN@OXqMSMQUhM)7 z=d=}8D{6TA8O4*M;L8jqFk>6GUkuGwC!Wych3r7y`$Ut0m3STZcCXA4>BXo)37_d~ z^&Fk-st-H#*tED+>a@J%Sn0LYP_)yd@rud5Y<7U7ZoFs$Hg4`>72CZZAg}~HzNu=L zyk}te#!k$aBCpqki?|{o_ze}pCe;Oe_%8lX`aF95$jNTj*Ip5+n|3Am1q1$JxM_KO zcFG|i3O+|gyHu;hsyijOx|g{i$TilzwA9K_kPl2KU-KxFo>s@^n};|YrKd|*^c_!` zNc7%pELY5_`)BS8>}3&7lLpfNk|c4&U5jw0iAI+(9|t9AKk!LLN0NGsxbrJhpa)Qs zRrPIbb>DmH>|ErieEX=sK@GovPKT@G#Tk}YCnk50*gGFX`p~~69 zG;$|clO{ZAQZPpc4S&Up4XDCeb#(P>(Yp8MGgI+NMrzh2aB$G}71UVa7@ui|A3p7K z&DzA@*~|R(3x4F`S`o6T!!^Y}uAE->T?K#GuRymzp#hUj3SQ*R?pzwN6C}Llo?czq z1P9^O$IM7>9gP5;+y!yk@raW5CV{{vZ~5_gWe{YX>ElP~Gt(=Fj+36#n811J5ac~TRMZt9Z&2KU_W`5>aE9UIr+ipoMV1tp~SZo*DU#T{$i%d)08^* z5drtbp#H!X`N<_^b8{|(z5Tnt1h&nK+M0goO|GlL_jj~3*QV_d*RwvmWr|U2e8l6^ zdC$BcNi-rpE%uZT2QMTZE<$)`A45VwV}1=XVB zR1wo6F}1s++sT3b+0`Z#)yF=?DzNEde%qVu`v)57gqtdf)$F;b+j zL)5R4`VKhcEU(uJQoG+j&!{HJXC%30UYM!^<@(&PXke05QVkEXSMikn7P+dN%(8eO z+zR^Y{$d(3d!hi7_nfx;T!g3BxYErZPvf&Yc}d(GFb?$$ymy8gEb&PjN;8meDLNZw+LDY^87=uOd^v8iJJ!B zm4ztG#Yw?39ajGHJF6NrE+hPAG|BWWA7rtsY}y6*oE~44uLkbsa)l3fou3fb_vKLx zw>yWu3LSaMq$kz4hcTSxRruwc%^;BlvBDTK`HLHED=WCo;^K|suMaMnuj}5O_%&7< z%fGy;Y&)gj5kp}TY0h_G11-*Tir}ug^uxybdFqi&r0*WMTlJU3?Pxo})*oSq+{_^h zV&)J`CuDFZ1e`n?tV@oGT0!wY!IsT zeA_tXnK5pSO)UT9Qm5Z>Ue4GPZW9?DJFAbH{9O2HeA4VA+rV9Cj!aYjyv#2J?;c3S!v9y;}z-0_}9w>-78lUd3ryzT=y)rL+ zvnTHIJ}9ypX_E_28!I%i%-mI)*ds}4ws_%Mf#?1FgHl(6t7eJ%;k;TCe_fwKWMDmW zhZ@`~y9f52O$|P&Yn~}e*E{(k_yE&+YocRnx?}(#L9XC}OVTrTW0v7?HucUbx=DePG=7^`yPm5JDrwfYSKW8aEvDTvvJ~!! zv+TW6UySX(1PZo0@{+;cIDjKDFo$H{Khyh|7ev*d7RCB`U3tlW?f=mF~lP z8eEIZ71V9D<`ecqYX+%@dGMuT`dlPk%jD}~)Z&+RFv*9RC2)$LYXwodVYICl@-*2i zyV3V)N1pMUrOLu{HTP;~Kl*uZ7ytUT;^e~dszf7tD}RW$!D)xOn09_B`Y>%zGxY?& zND$fI2F=}R=&BHOj)Q-u_7e6EHZ6KYkE&Ny#f4xwrBMWpeO&$U41bCCSOXAMv zkg8^Ip9;WzTGsU3)%Jobdb2CPn)06ck#dkZOP{!=v{qqKM02PP$G%8HdcN&57phAH zwc&1?>r~Xp>2gFR+ddWWGOQ|gvX$ul$IJhcJ(S5)9ktxpbuZ1SoQ zrIz8}3Vw<6dJLF-J(Z|sS$Uo3j5u)Qt%GQMYzbx{hmhD@5E=5Dv=?hdtOLT~8p0;} z8*`JV(nclKP&joHoQO2AWVL}{#;q7Xx4Ok3qOEEAJSJwe$jmr-p|k0)*Ci-yt(OZ^ z_KK;*_GwY?aw9&irk1srHs+4!zNMm7n?`*jq`QLreAg8~rlo*sT3 z$r>w7K5_*WzChRNaFu6mw$@r^Z_6BcmfZKY4saRDCS=Ka1vi~tZPzwtD=Qc5Osr`R zH(|UoC1&&8NlEcSW=TGH;IwssJ0l4;j0U1=PC__=(HMNpDF`eqH@LnkO>C0Ew=Wr9 zXxZmNv#;aExYPTTaVPUFpYIEktru=+8@}n8>`(q*R*nKX7FQfGp{8csyCdv~Q&GX0 zd%jWfk4;GX$)Y9qe^fS>v^(5gUE7lCL339SOl?o~FmIXj%U7cXLO zr+-Osoa<4Szx*KvF)(9%Ywah9&n>v`%qV3ktwkI6E;L|_-zHUe$!veXq6a|& zJ$SF=W<*wSUkJ3bhLtIO@=w~07~u=d(lu922j4a7dtFKyySF?$TTn+oP_dyHu^Mgs zr94wPu=>2Sol*q1W2D6urS%(3e#I8$c4F>jmcLkJSZO=XI--$3enhfwxQEN%z?lRo z^S(Cl418^WCFOT(MSfn&6g7%DaRis%!3%Cwi9Yh{sat;m=h2tOXKe{f1G!}eGUL5w z_B5KJU#ZVb1fyHtwBB*j>ih9T(D7l*g_aKY;n<|HeY{OEb#&K#fglk% z32W9!gMQGY%YLIJcvmz)S5RobTd4Ws_QoW1Ya!N-USw>$?@|>8Z)y`!RNJTW8fBCm z)neR8^z2*K?N^-NZ%K27Pa&u8KX@Q4{GcD?&#T%Ah6$M2)4Y(wCW50tz%SoAJBHrS zxJh(=&^_%+o|#XahBEK{eAkStuV@wfPDD%Vc zuPFg5E!u@T8XV42iXnf6wz&p%qG}P#wLMd+_d!$<9Z~&20U8EgkbVZqA7E0Swx2sYYtMwjE=A zRlM#-T;aJ_0e?@o4Lmm_HK9$xtY?iHcF7fSZu}j^Es@6$3{8-_*=@eRRODxcrD|%3 zFbQ|9D`qZ`+ZdJd%(FXyzT#_|5F~&gPw~fLK_Wcn)Op}GM}TDw+tfUXD(jGs&7R;j zIOO`X(p*0`*B%}oZo|M(BTK58iZm84X~6J?+hi2f@CojXS!qm_R~KX!O+InylP>#+ zPeH537oM~6|9hHE-e@E4AGG;vp{KH^NbFSU1OXy}GLb86J?_gMiSlZ)8re}HOY7`Yb0fBDRji`7dyNZ#!NlMb?Ac)Kv30ck&SgnTBS`8V3D2dDR5(C6UI!2TlA09_}l= zok#_yf!7L)8P!G(5lsWS@)V!Ux-J*Y2cw)}iMoaOse*Q1(yUF?^J{yiTH>lH>uFek zSuVwfyn6c(m(C9Swg@1Nz=4=DCb?y14iO9k;B7Af^rcltcSp8qMN3+0DfrO|3y6CC zNo-KtoG~VBtcOjd%j)m0y1RDD_PR+2^oZ+AVs5bS4jNH+i>Usf(n8qenP}ao4^cG7>@md+A6b-oPsg~nMAdJy9U5uWp-Z@X8}8W2xw*BCG3;5(Z+U! zAk&=OXdb|2RS$W|)$CBI6A!a=!JW8#xEXzpRsYg;dj?gzKEjRos$OPezV5y@_u~=X zuCFGDC+&-k8fyPm#}W6TM2A_;p8JZ#k^g$pXkzvP5F}#C)w+C2Yh715a_T2Bc`~yj zpjB8+Hg49^!`EX#8>{+Zs>?IxM+S*gH@^Y!B~zb*Cv`%4^*SNy02m<$I=~C$VHe~g ze;KG`ADPdUE7H)=(9_er$`SU>MK-Ite0#9(ZYsl3w^?LBL@B>|Pnc>Vw_ZqRTi0Fd zxU|75)YXR*aYL`iuF?piAc0e9Cwr___+7--yO!VCYX{yIXeCZ0Ln)yi820Jgu z-<@_|e2E{eY~cd{OYJX@4WuG5n5JyYVl%5)sHj;h``k9Fr^BRxNHuU zrl^-Znok`gz8E;Y8X>^nHr&?NCj*%8o!SfiH#w6whXkbdBEFqk$iEJDb_!%R7Tyu@ zXLfk!k}YzJZKnU>$4Q>VxHw_ABG(ut-fRW(re<*$WD!y%dNteONORn5E_SgYKppjy z)xF24d`&i}C)p!>NBg-B`~2H8e91S2f#Sg~kNV;%b)F!p_ETD1$_oY%Q8`74U+8cd zmDP0Pe;&DA)a-M2PFT(%FTAI%&KJnlTks!-2^f6t2WLJ;$^s=XB+GLEs8%ywTW~+cc{?GgD6pB@1rIGw5T0)p#q-2` zUI0l0FE15$cc&9{G@inZs>mN@Qm>Bwl8ET9{K-A`BFA9JZw zjh+Csva6hZD$Ew^ zOt`c#|1Fj^CUO~1_=|gEKR7=0Y!Gt*o1xK0s6O`D;D@qIHOX3{Xr;;TB@r=U%>Q38B@ zjYpH)?a~fne{GrbslM@$$P0r4ww=dm-aOCM0n~`5kr+!8wYV%vT2|<&I(2P1=4=p@ z!${r5=+Vooy7{$Lj)#KViMBjxq68o}aA}j^!%W(G_LZv}J!s!{R>3Gud0vH$1p61O zBn%ZGWn5Vo*1l+Kdbpg#EGmFx6>1emH=TeXy zKw{^cE5!lCV~5}Ivz{ zE~XhVKcoeE8exYHAY9C6H={2&N8Fh@i6e5NQ$^8@3&aewyVcBtB{#2KdN33d@9OMa z<6K0(@+6P>DY3TlAqk z^{_LPDTgxNF{_S9_aB1Ku3_CwOycKe}ba{2OZyAnINjnbw#0wUh!ItwK zyO`>Ifi+f>KwmCK(m)nSW2f+IM*KC0{U6p3H|43uxa0@k^Ifoc|FP*Ob9zevm%qvA z4d0*CTk~*oD(VehvD6R=ZI-OOj&t756-4LppjRyZ%V5R=wFxzyT3S~`DxzQKj(i4X zn%bA(#Yp;GQTst6F7Rf1WAs9Ht^2(cI_X%XXhL_yDc0r4s8}V^uS!ps1`13_1<9~U zUoTH)Z&5o(XKcERu-91e8aqXZkS<14XSQ}2M$S>)P8JKQ)6@y*FkcMR>i8JnoMRTP zFebusJC^Fs(a3K*IeCe26CVs;|AFNlDUne6qV}EDe5V$N#RyXq51=j*sj$yF6f^ul zZ3Py%-U&25*rmv+QW|pz%$@|Y*A4)0+t&z4ed4F~f?XPDc@y1Zl!UU2rJg762WkgJ zr3B%}k9BrLyir28a*hgm#JsmxHhU0=yJ~`W7o}VV{8pO93#Qp0^SS)2*A?jz#s6$* z^m`SD`>e$IEaQ&!Udp*;DHq4g6%{H)4JUd1ORZmfl-!z_Y~U2CBMhAU@$?Vkzc?iu zwbBpXjzwG59F>Qd`8y#87;3DbJu=(W%cJSVdYE!uXUK!t5u$vgVgAuTnQJ zU3I?qn#`V0mEGn7A0Ns2$yM(nH+wUWm?O4{%B(hGleUlg#X&R=U7j3HjhACRE2y57nJfwtJ`B zE`5O+Z08s3M$F>DSPIj?vNRVl%Ojc+dXA8XWd@&^UNaXwjcZSlo%R4YGJfH%gO((P|FSuHCO* zkY{n>2gwMY>WhKOHnz4ef2}{EWikSDPL2r=PRL6T@j3!A zvXmV;sU0B%m~f*p+AY*K_BKNOK3<`}$sp!>-|GtL(TYW5BkpSX5*R{(X(2N7T6@Yu zV;&Gq@En5})b{n#MCI|q>RZ48P@g62r%MU8KRqk^I7orE3X0pKBv3R7d?=rW;|6m;hI?Dm z()7xsqKcwbEqXwcUJ*jh4J7jckFQo|Vj1PaO49KUne2k)JNk4tSd5p3=XjQOa%Ms~ zx4XRRMA`LU_BISWW{gHAWun%Tz@s0|!#q0uynDCWt!4hw6%mEat^0>ybLxLy1)Fo( z|3k?MZ%K`sY-P8Fwxh$Y`d@6U>$|c%DgdJJ!KeDYwLM8ptks%xgg14`ZG5V)*lNC} z?!~@Tf#S&T7y~@S#RsD2mQujq0{rg20EltKQy5)&D2IvyZNjJloiGLT&;x(>Clo`A zJmVBdPf~>@CW$y1dZV;Fd1tiyKcRyQ3lH~OTV^cATc3TcC_{=~gMY(Q`mlKzL9l&| z;vgJ>P~V*PI;LRsjC08h)Oz7M*1ZogT5!)w%#ejm^j|&_*P~^?m}^n|-ZBbI6?@(2 z`2zGU7@qN0DrpyCbw;KN0fd`d-QZip0U$Z-F>@gn2C+?--)a}b;S4Ibtmu2&Nkg%z%| zAMXZ7G&2vPf*XGj(*dGF1S*sX3FO69f{+YL-8djH4siQ{IVajEc^^oj4`bLV5k$2; zkxB-V%K4%&^@lBnx?t@0tHqxU8ymjon1s0ysVbiPY-3;QJ_bnVY7f6gIDz}+1PAv^ z;>DBKpufoH_XoqT3eQ7{RgA|v1E$QOB6a2C%B+x!^>nR?EQi4yZ@Pt}a1-yi=oMts zBXxb{W~L#2IzNwuTL9bH6h|(+ay2>eE`EfAQE(Q3INEw zlw%W3pn$!y_jFq5#UFyVI4Li=OXT)q{%`$0!upRtKNgpOgYV+L{=4Bi3RrvhMDk3T-PgnGDopFLuJGya%Zn-1vC#Lpjopu|A~6s5Vl z!4qjG_kWJT2B8mG?PWhc=u{a(4c!?wNy^&Ynf7q*+E22KiMNX$Pt_o*J$BZJ#AV{u z8gp?=QY+lp>r1h#b=$HW++HzaikLs#nRZ`4+<~ss(}RDria9+fUqIm4frKitX^WgR zpZq|jqng@$HxXx=cBxEMBdZ0Ynh2110_pVM8?HD1OcYDCL)^}KLGDD|v`f~hrDUyRX1E;u zpWOnBI%yZZHv!`f4GFqL8X8n$@Y<(pfdXt31k!)pr!}#MNbZ~+bK;k7kEF%Ujk?!-Ji5li&m zvUJAwrMP+!$?OEeBy`>O*wujaiUC6hD7Qr8ZOkL!HAUo;_HCNHbUluAz$7sPgv&Z; z{rTpFwe|8ar(T(@&A#Q>7LgF%jB@|;^H*bOGgcr+`iQ+-FRo|Akd_Y2wE}eUn~%MO zy+!ALQ&Rr^P2`_{n(sWuM91fNvN^G!PbR83Lpi0}l%ahXPABy7qsz)C<+|E0O+x>$ z8$zWfyFdEh6a&p)H~!J}SS>><0S&9c-vg^dXf3>;O!SUGwK-37oVSv~xw^e2&;qLVM(B0LlvXLXvy=)- z34)Ivqvi&2M0Pr5Bxh9N37t6v8d@AJ0>?R0|9U2^wj}*h*`9fJp6Jn&4UTr&n_<;# zUTF;u?H&#&YiWrcmMnNsE*?0B?oq;D39Hr$Yu-U#MYF<`ZcY*j<0^iEl<)wl9Di#) z){!WI{PXdwcC5=3oa|ADOIM}Z|_L8wD5oEQ9V3W*5IH!U(IC@bDiajKdUiSYZ6*8Sz zb>I7lXY%GRdmJk!W{rT_IXDy@bmL=8>xjPk!F88|0wM`AcyYEWG$*gOC5Aa95x!E` z>g1;KyV=3scJ`6U13}T~gLw_8rAggzl#9(&waxhNuJ~%ndAPgB1=O45U{<3~$nubt zFRG?G9VGiQbb&Yha94kQHn;KbC-MqaN;)ZPB>1RIYHOP2zle}-$ZbEGJ(9^Et||_l zYy60jy?!bhT}d+gX@)~OQ>|9aUzt}FwZ?UM8)g%VZd^)&t9;z0f$Q$ADiy?FGK!}h zwI7_p3i&WE8=9U@FA^|fw|~<~jpMOYukv=B6aBYvk@7Bu==m&8YXy&QWOG!3 zPtg)A-gZt-oG;db`J^L5_U)2qeibwrQ6*3y&xb{XFxW>NFgeDWQX(TuJGE)E z+6$$w%Retnbt2Xu8!|4zN}>b1 z*OO~PM2+X^-jKaM&!RU_bTN2x=^RN?nT3@J zcbQO)G4@*eFuNuLuli$mE1igLI6S@!4qsp>k_grijZXf#zH9XEhP(WFhzj<;y=Gu{ zJ*>bfxx_vdrL}wqYPZcBFp}l9Eax&9&DqoP>_=i>F?tcTWew^M7ztL_`nhQF3r&}S zx$bFp!#v9{0gx7Y=~VEEb^YK+<4Ii;wq_mXm}(O=)*n@BBgnKokK1gP=XMShETep& zT6R8y3Up)1(jXONrJ7H*JY~+A@+(bV5&f`>uftzJ`qAM-AR!%Wcdcy2V*9^s%ndG z$XiOUt3rKDw&ik!dE>|EvA5~7Qttedr+wC+2jxiC(zkoMgm)^E)!K^dE>@Ji`UKa; zQrB$Sx7>v5`K?-nH7%fy1M-}1IlfnB>btTTD(m{(Q$lWwpYSGmGz5dQ#02Ube4I0j z=JbuC8qQ?e{1i(qoU}#cR@aLLPaDAh=K}&NK%lDl9Z_AjM6|W z&>({3WF{jbNuorFEjczA=qA%-XrhucHqhj7Pj$7w`+YO_u9>xF*4$a^zV(O8rn{<6 z?K%kzohOj(r`k+9F118`1fiHR!eGi^`qVEOZT zgTmm=2-(cBAQ#xa5epXhG2Oth!9G`7vB5jynYAW5OT88FFcCDVXtZnQvt@;n7VYi+ z#0B<_g6R0IkTSJKAO%rBkBgFE|D@ zss`0RetqoFb;A>|R?Va*Ot6I<+`YokSwSb2THIRJTMnjHpYh2Iz68^7kB1P*H877b zUHo7|n$)eY)2OAfM7LK?L30_`Ph8P{Aw-j=SM3PoDEEx|(eiOcJd$Rm_sKQ^QzfRK3ctuf4C5y~EJc!q+Vd6PH+Tpt#po z5Z+O6P?DL73IhRjRD7pDpC1>cVwd?`Pp#1S3ejh9VK@q&wAVJRUkl^X?>pbi-*_*) z`XrOrQI}P;eCgAv;tE*DFHLHl;u?rr8wT{QYr}kYesFn=&e7WeI;KOK&z&2$f(Yc? zLcPA`)(AC}UD~DbVb8Po(&HU<8(hUSuQPWsCF``IB~^4>=_l{6<(vdltFny#;n~m9 zs@-Cne(^cSBn?}E;vfgTfm5QrkjchGD9R%EgfIR1xKzK>sikidMMlX+dh=*GgITi{ zr`W51VBAaWhATwiQ1-0n!J;<{R#zAoC}9%I;K#f!3)nL48I$t?Jt3=7qM$+EnE3%s z++q)!@488~iDYd(v;z2!Rogm$N4qg)vM95*XUvb6IX%`DBOP!H5U`?q)$vn{>365S zwZ50^5L839u;%fsQxUK8CZ1Or7*8^MzOLDwMYSp?d1gE zy1>P@p&fY@8?V_{kO=4AEFk1P3%4)vG+(1F>K?OBwy3HCw+Y7bwyTDI6?5})y-_b{ zXt!MX(#%(p_S8#*=cua`YcC_BllxHmyNokVnzcD62|a<3CnXLsDkoiHb zouuN@@+>dguJXwARe92tKFF~S`W8baM;uM|kEd#-&z}=>dFrwK>${6YgxIH}tq%;R za=n_F*C2GiYhP%L>Dz&*sYjUy^wPi9Xk$Wh^0fwd!9+YO#o?c#sXp`m+2CyWP!v1v zi}s51sW685^6f8(*BygY*YumG=p_y5+=?a0v+pi6``%rs<^Lu9t6`8bkUedsrwK8W$ zvsWqPyViJ&7*4b~5(PO+8q^gj#}HcnP?q%GXIa49<22u$W8DKt2W-NmJT~kYNL@ zV~1*7?UxlAm|pfM;}vKi3GOJ8CD?vyj%(N|z2wKMd^15evXN>V7j?_i%`7QcMV|O6 zbXZ`a*n4khZODCZCy00B3-A5h91nzzO2zR%{EtV(#RQef7 zvx8{94FVK=m!9K&(x1r31p;Q%9`4P0fqdsMa<@hy#9-ERj`Jq&%g2DHu?z>Pj4|bH zREg`=ME5|M&8yBAP;CVd)S%@jy+K9+sTyiV+^(&nbG^2SGgjN@UxYk2KjR0P2!9m&pCk`IPs@q+)Dij3nOq}%z0zl?Q)rqa?XFap-&Z}J9*-HUt$T!EinF-6 z|5G}0KWM-MP*(t^ROD*Xz6cCew}@-ecUzrXDSp`7Ig5i0-ffV|gi9j)4ef(ldWpbdHBHj3@? zO0ie-?s%z>*vEIOa2+%ew8cnQPtZ9k8~8wV8=ds^?(HD{I`$F*p`gDcLDKN!py-_} zj_vY>99DQQ<_CK*#zppSiH%K1zb&+#*Odl;PlkjbO%}p^k_wgJ!_;fc>x{}uG!u-E z0>&Q&ccqtg(SKpzH_;#>N2wA(RitQ>W;JxGEeRlx@subTHdR@P4h!Q{S5w33bP{`t z?qi`kOA12g(M7S>h+@Rp3g^hk#UUsM+TqDn>3x1}jc<-m;!R60?ANIlR7fp9vfqC< zeM;2cQ|tT}21#w<_hOrQ#UW@_dwCH0*nAfsG()zs#UQfm`;X1ddvZfVx$)T_4!&m> zxLx@lxoyv@5!d!=g}b%$Y2{^PHXNYcM@jkWDGwFZ6;ADVf?@6Dx}$w(E(Q|aog8Mu z!$YJD^!*0>3H0Yi{q?>|!z*K*36ZIni9h5dho(c*A}-Ejs+p2C28pAfAa<&O>qkbG zH0LG@A#wV;By$&ecP85}TxUBzYG2{8*f%97ENj?y#gCMczccdc@F}*c$D(qm7u=bL zTUQzc%~>9V4+KGYelcqwqo_47a$~lrA1ef&W@i*Q?QVyYPbTcX4~ln5<#k8Rmfxqbsrc7{Zm`;|SyK_xlwUs=+#ngZ4(KhKS0C zfh&QE`Y#N4Z%dK%E5_C@`>wn)N0&oR2G~`Un2%z*G;p|+EG}*Ksqc&O8b)kiKheP} z9CcI{R(1>*W*m(;q7fLX@|s`akJ)Pp+-Tcc1MiKxzy}!%klAT+Hs4{sFUPPacbqMx zHGy%t#`n&Jo20*^fRcv5D8A&os+uz>Uu2Up-}DPQqIo2l5CzW-)DkPgt=z3uo2KYs zgZx9)@b_^OTQ4g0YW!7=8~g8uSF)}xN_+IK{;&!+iqigZhg?^pE$EAJGf#478#i0P>na$uY>ta_6-Tq%|#s%l;YaiM^4F;L+#teou8Mf z^}tjg`HRj>$~D_KGcCbNHCd6{j)Nv!|9n1^a@72FbDe2pPz)X)8*7vGn!hgpdu=oF zYZ{ms`6k*gaqK6!yh3QyoKhZ)(*cS-;qL!#M95s$PU8y@Mn6$d2}QoLq|jJ#t8@>j5jJajJP1CgN8inHv^~@8RS4UC2wsK#MB`5%b}b$&<<*IxzqAV+5ARPA zzK}O{*rpPb>s9pnEy^RzkIL5)Ae^H`3q0s&>-SSgOS&Rr$Loe(l1XK^I{q<0=Gzi5 zqrxG;TbS-wl(%k8M)V!CURT zWa)fFnvFC?`}s|wGhb8RX8k)W$Sl_n2mdC9oqpe|P!@e}Im!~&yzJ+vtLNonjJ)dX zG`lt;WUY=*Tl09|^R=j3YR=#|{dGZghQo%7K5r!SNCg;=->TOrZe%2gSsqm#&2daI zq-tZLEm__>2r-~X<;-VQQjDbdFU7{ks`L#>m)w4MRMU83DP$;~T7fJL-U`xT)IHZX zuY$~TrsU$wadt`EnMTwWcBg1JWh+)}fK~+Xi)YmiRD6z4C@~I9^W0gA9Nya;M@wxz z8_&Y`T|Z7GgZ6(yT#^ujHFLKB^|?jH#q*c_>It(w3=C+~#RYc?MhAvhc?6VvS9vZD z;Y_5jd_P4Fi+`OGCB@Y*3D($$PkMbZdupY;*)Z!6hD@ux^qYc{IVqn}mV)zwG)$S6 ziVDA6YU7@*<8zto^`or~mbVRmOd9{~r(KhUO|q<-cb+x&mph{Mpb9sIN}8Nr*qR_I zq14kaJz#@#n2H8JOwldREYKCa@O#eo#KdEyO&p8=e$?~4$}1s&4Lh>CrEvf2(Ss!~ z>Y*gvDMCKk`33%u#s7efCYy}G2wzHl>@O6fpKiFw95xDCUH6wD)wj=Ij!vldaP_LH zsMWL3muxuwTYS)SEqjS7PEBruBjxIri@D&e#20}RcTo3C!#7!O=RTio&)U7(!`i2j zdP%!jXu(LhZg#tpY!MM0#Xgzw^@nCBrgrVTGdBN{0Mix5|Bl zUDv9mlEye`)B$*%ZrI9lckDv2OgE1d@fN-3f2gUb5zR395R~zuBs*;ALPR%I*!n5j51`cO zY>qFFQHJG^-uExInO&f<*gJ<7h8?%ThVR|OMUrtJY_dUkKy%kI(^mugb}8fFSp_!j z8@b8Oq!C8#Qyu2tR5zL}pCbwHg4zl!8k3QY9pKql{BO4aI$bE>EE9=!bUiSBR&@z? zz*gmxqKwS0sDQrFN7}q)`El-k8E4%#g}d^X>($sNf1XCTUf4A_uNCSUIk(aD))8>% zzViPh%8Ai1Z0|Amye^&4T$0_}cg=JUN-G-SN(wsr=8^msD;%DIV>K;&Z{@2)Q5*z% z28&=1?&br#xDkC8U9XtRIhy%ifg6PgMK1z++c&?Q?xe}ebjtnOk15bJ?lJtpom8ls zd+nN0nTE6NT#(;|=_64K)rME5!Ue3v=EdOjv*g2iFeGZJw2wI%$&-YHo;Stry)3f` znIv&j_1X#!f5gPkeW2jBC?Vs|O#gm)#$h&~9`la!tuEjz3t<@n+ zn?f$k1~E5z=C;@D;IC^Y8`8x5X@;bb4OWEqtON-aJ(kJg(v8qi)zu z!1TOIfv~n+TMijI&VDJlY`VzF{GG+H1kfTqw-#a^=t$XROipXf)#SOyna^BciC!NQ zN(L*L67y;2v`&WBEw$gh_M(9ofXcBOneneu9;%oOOHe7o4 zdiNSknIf40TQqFm&GGnkQ+u6JWJ$4k&SETZlP>iuKrW`|3RjOw%l^{@9_z>1F9ueI~Vko6SPLW+4ogDOA0Y6wceu1tYS-v)?O*=;*RZHb_fRs+6WpY-)o)e`Xb53XvwKe%&VL zkO8Ng+&Tcz7u`j_qgoh}l&ExfhS}$QcF7@Q_^U(oZ!B?d`zWMln-KX7I1(?)Gm8zA z$y;{+2GW@t&%O~3{x^;^szmRJ|9ref~Z_$CR9o$aBp>BeoYLRwVXd&o)P(<5dvCDP zwZFJIreEsi_a!WYdb|fw9u~(`u&SN+8aigkYx>99?ndfyRAedzdiJO`VCOS zv6Q(xgIu;pw%50Mj<5$l{0>vXpt54e!6fkzrIdA``C+ija2({|K)Lwv>H#nAQ*W$T zaZ=0P%{%)?Cn+7;RWjJoV{p1VA*ew1fRVxM2Bc$jpG`}f}g$Q>Jl&L=w7nrC(V1h36 z7<7Pwf;1qb|ZAYC_F;wWIp?%*DNZ)2rnZ`nIj zVi`soKvfLLRRsRZp53C?on)ouHzCN0h7-J%3elFr)B<;e0*=$yFU zyeuH$v%0furh4kGgu1q;)bY@iY`2-Q*i0GE9Z9}-&6CpcQO7eW^YqlX`wIO;@~~l&BUgPg;^lo zMEP8(&?ktj5GoS@5o>UGvj3>+{U}N@C~qE6g8neY*-o}brf3Dq_=PRhFcAxW`r!g4 zf+Vs5=Ck@9rq?s|BLsztrkN-su^vh>_xFQSW%feh@7T`0rv8MG8owYH3tf``F=deN z%wA1+4aSN+Q&>JnR%E94ptM7(L_7Iy24)d3Kym6HD&%1>np1{hNOQBL36wwzg}L$0 z3y8X|OpTd0rKrG@+3PjJ+iuRI1RMLBzp(wl)mjixyLYkkB1j(COR{8^Bia% zuMSW2LF1Br@IH8(m0yTZ4YL$cs&UhbnKP9zYE@r50%#?OA#mJ+a%R5WYM@$PEiCRaCBrWxQI|Q zpFy60&KQ%2AG7lIWc$yB6j)@7AINiqDpic;0{lWd;;7N&OPO4P2c;Ag)MbT0&~Oxc z*$M2=H(n#(e}xBr{y#!}H<4{uKzINQn2vIHq+DmOQWT@Td6SRI4oEry&prUZZy%+! zG_|--pZrx*v-Yy;*inG?6dYQd@V!2@x;Ty6d2g-uSMQ*9<_(9N&Ou)@vcMNkJerr` zNH~iF9evsM*0+Y87rkb41}hL%h$c4`3^~a%G4d{X^7^F7_vr}-5c}=+W~-}qiisYI z(3N}yDOY+zb29=PKqH0osO{tdS-xOhhQRBx_j9j}m~ebBT5d_N*qEFO{rABVFJlQo z#I^jOOh`ghnd{hY=+K$5aO_{JsCM1j8XWfAX~sF5rbE+CrYa&*S2r?JS4T%zS4RVV z&}U$0-fnLr!F#J8G!9s(sNOOL+sgZjwzkmFwzjC<7&l&U*^NFD0bM;vA{gp=EiV}A zYM@hR@scEHde!O51uZY@qFeCL4KTm)-n%8{dQkPo z$r(dU%Yl^eSl?MY#THy$O_{n!LaZERq%F(M2MhGivpFOK42 zb!6&4-hj5NI8PJuMdwPlt2hzKq5IVXYbjn7+zB|xpthEBiP8$ zecJ`an*^$!4xfsM7?~B=5!j857Ik;Bys|q`;p_tVh(~tLzN+i2C2xyViOl3p{*PP5 zWJkL6?)u7f9d*DkE9+pjx1-~u-|e79GdC@Lz#jK*;Mr%k3qJtqYw2ENB-#YyA-a2> zV79#jyYM`Rfzv2|Q%ODFH#=h)C)xLOYMwX}$g z5Nc6>ha#8a&fzO&wWZC;iU1UIJDZ1?DI&{OM!Q-C1IhRYZd_3t%x z@z~0<)*I-T;w}WtH)h2$+sABdSocg!YNE;eJ=G@;NFRV^k17>+Yi6n|zCNAS*eXuQ z&1Jj=$s~(F-GDG$?#!}${P^+IIuSInW*aV;jB{R{%WIQ(Rj&=*ROUJP#U4^4uvGXS z=e%-~^;6W{qGCP3CqpO@61o}!d%`ke@XzixSsNKux;ohB?N)?^DTb)2rJ0Pjf@5Id z&2a~ZyqiulMeidmJNk)41C)mZDU<2kxpQtyV}t@ubq(}Bjc0VwP4AP9JD-2$m%7Y$ z$0Xt2JDDIb(b4Ks0u`Zb0dqwSZ>Q~z0;gW|p22oDo7(oz~mm zJ3BKpH8r)_g%2qU(RaJR;Rq<|g?>}TgxAn1{S0YP#D&9nehy7`6 zLbcm-FDa5ioxFJ|Gc#xgi&HpIVQ*X8;}9e5uj=w0mQaBTSuMz$;A?{nA6e z4rIWu&NT&o5m3i4YRkHp9fNN2ID)JoP27ni?6~%Ijf5HK#vnyRFTPcta6z#_Q{w+M>II>mx&6V9cDIl%(28wc%1`{NqJ=yhhhICN5{S0wzl;GA}phoV{OS{VR7pj zUDA>3N&-04YT2Ah?0!;hipxnARnPU`om!;P(`)+P((?8#fl!kOsrc1$E?e`2NvGAF z^n~1ToS9@^@p{3oKTluPb*myGF)t6`Zn!I)((4bO<#*6}Z!L274tQ)jIxN?RA@T7N zSG_Rd4%yDS?drNl5(~?KZ9&T0MB+zG3_c!Zi_`t&i9d!RwLs#%=Gb`xBMMjx?YDDM)TZKbcnY?H6?F(xhJ<7 zuPeIGzWCZTkKK(QcSzqTn@mWxVrPDVP%+VAdWCo*?`?i`M(C@7GEfW@Ck{`~aQTZC z?5F)kD+?~A>SMW($euf~%Ui3|rxyEr8kE>87t~k=RCgSC7w5`Eq|h_?g#0mqZYFdV z%~Xl?IL4i>reX9}t?oI+4GRPBpl0Od9eZpS8y`nl1;nu>Tk~!A*_BNiLJf1E#Sh1G)cO27FBgS8x?iM5qjxl z%MfIEhNUq8swZMP^ftTq{SaypHXP(+((w3<6!g!;@q(ODUz)DAv5q4F)abvlo zgH*4ZjkKHy1H`oF)E@=XnN)i%PmpHq6R?*g!H)mrfcLJ zjNH*aJsYwX7iFTqJ)gQRu1-91cDT%-S{SlrVUHHnWes89)skW^>d>%t_1M|4vXwK4 z=r2sXZJejW3rPLo>;EAhZ%jI2?|!KLvuyuWb5AGQ2X|}^p1&%4QW0RyK+muStS8Uu z*OqBHW&3xej0~rFDr9sb^RXGvx{%j#t{Sm7bhd4`&zgOR(0iRU02WBz!=`5ecS{;D zSl9F3I8S=cUM!vS=1MJFf)u9A_4khn%K`6pwj-X`OL%ONGy9v(o^CsIqP|}eqJC&Q?@@PXe&Wy>{`Gx_#E-Ky}Jk< zZvm@{gv(b}m6~@I92}tI8wZC)|B{nw<`;W+p>B2QxcKW*w)W4=dDhr2K6!nliz9x7 zC1WHzBfnpF*r>?v_IVcm@qt=j-Vqm_=uP_mK>B2y-d+&1sfYcj!mO@oc8Yy5 z;{!E&>XoG3(gx=`+tz{HC!{@nQTO~=&kN&a!6$?OK=wJSz( zF=uoy9|;IAJm2`?l}Tj;;v}s(ef=q^I)wNy58*2dSe{%H(kPQC z!kielu6{5`w|!_B6di+PBUn2>|Hat97v9`99lXwk(b?Krv@YT{+$mPJzy5%L5~CT= z3Tf)Qshh3z?cBZF2hu_twCk^C9-!5>WYLLK|D`AIj%*9}D<~x5d2+M!KSr3e8TLFA z2;;73E^`1Iy?Bowf(~Ck=ZYmxPl&7SmhYLeF7+7d8qKnA44%BvFD+isAnsxlGSrNO z5P&j@H9-8A$AMs)yQx9U>|3)d5;;+Ay|=9VWIkiskP@~eKe;ZEY@5N6ksQN-0AdF^ zUh}NT#Rv4^HC%}6C4VELiPXhffj>42WfuTH7OKK95K zTbn9mpZ6~Ray@}3m(F=X!sY3t+JR4TTd1!Mc%Y^T1!sjS!Jct}jcgaXyjth-M2w=n z2YbH=9?o?zJESqWua?_^Ha$&h`|DuX>L==hx3*FS2AWJlQB}07!*{!ZGqs}4UB%; zC1Pk}?9nK4L83yfDZt-7*J~xyMvqWFa-y0;@g%F*vy_gIyOUb=SpWX? zG!wr0kN6^^%k+U~v&Cnh#BS$iXXoZ-J1$PO@-3B(^j$`ELU<&Mb=|nAQMvw2d|$2Y zm5@A?<0xZZ`D~Bf5a^LeM2WGU_1YLu*j*Wq3_^-yAI$G|HZ9bAbW}6))4uW!6b@JD zR9zIuT<-!miw)8(UQkJ;4$>q!VXFD-Hb(~w#9vsVZEix{q%!6xmh9pF`qAAT>3AaT z1LYd_+1U^_G`e=!F!~HbjHaSWS%JInw34=^-}6n`wbDJnZ6O81WcI}FocR-6!N}JK zGe9ED1V_85z$rfVg8Y6Eh_9|LJWGR`A+@VLL639Oo`yoMENqJtnF@-E3JpT`v?@>J zr7`ZoAw5=c)xyieRxDsR?Xa*9hhGkvon>a8ouyg68Js&K?{;R-3dwv$p-y`|VzAzJ zgsZvPxD<5Vny$_KaW>UUP}u3uFZ|ApfIK_zCT5NlOhne$=wO1P;W!P)_r-LTpoOV) zXS{8T24v|pkRchWGrl2v0fq)5%m)hd-`F;VWIteaC2K zmj~rsOt9d0U4z1(eT`5~FV*|dsL|5m;=H_Koy)=c`ufJk`cIsG+&2#mvUB+Lt*-9H z{)~{rg^CBvgYesNZ+_h8E53z()(G3se5U8mS9yq%$Y|IFKJt30Yk^B5bh9vPCOO@= zLj4%6ZEYVvw#{vj)IDXJm>`#FczNbc_qcc3&66Xk^I!;6V=`nrTE@54+--AvV|p6Y zSDcr$PulFp50=2@w~!DL-_lg0g3dP-t^mS$rLwbq!8CLacQeY3cO`R%W&?K9n$?CN ziTd3B)7S|Czdp8rhZAzq%y7Ts#6cI9vM@J!X})$Gw_kXW<7eZggc2w`4Ja|lKJWXY zRkg^%X098;!a7+&K>!@XH0wjsqRI(_fKvXa@lc7x25>#UWMcf9rlxjxl9MwMXdMmh zj7iY%sK{6w<1KiyWavKVT3?6id*?BD?iUD~-*KA+S$D0n8HzuOY0tDo@|ncO4Z`lL zUv;_(jr5oJlxi3)$>{0Nf}M>WRkDzLX2OAfb)e50`vAi#XJkSybAP{?(v*lyYRkiL1o zwwb;rbHmPw+~{)p2(e|bkC-)(pRn)#Ax#pG=d-0(r!tKh?#qYk8ldx_ z=fAH2kJAA5p!V+F4Vt_nERGhCJFP!HJaO6wr#`bX9U?8|DYlouP^SMEEy~EyOixQ& zU0^*wR3Sogn`YD0Wb?Xpvujw0tv+6f}HEk%Z4FCs&SQ54}$zL&eAgV=BZ62miIOE!=v)^GRiS26xxygLGVV!~JJx*QR%G$8e&4cEKb!9DQB;xm z&LuiLki($lP~cHJ`oWNu}uX?DP)Wbu%?>V)|*k zE?{aKL7tNyZjn@~c@%_izm;qzyV{bILhLFmUR@{l^|dUBJ}X>z5BCZ5Fe-aC-5$_-FmYzMevc<8OSrjn(%eC8M$+WPn0tm!ImF**H;$na*e-i{YT;yn=*Z zu-m_l>3Ct;;zl$G?kR+slhN0(0S=`O{il1XKPZ0d@taKoLwA|CV!3r4(tE7>#mbeZ zaO&v|_{+A0=>iM`x-_nS5<`N0@_QHv-?N+;R8_I7^0t60A1oN|=L zs|z0+P=uu~k291_5e^;vh-IO}w-Q@?N<-7%?hZBy! zotQ)$$0XoI+rH$By2O^m#pmnGzg8q7vpK!G=Z~3ZH5mq3dqAM5n#pZ>`-3-wldhXq zxsjH}XV#dcJ6x*ljpeclambG$Ce;xEFA#)}HMbJ_#lH*WR*hsiXJ1NMRg>u0Ej`u} zCk_hE%kn{*4cQZ)cgtQctKE87X{8;_!ZcztJ$AO<1T!)dq7iPWAEBE zo_w}<%&eh6=+Q~c*7S=iFwvL?<&wHpsh-lZia48tnWoi zuwFZOeAfU?^j*Ri_KvWmf4>d2XJV|{t7PzvKBE;G~StqGr3u)SqOT!Q^|`O~ph z<&UxpV6Sj)Ab>y1-ib-^TCmB0ItzY5C2_6_csNk}$-w4XI=o`JCz!?nSJ;eSD32v5Qn>+;0F)gk#)s1-{OB%YsH6 zKFMY$PVb(CQe}v)={?(#k~bsXO#Tie7+F!phyihX%x}*13@>cXOvdfI%%eX$H}x)^3A?qA#3rqHonLh0`{A%KNJj++1p%{cbW~fT(s#ev@y4iC zEnzxAv+N@U**)X@xSv)Y_5y;r{m5g^EkY)Kjk{6V4r|DEbVL4~7u2Q_AB}pgP zX;km4vB9ZVndw1T#SCwhU$XN^ObSp z-iTC7fVM7D5w~PUC%bI=^wQ=|EQavp#w^AM^(5exo{<6W-sszE%+4?bn9}BwRa=NT@};~kp4}Ts8y|o3Jkm;{CS9Qs6pU%rpTnK7#GQAD^@?#W>6>6}g%4fH?S-m|SCLMyyaI&XN>4sx-F!M260sE3fZV024A! z02(&crcuoSEYk~dWl46)Kxw-H#&hNN2yqcYubwHNZK;vdDH;(u$1}6jgTEevzgBx& zagz+DP}o*ZkfCwW#Cs(C5`fx`tE(WD*&2ro?Z1YEbk+g|l4hr#uF;B`Z~o+NXA3zM zKw|M;(Qahen*5PU8qVK z-QK>lrL0UbD^2*w_s2Yf<9bx@lVJ-?L?l6S%l-pn=ty7RXsxxi_DgJ3pUe86twlB| z2`Q7TppXzU`b;W`jj4DFvP4PCV50>)DJ8AL?&)O9tHTh95<2%fUx}9;%`F@zd@vmj zzEd(G5-$t9u^+Db{o8YErn*1()*0vuKHuS-hhNkSLm1Dkg~3j8Bmq#U`lz zN&ZylOGYNSp2ONVg`78G2SEo4iXfN-DYyR%TY~ZeysUbV)w@nTNIT<=<_bIGdzc8@ z|DfEXffpwE=kNddK=qFd08I1GJorZjU`P089{eK%NHP2~5B?X)U<|p}y2sp?;70T0 z-?Az%NUOiw-D~aW47e)`!Jm}FeC*#+81{A+hgiERYmg@g>f`f+QoeTG5WiQ-+W@q`t x#)9_5zutnAX`z#!np=wLSUhupPP{Ltz$$Kav0d literal 0 HcmV?d00001 From c843a08b25c62578526e3af859b0340029e2c6d6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 17 May 2024 13:59:43 +0100 Subject: [PATCH 020/169] Added docker setup command for CM, extra run options --- main.py | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/main.py b/main.py index 80643f4d3..b6d8b9c8d 100644 --- a/main.py +++ b/main.py @@ -68,20 +68,32 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): content += f"{cur_space1}=== \"{device}\"\n" content += f"{cur_space2}###### {device} device\n\n" + content += f"{cur_space2}###### Docker Setup Command\n\n" + test_query_count=100 + content += mlperf_inference_run_command(spaces+12, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True) + content += f"{cur_space2}The above command should get you to an interactive shell inside the docker container and do a quick test run for the Offline scenario. Please see [here]() for more options to customize the docker container. Once inside the docker container please do the below commands to do the accuracy + performance runs for each scenario.\n\n" + run_suffix = "" + run_suffix += f"\n{cur_space2} ###### Run Options\n\n" + run_suffix += f"{cur_space2} * Use `--division=closed` to do a closed division submission which includes compliance runs\n\n" + run_suffix += f"{cur_space2} * Use `--rerun` to do a rerun even when a valid run exists\n\n" + for scenario in scenarios: cur_space3 = cur_space2 + " " content += f"{cur_space2}=== \"{scenario}\"\n{cur_space3}####### {scenario}\n" run_cmd = mlperf_inference_run_command(spaces+16, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), "valid") content += run_cmd + content += run_suffix + content += f"{cur_space2}=== \"All Scenarios\"\n{cur_space3}####### All Scenarios\n" run_cmd = mlperf_inference_run_command(spaces+16, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), "valid") content += run_cmd + content += run_suffix return content @env.macro - def mlperf_inference_run_command(spaces, model, implementation, framework, category, scenario, device="cpu", execution_mode="test", test_query_count="20"): + def mlperf_inference_run_command(spaces, model, implementation, framework, category, scenario, device="cpu", execution_mode="test", test_query_count="20", docker=False): pre_space = "" for i in range(1,spaces): pre_space = pre_space + " " @@ -95,9 +107,20 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ scenario_variation_tag = "" scenario_option = f"\\\n {pre_space} --scenario={scenario}" - cmd_suffix = f" \\\n {pre_space} --docker" - #cmd_suffix = f"" - if execution_mode == "test": - cmd_suffix += f" \\\n {pre_space} --test_query_count={test_query_count}" + if docker: + docker_cmd_suffix = f" \\\n {pre_space} --docker" + docker_cmd_suffix += f" \\\n {pre_space} --test_query_count={test_query_count}" + + docker_setup_cmd = f"\n{f_pre_space} ```bash\n{f_pre_space} cm run script --tags=run-mlperf,inference,_full{scenario_variation_tag} \\\n {pre_space} --model={model} \\\n {pre_space} --implementation={implementation} \\\n {pre_space} --framework={framework} \\\n {pre_space} --category={category} {scenario_option} \\\n {pre_space} --execution-mode=test \\\n {pre_space} --device={device} {docker_cmd_suffix}\n{f_pre_space} ```\n" + + return docker_setup_cmd + + else: + cmd_suffix = f"" + + if execution_mode == "test": + cmd_suffix += f" \\\n {pre_space} --test_query_count={test_query_count}" + + run_cmd = f"\n{f_pre_space} ```bash\n{f_pre_space} cm run script --tags=run-mlperf,inference,_full{scenario_variation_tag} \\\n {pre_space} --model={model} \\\n {pre_space} --implementation={implementation} \\\n {pre_space} --framework={framework} \\\n {pre_space} --category={category} {scenario_option} \\\n {pre_space} --execution-mode={execution_mode} \\\n {pre_space} --device={device} {cmd_suffix}\n{f_pre_space} ```\n" - return f"\n{f_pre_space} ```bash\n{f_pre_space} cm run script --tags=run-mlperf,inference,_full{scenario_variation_tag} \\\n {pre_space} --model={model} \\\n {pre_space} --implementation={implementation} \\\n {pre_space} --framework={framework} \\\n {pre_space} --category={category} {scenario_option} \\\n {pre_space} --execution-mode={execution_mode} \\\n {pre_space} --device={device} {cmd_suffix}\n{f_pre_space} ```\n" + return run_cmd From 664a37acf776cb206f379e96d3be6a26124d7f6f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 17 May 2024 20:40:27 +0100 Subject: [PATCH 021/169] Added support for docker options in the docs --- main.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index b6d8b9c8d..47e902a79 100644 --- a/main.py +++ b/main.py @@ -71,7 +71,17 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): content += f"{cur_space2}###### Docker Setup Command\n\n" test_query_count=100 content += mlperf_inference_run_command(spaces+12, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True) - content += f"{cur_space2}The above command should get you to an interactive shell inside the docker container and do a quick test run for the Offline scenario. Please see [here]() for more options to customize the docker container. Once inside the docker container please do the below commands to do the accuracy + performance runs for each scenario.\n\n" + content += f"{cur_space2}The above command should get you to an interactive shell inside the docker container and do a quick test run for the Offline scenario. Once inside the docker container please do the below commands to do the accuracy + performance runs for each scenario.\n\n" + content += f"{cur_space2}
    \n" + content += f"{cur_space2} Please click here to see more options for the docker launch \n\n" + content += f"{cur_space2}* `--docker_cm_repo `: to use a custom fork of cm4mlops repository inside the docker image\n\n" + content += f"{cur_space2}* `--docker_cache=no`: to not use docker cache during the image build\n" + + if device.lower() not in [ "cuda" ]: + content += f"{cur_space2}* `--docker_os=ubuntu`: ubuntu and rhel are supported. \n" + content += f"{cur_space2}* `--docker_os_version=20.04`: [20.04, 22.04] are supported for Ubuntu and [8, 9] for RHEL\n" + + content += f"{cur_space2}
    \n" run_suffix = "" run_suffix += f"\n{cur_space2} ###### Run Options\n\n" run_suffix += f"{cur_space2} * Use `--division=closed` to do a closed division submission which includes compliance runs\n\n" From a97fc5f2cda0d7892c19992aa3cbdccdab1b2274 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 18 May 2024 15:42:41 +0100 Subject: [PATCH 022/169] Added --quiet to the CM run_cmds in docs --- main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/main.py b/main.py index 47e902a79..a83315334 100644 --- a/main.py +++ b/main.py @@ -118,19 +118,19 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ scenario_option = f"\\\n {pre_space} --scenario={scenario}" if docker: - docker_cmd_suffix = f" \\\n {pre_space} --docker" + docker_cmd_suffix = f" \\\n {pre_space} --docker --quiet" docker_cmd_suffix += f" \\\n {pre_space} --test_query_count={test_query_count}" - docker_setup_cmd = f"\n{f_pre_space} ```bash\n{f_pre_space} cm run script --tags=run-mlperf,inference,_full{scenario_variation_tag} \\\n {pre_space} --model={model} \\\n {pre_space} --implementation={implementation} \\\n {pre_space} --framework={framework} \\\n {pre_space} --category={category} {scenario_option} \\\n {pre_space} --execution-mode=test \\\n {pre_space} --device={device} {docker_cmd_suffix}\n{f_pre_space} ```\n" + docker_setup_cmd = f"\n{f_pre_space} ```bash\n{f_pre_space} cm run script --tags=run-mlperf,inference,_find-performance,_full{scenario_variation_tag} \\\n {pre_space} --model={model} \\\n {pre_space} --implementation={implementation} \\\n {pre_space} --framework={framework} \\\n {pre_space} --category={category} {scenario_option} \\\n {pre_space} --execution-mode=test \\\n {pre_space} --device={device} {docker_cmd_suffix}\n{f_pre_space} ```\n" return docker_setup_cmd else: - cmd_suffix = f"" + cmd_suffix = f"\\\n {pre_space} --quiet" if execution_mode == "test": cmd_suffix += f" \\\n {pre_space} --test_query_count={test_query_count}" - run_cmd = f"\n{f_pre_space} ```bash\n{f_pre_space} cm run script --tags=run-mlperf,inference,_full{scenario_variation_tag} \\\n {pre_space} --model={model} \\\n {pre_space} --implementation={implementation} \\\n {pre_space} --framework={framework} \\\n {pre_space} --category={category} {scenario_option} \\\n {pre_space} --execution-mode={execution_mode} \\\n {pre_space} --device={device} {cmd_suffix}\n{f_pre_space} ```\n" + run_cmd = f"\n{f_pre_space} ```bash\n{f_pre_space} cm run script --tags=run-mlperf,inference{scenario_variation_tag} \\\n {pre_space} --model={model} \\\n {pre_space} --implementation={implementation} \\\n {pre_space} --framework={framework} \\\n {pre_space} --category={category} {scenario_option} \\\n {pre_space} --execution-mode={execution_mode} \\\n {pre_space} --device={device} {cmd_suffix}\n{f_pre_space} ```\n" return run_cmd From 18ff1a8a663094db1ce550633f1dff3c04071f1e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 22 May 2024 00:58:24 +0100 Subject: [PATCH 023/169] Fix the test query count for cm commands --- main.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index a83315334..c97b219c0 100644 --- a/main.py +++ b/main.py @@ -69,7 +69,8 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): content += f"{cur_space2}###### {device} device\n\n" content += f"{cur_space2}###### Docker Setup Command\n\n" - test_query_count=100 + test_query_count=get_test_query_count(model, implementation, device) + content += mlperf_inference_run_command(spaces+12, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True) content += f"{cur_space2}The above command should get you to an interactive shell inside the docker container and do a quick test run for the Offline scenario. Once inside the docker container please do the below commands to do the accuracy + performance runs for each scenario.\n\n" content += f"{cur_space2}
    \n" @@ -101,6 +102,21 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): return content + def get_test_query_count(model, implementation, device, num_devices=1): + + if model == "resnet50": + p_range = 1000 + elif model in [ "retinanet", "bert-99", "bert-99.9" ]: + p_range = 100 + else: + p_range = 50 + if device == "cuda": + p_range *= 10 + p_range *= num_devices + + return p_range + + @env.macro def mlperf_inference_run_command(spaces, model, implementation, framework, category, scenario, device="cpu", execution_mode="test", test_query_count="20", docker=False): From ac6d20a19eeef46f9296ec6d8512e885e197cf1f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 31 May 2024 22:32:33 +0100 Subject: [PATCH 024/169] Support ctuning-cpp implementation --- main.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index c97b219c0..7670d46b3 100644 --- a/main.py +++ b/main.py @@ -9,6 +9,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): pre_space += " " content="" + scenarios = [] if implementation == "reference": devices = [ "CPU", "CUDA", "ROCm" ] if model.lower() == "resnet50": @@ -31,6 +32,13 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): elif implementation == "cpp": devices = [ "CPU", "CUDA" ] frameworks = [ "Onnxruntime" ] + elif implementation == "ctuning-cpp": + scenarios = [ "SingleStream" ] + devices = [ "CPU" ] + if model.lower() == "resnet50": + frameworks = [ "TFLite" ] + else: + frameworks = [] if model.lower() == "bert-99.9": categories = [ "Datacenter" ] @@ -40,11 +48,11 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): categories = [ "Edge", "Datacenter" ] for category in categories: - if category == "Edge": + if category == "Edge" and not scenarios: scenarios = [ "Offline", "SingleStream" ] if model.lower() in [ "resnet50", "retinanet" ]: - scenarios.append("Multistream") - elif category == "Datacenter": + scenarios.append("MultiStream") + elif category == "Datacenter" and not scenarios: scenarios = [ "Offline", "Server" ] content += f"{pre_space}=== \"{category.lower()}\"\n\n" From 6b2264ba25ddf0637e8e56968ded0be87f5966e8 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 31 May 2024 23:53:18 +0100 Subject: [PATCH 025/169] Added commands for mobilenet models --- .../image_classification/mobilenets.md | 59 +++++++++++++++++++ .../image_classification/resnet50.md | 2 + docs/submission/index.md | 2 +- 3 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 docs/benchmarks/image_classification/mobilenets.md diff --git a/docs/benchmarks/image_classification/mobilenets.md b/docs/benchmarks/image_classification/mobilenets.md new file mode 100644 index 000000000..84e911076 --- /dev/null +++ b/docs/benchmarks/image_classification/mobilenets.md @@ -0,0 +1,59 @@ +# Image Classification using Mobilenet models + +Mobilenet models are not official MLPerf models and so cannot be used for a Closed division MLPerf inference submission. But since they can be run with Imagenet dataset, we are allowed to use them for Open division submission. Only CPU runs are supported now. + +## TFLite Backend + +=== "Mobilenet-V1" + ### Mobilenet V1 + ```bash + cm run script --tags=run,mobilenet-models,_tflite,_mobilenet-v1 + ``` +=== "Mobilenet-V2" + ### Mobilenet V2 + ```bash + cm run script --tags=run,mobilenet-models,_tflite,_mobilenet-v2 + ``` +=== "Mobilenet-V2" + ### Mobilenet V2 + ```bash + cm run script --tags=run,mobilenet-models,_tflite,_mobilenet-v2 + ``` +=== "Mobilenets" + ### Mobilenet V1,V2,V3 + ```bash + cm run script --tags=run,mobilenet-models,_tflite,_mobilenet + ``` +=== "Efficientnet" + ### Efficientnet + ```bash + cm run script --tags=run,mobilenet-models,_tflite,_efficientnet + ``` + +## ARMNN Backend +=== "Mobilenet-V1" + ### Mobilenet V1 + ```bash + cm run script --tags=run,mobilenet-models,_tflite,_armnn,_mobilenet-v1 + ``` +=== "Mobilenet-V2" + ### Mobilenet V2 + ```bash + cm run script --tags=run,mobilenet-models,_tflite,_armnn,_mobilenet-v2 + ``` +=== "Mobilenet-V2" + ### Mobilenet V2 + ```bash + cm run script --tags=run,mobilenet-models,_tflite,_armnn,_mobilenet-v2 + ``` +=== "Mobilenets" + ### Mobilenet V1,V2,V3 + ```bash + cm run script --tags=run,mobilenet-models,_tflite,_armnn,_mobilenet + ``` +=== "Efficientnet" + ### Efficientnet + ```bash + cm run script --tags=run,mobilenet-models,_tflite,_armnn,_efficientnet + ``` + diff --git a/docs/benchmarks/image_classification/resnet50.md b/docs/benchmarks/image_classification/resnet50.md index 1a77db65a..1d37dcf13 100644 --- a/docs/benchmarks/image_classification/resnet50.md +++ b/docs/benchmarks/image_classification/resnet50.md @@ -66,3 +66,5 @@ Get the Official MLPerf ResNet50 Model ### MLPerf Modular Implementation in C++ {{ mlperf_inference_implementation_readme (4, "resnet50", "cpp") }} + +Please see [mobilenets.md](mobilenets.md) for running mobilenet models for Image Classification. diff --git a/docs/submission/index.md b/docs/submission/index.md index b5ff53033..94287f5d2 100644 --- a/docs/submission/index.md +++ b/docs/submission/index.md @@ -1,4 +1,4 @@ -If you follow the `cm run` commands under the individual model pages in the [benchmarks](../benchmarks) directory, all the valid results will get aggregated to the `cm cache` folder. Once all the results across all the modelsare ready you can use the following command to generate a valid submission tree compliant with the [MLPerf requirements](https://github.com/mlcommons/policies/blob/master/submission_rules.adoc#inference-1). +If you follow the `cm run` commands under the individual model pages in the [benchmarks](../benchmarks/index.md) directory, all the valid results will get aggregated to the `cm cache` folder. Once all the results across all the modelsare ready you can use the following command to generate a valid submission tree compliant with the [MLPerf requirements](https://github.com/mlcommons/policies/blob/master/submission_rules.adoc#inference-1). ## Generate actual submission tree From 9b896a6f93a19b8d97e54cc5c953314479fa4c3f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 31 May 2024 23:59:57 +0100 Subject: [PATCH 026/169] Docs cleanup --- docs/benchmarks/text_to_image/sdxl.md | 2 -- main.py | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/benchmarks/text_to_image/sdxl.md b/docs/benchmarks/text_to_image/sdxl.md index 2e9c95c66..32173d49b 100644 --- a/docs/benchmarks/text_to_image/sdxl.md +++ b/docs/benchmarks/text_to_image/sdxl.md @@ -37,13 +37,11 @@ Get the Official MLPerf Stable Diffusion Model === "Intel" ### Intel MLPerf Implementation - GPTJ-99 {{ mlperf_inference_implementation_readme (4, "sdxl", "intel") }} === "Qualcomm" ### Qualcomm AI100 MLPerf Implementation - GPTJ-99 {{ mlperf_inference_implementation_readme (4, "sdxl", "qualcomm") }} diff --git a/main.py b/main.py index 7670d46b3..f8421b30a 100644 --- a/main.py +++ b/main.py @@ -24,6 +24,8 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): devices = [ "CUDA" ] frameworks = [ "TensorRT" ] elif implementation == "intel": + if model not in [ "bert-99", "bert-99.9", "gptj-99", "gptj-99.9" ]: + return pre_space+" WIP" devices = [ "CPU" ] frameworks = [ "Pytorch" ] elif implementation == "qualcomm": From ce5a0b03ab6bf1795bdd21ed1d323428146c63c9 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 1 Jun 2024 00:06:50 +0100 Subject: [PATCH 027/169] Docs cleanup --- .../image_classification/mobilenets.md | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/benchmarks/image_classification/mobilenets.md b/docs/benchmarks/image_classification/mobilenets.md index 84e911076..1907c2a41 100644 --- a/docs/benchmarks/image_classification/mobilenets.md +++ b/docs/benchmarks/image_classification/mobilenets.md @@ -7,53 +7,53 @@ Mobilenet models are not official MLPerf models and so cannot be used for a Clos === "Mobilenet-V1" ### Mobilenet V1 ```bash - cm run script --tags=run,mobilenet-models,_tflite,_mobilenet-v1 + cm run script --tags=run,mobilenet-models,_tflite,_mobilenet-v1 --adr.compiler.tags=gcc ``` === "Mobilenet-V2" ### Mobilenet V2 ```bash - cm run script --tags=run,mobilenet-models,_tflite,_mobilenet-v2 + cm run script --tags=run,mobilenet-models,_tflite,_mobilenet-v2 --adr.compiler.tags=gcc ``` === "Mobilenet-V2" ### Mobilenet V2 ```bash - cm run script --tags=run,mobilenet-models,_tflite,_mobilenet-v2 + cm run script --tags=run,mobilenet-models,_tflite,_mobilenet-v2 --adr.compiler.tags=gcc ``` === "Mobilenets" ### Mobilenet V1,V2,V3 ```bash - cm run script --tags=run,mobilenet-models,_tflite,_mobilenet + cm run script --tags=run,mobilenet-models,_tflite,_mobilenet --adr.compiler.tags=gcc ``` === "Efficientnet" ### Efficientnet ```bash - cm run script --tags=run,mobilenet-models,_tflite,_efficientnet + cm run script --tags=run,mobilenet-models,_tflite,_efficientnet --adr.compiler.tags=gcc ``` ## ARMNN Backend === "Mobilenet-V1" ### Mobilenet V1 ```bash - cm run script --tags=run,mobilenet-models,_tflite,_armnn,_mobilenet-v1 + cm run script --tags=run,mobilenet-models,_tflite,_armnn,_mobilenet-v1 --adr.compiler.tags=gcc ``` === "Mobilenet-V2" ### Mobilenet V2 ```bash - cm run script --tags=run,mobilenet-models,_tflite,_armnn,_mobilenet-v2 + cm run script --tags=run,mobilenet-models,_tflite,_armnn,_mobilenet-v2 --adr.compiler.tags=gcc ``` === "Mobilenet-V2" ### Mobilenet V2 ```bash - cm run script --tags=run,mobilenet-models,_tflite,_armnn,_mobilenet-v2 + cm run script --tags=run,mobilenet-models,_tflite,_armnn,_mobilenet-v2 --adr.compiler.tags=gcc ``` === "Mobilenets" ### Mobilenet V1,V2,V3 ```bash - cm run script --tags=run,mobilenet-models,_tflite,_armnn,_mobilenet + cm run script --tags=run,mobilenet-models,_tflite,_armnn,_mobilenet --adr.compiler.tags=gcc ``` === "Efficientnet" ### Efficientnet ```bash - cm run script --tags=run,mobilenet-models,_tflite,_armnn,_efficientnet + cm run script --tags=run,mobilenet-models,_tflite,_armnn,_efficientnet --adr.compiler.tags=gcc ``` From 1f3eacdb33e26df517b06134c9e6a5d1917e52ba Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 5 Jun 2024 13:41:31 +0100 Subject: [PATCH 028/169] Added separate files for dataset and models in the docs --- .../image_classification/get-resnet50-data.md | 43 +++ .../image_classification/resnet50.md | 43 --- docs/benchmarks/language/bert.md | 36 -- docs/benchmarks/language/get-bert-data.md | 38 ++ docs/benchmarks/language/get-gptj-data.md | 25 ++ .../language/get-llama2-70b-data.md | 26 ++ docs/benchmarks/language/gpt-j.md | 23 -- docs/benchmarks/language/llama2-70b.md | 23 -- docs/benchmarks/medical_imaging/3d-unet.md | 35 -- .../medical_imaging/get-3d-unet-data.md | 38 ++ .../object_detection/get-retinanet-data.md | 38 ++ docs/benchmarks/object_detection/retinanet.md | 36 -- docs/benchmarks/recommendation/dlrm-v2.md | 32 +- .../recommendation/get-dlrm_v2-data.md | 25 ++ .../benchmarks/text_to_image/get-sdxl-data.md | 26 ++ docs/benchmarks/text_to_image/sdxl.md | 23 -- docs/submission/tools-readme.md | 1 - main.py | 345 ++++++++++-------- mkdocs.yml | 3 +- 19 files changed, 455 insertions(+), 404 deletions(-) create mode 100644 docs/benchmarks/image_classification/get-resnet50-data.md create mode 100644 docs/benchmarks/language/get-bert-data.md create mode 100644 docs/benchmarks/language/get-gptj-data.md create mode 100644 docs/benchmarks/language/get-llama2-70b-data.md create mode 100644 docs/benchmarks/medical_imaging/get-3d-unet-data.md create mode 100644 docs/benchmarks/object_detection/get-retinanet-data.md create mode 100644 docs/benchmarks/recommendation/get-dlrm_v2-data.md create mode 100644 docs/benchmarks/text_to_image/get-sdxl-data.md delete mode 120000 docs/submission/tools-readme.md diff --git a/docs/benchmarks/image_classification/get-resnet50-data.md b/docs/benchmarks/image_classification/get-resnet50-data.md new file mode 100644 index 000000000..d1f83ae9a --- /dev/null +++ b/docs/benchmarks/image_classification/get-resnet50-data.md @@ -0,0 +1,43 @@ +# Image Classification using ResNet50 + +## Dataset + +The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. + +=== "Validation" + ResNet50 validation run uses the Imagenet 2012 validation dataset consisting of 50,000 images. + + ### Get Validation Dataset + ``` + cm run script --tags=get,dataset,imagenet,validation -j + ``` +=== "Calibration" + ResNet50 calibration dataset consist of 500 images selected from the Imagenet 2012 validation dataset. There are 2 alternative options for the calibration dataset. + + ### Get Calibration Dataset Using Option 1 + ``` + cm run script --tags=get,dataset,imagenet,calibration,_mlperf.option1 -j + ``` + ### Get Calibration Dataset Using Option 2 + ``` + cm run script --tags=get,dataset,imagenet,calibration,_mlperf.option2 -j + ``` + +## Model +The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. + +Get the Official MLPerf ResNet50 Model + +=== "Tensorflow" + + ### Tensorflow + ``` + cm run script --tags=get,ml-model,resnet50,_tensorflow -j + ``` +=== "Onnx" + + ### Onnx + ``` + cm run script --tags=get,ml-model,resnet50,_onnx -j + ``` + diff --git a/docs/benchmarks/image_classification/resnet50.md b/docs/benchmarks/image_classification/resnet50.md index 1d37dcf13..17e682d0e 100644 --- a/docs/benchmarks/image_classification/resnet50.md +++ b/docs/benchmarks/image_classification/resnet50.md @@ -1,46 +1,5 @@ # Image Classification using ResNet50 -## Dataset - -The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. - -=== "Validation" - ResNet50 validation run uses the Imagenet 2012 validation dataset consisting of 50,000 images. - - ### Get Validation Dataset - ``` - cm run script --tags=get,dataset,imagenet,validation -j - ``` -=== "Calibration" - ResNet50 calibration dataset consist of 500 images selected from the Imagenet 2012 validation dataset. There are 2 alternative options for the calibration dataset. - - ### Get Calibration Dataset Using Option 1 - ``` - cm run script --tags=get,dataset,imagenet,calibration,_mlperf.option1 -j - ``` - ### Get Calibration Dataset Using Option 2 - ``` - cm run script --tags=get,dataset,imagenet,calibration,_mlperf.option2 -j - ``` - -## Model -The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. - -Get the Official MLPerf ResNet50 Model - -=== "Tensorflow" - - ### Tensorflow - ``` - cm run script --tags=get,ml-model,resnet50,_tensorflow -j - ``` -=== "Onnx" - - ### Onnx - ``` - cm run script --tags=get,ml-model,resnet50,_onnx -j - ``` - ## Benchmark Implementations === "MLCommons-Python" ### MLPerf Reference Implementation in Python @@ -66,5 +25,3 @@ Get the Official MLPerf ResNet50 Model ### MLPerf Modular Implementation in C++ {{ mlperf_inference_implementation_readme (4, "resnet50", "cpp") }} - -Please see [mobilenets.md](mobilenets.md) for running mobilenet models for Image Classification. diff --git a/docs/benchmarks/language/bert.md b/docs/benchmarks/language/bert.md index e2aa0995d..8cb3badd5 100644 --- a/docs/benchmarks/language/bert.md +++ b/docs/benchmarks/language/bert.md @@ -1,41 +1,5 @@ # Question Answering using Bert-Large -## Dataset - -The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. - -=== "Validation" - BERT validation run uses the SQuAD v1.1 dataset. - - ### Get Validation Dataset - ``` - cm run script --tags=get,dataset,squad,validation -j - ``` - -## Model -The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. - -Get the Official MLPerf Bert-Large Model - -=== "Pytorch" - - ### Pytorch - ``` - cm run script --tags=get,ml-model,bert-large,_pytorch -j - ``` -=== "Onnx" - - ### Onnx - ``` - cm run script --tags=get,ml-model,bert-large,_onnx -j - ``` -=== "Tensorflow" - - ### Tensorflow - ``` - cm run script --tags=get,ml-model,bert-large,_tensorflow -j - ``` - ## Benchmark Implementations === "MLCommons-Python" ### MLPerf Reference Implementation in Python diff --git a/docs/benchmarks/language/get-bert-data.md b/docs/benchmarks/language/get-bert-data.md new file mode 100644 index 000000000..f5462b181 --- /dev/null +++ b/docs/benchmarks/language/get-bert-data.md @@ -0,0 +1,38 @@ +# Question Answering using Bert-Large + +## Dataset + +The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. + +=== "Validation" + BERT validation run uses the SQuAD v1.1 dataset. + + ### Get Validation Dataset + ``` + cm run script --tags=get,dataset,squad,validation -j + ``` + +## Model +The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. + +Get the Official MLPerf Bert-Large Model + +=== "Pytorch" + + ### Pytorch + ``` + cm run script --tags=get,ml-model,bert-large,_pytorch -j + ``` +=== "Onnx" + + ### Onnx + ``` + cm run script --tags=get,ml-model,bert-large,_onnx -j + ``` +=== "Tensorflow" + + ### Tensorflow + ``` + cm run script --tags=get,ml-model,bert-large,_tensorflow -j + ``` + diff --git a/docs/benchmarks/language/get-gptj-data.md b/docs/benchmarks/language/get-gptj-data.md new file mode 100644 index 000000000..9ea31feb4 --- /dev/null +++ b/docs/benchmarks/language/get-gptj-data.md @@ -0,0 +1,25 @@ +# Text Summarization using GPT-J + +## Dataset + +The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. + +=== "Validation" + GPT-J validation run uses the CNNDM dataset. + + ### Get Validation Dataset + ``` + cm run script --tags=get,dataset,cnndm,validation -j + ``` + +## Model +The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. + +Get the Official MLPerf GPT-J Model + +=== "Pytorch" + + ### Pytorch + ``` + cm run script --tags=get,ml-model,gptj,_pytorch -j + ``` diff --git a/docs/benchmarks/language/get-llama2-70b-data.md b/docs/benchmarks/language/get-llama2-70b-data.md new file mode 100644 index 000000000..4b04f7068 --- /dev/null +++ b/docs/benchmarks/language/get-llama2-70b-data.md @@ -0,0 +1,26 @@ +# Text Summarization using LLAMA2-70b + +## Dataset + +The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. + +=== "Validation" + LLAMA2-70b validation run uses the Open ORCA dataset. + + ### Get Validation Dataset + ``` + cm run script --tags=get,dataset,openorca,validation -j + ``` + +## Model +The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. + +Get the Official MLPerf LLAMA2-70b Model + +=== "Pytorch" + + ### Pytorch + ``` + cm run script --tags=get,ml-model,llama2-70b,_pytorch -j + ``` + diff --git a/docs/benchmarks/language/gpt-j.md b/docs/benchmarks/language/gpt-j.md index d1c351214..4ed46a58b 100644 --- a/docs/benchmarks/language/gpt-j.md +++ b/docs/benchmarks/language/gpt-j.md @@ -1,28 +1,5 @@ # Text Summarization using GPT-J -## Dataset - -The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. - -=== "Validation" - GPT-J validation run uses the CNNDM dataset. - - ### Get Validation Dataset - ``` - cm run script --tags=get,dataset,cnndm,validation -j - ``` - -## Model -The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. - -Get the Official MLPerf GPT-J Model - -=== "Pytorch" - - ### Pytorch - ``` - cm run script --tags=get,ml-model,gptj,_pytorch -j - ``` ## Benchmark Implementations === "MLCommons-Python" diff --git a/docs/benchmarks/language/llama2-70b.md b/docs/benchmarks/language/llama2-70b.md index 7f8052aef..b131eb437 100644 --- a/docs/benchmarks/language/llama2-70b.md +++ b/docs/benchmarks/language/llama2-70b.md @@ -1,28 +1,5 @@ # Text Summarization using LLAMA2-70b -## Dataset - -The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. - -=== "Validation" - LLAMA2-70b validation run uses the Open ORCA dataset. - - ### Get Validation Dataset - ``` - cm run script --tags=get,dataset,openorca,validation -j - ``` - -## Model -The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. - -Get the Official MLPerf LLAMA2-70b Model - -=== "Pytorch" - - ### Pytorch - ``` - cm run script --tags=get,ml-model,llama2-70b,_pytorch -j - ``` ## Benchmark Implementations === "MLCommons-Python" diff --git a/docs/benchmarks/medical_imaging/3d-unet.md b/docs/benchmarks/medical_imaging/3d-unet.md index bd3ccae40..dbe233993 100644 --- a/docs/benchmarks/medical_imaging/3d-unet.md +++ b/docs/benchmarks/medical_imaging/3d-unet.md @@ -1,40 +1,5 @@ # Medical Imaging using 3d-unet (KiTS 2019 kidney tumor segmentation task) -## Dataset - -The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. - -=== "Validation" - 3d-unet validation run uses the KiTS19 dataset performing [KiTS 2019](https://kits19.grand-challenge.org/) kidney tumor segmentation task - - ### Get Validation Dataset - ``` - cm run script --tags=get,dataset,kits19,validation -j - ``` - -## Model -The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. - -Get the Official MLPerf 3d-unet Model - -=== "Pytorch" - - ### Pytorch - ``` - cm run script --tags=get,ml-model,3d-unet,_pytorch -j - ``` -=== "Onnx" - - ### Onnx - ``` - cm run script --tags=get,ml-model,3d-unet,_onnx -j - ``` -=== "Tensorflow" - - ### Tensorflow - ``` - cm run script --tags=get,ml-model,3d-unet,_tensorflow -j - ``` ## Benchmark Implementations === "MLCommons-Python" diff --git a/docs/benchmarks/medical_imaging/get-3d-unet-data.md b/docs/benchmarks/medical_imaging/get-3d-unet-data.md new file mode 100644 index 000000000..efc6ce6ed --- /dev/null +++ b/docs/benchmarks/medical_imaging/get-3d-unet-data.md @@ -0,0 +1,38 @@ +# Medical Imaging using 3d-unet (KiTS 2019 kidney tumor segmentation task) + +## Dataset + +The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. + +=== "Validation" + 3d-unet validation run uses the KiTS19 dataset performing [KiTS 2019](https://kits19.grand-challenge.org/) kidney tumor segmentation task + + ### Get Validation Dataset + ``` + cm run script --tags=get,dataset,kits19,validation -j + ``` + +## Model +The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. + +Get the Official MLPerf 3d-unet Model + +=== "Pytorch" + + ### Pytorch + ``` + cm run script --tags=get,ml-model,3d-unet,_pytorch -j + ``` +=== "Onnx" + + ### Onnx + ``` + cm run script --tags=get,ml-model,3d-unet,_onnx -j + ``` +=== "Tensorflow" + + ### Tensorflow + ``` + cm run script --tags=get,ml-model,3d-unet,_tensorflow -j + ``` + diff --git a/docs/benchmarks/object_detection/get-retinanet-data.md b/docs/benchmarks/object_detection/get-retinanet-data.md new file mode 100644 index 000000000..f2d432210 --- /dev/null +++ b/docs/benchmarks/object_detection/get-retinanet-data.md @@ -0,0 +1,38 @@ +# Object Detection using Retinanet + +## Dataset + +The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. + +=== "Validation" + Retinanet validation run uses the OpenImages v6 MLPerf validation dataset resized to 800x800 and consisting of 24,576 images. + + ### Get Validation Dataset + ``` + cm run script --tags=get,dataset,openimages,_validation -j + ``` +=== "Calibration" + Retinanet calibration dataset consist of 500 images selected from the OpenImages v6 dataset. + + ``` + cm run script --tags=get,dataset,openimages,_calibration -j + ``` + +## Model +The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. + +Get the Official MLPerf Retinanet Model + +=== "Pytorch" + + ### Pytorch + ``` + cm run script --tags=get,ml-model,retinanet,_pytorch -j + ``` +=== "Onnx" + + ### Onnx + ``` + cm run script --tags=get,ml-model,retinanet,_onnx -j + ``` + diff --git a/docs/benchmarks/object_detection/retinanet.md b/docs/benchmarks/object_detection/retinanet.md index f500f616d..a6b469e8f 100644 --- a/docs/benchmarks/object_detection/retinanet.md +++ b/docs/benchmarks/object_detection/retinanet.md @@ -1,41 +1,5 @@ # Object Detection using Retinanet -## Dataset - -The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. - -=== "Validation" - Retinanet validation run uses the OpenImages v6 MLPerf validation dataset resized to 800x800 and consisting of 24,576 images. - - ### Get Validation Dataset - ``` - cm run script --tags=get,dataset,openimages,_validation -j - ``` -=== "Calibration" - Retinanet calibration dataset consist of 500 images selected from the OpenImages v6 dataset. - - ``` - cm run script --tags=get,dataset,openimages,_calibration -j - ``` - -## Model -The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. - -Get the Official MLPerf Retinanet Model - -=== "Pytorch" - - ### Pytorch - ``` - cm run script --tags=get,ml-model,retinanet,_pytorch -j - ``` -=== "Onnx" - - ### Onnx - ``` - cm run script --tags=get,ml-model,retinanet,_onnx -j - ``` - ## Benchmark Implementations === "MLCommons-Python" ### MLPerf Reference Implementation in Python diff --git a/docs/benchmarks/recommendation/dlrm-v2.md b/docs/benchmarks/recommendation/dlrm-v2.md index 1294b008b..750f18e3e 100644 --- a/docs/benchmarks/recommendation/dlrm-v2.md +++ b/docs/benchmarks/recommendation/dlrm-v2.md @@ -1,36 +1,22 @@ # Recommendation using DLRM v2 -## Dataset - -The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. - -=== "Validation" - DLRM validation run uses the Criteo dataset (Day 23). - - ### Get Validation Dataset - ``` - cm run script --tags=get,dataset,criteo,validation -j - ``` -## Model -The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. - -Get the Official MLPerf DLRM v2 Model - -=== "Pytorch" - - ### Pytorch - ``` - cm run script --tags=get,ml-model,dlrm_v2,_pytorch -j - ``` ## Benchmark Implementations === "MLCommons-Python" ### MLPerf Reference Implementation in Python - + + DLRM-v2-99 +{{ mlperf_inference_implementation_readme (4, "dlrm_v2-99", "reference") }} + + DLRM-v2-99.9 {{ mlperf_inference_implementation_readme (4, "dlrm_v2-99.9", "reference") }} === "Nvidia" ### Nvidia MLPerf Implementation + DLRM-v2-99 +{{ mlperf_inference_implementation_readme (4, "dlrm_v2-99", "nvidia") }} + + DLRM-v2-99.9 {{ mlperf_inference_implementation_readme (4, "dlrm_v2-99.9", "nvidia") }} diff --git a/docs/benchmarks/recommendation/get-dlrm_v2-data.md b/docs/benchmarks/recommendation/get-dlrm_v2-data.md new file mode 100644 index 000000000..97464a164 --- /dev/null +++ b/docs/benchmarks/recommendation/get-dlrm_v2-data.md @@ -0,0 +1,25 @@ +# Recommendation using DLRM v2 + +## Dataset + +The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. + +=== "Validation" + DLRM validation run uses the Criteo dataset (Day 23). + + ### Get Validation Dataset + ``` + cm run script --tags=get,dataset,criteo,validation -j + ``` +## Model +The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. + +Get the Official MLPerf DLRM v2 Model + +=== "Pytorch" + + ### Pytorch + ``` + cm run script --tags=get,ml-model,dlrm_v2,_pytorch -j + ``` + diff --git a/docs/benchmarks/text_to_image/get-sdxl-data.md b/docs/benchmarks/text_to_image/get-sdxl-data.md new file mode 100644 index 000000000..830465d44 --- /dev/null +++ b/docs/benchmarks/text_to_image/get-sdxl-data.md @@ -0,0 +1,26 @@ +# Text to Image using Stable Diffusion + +## Dataset + +The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. + +=== "Validation" + Stable Diffusion validation run uses the Coco 2014 dataset. + + ### Get Validation Dataset + ``` + cm run script --tags=get,dataset,coco2014,_validation -j + ``` + +## Model +The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. + +Get the Official MLPerf Stable Diffusion Model + +=== "Pytorch" + + ### Pytorch + ``` + cm run script --tags=get,ml-model,sdxl,_pytorch -j + ``` + diff --git a/docs/benchmarks/text_to_image/sdxl.md b/docs/benchmarks/text_to_image/sdxl.md index 32173d49b..6010a208e 100644 --- a/docs/benchmarks/text_to_image/sdxl.md +++ b/docs/benchmarks/text_to_image/sdxl.md @@ -1,28 +1,5 @@ # Text to Image using Stable Diffusion -## Dataset - -The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. - -=== "Validation" - Stable Diffusion validation run uses the Coco 2014 dataset. - - ### Get Validation Dataset - ``` - cm run script --tags=get,dataset,coco2014,_validation -j - ``` - -## Model -The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. - -Get the Official MLPerf Stable Diffusion Model - -=== "Pytorch" - - ### Pytorch - ``` - cm run script --tags=get,ml-model,sdxl,_pytorch -j - ``` ## Benchmark Implementations === "MLCommons-Python" diff --git a/docs/submission/tools-readme.md b/docs/submission/tools-readme.md deleted file mode 120000 index d6f026eab..000000000 --- a/docs/submission/tools-readme.md +++ /dev/null @@ -1 +0,0 @@ -../../tools/submission/README.md \ No newline at end of file diff --git a/main.py b/main.py index ccf76b3c1..1ae366f7e 100644 --- a/main.py +++ b/main.py @@ -1,162 +1,187 @@ def define_env(env): - @env.macro - def mlperf_inference_implementation_readme(spaces, model, implementation): - pre_space = "" - for i in range(1,spaces): - pre_space = pre_space + " " - f_pre_space = pre_space - pre_space += " " - - content="" - scenarios = [] - if implementation == "reference": - devices = [ "CPU", "CUDA", "ROCm" ] - if model.lower() == "resnet50": - frameworks = [ "Onnxruntime", "Tensorflow", "Deepsparse" ] - elif model.lower() == "retinanet": - frameworks = [ "Onnxruntime", "Pytorch" ] - elif "bert" in model.lower(): - frameworks = [ "Onnxruntime", "Pytorch", "Tensorflow" ] - else: - frameworks = [ "Pytorch" ] - elif implementation == "nvidia": - devices = [ "CUDA" ] - frameworks = [ "TensorRT" ] - elif implementation == "intel": - if model not in [ "bert-99", "bert-99.9", "gptj-99", "gptj-99.9" ]: - return pre_space+" WIP" - devices = [ "CPU" ] - frameworks = [ "Pytorch" ] - elif implementation == "qualcomm": - devices = [ "QAIC" ] - frameworks = [ "Glow" ] - elif implementation == "cpp": - devices = [ "CPU", "CUDA" ] - frameworks = [ "Onnxruntime" ] - elif implementation == "ctuning-cpp": - scenarios = [ "SingleStream" ] - devices = [ "CPU" ] - if model.lower() == "resnet50": - frameworks = [ "TFLite" ] - else: - frameworks = [] - - if model.lower() == "bert-99.9": - categories = [ "Datacenter" ] - elif "dlrm" in model.lower() or "llama2" in model.lower(): - categories = [ "Datacenter" ] - else: - categories = [ "Edge", "Datacenter" ] - - for category in categories: - if category == "Edge" and not scenarios: - scenarios = [ "Offline", "SingleStream" ] - if model.lower() in [ "resnet50", "retinanet" ]: - scenarios.append("MultiStream") - elif category == "Datacenter" and not scenarios: - scenarios = [ "Offline", "Server" ] - - content += f"{pre_space}=== \"{category.lower()}\"\n\n" - - cur_space = pre_space + " " - scenarios_string = ", ".join(scenarios) - - content += f"{cur_space}#### {category} category \n\n{cur_space} In the {category.lower()} category, {model} has {scenarios_string} scenarios and all the scenarios are mandatory for a closed division submission.\n\n" - - - for framework in frameworks: - cur_space1 = cur_space + " " - content += f"{cur_space}=== \"{framework}\"\n" - content += f"{cur_space1}##### {framework} framework\n\n" - - for device in devices: - if framework.lower() == "deepsparse": - if device.lower() != "cpu": - continue - cur_space2 = cur_space1 + " " - content += f"{cur_space1}=== \"{device}\"\n" - content += f"{cur_space2}###### {device} device\n\n" - - content += f"{cur_space2}###### Docker Setup Command\n\n" - test_query_count=get_test_query_count(model, implementation, device) - - content += mlperf_inference_run_command(spaces+12, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True) - content += f"{cur_space2}The above command should get you to an interactive shell inside the docker container and do a quick test run for the Offline scenario. Once inside the docker container please do the below commands to do the accuracy + performance runs for each scenario.\n\n" - content += f"{cur_space2}
    \n" - content += f"{cur_space2} Please click here to see more options for the docker launch \n\n" - content += f"{cur_space2}* `--docker_cm_repo `: to use a custom fork of cm4mlops repository inside the docker image\n\n" - content += f"{cur_space2}* `--docker_cache=no`: to not use docker cache during the image build\n" - - if device.lower() not in [ "cuda" ]: - content += f"{cur_space2}* `--docker_os=ubuntu`: ubuntu and rhel are supported. \n" - content += f"{cur_space2}* `--docker_os_version=20.04`: [20.04, 22.04] are supported for Ubuntu and [8, 9] for RHEL\n" - - content += f"{cur_space2}
    \n" - run_suffix = "" - run_suffix += f"\n{cur_space2} ###### Run Options\n\n" - run_suffix += f"{cur_space2} * Use `--division=closed` to do a closed division submission which includes compliance runs\n\n" - run_suffix += f"{cur_space2} * Use `--rerun` to do a rerun even when a valid run exists\n\n" - - for scenario in scenarios: - cur_space3 = cur_space2 + " " - content += f"{cur_space2}=== \"{scenario}\"\n{cur_space3}####### {scenario}\n" - run_cmd = mlperf_inference_run_command(spaces+16, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), "valid") - content += run_cmd - content += run_suffix - - content += f"{cur_space2}=== \"All Scenarios\"\n{cur_space3}####### All Scenarios\n" - run_cmd = mlperf_inference_run_command(spaces+16, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), "valid") - content += run_cmd - content += run_suffix - - return content - - def get_test_query_count(model, implementation, device, num_devices=1): - - if model == "resnet50": - p_range = 1000 - elif model in [ "retinanet", "bert-99", "bert-99.9" ]: - p_range = 100 - else: - p_range = 50 - if device == "cuda": - p_range *= 40 - p_range *= num_devices - - return p_range - - - - @env.macro - def mlperf_inference_run_command(spaces, model, implementation, framework, category, scenario, device="cpu", execution_mode="test", test_query_count="20", docker=False): - pre_space = "" - for i in range(1,spaces): - pre_space = pre_space + " " - f_pre_space = pre_space - pre_space += " " - - if scenario == "All Scenarios": - scenario_variation_tag = ",_all-scenarios" - scenario_option = "" - else: - scenario_variation_tag = "" - scenario_option = f"\\\n {pre_space} --scenario={scenario}" - - if docker: - docker_cmd_suffix = f" \\\n {pre_space} --docker --quiet" - docker_cmd_suffix += f" \\\n {pre_space} --test_query_count={test_query_count}" - - docker_setup_cmd = f"\n{f_pre_space} ```bash\n{f_pre_space} cm run script --tags=run-mlperf,inference,_find-performance,_full{scenario_variation_tag} \\\n {pre_space} --model={model} \\\n {pre_space} --implementation={implementation} \\\n {pre_space} --framework={framework} \\\n {pre_space} --category={category} {scenario_option} \\\n {pre_space} --execution-mode=test \\\n {pre_space} --device={device} {docker_cmd_suffix}\n{f_pre_space} ```\n" - - return docker_setup_cmd - - else: - cmd_suffix = f"\\\n {pre_space} --quiet" - - if execution_mode == "test": - cmd_suffix += f" \\\n {pre_space} --test_query_count={test_query_count}" - - run_cmd = f"\n{f_pre_space} ```bash\n{f_pre_space} cm run script --tags=run-mlperf,inference{scenario_variation_tag} \\\n {pre_space} --model={model} \\\n {pre_space} --implementation={implementation} \\\n {pre_space} --framework={framework} \\\n {pre_space} --category={category} {scenario_option} \\\n {pre_space} --execution-mode={execution_mode} \\\n {pre_space} --device={device} {cmd_suffix}\n{f_pre_space} ```\n" - - return run_cmd + @env.macro + def mlperf_inference_implementation_readme(spaces, model, implementation): + pre_space = "" + + for i in range(1,spaces): + pre_space = pre_space + " " + f_pre_space = pre_space + pre_space += " " + + content="" + scenarios = [] + + if implementation == "reference": + devices = [ "CPU", "CUDA", "ROCm" ] + if model.lower() == "resnet50": + frameworks = [ "Onnxruntime", "Tensorflow", "Deepsparse" ] + elif model.lower() == "retinanet": + frameworks = [ "Onnxruntime", "Pytorch" ] + elif "bert" in model.lower(): + frameworks = [ "Onnxruntime", "Pytorch", "Tensorflow" ] + else: + frameworks = [ "Pytorch" ] + + elif implementation == "nvidia": + devices = [ "CUDA" ] + frameworks = [ "TensorRT" ] + elif implementation == "intel": + if model not in [ "bert-99", "bert-99.9", "gptj-99", "gptj-99.9" ]: + return pre_space+" WIP" + devices = [ "CPU" ] + frameworks = [ "Pytorch" ] + + elif implementation == "qualcomm": + devices = [ "QAIC" ] + frameworks = [ "Glow" ] + + elif implementation == "cpp": + devices = [ "CPU", "CUDA" ] + frameworks = [ "Onnxruntime" ] + + elif implementation == "ctuning-cpp": + scenarios = [ "SingleStream" ] + devices = [ "CPU" ] + if model.lower() == "resnet50": + frameworks = [ "TFLite" ] + else: + frameworks = [] + + if model.lower() == "bert-99.9": + categories = [ "Datacenter" ] + elif "dlrm" in model.lower() or "llama2" in model.lower(): + categories = [ "Datacenter" ] + else: + categories = [ "Edge", "Datacenter" ] + + for category in categories: + if category == "Edge" and not scenarios: + scenarios = [ "Offline", "SingleStream" ] + if model.lower() in [ "resnet50", "retinanet" ]: + scenarios.append("MultiStream") + elif category == "Datacenter" and not scenarios: + scenarios = [ "Offline", "Server" ] + + content += f"{pre_space}=== \"{category.lower()}\"\n\n" + + cur_space = pre_space + " " + scenarios_string = ", ".join(scenarios) + + content += f"{cur_space}#### {category} category \n\n{cur_space} In the {category.lower()} category, {model} has {scenarios_string} scenarios and all the scenarios are mandatory for a closed division submission.\n\n" + + + for framework in frameworks: + cur_space1 = cur_space + " " + content += f"{cur_space}=== \"{framework}\"\n" + content += f"{cur_space1}##### {framework} framework\n\n" + + for device in devices: + if framework.lower() == "deepsparse": + if device.lower() != "cpu": + continue + cur_space2 = cur_space1 + " " + content += f"{cur_space1}=== \"{device}\"\n" + content += f"{cur_space2}###### {device} device\n\n" + + if "99.9" not in model: #not showing docker command as it is already done for the 99% variant + content += f"{cur_space2}###### Docker Setup Command\n\n" + test_query_count=get_test_query_count(model, implementation, device) + + content += mlperf_inference_run_command(spaces+12, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True) + content += f"{cur_space2}The above command should get you to an interactive shell inside the docker container and do a quick test run for the Offline scenario. Once inside the docker container please do the below commands to do the accuracy + performance runs for each scenario.\n\n" + content += f"{cur_space2}
    \n" + content += f"{cur_space2} Please click here to see more options for the docker launch \n\n" + content += f"{cur_space2}* `--docker_cm_repo `: to use a custom fork of cm4mlops repository inside the docker image\n\n" + content += f"{cur_space2}* `--docker_cache=no`: to not use docker cache during the image build\n" + else: + content += f"{cur_space2}Use the same docker container as for the {model.replace('99.9', '99')} model.\n\n" + + if device.lower() not in [ "cuda" ]: + content += f"{cur_space2}* `--docker_os=ubuntu`: ubuntu and rhel are supported. \n" + content += f"{cur_space2}* `--docker_os_version=20.04`: [20.04, 22.04] are supported for Ubuntu and [8, 9] for RHEL\n" + + content += f"{cur_space2}
    \n" + run_suffix = "" + run_suffix += f"\n{cur_space2} ###### Run Options\n\n" + run_suffix += f"{cur_space2} * Use `--division=closed` to do a closed division submission which includes compliance runs\n\n" + run_suffix += f"{cur_space2} * Use `--rerun` to do a rerun even when a valid run exists\n\n" + + for scenario in scenarios: + cur_space3 = cur_space2 + " " + content += f"{cur_space2}=== \"{scenario}\"\n{cur_space3}####### {scenario}\n" + run_cmd = mlperf_inference_run_command(spaces+16, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), "valid") + content += run_cmd + content += run_suffix + + content += f"{cur_space2}=== \"All Scenarios\"\n{cur_space3}####### All Scenarios\n" + run_cmd = mlperf_inference_run_command(spaces+16, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), "valid") + content += run_cmd + content += run_suffix + + readme_suffix = get_readme_suffix(spaces, model, implementation) + + return content + readme_suffix + + def get_test_query_count(model, implementation, device, num_devices=1): + + if model == "resnet50": + p_range = 1000 + elif model in [ "retinanet", "bert-99", "bert-99.9" ]: + p_range = 100 + else: + p_range = 50 + + if device == "cuda": + p_range *= 40 + p_range *= num_devices + + return p_range + + def get_readme_suffix(spaces, model, implementation): + readme_suffix = "" + pre_space="" + for i in range(1,spaces): + pre_space = pre_space + " " + pre_space += " " + + if implementation == "reference": + model_base_name = model.replace("-99.9","").replace("-99","") + readme_suffix+= f"{pre_space}* If you want to download the official MLPerf model and dataset for {model} you can follow [this README](get-{model_base_name}-data.md).\n" + if model == "resnet50": + readme_suffix += f"{pre_space}* Please see [mobilenets.md](mobilenets.md) for running mobilenet models for Image Classification." + return readme_suffix + + + @env.macro + def mlperf_inference_run_command(spaces, model, implementation, framework, category, scenario, device="cpu", execution_mode="test", test_query_count="20", docker=False): + pre_space = "" + for i in range(1,spaces): + pre_space = pre_space + " " + f_pre_space = pre_space + pre_space += " " + + if scenario == "All Scenarios": + scenario_variation_tag = ",_all-scenarios" + scenario_option = "" + else: + scenario_variation_tag = "" + scenario_option = f"\\\n {pre_space} --scenario={scenario}" + + if docker: + docker_cmd_suffix = f" \\\n {pre_space} --docker --quiet" + docker_cmd_suffix += f" \\\n {pre_space} --test_query_count={test_query_count}" + + docker_setup_cmd = f"\n{f_pre_space} ```bash\n{f_pre_space} cm run script --tags=run-mlperf,inference,_find-performance,_full{scenario_variation_tag} \\\n {pre_space} --model={model} \\\n {pre_space} --implementation={implementation} \\\n {pre_space} --framework={framework} \\\n {pre_space} --category={category} {scenario_option} \\\n {pre_space} --execution-mode=test \\\n {pre_space} --device={device} {docker_cmd_suffix}\n{f_pre_space} ```\n" + + return docker_setup_cmd + + else: + cmd_suffix = f"\\\n {pre_space} --quiet" + + if execution_mode == "test": + cmd_suffix += f" \\\n {pre_space} --test_query_count={test_query_count}" + + run_cmd = f"\n{f_pre_space} ```bash\n{f_pre_space} cm run script --tags=run-mlperf,inference{scenario_variation_tag} \\\n {pre_space} --model={model} \\\n {pre_space} --implementation={implementation} \\\n {pre_space} --framework={framework} \\\n {pre_space} --category={category} {scenario_option} \\\n {pre_space} --execution-mode={execution_mode} \\\n {pre_space} --device={device} {cmd_suffix}\n{f_pre_space} ```\n" + + return run_cmd diff --git a/mkdocs.yml b/mkdocs.yml index 0d0f64152..758f23c25 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -27,7 +27,8 @@ nav: - Benchmarks: - benchmarks/index.md - Image Classification: - - ResNet50: benchmarks/image_classification/resnet50.md + - ResNet50: + - benchmarks/image_classification/resnet50.md - Text to Image: - Stable Diffusion: benchmarks/text_to_image/sdxl.md - Object Detection: From 63fcc600d1066ea917cbd2b910a7e2932d0d0be8 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 5 Jun 2024 13:48:13 +0100 Subject: [PATCH 029/169] Remove redundant tab in the docs --- mkdocs.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index 758f23c25..2095557db 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -19,8 +19,6 @@ theme: - navigation.top - toc.follow nav: - - Inference: - - index.md - Install: - install/index.md - Quick Start: install/quick-start.md From 5e49864ae6341473276e050f45aad1fe6ed0c74b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 5 Jun 2024 13:56:04 +0100 Subject: [PATCH 030/169] Fixes some WIP models in the docs --- main.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/main.py b/main.py index 1ae366f7e..56f8cc35f 100644 --- a/main.py +++ b/main.py @@ -24,8 +24,11 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): frameworks = [ "Pytorch" ] elif implementation == "nvidia": + if model in [ "sdxl", "llama2-99", "llama2-99.9" ]: + return pre_space+" WIP" devices = [ "CUDA" ] frameworks = [ "TensorRT" ] + elif implementation == "intel": if model not in [ "bert-99", "bert-99.9", "gptj-99", "gptj-99.9" ]: return pre_space+" WIP" @@ -33,6 +36,9 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): frameworks = [ "Pytorch" ] elif implementation == "qualcomm": + if model not in [ "resnet50", "retinanet", "bert-99", "bert-99.9" ]: + return pre_space+" WIP" + devices = [ "QAIC" ] frameworks = [ "Glow" ] From a33c2a772ab28a6f5f2f637d13e27d1ea34b6913 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 5 Jun 2024 14:45:51 +0100 Subject: [PATCH 031/169] Use the official docs page for CM installation --- docs/install/index.md | 96 +------------------------------------------ 1 file changed, 2 insertions(+), 94 deletions(-) diff --git a/docs/install/index.md b/docs/install/index.md index ec76718d9..10ed6e325 100644 --- a/docs/install/index.md +++ b/docs/install/index.md @@ -1,100 +1,7 @@ # Installation We use MLCommons CM Automation framework to run MLPerf inference benchmarks. -## CM Install - -We have successfully tested CM on - -* Ubuntu 18.x, 20.x, 22.x , 23.x, -* RedHat 8, RedHat 9, CentOS 8 -* macOS -* Wndows 10, Windows 11 - -=== "Ubuntu" - ### Ubuntu, Debian - - - ```bash - sudo apt update && sudo apt upgrade - sudo apt install python3 python3-pip python3-venv git wget curl - ``` - - **Note that you must set up virtual env on Ubuntu 23+ before using any Python project:** - ```bash - python3 -m venv cm - source cm/bin/activate - ``` - - You can now install CM via PIP: - - ```bash - python3 -m pip install cmind - ``` - - You might need to do the following command to update the `PATH` to include the BIN paths from pip installs - - ```bash - source $HOME/.profile - ``` - - You can check that CM is available by checking the `cm` command - - -=== "Red Hat" - ### Red Hat - - ```bash - sudo dnf update - sudo dnf install python3 python-pip git wget curl - python3 -m pip install cmind --user - ``` - -=== "macOS" - ### macOS - - *Note that CM currently does not work with Python installed from the Apple Store. - Please install Python via brew as described below.* - - If `brew` package manager is not installed, please install it as follows (see details [here](https://brew.sh/)): - ```bash - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - ``` - - Don't forget to add brew to PATH environment as described in the end of the installation output. - - Then install python, pip, git and wget: - - ```bash - brew install python3 git wget curl - python3 -m pip install cmind - ``` - -=== "Windows" - - ### Windows - * Configure Windows 10+ to [support long paths](https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=registry#enable-long-paths-in-windows-10-version-1607-and-later) from command line as admin: - - ```bash - reg add "HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem" /v LongPathsEnabled /t REG_DWORD /d 1 /f - ``` - - * Download and install Git from [git-for-windows.github.io](https://git-for-windows.github.io). - * Configure Git to accept long file names: `git config --system core.longpaths true` - * Download and install Python 3+ from [www.python.org/downloads/windows](https://www.python.org/downloads/windows). - * Don't forget to select option to add Python binaries to PATH environment! - * Configure Windows to accept long fie names during Python installation! - - * Install CM via PIP: - - ```bash - python -m pip install cmind - ``` - - *Note that we [have reports](https://github.com/mlcommons/ck/issues/844) - that CM does not work when Python was first installed from the Microsoft Store. - If CM fails to run, you can find a fix [here](https://stackoverflow.com/questions/57485491/python-python3-executes-in-command-prompt-but-does-not-run-correctly)*. - -Please visit the [official CM installation page](https://github.com/mlcommons/ck/blob/master/docs/installation.md) for more details +Please follow the [official installation page](https://docs.mlcommons.org/ck/install) to install CM ## Download the CM MLOps Repository @@ -102,4 +9,5 @@ Please visit the [official CM installation page](https://github.com/mlcommons/ck cm pull repo gateoverflow@cm4mlops ``` + Now, you are ready to use the `cm` commands to run MLPerf inference as given in the [benchmarks](../benchmarks/index.md) page From 16a80098fd350e8f802af7e2b17ec2546e15b10b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 5 Jun 2024 14:49:18 +0100 Subject: [PATCH 032/169] Fix the deadlink in docs --- mkdocs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index 2095557db..1120ee660 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -21,7 +21,6 @@ theme: nav: - Install: - install/index.md - - Quick Start: install/quick-start.md - Benchmarks: - benchmarks/index.md - Image Classification: From 6ad8a0e57a169556d08278804fbee6dc3602a872 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 6 Jun 2024 14:30:21 +0100 Subject: [PATCH 033/169] Fix indendation issue in docs --- main.py | 106 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/main.py b/main.py index 56f8cc35f..969a3d61d 100644 --- a/main.py +++ b/main.py @@ -69,62 +69,62 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): elif category == "Datacenter" and not scenarios: scenarios = [ "Offline", "Server" ] - content += f"{pre_space}=== \"{category.lower()}\"\n\n" - - cur_space = pre_space + " " - scenarios_string = ", ".join(scenarios) - - content += f"{cur_space}#### {category} category \n\n{cur_space} In the {category.lower()} category, {model} has {scenarios_string} scenarios and all the scenarios are mandatory for a closed division submission.\n\n" - - - for framework in frameworks: - cur_space1 = cur_space + " " - content += f"{cur_space}=== \"{framework}\"\n" - content += f"{cur_space1}##### {framework} framework\n\n" - - for device in devices: - if framework.lower() == "deepsparse": - if device.lower() != "cpu": - continue - cur_space2 = cur_space1 + " " - content += f"{cur_space1}=== \"{device}\"\n" - content += f"{cur_space2}###### {device} device\n\n" - - if "99.9" not in model: #not showing docker command as it is already done for the 99% variant - content += f"{cur_space2}###### Docker Setup Command\n\n" - test_query_count=get_test_query_count(model, implementation, device) - - content += mlperf_inference_run_command(spaces+12, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True) - content += f"{cur_space2}The above command should get you to an interactive shell inside the docker container and do a quick test run for the Offline scenario. Once inside the docker container please do the below commands to do the accuracy + performance runs for each scenario.\n\n" - content += f"{cur_space2}
    \n" - content += f"{cur_space2} Please click here to see more options for the docker launch \n\n" - content += f"{cur_space2}* `--docker_cm_repo `: to use a custom fork of cm4mlops repository inside the docker image\n\n" - content += f"{cur_space2}* `--docker_cache=no`: to not use docker cache during the image build\n" - else: - content += f"{cur_space2}Use the same docker container as for the {model.replace('99.9', '99')} model.\n\n" - - if device.lower() not in [ "cuda" ]: - content += f"{cur_space2}* `--docker_os=ubuntu`: ubuntu and rhel are supported. \n" - content += f"{cur_space2}* `--docker_os_version=20.04`: [20.04, 22.04] are supported for Ubuntu and [8, 9] for RHEL\n" - - content += f"{cur_space2}
    \n" - run_suffix = "" - run_suffix += f"\n{cur_space2} ###### Run Options\n\n" - run_suffix += f"{cur_space2} * Use `--division=closed` to do a closed division submission which includes compliance runs\n\n" - run_suffix += f"{cur_space2} * Use `--rerun` to do a rerun even when a valid run exists\n\n" - - for scenario in scenarios: - cur_space3 = cur_space2 + " " - content += f"{cur_space2}=== \"{scenario}\"\n{cur_space3}####### {scenario}\n" - run_cmd = mlperf_inference_run_command(spaces+16, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), "valid") + content += f"{pre_space}=== \"{category.lower()}\"\n\n" + + cur_space = pre_space + " " + scenarios_string = ", ".join(scenarios) + + content += f"{cur_space}#### {category} category \n\n{cur_space} In the {category.lower()} category, {model} has {scenarios_string} scenarios and all the scenarios are mandatory for a closed division submission.\n\n" + + + for framework in frameworks: + cur_space1 = cur_space + " " + content += f"{cur_space}=== \"{framework}\"\n" + content += f"{cur_space1}##### {framework} framework\n\n" + + for device in devices: + if framework.lower() == "deepsparse": + if device.lower() != "cpu": + continue + cur_space2 = cur_space1 + " " + content += f"{cur_space1}=== \"{device}\"\n" + content += f"{cur_space2}###### {device} device\n\n" + + if "99.9" not in model: #not showing docker command as it is already done for the 99% variant + content += f"{cur_space2}###### Docker Setup Command\n\n" + test_query_count=get_test_query_count(model, implementation, device) + + content += mlperf_inference_run_command(spaces+12, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True) + content += f"{cur_space2}The above command should get you to an interactive shell inside the docker container and do a quick test run for the Offline scenario. Once inside the docker container please do the below commands to do the accuracy + performance runs for each scenario.\n\n" + content += f"{cur_space2}
    \n" + content += f"{cur_space2} Please click here to see more options for the docker launch \n\n" + content += f"{cur_space2}* `--docker_cm_repo `: to use a custom fork of cm4mlops repository inside the docker image\n\n" + content += f"{cur_space2}* `--docker_cache=no`: to not use docker cache during the image build\n" + else: + content += f"{cur_space2}Use the same docker container as for the {model.replace('99.9', '99')} model.\n\n" + + if device.lower() not in [ "cuda" ]: + content += f"{cur_space2}* `--docker_os=ubuntu`: ubuntu and rhel are supported. \n" + content += f"{cur_space2}* `--docker_os_version=20.04`: [20.04, 22.04] are supported for Ubuntu and [8, 9] for RHEL\n" + + content += f"{cur_space2}
    \n" + run_suffix = "" + run_suffix += f"\n{cur_space2} ###### Run Options\n\n" + run_suffix += f"{cur_space2} * Use `--division=closed` to do a closed division submission which includes compliance runs\n\n" + run_suffix += f"{cur_space2} * Use `--rerun` to do a rerun even when a valid run exists\n\n" + + for scenario in scenarios: + cur_space3 = cur_space2 + " " + content += f"{cur_space2}=== \"{scenario}\"\n{cur_space3}####### {scenario}\n" + run_cmd = mlperf_inference_run_command(spaces+16, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), "valid") + content += run_cmd + content += run_suffix + + content += f"{cur_space2}=== \"All Scenarios\"\n{cur_space3}####### All Scenarios\n" + run_cmd = mlperf_inference_run_command(spaces+16, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), "valid") content += run_cmd content += run_suffix - content += f"{cur_space2}=== \"All Scenarios\"\n{cur_space3}####### All Scenarios\n" - run_cmd = mlperf_inference_run_command(spaces+16, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), "valid") - content += run_cmd - content += run_suffix - readme_suffix = get_readme_suffix(spaces, model, implementation) return content + readme_suffix From cf0ca4f61983f217f1cbd396516a76b63213777b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 6 Jun 2024 16:41:46 +0100 Subject: [PATCH 034/169] Added dockerinfo for nvidia implementation --- main.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 969a3d61d..7543812d1 100644 --- a/main.py +++ b/main.py @@ -92,6 +92,10 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): if "99.9" not in model: #not showing docker command as it is already done for the 99% variant content += f"{cur_space2}###### Docker Setup Command\n\n" + + docker_info = get_docker_info(spaces+12, model, implementation, device) + content += docker_info + test_query_count=get_test_query_count(model, implementation, device) content += mlperf_inference_run_command(spaces+12, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True) @@ -125,9 +129,11 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): content += run_cmd content += run_suffix + readme_prefix = get_readme_prefix(spaces, model, implementation) + readme_suffix = get_readme_suffix(spaces, model, implementation) - return content + readme_suffix + return readme_prefix + content + readme_suffix def get_test_query_count(model, implementation, device, num_devices=1): @@ -144,6 +150,27 @@ def get_test_query_count(model, implementation, device, num_devices=1): return p_range + def get_readme_prefix(spaces, model, implementation): + readme_prefix = "" + pre_space=" " + #for i in range(1,spaces): + # pre_space = pre_space + " " + #pre_space += " " + + return readme_prefix + + def get_docker_info(spaces, model, implementation, device): + info = "" + pre_space="" + for i in range(1,spaces): + pre_space = pre_space + " " + pre_space += " " + #pre_space = " " + if implementation == "nvidia": + info += f"\n{pre_space}!!! tip\n\n" + info+= f"{pre_space} All the Nvidia benchmarks use the same docker container. So, if you have already done the docker setup command for any benchmark, you can skip the docker setup command below and do the Run commands inside the already built docker container.\n\n" + return info + def get_readme_suffix(spaces, model, implementation): readme_suffix = "" pre_space="" From 49013209f80f7de187a058964a529753c5d5d0cd Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 6 Jun 2024 17:05:40 +0100 Subject: [PATCH 035/169] Added run options for gptj --- docs/benchmarks/language/gpt-j.md | 6 ++++++ main.py | 14 ++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/docs/benchmarks/language/gpt-j.md b/docs/benchmarks/language/gpt-j.md index 4ed46a58b..b108f28cb 100644 --- a/docs/benchmarks/language/gpt-j.md +++ b/docs/benchmarks/language/gpt-j.md @@ -6,23 +6,28 @@ ### MLPerf Reference Implementation in Python GPT-J-99 + {{ mlperf_inference_implementation_readme (4, "gptj-99", "reference") }} GPTJ-99.9 + {{ mlperf_inference_implementation_readme (4, "gptj-99.9", "reference") }} === "Nvidia" ### Nvidia MLPerf Implementation GPTJ-99 + {{ mlperf_inference_implementation_readme (4, "gptj-99", "nvidia") }} GPTJ-99.9 + {{ mlperf_inference_implementation_readme (4, "gptj-99.9", "nvidia") }} === "Intel" ### Intel MLPerf Implementation GPTJ-99 + {{ mlperf_inference_implementation_readme (4, "gptj-99", "intel") }} @@ -30,5 +35,6 @@ ### Qualcomm AI100 MLPerf Implementation GPTJ-99 + {{ mlperf_inference_implementation_readme (4, "gptj-99", "qualcomm") }} diff --git a/main.py b/main.py index 7543812d1..785bc7e80 100644 --- a/main.py +++ b/main.py @@ -185,6 +185,14 @@ def get_readme_suffix(spaces, model, implementation): readme_suffix += f"{pre_space}* Please see [mobilenets.md](mobilenets.md) for running mobilenet models for Image Classification." return readme_suffix + def get_run_cmd_extra(f_pre_space, model, implementation, device): + extra_content = "" + f_pre_space += " " + if "gptj" in model and device == "cuda" and implementation == "reference": + extra_content += f"{f_pre_space}!!! tip\n\n" + extra_content += f"{f_pre_space} * `--precision=[float16|bfloat16]` can help run on GPUs with less RAM \n" + extra_content += f"{f_pre_space} * `--beam-size=1` Beam size of 4 is mandatory for a closed division submission but reducing the beam size can help in running the model on GPUs with lower device memory\n" + return extra_content @env.macro def mlperf_inference_run_command(spaces, model, implementation, framework, category, scenario, device="cpu", execution_mode="test", test_query_count="20", docker=False): @@ -201,13 +209,15 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ scenario_variation_tag = "" scenario_option = f"\\\n {pre_space} --scenario={scenario}" + run_cmd_extra = get_run_cmd_extra(f_pre_space, model, implementation, device) + if docker: docker_cmd_suffix = f" \\\n {pre_space} --docker --quiet" docker_cmd_suffix += f" \\\n {pre_space} --test_query_count={test_query_count}" docker_setup_cmd = f"\n{f_pre_space} ```bash\n{f_pre_space} cm run script --tags=run-mlperf,inference,_find-performance,_full{scenario_variation_tag} \\\n {pre_space} --model={model} \\\n {pre_space} --implementation={implementation} \\\n {pre_space} --framework={framework} \\\n {pre_space} --category={category} {scenario_option} \\\n {pre_space} --execution-mode=test \\\n {pre_space} --device={device} {docker_cmd_suffix}\n{f_pre_space} ```\n" - return docker_setup_cmd + return docker_setup_cmd + run_cmd_extra else: cmd_suffix = f"\\\n {pre_space} --quiet" @@ -217,4 +227,4 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ run_cmd = f"\n{f_pre_space} ```bash\n{f_pre_space} cm run script --tags=run-mlperf,inference{scenario_variation_tag} \\\n {pre_space} --model={model} \\\n {pre_space} --implementation={implementation} \\\n {pre_space} --framework={framework} \\\n {pre_space} --category={category} {scenario_option} \\\n {pre_space} --execution-mode={execution_mode} \\\n {pre_space} --device={device} {cmd_suffix}\n{f_pre_space} ```\n" - return run_cmd + return run_cmd + run_cmd_extra From c007322718f1ef9a901e839573f14b0082bf3d6a Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 7 Jun 2024 01:15:58 +0530 Subject: [PATCH 036/169] Added execution environment tabs --- docs/benchmarks/medical_imaging/3d-unet.md | 3 + main.py | 102 +++++++++++++-------- 2 files changed, 66 insertions(+), 39 deletions(-) diff --git a/docs/benchmarks/medical_imaging/3d-unet.md b/docs/benchmarks/medical_imaging/3d-unet.md index dbe233993..252ba3d07 100644 --- a/docs/benchmarks/medical_imaging/3d-unet.md +++ b/docs/benchmarks/medical_imaging/3d-unet.md @@ -5,6 +5,9 @@ === "MLCommons-Python" ### MLPerf Reference Implementation in Python + 3d-unet-99 +{{ mlperf_inference_implementation_readme (4, "3d-unet-99", "reference") }} + 3d-unet-99.9 {{ mlperf_inference_implementation_readme (4, "3d-unet-99.9", "reference") }} diff --git a/main.py b/main.py index 785bc7e80..6fe136880 100644 --- a/main.py +++ b/main.py @@ -11,6 +11,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): content="" scenarios = [] + execution_envs = ["Docker","Native"] if implementation == "reference": devices = [ "CPU", "CUDA", "ROCm" ] @@ -24,7 +25,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): frameworks = [ "Pytorch" ] elif implementation == "nvidia": - if model in [ "sdxl", "llama2-99", "llama2-99.9" ]: + if model in [ "sdxl", "llama2-70b-99", "llama2-70b-99.9" ]: return pre_space+" WIP" devices = [ "CUDA" ] frameworks = [ "TensorRT" ] @@ -64,10 +65,12 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): for category in categories: if category == "Edge" and not scenarios: scenarios = [ "Offline", "SingleStream" ] - if model.lower() in [ "resnet50", "retinanet" ]: + if model.lower() in [ "resnet50", "retinanet" ] and not "MultiStream" in scenarios:#MultiStream was duplicating scenarios.append("MultiStream") elif category == "Datacenter" and not scenarios: scenarios = [ "Offline", "Server" ] + if "99.9" in model: # Directly refer the commands of -99 + return f"\n{pre_space}Follow the same procedure described in {model.split('.')[0]}. Change the `--model` tag value to `{model.lower()}`." content += f"{pre_space}=== \"{category.lower()}\"\n\n" @@ -87,47 +90,60 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): if device.lower() != "cpu": continue cur_space2 = cur_space1 + " " + cur_space3 = cur_space2 + " " + cur_space4 = cur_space3 + " " + content += f"{cur_space1}=== \"{device}\"\n" content += f"{cur_space2}###### {device} device\n\n" if "99.9" not in model: #not showing docker command as it is already done for the 99% variant - content += f"{cur_space2}###### Docker Setup Command\n\n" - - docker_info = get_docker_info(spaces+12, model, implementation, device) - content += docker_info - - test_query_count=get_test_query_count(model, implementation, device) - - content += mlperf_inference_run_command(spaces+12, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True) - content += f"{cur_space2}The above command should get you to an interactive shell inside the docker container and do a quick test run for the Offline scenario. Once inside the docker container please do the below commands to do the accuracy + performance runs for each scenario.\n\n" - content += f"{cur_space2}
    \n" - content += f"{cur_space2} Please click here to see more options for the docker launch \n\n" - content += f"{cur_space2}* `--docker_cm_repo `: to use a custom fork of cm4mlops repository inside the docker image\n\n" - content += f"{cur_space2}* `--docker_cache=no`: to not use docker cache during the image build\n" - else: - content += f"{cur_space2}Use the same docker container as for the {model.replace('99.9', '99')} model.\n\n" - - if device.lower() not in [ "cuda" ]: - content += f"{cur_space2}* `--docker_os=ubuntu`: ubuntu and rhel are supported. \n" - content += f"{cur_space2}* `--docker_os_version=20.04`: [20.04, 22.04] are supported for Ubuntu and [8, 9] for RHEL\n" - - content += f"{cur_space2}
    \n" - run_suffix = "" - run_suffix += f"\n{cur_space2} ###### Run Options\n\n" - run_suffix += f"{cur_space2} * Use `--division=closed` to do a closed division submission which includes compliance runs\n\n" - run_suffix += f"{cur_space2} * Use `--rerun` to do a rerun even when a valid run exists\n\n" - - for scenario in scenarios: - cur_space3 = cur_space2 + " " - content += f"{cur_space2}=== \"{scenario}\"\n{cur_space3}####### {scenario}\n" - run_cmd = mlperf_inference_run_command(spaces+16, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), "valid") - content += run_cmd - content += run_suffix - - content += f"{cur_space2}=== \"All Scenarios\"\n{cur_space3}####### All Scenarios\n" - run_cmd = mlperf_inference_run_command(spaces+16, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), "valid") - content += run_cmd - content += run_suffix + # to select the execution environments(currently Docker and Native) + for execution_env in execution_envs: + if (device == "ROCm" or implementation == "qualcomm") and execution_env == "Docker": + continue # docker not currently supported for Qualcomm implementation and ROCm device + if implementation == "nvidia" and execution_env == "Native": + continue # Nvidia implementation only supports execution through docker + content += f"{cur_space2}=== \"{execution_env}\"\n" + content += f"{cur_space3}###### {execution_env} Environment\n\n" + test_query_count=get_test_query_count(model, implementation, device) + if execution_env == "Native": # Native implementation steps through virtual environment + content += f"{cur_space3}##### Setup a virtual environment for Python\n" + content += get_venv_command(spaces+16) + content += f"{cur_space3}##### Execute the CM command\n" + content += mlperf_inference_run_command(spaces+16, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True).replace("--docker ","") + content += f"{cur_space3}The above command should do a test run of Offline scenario and record the estimated offline_target_qps.\n\n" + else: # Docker implementation steps + docker_info = get_docker_info(spaces+16, model, implementation, device) + content += docker_info + content += mlperf_inference_run_command(spaces+16, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True) + content += f"{cur_space3}The above command should get you to an interactive shell inside the docker container and do a quick test run for the Offline scenario. Once inside the docker container please do the below commands to do the accuracy + performance runs for each scenario.\n\n" + content += f"{cur_space3}
    \n" + content += f"{cur_space3} Please click here to see more options for the docker launch \n\n" + content += f"{cur_space3}* `--docker_cm_repo `: to use a custom fork of cm4mlops repository inside the docker image\n\n" + content += f"{cur_space3}* `--docker_cache=no`: to not use docker cache during the image build\n" + + if device.lower() not in [ "cuda" ]: + content += f"{cur_space3}* `--docker_os=ubuntu`: ubuntu and rhel are supported. \n" + content += f"{cur_space3}* `--docker_os_version=20.04`: [20.04, 22.04] are supported for Ubuntu and [8, 9] for RHEL\n" + + content += f"{cur_space3}
    \n" + run_suffix = "" + run_suffix += f"\n{cur_space3} ###### Run Options\n\n" + run_suffix += f"{cur_space3} * Use `--division=closed` to do a closed division submission which includes compliance runs\n\n" + run_suffix += f"{cur_space3} * Use `--rerun` to do a rerun even when a valid run exists\n\n" + + for scenario in scenarios: + content += f"{cur_space3}=== \"{scenario}\"\n{cur_space4}####### {scenario}\n\n" + run_cmd = mlperf_inference_run_command(spaces+20, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), "valid") + content += run_cmd + content += run_suffix + + content += f"{cur_space3}=== \"All Scenarios\"\n{cur_space4}####### All Scenarios\n\n" + run_cmd = mlperf_inference_run_command(spaces+20, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), "valid") + content += run_cmd + content += run_suffix + + readme_prefix = get_readme_prefix(spaces, model, implementation) @@ -158,6 +174,14 @@ def get_readme_prefix(spaces, model, implementation): #pre_space += " " return readme_prefix + + def get_venv_command(spaces): + pre_space = " "*spaces + return f"""\n +{pre_space}```bash +{pre_space}cm run script --tags=\"install python-venv\" --name=mlperf +{pre_space}export CM_SCRIPT_EXTRA_CMD=\"--adr.python.name=mlperf\" +{pre_space}```\n""" def get_docker_info(spaces, model, implementation, device): info = "" From 1fba83ef116715952811708db9330c132a530164 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 6 Jun 2024 23:40:06 +0100 Subject: [PATCH 037/169] Cleanup of the docs --- main.py | 67 ++++++++++++++++++++++++++++++++------------------------- 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/main.py b/main.py index 6fe136880..49a6e7cf8 100644 --- a/main.py +++ b/main.py @@ -69,8 +69,6 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): scenarios.append("MultiStream") elif category == "Datacenter" and not scenarios: scenarios = [ "Offline", "Server" ] - if "99.9" in model: # Directly refer the commands of -99 - return f"\n{pre_space}Follow the same procedure described in {model.split('.')[0]}. Change the `--model` tag value to `{model.lower()}`." content += f"{pre_space}=== \"{category.lower()}\"\n\n" @@ -96,23 +94,26 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): content += f"{cur_space1}=== \"{device}\"\n" content += f"{cur_space2}###### {device} device\n\n" - if "99.9" not in model: #not showing docker command as it is already done for the 99% variant - # to select the execution environments(currently Docker and Native) - for execution_env in execution_envs: - if (device == "ROCm" or implementation == "qualcomm") and execution_env == "Docker": - continue # docker not currently supported for Qualcomm implementation and ROCm device - if implementation == "nvidia" and execution_env == "Native": - continue # Nvidia implementation only supports execution through docker - content += f"{cur_space2}=== \"{execution_env}\"\n" - content += f"{cur_space3}###### {execution_env} Environment\n\n" - test_query_count=get_test_query_count(model, implementation, device) + # to select the execution environments(currently Docker and Native) + for execution_env in execution_envs: + if (device == "ROCm" or implementation == "qualcomm") and execution_env == "Docker": + continue # docker not currently supported for Qualcomm implementation and ROCm device + if implementation == "nvidia" and execution_env == "Native": + continue # Nvidia implementation only supports execution through docker + content += f"{cur_space2}=== \"{execution_env}\"\n" + content += f"{cur_space3}###### {execution_env} Environment\n\n" + test_query_count=get_test_query_count(model, implementation, device) + + if "99.9" not in model: #not showing docker command as it is already done for the 99% variant if execution_env == "Native": # Native implementation steps through virtual environment content += f"{cur_space3}##### Setup a virtual environment for Python\n" content += get_venv_command(spaces+16) - content += f"{cur_space3}##### Execute the CM command\n" + content += f"{cur_space3}##### Performance Estimation for Offline Scenario\n" content += mlperf_inference_run_command(spaces+16, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True).replace("--docker ","") content += f"{cur_space3}The above command should do a test run of Offline scenario and record the estimated offline_target_qps.\n\n" + else: # Docker implementation steps + content += f"{cur_space3}##### Docker Container Build and Performance Estimation for Offline Scenario\n" docker_info = get_docker_info(spaces+16, model, implementation, device) content += docker_info content += mlperf_inference_run_command(spaces+16, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True) @@ -127,21 +128,28 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): content += f"{cur_space3}* `--docker_os_version=20.04`: [20.04, 22.04] are supported for Ubuntu and [8, 9] for RHEL\n" content += f"{cur_space3}
    \n" - run_suffix = "" - run_suffix += f"\n{cur_space3} ###### Run Options\n\n" - run_suffix += f"{cur_space3} * Use `--division=closed` to do a closed division submission which includes compliance runs\n\n" - run_suffix += f"{cur_space3} * Use `--rerun` to do a rerun even when a valid run exists\n\n" - - for scenario in scenarios: - content += f"{cur_space3}=== \"{scenario}\"\n{cur_space4}####### {scenario}\n\n" - run_cmd = mlperf_inference_run_command(spaces+20, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), "valid") - content += run_cmd - content += run_suffix - - content += f"{cur_space3}=== \"All Scenarios\"\n{cur_space4}####### All Scenarios\n\n" - run_cmd = mlperf_inference_run_command(spaces+20, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), "valid") + else: + content += f"{cur_space3} You can reuse the same environment as described for {model.split('.')[0]}.\n" + content += f"{cur_space3}##### Performance Estimation for Offline Scenario\n" + content += mlperf_inference_run_command(spaces+16, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True).replace("--docker ","") + content += f"{cur_space3}The above command should do a test run of Offline scenario and record the estimated offline_target_qps.\n\n" + + + run_suffix = "" + run_suffix += f"\n{cur_space3} ###### Run Options\n\n" + run_suffix += f"{cur_space3} * Use `--division=closed` to do a closed division submission which includes compliance runs\n\n" + run_suffix += f"{cur_space3} * Use `--rerun` to do a rerun even when a valid run exists\n\n" + + for scenario in scenarios: + content += f"{cur_space3}=== \"{scenario}\"\n{cur_space4}####### {scenario}\n\n" + run_cmd = mlperf_inference_run_command(spaces+20, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), "valid") content += run_cmd content += run_suffix + + content += f"{cur_space3}=== \"All Scenarios\"\n{cur_space4}####### All Scenarios\n\n" + run_cmd = mlperf_inference_run_command(spaces+20, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), "valid") + content += run_cmd + content += run_suffix @@ -192,7 +200,7 @@ def get_docker_info(spaces, model, implementation, device): #pre_space = " " if implementation == "nvidia": info += f"\n{pre_space}!!! tip\n\n" - info+= f"{pre_space} All the Nvidia benchmarks use the same docker container. So, if you have already done the docker setup command for any benchmark, you can skip the docker setup command below and do the Run commands inside the already built docker container.\n\n" + info+= f"{pre_space} All the Nvidia benchmarks, except GPT-J and LLAMA2-70B, use the same Docker container. Therefore, if you have already executed the Docker setup command for any benchmark, you can skip the Docker setup command below and run the commands inside the existing Docker container. The Docker container for GPT-J and LLAMA2-70B is the same and can be used for the other benchmarks, but not vice versa. This is because TensorRT-LLM is built specifically for the LLM benchmarks. If you are already inside a Docker container, execute the below Docker setup command without the --docker option for performance estimation.\n\n" return info def get_readme_suffix(spaces, model, implementation): @@ -203,8 +211,9 @@ def get_readme_suffix(spaces, model, implementation): pre_space += " " if implementation == "reference": - model_base_name = model.replace("-99.9","").replace("-99","") - readme_suffix+= f"{pre_space}* If you want to download the official MLPerf model and dataset for {model} you can follow [this README](get-{model_base_name}-data.md).\n" + if not model.endswith("-99"): + model_base_name = model.replace("-99.9","").replace("-99","") + readme_suffix+= f"{pre_space}* If you want to download the official MLPerf model and dataset for {model} you can follow [this README](get-{model_base_name}-data.md).\n" if model == "resnet50": readme_suffix += f"{pre_space}* Please see [mobilenets.md](mobilenets.md) for running mobilenet models for Image Classification." return readme_suffix From ee35e73c6a0572f923299e42b4f358cca3b0df80 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 6 Jun 2024 23:43:08 +0100 Subject: [PATCH 038/169] Cleanup of the docs --- docs/benchmarks/medical_imaging/3d-unet.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/benchmarks/medical_imaging/3d-unet.md b/docs/benchmarks/medical_imaging/3d-unet.md index 252ba3d07..d001d317e 100644 --- a/docs/benchmarks/medical_imaging/3d-unet.md +++ b/docs/benchmarks/medical_imaging/3d-unet.md @@ -5,24 +5,30 @@ === "MLCommons-Python" ### MLPerf Reference Implementation in Python - 3d-unet-99 + 3d-unet-99 + {{ mlperf_inference_implementation_readme (4, "3d-unet-99", "reference") }} - 3d-unet-99.9 + 3d-unet-99.9 + {{ mlperf_inference_implementation_readme (4, "3d-unet-99.9", "reference") }} === "Nvidia" ### Nvidia MLPerf Implementation - 3d-unet-99 + 3d-unet-99 + {{ mlperf_inference_implementation_readme (4, "3d-unet-99", "nvidia") }} - 3d-unet-99.9 + 3d-unet-99.9 + {{ mlperf_inference_implementation_readme (4, "3d-unet-99.9", "nvidia") }} === "Intel" ### Intel MLPerf Implementation 3d-unet-99 + {{ mlperf_inference_implementation_readme (4, "3d-unet-99", "intel") }} 3d-unet-99.9 + {{ mlperf_inference_implementation_readme (4, "3d-unet-99.9", "intel") }} From 2d47e060149b28b372060cdc9e4b06fdf74a3bae Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 7 Jun 2024 10:36:10 +0100 Subject: [PATCH 039/169] Reordered the sections of the docs page --- main.py | 78 ++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 49 insertions(+), 29 deletions(-) diff --git a/main.py b/main.py index 49a6e7cf8..1c6e86808 100644 --- a/main.py +++ b/main.py @@ -101,22 +101,22 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): if implementation == "nvidia" and execution_env == "Native": continue # Nvidia implementation only supports execution through docker content += f"{cur_space2}=== \"{execution_env}\"\n" - content += f"{cur_space3}###### {execution_env} Environment\n\n" + content += f"{cur_space3}####### {execution_env} Environment\n\n" test_query_count=get_test_query_count(model, implementation, device) if "99.9" not in model: #not showing docker command as it is already done for the 99% variant if execution_env == "Native": # Native implementation steps through virtual environment - content += f"{cur_space3}##### Setup a virtual environment for Python\n" + content += f"{cur_space3}######## Setup a virtual environment for Python\n" content += get_venv_command(spaces+16) - content += f"{cur_space3}##### Performance Estimation for Offline Scenario\n" - content += mlperf_inference_run_command(spaces+16, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True).replace("--docker ","") + content += f"{cur_space3}######## Performance Estimation for Offline Scenario\n" + content += mlperf_inference_run_command(spaces+17, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True).replace("--docker ","") content += f"{cur_space3}The above command should do a test run of Offline scenario and record the estimated offline_target_qps.\n\n" else: # Docker implementation steps - content += f"{cur_space3}##### Docker Container Build and Performance Estimation for Offline Scenario\n" + content += f"{cur_space3}######## Docker Container Build and Performance Estimation for Offline Scenario\n" docker_info = get_docker_info(spaces+16, model, implementation, device) content += docker_info - content += mlperf_inference_run_command(spaces+16, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True) + content += mlperf_inference_run_command(spaces+17, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True) content += f"{cur_space3}The above command should get you to an interactive shell inside the docker container and do a quick test run for the Offline scenario. Once inside the docker container please do the below commands to do the accuracy + performance runs for each scenario.\n\n" content += f"{cur_space3}
    \n" content += f"{cur_space3} Please click here to see more options for the docker launch \n\n" @@ -130,26 +130,28 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): content += f"{cur_space3}
    \n" else: content += f"{cur_space3} You can reuse the same environment as described for {model.split('.')[0]}.\n" - content += f"{cur_space3}##### Performance Estimation for Offline Scenario\n" - content += mlperf_inference_run_command(spaces+16, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True).replace("--docker ","") + content += f"{cur_space3}####### Performance Estimation for Offline Scenario\n" + content += mlperf_inference_run_command(spaces+17, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True).replace("--docker ","") content += f"{cur_space3}The above command should do a test run of Offline scenario and record the estimated offline_target_qps.\n\n" - run_suffix = "" - run_suffix += f"\n{cur_space3} ###### Run Options\n\n" - run_suffix += f"{cur_space3} * Use `--division=closed` to do a closed division submission which includes compliance runs\n\n" - run_suffix += f"{cur_space3} * Use `--rerun` to do a rerun even when a valid run exists\n\n" + run_suffix = "" + run_suffix += f"{cur_space3}
    \n" + run_suffix += f"{cur_space3} Please click here to see more options for the RUN command\n\n" + run_suffix += f"{cur_space3}* Use `--division=closed` to do a closed division submission which includes compliance runs\n\n" + run_suffix += f"{cur_space3}* Use `--rerun` to do a rerun even when a valid run exists\n" + run_suffix += f"{cur_space3}
    \n" - for scenario in scenarios: - content += f"{cur_space3}=== \"{scenario}\"\n{cur_space4}####### {scenario}\n\n" - run_cmd = mlperf_inference_run_command(spaces+20, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), "valid") - content += run_cmd - content += run_suffix - - content += f"{cur_space3}=== \"All Scenarios\"\n{cur_space4}####### All Scenarios\n\n" - run_cmd = mlperf_inference_run_command(spaces+20, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), "valid") + for scenario in scenarios: + content += f"{cur_space3}=== \"{scenario}\"\n{cur_space4}####### {scenario}\n\n" + run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), "valid") content += run_cmd - content += run_suffix + #content += run_suffix + + content += f"{cur_space3}=== \"All Scenarios\"\n{cur_space4}####### All Scenarios\n\n" + run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), "valid") + content += run_cmd + content += run_suffix @@ -187,7 +189,7 @@ def get_venv_command(spaces): pre_space = " "*spaces return f"""\n {pre_space}```bash -{pre_space}cm run script --tags=\"install python-venv\" --name=mlperf +{pre_space}cm run script --tags=install,python-venv --name=mlperf {pre_space}export CM_SCRIPT_EXTRA_CMD=\"--adr.python.name=mlperf\" {pre_space}```\n""" @@ -240,24 +242,42 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ scenario_option = "" else: scenario_variation_tag = "" - scenario_option = f"\\\n {pre_space} --scenario={scenario}" + scenario_option = f"\\\n{pre_space} --scenario={scenario}" run_cmd_extra = get_run_cmd_extra(f_pre_space, model, implementation, device) if docker: - docker_cmd_suffix = f" \\\n {pre_space} --docker --quiet" - docker_cmd_suffix += f" \\\n {pre_space} --test_query_count={test_query_count}" - - docker_setup_cmd = f"\n{f_pre_space} ```bash\n{f_pre_space} cm run script --tags=run-mlperf,inference,_find-performance,_full{scenario_variation_tag} \\\n {pre_space} --model={model} \\\n {pre_space} --implementation={implementation} \\\n {pre_space} --framework={framework} \\\n {pre_space} --category={category} {scenario_option} \\\n {pre_space} --execution-mode=test \\\n {pre_space} --device={device} {docker_cmd_suffix}\n{f_pre_space} ```\n" + docker_cmd_suffix = f" \\\n{pre_space} --docker --quiet" + docker_cmd_suffix += f" \\\n{pre_space} --test_query_count={test_query_count}" + + docker_setup_cmd = f"""\n +{f_pre_space}```bash +{f_pre_space}cm run script --tags=run-mlperf,inference,_find-performance,_full{scenario_variation_tag} +{pre_space} --model={model} +{pre_space} --implementation={implementation} +{pre_space} --framework={framework} +{pre_space} --category={category} {scenario_option} +{pre_space} --execution-mode=test +{pre_space} --device={device} {docker_cmd_suffix} +{f_pre_space}```\n""" return docker_setup_cmd + run_cmd_extra else: - cmd_suffix = f"\\\n {pre_space} --quiet" + cmd_suffix = f"\\\n{pre_space} --quiet" if execution_mode == "test": cmd_suffix += f" \\\n {pre_space} --test_query_count={test_query_count}" - run_cmd = f"\n{f_pre_space} ```bash\n{f_pre_space} cm run script --tags=run-mlperf,inference{scenario_variation_tag} \\\n {pre_space} --model={model} \\\n {pre_space} --implementation={implementation} \\\n {pre_space} --framework={framework} \\\n {pre_space} --category={category} {scenario_option} \\\n {pre_space} --execution-mode={execution_mode} \\\n {pre_space} --device={device} {cmd_suffix}\n{f_pre_space} ```\n" + run_cmd = f"""\n +{f_pre_space}```bash +{f_pre_space}cm run script --tags=run-mlperf,inference{scenario_variation_tag} +{pre_space} --model={model} +{pre_space} --implementation={implementation} +{pre_space} --framework={framework} +{pre_space} --category={category} {scenario_option} +{pre_space} --execution-mode={execution_mode} +{pre_space} --device={device} {cmd_suffix} +{f_pre_space}```\n""" return run_cmd + run_cmd_extra From e43bb87ae4e24a95c27847a0d7b2611429ca13c7 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 7 Jun 2024 11:40:08 +0100 Subject: [PATCH 040/169] Removed an unnecessary heading in the docs --- .../image_classification/resnet50.md | 13 ++++++------ docs/benchmarks/language/bert.md | 9 ++++----- docs/benchmarks/language/gpt-j.md | 9 ++++----- docs/benchmarks/language/llama2-70b.md | 7 +++---- docs/benchmarks/medical_imaging/3d-unet.md | 7 +++---- docs/benchmarks/object_detection/retinanet.md | 13 ++++++------ docs/benchmarks/recommendation/dlrm-v2.md | 4 ++-- docs/benchmarks/text_to_image/sdxl.md | 9 ++++----- main.py | 20 +++++++++---------- 9 files changed, 42 insertions(+), 49 deletions(-) diff --git a/docs/benchmarks/image_classification/resnet50.md b/docs/benchmarks/image_classification/resnet50.md index 17e682d0e..26875258d 100644 --- a/docs/benchmarks/image_classification/resnet50.md +++ b/docs/benchmarks/image_classification/resnet50.md @@ -1,27 +1,26 @@ # Image Classification using ResNet50 -## Benchmark Implementations === "MLCommons-Python" - ### MLPerf Reference Implementation in Python + ## MLPerf Reference Implementation in Python {{ mlperf_inference_implementation_readme (4, "resnet50", "reference") }} === "Nvidia" - ### Nvidia MLPerf Implementation + ## Nvidia MLPerf Implementation {{ mlperf_inference_implementation_readme (4, "resnet50", "nvidia") }} === "Intel" - ### Intel MLPerf Implementation + ## Intel MLPerf Implementation {{ mlperf_inference_implementation_readme (4, "resnet50", "intel") }} === "Qualcomm" - ### Qualcomm AI100 MLPerf Implementation + ## Qualcomm AI100 MLPerf Implementation {{ mlperf_inference_implementation_readme (4, "resnet50", "qualcomm") }} -=== "MLCommon-C++" - ### MLPerf Modular Implementation in C++ +=== "MLCommons-C++" + ## MLPerf Modular Implementation in C++ {{ mlperf_inference_implementation_readme (4, "resnet50", "cpp") }} diff --git a/docs/benchmarks/language/bert.md b/docs/benchmarks/language/bert.md index 8cb3badd5..782340b3d 100644 --- a/docs/benchmarks/language/bert.md +++ b/docs/benchmarks/language/bert.md @@ -1,8 +1,7 @@ # Question Answering using Bert-Large -## Benchmark Implementations === "MLCommons-Python" - ### MLPerf Reference Implementation in Python + ## MLPerf Reference Implementation in Python BERT-99 {{ mlperf_inference_implementation_readme (4, "bert-99", "reference") }} @@ -11,7 +10,7 @@ {{ mlperf_inference_implementation_readme (4, "bert-99.9", "reference") }} === "Nvidia" - ### Nvidia MLPerf Implementation + ## Nvidia MLPerf Implementation BERT-99 {{ mlperf_inference_implementation_readme (4, "bert-99", "nvidia") }} @@ -20,7 +19,7 @@ {{ mlperf_inference_implementation_readme (4, "bert-99.9", "nvidia") }} === "Intel" - ### Intel MLPerf Implementation + ## Intel MLPerf Implementation BERT-99 {{ mlperf_inference_implementation_readme (4, "bert-99", "intel") }} @@ -28,7 +27,7 @@ {{ mlperf_inference_implementation_readme (4, "bert-99.9", "intel") }} === "Qualcomm" - ### Qualcomm AI100 MLPerf Implementation + ## Qualcomm AI100 MLPerf Implementation BERT-99 {{ mlperf_inference_implementation_readme (4, "bert-99", "qualcomm") }} diff --git a/docs/benchmarks/language/gpt-j.md b/docs/benchmarks/language/gpt-j.md index b108f28cb..2eefbbc79 100644 --- a/docs/benchmarks/language/gpt-j.md +++ b/docs/benchmarks/language/gpt-j.md @@ -1,9 +1,8 @@ # Text Summarization using GPT-J -## Benchmark Implementations === "MLCommons-Python" - ### MLPerf Reference Implementation in Python + ## MLPerf Reference Implementation in Python GPT-J-99 @@ -14,7 +13,7 @@ {{ mlperf_inference_implementation_readme (4, "gptj-99.9", "reference") }} === "Nvidia" - ### Nvidia MLPerf Implementation + ## Nvidia MLPerf Implementation GPTJ-99 @@ -25,14 +24,14 @@ {{ mlperf_inference_implementation_readme (4, "gptj-99.9", "nvidia") }} === "Intel" - ### Intel MLPerf Implementation + ## Intel MLPerf Implementation GPTJ-99 {{ mlperf_inference_implementation_readme (4, "gptj-99", "intel") }} === "Qualcomm" - ### Qualcomm AI100 MLPerf Implementation + ## Qualcomm AI100 MLPerf Implementation GPTJ-99 diff --git a/docs/benchmarks/language/llama2-70b.md b/docs/benchmarks/language/llama2-70b.md index b131eb437..f1785dcb3 100644 --- a/docs/benchmarks/language/llama2-70b.md +++ b/docs/benchmarks/language/llama2-70b.md @@ -1,9 +1,8 @@ # Text Summarization using LLAMA2-70b -## Benchmark Implementations === "MLCommons-Python" - ### MLPerf Reference Implementation in Python + ## MLPerf Reference Implementation in Python LLAMA2-70b-99 {{ mlperf_inference_implementation_readme (4, "llama2-70b-99", "reference") }} @@ -12,7 +11,7 @@ {{ mlperf_inference_implementation_readme (4, "llama2-70b-99.9", "reference") }} === "Nvidia" - ### Nvidia MLPerf Implementation + ## Nvidia MLPerf Implementation LLAMA2-70b-99 {{ mlperf_inference_implementation_readme (4, "llama2-70b-99", "nvidia") }} @@ -22,7 +21,7 @@ === "Qualcomm" - ### Qualcomm AI100 MLPerf Implementation + ## Qualcomm AI100 MLPerf Implementation LLAMA2-70b-99 {{ mlperf_inference_implementation_readme (4, "llama2-70b-99", "qualcomm") }} diff --git a/docs/benchmarks/medical_imaging/3d-unet.md b/docs/benchmarks/medical_imaging/3d-unet.md index d001d317e..b58ea7f2e 100644 --- a/docs/benchmarks/medical_imaging/3d-unet.md +++ b/docs/benchmarks/medical_imaging/3d-unet.md @@ -1,9 +1,8 @@ # Medical Imaging using 3d-unet (KiTS 2019 kidney tumor segmentation task) -## Benchmark Implementations === "MLCommons-Python" - ### MLPerf Reference Implementation in Python + ## MLPerf Reference Implementation in Python 3d-unet-99 @@ -14,7 +13,7 @@ {{ mlperf_inference_implementation_readme (4, "3d-unet-99.9", "reference") }} === "Nvidia" - ### Nvidia MLPerf Implementation + ## Nvidia MLPerf Implementation 3d-unet-99 {{ mlperf_inference_implementation_readme (4, "3d-unet-99", "nvidia") }} @@ -24,7 +23,7 @@ {{ mlperf_inference_implementation_readme (4, "3d-unet-99.9", "nvidia") }} === "Intel" - ### Intel MLPerf Implementation + ## Intel MLPerf Implementation 3d-unet-99 {{ mlperf_inference_implementation_readme (4, "3d-unet-99", "intel") }} diff --git a/docs/benchmarks/object_detection/retinanet.md b/docs/benchmarks/object_detection/retinanet.md index a6b469e8f..383a2ec1b 100644 --- a/docs/benchmarks/object_detection/retinanet.md +++ b/docs/benchmarks/object_detection/retinanet.md @@ -1,27 +1,26 @@ # Object Detection using Retinanet -## Benchmark Implementations === "MLCommons-Python" - ### MLPerf Reference Implementation in Python + ## MLPerf Reference Implementation in Python {{ mlperf_inference_implementation_readme (4, "retinanet", "reference") }} === "Nvidia" - ### Nvidia MLPerf Implementation + ## Nvidia MLPerf Implementation {{ mlperf_inference_implementation_readme (4, "retinanet", "nvidia") }} === "Intel" - ### Intel MLPerf Implementation + ## Intel MLPerf Implementation {{ mlperf_inference_implementation_readme (4, "retinanet", "intel") }} === "Qualcomm" - ### Qualcomm AI100 MLPerf Implementation + ## Qualcomm AI100 MLPerf Implementation {{ mlperf_inference_implementation_readme (4, "retinanet", "qualcomm") }} -=== "MLCommon-C++" - ### MLPerf Modular Implementation in C++ +=== "MLCommons-C++" + ## MLPerf Modular Implementation in C++ {{ mlperf_inference_implementation_readme (4, "retinanet", "cpp") }} diff --git a/docs/benchmarks/recommendation/dlrm-v2.md b/docs/benchmarks/recommendation/dlrm-v2.md index 750f18e3e..18266615f 100644 --- a/docs/benchmarks/recommendation/dlrm-v2.md +++ b/docs/benchmarks/recommendation/dlrm-v2.md @@ -3,7 +3,7 @@ ## Benchmark Implementations === "MLCommons-Python" - ### MLPerf Reference Implementation in Python + ## MLPerf Reference Implementation in Python DLRM-v2-99 {{ mlperf_inference_implementation_readme (4, "dlrm_v2-99", "reference") }} @@ -12,7 +12,7 @@ {{ mlperf_inference_implementation_readme (4, "dlrm_v2-99.9", "reference") }} === "Nvidia" - ### Nvidia MLPerf Implementation + ## Nvidia MLPerf Implementation DLRM-v2-99 {{ mlperf_inference_implementation_readme (4, "dlrm_v2-99", "nvidia") }} diff --git a/docs/benchmarks/text_to_image/sdxl.md b/docs/benchmarks/text_to_image/sdxl.md index 6010a208e..2d84838d4 100644 --- a/docs/benchmarks/text_to_image/sdxl.md +++ b/docs/benchmarks/text_to_image/sdxl.md @@ -1,24 +1,23 @@ # Text to Image using Stable Diffusion -## Benchmark Implementations === "MLCommons-Python" - ### MLPerf Reference Implementation in Python + ## MLPerf Reference Implementation in Python {{ mlperf_inference_implementation_readme (4, "sdxl", "reference") }} === "Nvidia" - ### Nvidia MLPerf Implementation + ## Nvidia MLPerf Implementation {{ mlperf_inference_implementation_readme (4, "sdxl", "nvidia") }} === "Intel" - ### Intel MLPerf Implementation + ## Intel MLPerf Implementation {{ mlperf_inference_implementation_readme (4, "sdxl", "intel") }} === "Qualcomm" - ### Qualcomm AI100 MLPerf Implementation + ## Qualcomm AI100 MLPerf Implementation {{ mlperf_inference_implementation_readme (4, "sdxl", "qualcomm") }} diff --git a/main.py b/main.py index 1c6e86808..18915eb95 100644 --- a/main.py +++ b/main.py @@ -75,13 +75,13 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): cur_space = pre_space + " " scenarios_string = ", ".join(scenarios) - content += f"{cur_space}#### {category} category \n\n{cur_space} In the {category.lower()} category, {model} has {scenarios_string} scenarios and all the scenarios are mandatory for a closed division submission.\n\n" + content += f"{cur_space}### {category} category \n\n{cur_space} In the {category.lower()} category, {model} has {scenarios_string} scenarios and all the scenarios are mandatory for a closed division submission.\n\n" for framework in frameworks: cur_space1 = cur_space + " " content += f"{cur_space}=== \"{framework}\"\n" - content += f"{cur_space1}##### {framework} framework\n\n" + content += f"{cur_space1}#### {framework} framework\n\n" for device in devices: if framework.lower() == "deepsparse": @@ -92,7 +92,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): cur_space4 = cur_space3 + " " content += f"{cur_space1}=== \"{device}\"\n" - content += f"{cur_space2}###### {device} device\n\n" + content += f"{cur_space2}##### {device} device\n\n" # to select the execution environments(currently Docker and Native) for execution_env in execution_envs: @@ -101,19 +101,19 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): if implementation == "nvidia" and execution_env == "Native": continue # Nvidia implementation only supports execution through docker content += f"{cur_space2}=== \"{execution_env}\"\n" - content += f"{cur_space3}####### {execution_env} Environment\n\n" + content += f"{cur_space3}###### {execution_env} Environment\n\n" test_query_count=get_test_query_count(model, implementation, device) if "99.9" not in model: #not showing docker command as it is already done for the 99% variant if execution_env == "Native": # Native implementation steps through virtual environment - content += f"{cur_space3}######## Setup a virtual environment for Python\n" + content += f"{cur_space3}####### Setup a virtual environment for Python\n" content += get_venv_command(spaces+16) - content += f"{cur_space3}######## Performance Estimation for Offline Scenario\n" + content += f"{cur_space3}####### Performance Estimation for Offline Scenario\n" content += mlperf_inference_run_command(spaces+17, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True).replace("--docker ","") content += f"{cur_space3}The above command should do a test run of Offline scenario and record the estimated offline_target_qps.\n\n" else: # Docker implementation steps - content += f"{cur_space3}######## Docker Container Build and Performance Estimation for Offline Scenario\n" + content += f"{cur_space3}####### Docker Container Build and Performance Estimation for Offline Scenario\n" docker_info = get_docker_info(spaces+16, model, implementation, device) content += docker_info content += mlperf_inference_run_command(spaces+17, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True) @@ -130,7 +130,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): content += f"{cur_space3}\n" else: content += f"{cur_space3} You can reuse the same environment as described for {model.split('.')[0]}.\n" - content += f"{cur_space3}####### Performance Estimation for Offline Scenario\n" + content += f"{cur_space3}###### Performance Estimation for Offline Scenario\n" content += mlperf_inference_run_command(spaces+17, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True).replace("--docker ","") content += f"{cur_space3}The above command should do a test run of Offline scenario and record the estimated offline_target_qps.\n\n" @@ -143,12 +143,12 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): run_suffix += f"{cur_space3}\n" for scenario in scenarios: - content += f"{cur_space3}=== \"{scenario}\"\n{cur_space4}####### {scenario}\n\n" + content += f"{cur_space3}=== \"{scenario}\"\n{cur_space4}###### {scenario}\n\n" run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), "valid") content += run_cmd #content += run_suffix - content += f"{cur_space3}=== \"All Scenarios\"\n{cur_space4}####### All Scenarios\n\n" + content += f"{cur_space3}=== \"All Scenarios\"\n{cur_space4}###### All Scenarios\n\n" run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), "valid") content += run_cmd content += run_suffix From 87471eec2d57ec40ab36c3b63e9f194082baa516 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 7 Jun 2024 12:41:04 +0100 Subject: [PATCH 041/169] Fixes the commands for datacenter --- main.py | 80 ++++++++++++++++++++++++++++++------------------------ mkdocs.yml | 3 +- 2 files changed, 45 insertions(+), 38 deletions(-) diff --git a/main.py b/main.py index 18915eb95..f73357d4e 100644 --- a/main.py +++ b/main.py @@ -64,10 +64,10 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): for category in categories: if category == "Edge" and not scenarios: - scenarios = [ "Offline", "SingleStream" ] - if model.lower() in [ "resnet50", "retinanet" ] and not "MultiStream" in scenarios:#MultiStream was duplicating - scenarios.append("MultiStream") - elif category == "Datacenter" and not scenarios: + scenarios = [ "Offline", "SingleStream" ] + if model.lower() in [ "resnet50", "retinanet" ] and not "MultiStream" in scenarios:#MultiStream was duplicating + scenarios.append("MultiStream") + elif category == "Datacenter": scenarios = [ "Offline", "Server" ] content += f"{pre_space}=== \"{category.lower()}\"\n\n" @@ -135,23 +135,23 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): content += f"{cur_space3}The above command should do a test run of Offline scenario and record the estimated offline_target_qps.\n\n" - run_suffix = "" - run_suffix += f"{cur_space3}
    \n" - run_suffix += f"{cur_space3} Please click here to see more options for the RUN command\n\n" - run_suffix += f"{cur_space3}* Use `--division=closed` to do a closed division submission which includes compliance runs\n\n" - run_suffix += f"{cur_space3}* Use `--rerun` to do a rerun even when a valid run exists\n" - run_suffix += f"{cur_space3}
    \n" + run_suffix = "" + run_suffix += f"{cur_space3}
    \n" + run_suffix += f"{cur_space3} Please click here to see more options for the RUN command\n\n" + run_suffix += f"{cur_space3}* Use `--division=closed` to do a closed division submission which includes compliance runs\n\n" + run_suffix += f"{cur_space3}* Use `--rerun` to do a rerun even when a valid run exists\n" + run_suffix += f"{cur_space3}
    \n" - for scenario in scenarios: - content += f"{cur_space3}=== \"{scenario}\"\n{cur_space4}###### {scenario}\n\n" - run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), "valid") - content += run_cmd - #content += run_suffix + for scenario in scenarios: + content += f"{cur_space3}=== \"{scenario}\"\n{cur_space4}###### {scenario}\n\n" + run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), "valid", scenarios) + content += run_cmd + #content += run_suffix - content += f"{cur_space3}=== \"All Scenarios\"\n{cur_space4}###### All Scenarios\n\n" - run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), "valid") - content += run_cmd - content += run_suffix + content += f"{cur_space3}=== \"All Scenarios\"\n{cur_space4}###### All Scenarios\n\n" + run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), "valid", scenarios) + content += run_cmd + content += run_suffix @@ -220,17 +220,22 @@ def get_readme_suffix(spaces, model, implementation): readme_suffix += f"{pre_space}* Please see [mobilenets.md](mobilenets.md) for running mobilenet models for Image Classification." return readme_suffix - def get_run_cmd_extra(f_pre_space, model, implementation, device): + def get_run_cmd_extra(f_pre_space, model, implementation, device, scenario, scenarios = []): extra_content = "" - f_pre_space += " " + f_pre_space += "" + if scenario == "Server" or (scenario == "All Scenarios" and "Server" in scenarios): + extra_content += f"{f_pre_space} * `` must be determined manually. It is usually around 80% of the Offline QPS, but on some systems, it can drop below 50%. If a higher value is specified, the latency constraint will not be met, and the run will be considered invalid.\n" + if "gptj" in model and device == "cuda" and implementation == "reference": - extra_content += f"{f_pre_space}!!! tip\n\n" extra_content += f"{f_pre_space} * `--precision=[float16|bfloat16]` can help run on GPUs with less RAM \n" extra_content += f"{f_pre_space} * `--beam-size=1` Beam size of 4 is mandatory for a closed division submission but reducing the beam size can help in running the model on GPUs with lower device memory\n" + if extra_content: + extra_content = f"{f_pre_space}!!! tip\n\n" + extra_content + return extra_content @env.macro - def mlperf_inference_run_command(spaces, model, implementation, framework, category, scenario, device="cpu", execution_mode="test", test_query_count="20", docker=False): + def mlperf_inference_run_command(spaces, model, implementation, framework, category, scenario, device="cpu", execution_mode="test", test_query_count="20", docker=False, scenarios = []): pre_space = "" for i in range(1,spaces): pre_space = pre_space + " " @@ -244,7 +249,10 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ scenario_variation_tag = "" scenario_option = f"\\\n{pre_space} --scenario={scenario}" - run_cmd_extra = get_run_cmd_extra(f_pre_space, model, implementation, device) + if scenario == "Server" or (scenario == "All Scenarios" and "Server" in scenarios): + scenario_option = f"\\\n{pre_space} --server_target_qps=" + + run_cmd_extra = get_run_cmd_extra(f_pre_space, model, implementation, device, scenario, scenarios) if docker: docker_cmd_suffix = f" \\\n{pre_space} --docker --quiet" @@ -252,12 +260,12 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ docker_setup_cmd = f"""\n {f_pre_space}```bash -{f_pre_space}cm run script --tags=run-mlperf,inference,_find-performance,_full{scenario_variation_tag} -{pre_space} --model={model} -{pre_space} --implementation={implementation} -{pre_space} --framework={framework} -{pre_space} --category={category} {scenario_option} -{pre_space} --execution-mode=test +{f_pre_space}cm run script --tags=run-mlperf,inference,_find-performance,_full{scenario_variation_tag} \\ +{pre_space} --model={model} \\ +{pre_space} --implementation={implementation} \\ +{pre_space} --framework={framework} \\ +{pre_space} --category={category} {scenario_option} \\ +{pre_space} --execution_mode=test \\ {pre_space} --device={device} {docker_cmd_suffix} {f_pre_space}```\n""" @@ -271,12 +279,12 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ run_cmd = f"""\n {f_pre_space}```bash -{f_pre_space}cm run script --tags=run-mlperf,inference{scenario_variation_tag} -{pre_space} --model={model} -{pre_space} --implementation={implementation} -{pre_space} --framework={framework} -{pre_space} --category={category} {scenario_option} -{pre_space} --execution-mode={execution_mode} +{f_pre_space}cm run script --tags=run-mlperf,inference{scenario_variation_tag} \\ +{pre_space} --model={model} \\ +{pre_space} --implementation={implementation} \\ +{pre_space} --framework={framework} \\ +{pre_space} --category={category} {scenario_option} \\ +{pre_space} --execution_mode={execution_mode} \\ {pre_space} --device={device} {cmd_suffix} {f_pre_space}```\n""" diff --git a/mkdocs.yml b/mkdocs.yml index 1120ee660..339a0657d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -24,8 +24,7 @@ nav: - Benchmarks: - benchmarks/index.md - Image Classification: - - ResNet50: - - benchmarks/image_classification/resnet50.md + - ResNet50: benchmarks/image_classification/resnet50.md - Text to Image: - Stable Diffusion: benchmarks/text_to_image/sdxl.md - Object Detection: From f40bc9c44bc500891016b97092fda484638ab4c2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sun, 16 Jun 2024 13:44:48 +0100 Subject: [PATCH 042/169] Fix the build --sdist for loadgen --- loadgen/pyproject.toml | 2 +- loadgen/setup.py | 6 ++++++ loadgen/version_generator.py | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/loadgen/pyproject.toml b/loadgen/pyproject.toml index f62b2c5f7..9be24eea6 100755 --- a/loadgen/pyproject.toml +++ b/loadgen/pyproject.toml @@ -4,4 +4,4 @@ build-backend = "setuptools.build_meta:__legacy__" [tool.cibuildwheel] environment = "CFLAGS='-std=c++14'" -build = "cp3{7,8,9,10,11}-*" +build = "cp3{7,8,9,10,11,12}-*" diff --git a/loadgen/setup.py b/loadgen/setup.py index d7b8224de..67f88b8ec 100644 --- a/loadgen/setup.py +++ b/loadgen/setup.py @@ -40,6 +40,9 @@ "query_sample_library.h", "system_under_test.h", "test_settings.h", + "issue_query_controller.h", + "early_stopping.h", + "query_dispatch_library.h", ] lib_headers = [ @@ -49,6 +52,8 @@ "utils.h", "version.h", "results.h", + "bindings/c_api.h", + "version_generator.py" ] lib_sources = [ @@ -63,6 +68,7 @@ ] lib_bindings = [ + "bindings/c_api.cc", "bindings/python_api.cc", ] diff --git a/loadgen/version_generator.py b/loadgen/version_generator.py index 4de930a63..2c5d11331 100644 --- a/loadgen/version_generator.py +++ b/loadgen/version_generator.py @@ -69,7 +69,6 @@ def generate_loadgen_version_definitions_sha1(ofile, loadgen_root): sha1s = "" loadgen_files = ( ["/bindings/" + s for s in os.listdir(loadgen_root + "/bindings")] + - ["/demos/" + s for s in os.listdir(loadgen_root + "/demos")] + ["/" + s for s in os.listdir(loadgen_root)]) for fn in sorted(loadgen_files): full_fn = loadgen_root + fn From 9dc997f19081e62ef505d695217a5b83656d451f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 2 Jul 2024 18:02:54 +0530 Subject: [PATCH 043/169] Fixes #1761, llama2 and mixtral runtime error on CPU systems --- language/llama2-70b/SUT.py | 5 ++++- language/mixtral-8x7b/SUT.py | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/language/llama2-70b/SUT.py b/language/llama2-70b/SUT.py index 17faced40..9f9e4bf7b 100644 --- a/language/llama2-70b/SUT.py +++ b/language/llama2-70b/SUT.py @@ -241,7 +241,10 @@ def load_model(self): self.model = self.model.to(self.device) # Force CPU if your system has GPU and you specifically want CPU-only run self.model.eval() - self.model = self.model.to(memory_format=torch.channels_last) + try: # for systems with low ram, the below command gives error as some part is offloaded to disk + self.model = self.model.to(memory_format=torch.channels_last) + except: + pass self.tokenizer = AutoTokenizer.from_pretrained( self.model_path, diff --git a/language/mixtral-8x7b/SUT.py b/language/mixtral-8x7b/SUT.py index aaf0eecee..b4a2dc705 100644 --- a/language/mixtral-8x7b/SUT.py +++ b/language/mixtral-8x7b/SUT.py @@ -301,8 +301,10 @@ def load_model(self): self.model = self.model.to(self.device) self.model.eval() - if self.device != "cpu": + try: # for systems with low ram, the below command gives error as some part is offloaded to disk self.model = self.model.to(memory_format=torch.channels_last) + except: + pass self.tokenizer = AutoTokenizer.from_pretrained( self.model_path, From 0718769f8f44168b589fb30c089c883095402d6a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 3 Jul 2024 14:59:00 +0100 Subject: [PATCH 044/169] Added mixtral to the benchmark list, improved benchmark docs --- docs/benchmarks/index.md | 25 ++++++++++++------------- main.py | 24 +++++++++++++++--------- mkdocs.yml | 2 +- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/docs/benchmarks/index.md b/docs/benchmarks/index.md index c4a970c81..39e3e72ac 100644 --- a/docs/benchmarks/index.md +++ b/docs/benchmarks/index.md @@ -2,27 +2,26 @@ Please visit the individual benchmark links to see the run commands using the unified CM interface. -1. [Image Classification](image_classification/resnet50.md) using ResNet50 model and Imagenet-2012 dataset +1. [Image Classification](image_classification/resnet50.md) using ResNet50-v1.5 model and Imagenet-2012 (224x224) validation dataset. Dataset size is 50,000 and QSL size is 1024. Reference model accuracy is 76.46%. Server scenario latency constraint is 15ms. -2. [Text to Image](text_to_image/sdxl.md) using Stable Diffusion model and Coco2014 dataset +2. [Text to Image](text_to_image/sdxl.md) using Stable Diffusion model and subset of Coco2014 dataset. Dataset size is 5000 amd QSL size is the same. Required accuracy for closed division is (23.01085758 <= FID <= 23.95007626, 32.68631873 <= CLIP <= 31.81331801). -3. [Object Detection](object_detection/retinanet.md) using Retinanet model and OpenImages dataset +3. [Object Detection](object_detection/retinanet.md) using Retinanet model and OpenImages dataset.Dataset size is 24781 and QSL size is 64. Reference model accuracy is 0.3755 mAP. Server scenario latency constraint is 100ms. -4. [Image Segmentation](medical_imaging/3d-unet.md) using 3d-unet model and KiTS19 dataset +4. [Medical Image Segmentation](medical_imaging/3d-unet.md) using 3d-unet model and KiTS2019 dataset. Dataset size is 42 and QSL size is the same. Reference model accuracy is 0.86330 mean DIXE score. Server scenario is not applicable. -5. [Question Answering](language/bert.md) using Bert-Large model and Squad v1.1 dataset +5. [Question Answering](language/bert.md) using Bert-Large model and Squad v1.1 dataset with 384 sequence length. Dataset size is 10833 and QSL size is the same. Reference model accuracy is f1 score = 90.874%. Server scenario latency constraint is 130ms. -6. [Text Summarization](language/gpt-j.md) using GPT-J model and CNN Daily Mail dataset +6. [Text Summarization](language/gpt-j.md) using GPT-J model and CNN Daily Mail v3.0.0 dataset. Dataset size is 13368 amd QSL size is the same. Reference model accuracy is (rouge1=42.9865, rouge2=20.1235, rougeL=29.9881, gen_len=4016878). Server scenario latency sconstraint is 20s. -7. [Text Summarization](language/llama2-70b.md) using LLAMA2-70b model and OpenORCA dataset +7. [Question Answering](language/llama2-70b.md) using LLAMA2-70b model and OpenORCA (GPT-4 split, max_seq_len=1024) dataset. Dataset size is 24576 and QSL size is the same. Reference model accuracy is (rouge1=44.4312, rouge2=22.0352, rougeL=28.6162, tokens_per_sample=294.45). Server scenario latency constraint is TTFT=2000ms, TPOT=200ms. -8. [Recommendation](recommendation/dlrm-v2.md) using DLRMv2 model and Criteo multihot dataset +8. [Question Answering, Math and Code Generation](language/mixtral-8x7b.md) using Mixtral-8x7B model and OpenORCA (5k samples of GPT-4 split, max_seq_len=2048), GSM8K (5k samples of the validation split, max_seq_len=2048), MBXP (5k samples of the validation split, max_seq_len=2048) datasets. Dataset size is 15000 and QSL size is the same. Reference model accuracy is (rouge1=45.4911, rouge2=23.2829, rougeL=30.3615, gsm8k accuracy = 73.78, mbxp accuracy = 60.12, tokens_per_sample=294.45). Server scenario latency constraint is TTFT=2000ms, TPOT=200ms. -All the eight benchmarks can participate in the datacenter category. -All the eight benchmarks except DLRMv2 and LLAMA2 and can participate in the edge category. +9. [Recommendation](recommendation/dlrm-v2.md) using DLRMv2 model and Synthetic Multihot Criteo dataset. Dataset size is 204800 and QSL size is the same. Reference model accuracy is AUC=80.31%. Server scenario latency constraint is 60 ms. + +All the nine benchmarks can participate in the datacenter category. +All the nine benchmarks except DLRMv2, LLAMA2 and Mixtral-8x7B and can participate in the edge category. `bert`, `llama2-70b`, `dlrm_v2` and `3d-unet` has a high accuracy (99.9%) variant, where the benchmark run must achieve a higher accuracy of at least `99.9%` of the FP32 reference model in comparison with the `99%` default accuracy requirement. - -The `dlrm_v2` benchmark has a high-accuracy variant only. If this accuracy is not met, the submission result can be submitted only to the open division. - diff --git a/main.py b/main.py index d208d07ce..2f1c036fe 100644 --- a/main.py +++ b/main.py @@ -12,6 +12,10 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): content="" scenarios = [] execution_envs = ["Docker","Native"] + code_version="r4.1" + + if model == "rnnt": + code_version="r4.0" if implementation == "reference": devices = [ "CPU", "CUDA", "ROCm" ] @@ -31,8 +35,10 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): frameworks = [ "TensorRT" ] elif implementation == "intel": - if model not in [ "bert-99", "bert-99.9", "gptj-99", "gptj-99.9" ]: + if model not in [ "bert-99", "bert-99.9", "gptj-99", "gptj-99.9", "resnet50", "retinanet", "3d-unet-99", "3d-unet-99.9" ]: return pre_space+" WIP" + if model in [ "bert-99", "bert-99.9", "retinanet", "3d-unet-99", "3d-unet-99.9" ]: + code_version="r4.0" devices = [ "CPU" ] frameworks = [ "Pytorch" ] @@ -109,14 +115,14 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): content += f"{cur_space3}####### Setup a virtual environment for Python\n" content += get_venv_command(spaces+16) content += f"{cur_space3}####### Performance Estimation for Offline Scenario\n" - content += mlperf_inference_run_command(spaces+17, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True).replace("--docker ","") + content += mlperf_inference_run_command(spaces+17, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True, scenarios, code_version).replace("--docker ","") content += f"{cur_space3}The above command should do a test run of Offline scenario and record the estimated offline_target_qps.\n\n" else: # Docker implementation steps content += f"{cur_space3}####### Docker Container Build and Performance Estimation for Offline Scenario\n" docker_info = get_docker_info(spaces+16, model, implementation, device) content += docker_info - content += mlperf_inference_run_command(spaces+17, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True) + content += mlperf_inference_run_command(spaces+17, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True, scenarios, code_version) content += f"{cur_space3}The above command should get you to an interactive shell inside the docker container and do a quick test run for the Offline scenario. Once inside the docker container please do the below commands to do the accuracy + performance runs for each scenario.\n\n" content += f"{cur_space3}
    \n" content += f"{cur_space3} Please click here to see more options for the docker launch \n\n" @@ -131,7 +137,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): else: content += f"{cur_space3} You can reuse the same environment as described for {model.split('.')[0]}.\n" content += f"{cur_space3}###### Performance Estimation for Offline Scenario\n" - content += mlperf_inference_run_command(spaces+17, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True).replace("--docker ","") + content += mlperf_inference_run_command(spaces+17, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True, scenarios, code_version).replace("--docker ","") content += f"{cur_space3}The above command should do a test run of Offline scenario and record the estimated offline_target_qps.\n\n" @@ -144,12 +150,12 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): for scenario in scenarios: content += f"{cur_space3}=== \"{scenario}\"\n{cur_space4}###### {scenario}\n\n" - run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), "valid", scenarios) + run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), "valid", 0, False, scenarios, code_version) content += run_cmd #content += run_suffix content += f"{cur_space3}=== \"All Scenarios\"\n{cur_space4}###### All Scenarios\n\n" - run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), "valid", scenarios) + run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), "valid", scenarios, code_version) content += run_cmd content += run_suffix @@ -235,7 +241,7 @@ def get_run_cmd_extra(f_pre_space, model, implementation, device, scenario, scen return extra_content @env.macro - def mlperf_inference_run_command(spaces, model, implementation, framework, category, scenario, device="cpu", execution_mode="test", test_query_count="20", docker=False, scenarios = []): + def mlperf_inference_run_command(spaces, model, implementation, framework, category, scenario, device="cpu", execution_mode="test", test_query_count="20", docker=False, scenarios = [], code_version="r4.1"): pre_space = "" for i in range(1,spaces): pre_space = pre_space + " " @@ -260,7 +266,7 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ docker_setup_cmd = f"""\n {f_pre_space}```bash -{f_pre_space}cm run script --tags=run-mlperf,inference,_find-performance,_full{scenario_variation_tag} \\ +{f_pre_space}cm run script --tags=run-mlperf,inference,_find-performance,_full,_{code_version}{scenario_variation_tag} \\ {pre_space} --model={model} \\ {pre_space} --implementation={implementation} \\ {pre_space} --framework={framework} \\ @@ -279,7 +285,7 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ run_cmd = f"""\n {f_pre_space}```bash -{f_pre_space}cm run script --tags=run-mlperf,inference{scenario_variation_tag} \\ +{f_pre_space}cm run script --tags=run-mlperf,inference,_{code_version}{scenario_variation_tag} \\ {pre_space} --model={model} \\ {pre_space} --implementation={implementation} \\ {pre_space} --framework={framework} \\ diff --git a/mkdocs.yml b/mkdocs.yml index a5f75efa6..90a3ce9fd 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -35,7 +35,7 @@ nav: - Bert-Large: benchmarks/language/bert.md - GPT-J: benchmarks/language/gpt-j.md - LLAMA2-70B: benchmarks/language/llama2-70b.md - - MIXTRAL-8x7b: benchmarks/language/mixtral-8x7b.md + - MIXTRAL-8x7B: benchmarks/language/mixtral-8x7b.md - Recommendation: - DLRM-v2: benchmarks/recommendation/dlrm-v2.md - Submission: From bc19ba154e6036d2cfa807a90b5d4bf2989abcbc Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 8 Jul 2024 14:05:34 +0100 Subject: [PATCH 045/169] Update docs for MLPerf inference v4.1 --- Submission_Guidelines.md | 16 +++---- language/bert/README.md | 2 +- language/bert/README_cm.md | 39 ----------------- language/gpt-j/README.md | 2 +- language/gpt-j/README_cm.md | 43 ------------------- language/llama2-70b/README.md | 11 +---- language/mixtral-8x7b/README.md | 3 +- recommendation/dlrm_v2/pytorch/README.md | 5 ++- .../3d-unet-brats19/.dockerignore | 0 .../3d-unet-brats19/.gitignore | 0 .../3d-unet-brats19/Dockerfile | 0 .../medical_imaging/3d-unet-brats19/Makefile | 0 .../medical_imaging/3d-unet-brats19/README.md | 0 .../3d-unet-brats19/Task043_BraTS_2019.py | 0 .../3d-unet-brats19/accuracy-brats.py | 0 .../3d-unet-brats19/brats_QSL.py | 0 .../folds/fold0_validation.txt | 0 .../folds/fold1_validation.txt | 0 .../folds/fold2_validation.txt | 0 .../folds/fold3_validation.txt | 0 .../folds/fold4_validation.txt | 0 .../3d-unet-brats19/onnxruntime_SUT.py | 0 .../medical_imaging/3d-unet-brats19/ov_SUT.py | 0 .../3d-unet-brats19/preprocess.py | 0 .../3d-unet-brats19/pytorch_SUT.py | 0 .../medical_imaging/3d-unet-brats19/run.py | 0 .../medical_imaging/3d-unet-brats19/tf_SUT.py | 0 .../3d-unet-brats19/unet_onnx_to_tf.py | 0 .../3d-unet-brats19/unet_pytorch_to_onnx.py | 0 .../medical_imaging/3d-unet-brats19/user.conf | 0 text_to_image/README.md | 13 ++---- vision/classification_and_detection/README.md | 9 ++-- .../README_cm_resnet50.md | 42 ------------------ .../README_cm_retinanet.md | 38 ---------------- vision/medical_imaging/3d-unet-brats19/nnUnet | 1 - .../medical_imaging/3d-unet-kits19/README.md | 3 +- .../3d-unet-kits19/README_cm.md | 39 ----------------- 37 files changed, 20 insertions(+), 246 deletions(-) delete mode 100644 language/bert/README_cm.md delete mode 100644 language/gpt-j/README_cm.md rename {vision => retired_benchmarks/vision}/medical_imaging/3d-unet-brats19/.dockerignore (100%) rename {vision => retired_benchmarks/vision}/medical_imaging/3d-unet-brats19/.gitignore (100%) rename {vision => retired_benchmarks/vision}/medical_imaging/3d-unet-brats19/Dockerfile (100%) rename {vision => retired_benchmarks/vision}/medical_imaging/3d-unet-brats19/Makefile (100%) rename {vision => retired_benchmarks/vision}/medical_imaging/3d-unet-brats19/README.md (100%) rename {vision => retired_benchmarks/vision}/medical_imaging/3d-unet-brats19/Task043_BraTS_2019.py (100%) rename {vision => retired_benchmarks/vision}/medical_imaging/3d-unet-brats19/accuracy-brats.py (100%) rename {vision => retired_benchmarks/vision}/medical_imaging/3d-unet-brats19/brats_QSL.py (100%) rename {vision => retired_benchmarks/vision}/medical_imaging/3d-unet-brats19/folds/fold0_validation.txt (100%) rename {vision => retired_benchmarks/vision}/medical_imaging/3d-unet-brats19/folds/fold1_validation.txt (100%) rename {vision => retired_benchmarks/vision}/medical_imaging/3d-unet-brats19/folds/fold2_validation.txt (100%) rename {vision => retired_benchmarks/vision}/medical_imaging/3d-unet-brats19/folds/fold3_validation.txt (100%) rename {vision => retired_benchmarks/vision}/medical_imaging/3d-unet-brats19/folds/fold4_validation.txt (100%) rename {vision => retired_benchmarks/vision}/medical_imaging/3d-unet-brats19/onnxruntime_SUT.py (100%) rename {vision => retired_benchmarks/vision}/medical_imaging/3d-unet-brats19/ov_SUT.py (100%) rename {vision => retired_benchmarks/vision}/medical_imaging/3d-unet-brats19/preprocess.py (100%) rename {vision => retired_benchmarks/vision}/medical_imaging/3d-unet-brats19/pytorch_SUT.py (100%) rename {vision => retired_benchmarks/vision}/medical_imaging/3d-unet-brats19/run.py (100%) rename {vision => retired_benchmarks/vision}/medical_imaging/3d-unet-brats19/tf_SUT.py (100%) rename {vision => retired_benchmarks/vision}/medical_imaging/3d-unet-brats19/unet_onnx_to_tf.py (100%) rename {vision => retired_benchmarks/vision}/medical_imaging/3d-unet-brats19/unet_pytorch_to_onnx.py (100%) rename {vision => retired_benchmarks/vision}/medical_imaging/3d-unet-brats19/user.conf (100%) delete mode 100644 vision/classification_and_detection/README_cm_resnet50.md delete mode 100644 vision/classification_and_detection/README_cm_retinanet.md delete mode 160000 vision/medical_imaging/3d-unet-brats19/nnUnet delete mode 100644 vision/medical_imaging/3d-unet-kits19/README_cm.md diff --git a/Submission_Guidelines.md b/Submission_Guidelines.md index 15818cc39..4e3ca2c1a 100644 --- a/Submission_Guidelines.md +++ b/Submission_Guidelines.md @@ -18,11 +18,7 @@ The MLPerf inference submission rules are spread between the [MLCommons policies ### Is there an automatic way to run the MLPerf inference benchmarks? -MLPerf inference submissions are expected on different hardware and related software stacks. For this reason, only reference implementations are provided by MLCommons and they can guide submitters to make their own optimal implementations for their software/hardware stack. Also, all the previous implementations are made available in the MLCommons Inference results repositories and they can also guide submitters in doing their own implementations. - -[The MLCommons taskforce on automation and reproducibility](https://github.com/mlcommons/ck/blob/master/docs/taskforce.md) has automated all the MLCommons inference tasks using the [MLCommons CM language](https://github.com/mlcommons/ck/blob/master/cm) and [this readme](https://github.com/mlcommons/ck/tree/master/docs/mlperf/inference) can guide you in running the reference implementations with very minimal effort. Currently, this automation supports MLCommons reference implementations, Nvidia implementations, and C++ implementations for Onnxruntime and TFLite. Feel free to join the [taskforce Discord channel](https://discord.gg/8jbEM4J6Ff) if you have any questions. - -The previous MLPerf inference results are aggregated in [Collective Knowledge platform (MLCommons CK playground)](platform) as [reproducible experiments](https://access.cknowledge.org/playground/?action=experiments) and can be used by submitters to compare their results with the previous ones while adding various derived metrics (such as performance/watt) and constraints. +MLPerf inference submissions are expected to be run on various hardware and supported software stacks. Therefore, MLCommons provides only reference implementations to guide submitters in creating optimal implementations for their specific software and hardware configurations. Additionally, all implementations used for MLPerf inference submissions are available in the MLCommons Inference results repositories, offering further guidance for submitters developing their own implementations. ### Expected time to do benchmark runs 1. Closed submission under data enter needs offline and server scenario runs with a minimum of ten minutes needed for both. @@ -41,17 +37,15 @@ The previous MLPerf inference results are aggregated in [Collective Knowledge pl 3. Any submission passing the submission checker is valid to go to the review discussions but submitters are still required to answer any queries and fix any issues being reported by other submitters. ### Reviewing other submissions -1. Ensure that the `system_desc_id.json` file is having meaningful responses - submission_checker only checks for the existence of the fields. +1. Ensure that the `system_desc_id.json` file is having meaningful responses - `submission_checker` only checks for the existence of the fields. 2. For power submissions, `power settings` and `analyzer table` files are to be submitted, and even though the submission checker checks for the existence of these files, the content of [these files](https://github.com/mlcommons/inference_policies/blob/master/power_measurement.adoc#64-power-management-settings) must be checked manually for validity. 3. README files in the submission directory must be checked to make sure that the instructions are reproducible. 4. For closed datacenter submissions, [ECC RAM and Networking requirements](https://github.com/mlcommons/inference_policies/blob/master/inference_rules.adoc#constraints-for-the-closed-division) must be ensured. 5. Submission checker might be reporting warnings and some of these warnings can warrant an answer from the submitter. -## Changes from MLCommons Inference 3.0 +## Changes from MLCommons Inference 4.0 -1. Two new benchmarks GPT-J and GPT-3 and DLRMv2 replacing DLRM -2. Submission checker is now checking for non-empty README files and mandatory system description and power-related fields -3. New script is provided which can be used to infer scenario results and low-accuracy results from a high-accuracy result -4. `min_query_count` is removed for all scenarios except offline due to early stopping. SingleStream now needs a minimum of 64 queries and MultiStream needs 662 queries as mandated by the early stopping criteria. +1. One new benchmark in the datacenter category: Mixtral-8x7B. No changes in the edge category. +2. For power submissions, there is no code change. diff --git a/language/bert/README.md b/language/bert/README.md index 144e6d9d0..0fa29e8e0 100644 --- a/language/bert/README.md +++ b/language/bert/README.md @@ -4,7 +4,7 @@ This is the reference implementation for MLPerf Inference benchmarks for Natural The chosen model is BERT-Large performing SQuAD v1.1 question answering task. -Please see [this readme](README_cm.md) file for an automated way to run this benchmark out of the box and do an end-to-end submission with or without docker using the [MLCommons CM](https://github.com/mlcommons/ck/tree/master/cm) language. +Please see the [new docs site](https://docs.mlcommons.org/inference/benchmarks/language/bert) for an automated way to run this benchmark across different available implementations and do an end-to-end submission with or without docker. ## Prerequisites diff --git a/language/bert/README_cm.md b/language/bert/README_cm.md deleted file mode 100644 index f0284e68d..000000000 --- a/language/bert/README_cm.md +++ /dev/null @@ -1,39 +0,0 @@ -## Setup - -Please follow the MLCommons CK [installation guide](https://github.com/mlcommons/ck/blob/master/docs/installation.md) to install CM. - -Install MLCommons CK repository with automation workflows for MLPerf. - -``` -cm pull repo mlcommons@ck -``` - -# Bert - -Bert has two variants - `bert-99` and `bert-99.9` where the `99` and `99.9` specifies the required accuracy constraint with respect to the reference floating point model. `bert-99.9` model is applicable only on a datacenter system. - -In the edge category, bert-99 has Offline and SingleStream scenarios and in the datacenter category, both `bert-99` and `bert-99.9` have Offline and Server scenarios. - - -## Run Command - -### One liner to do an end-to-end submission using the reference implementation - -The below command will automatically preprocess the dataset for a given backend, builds the loadgen, runs the inference for all the required scenarios and modes, and generate a submission folder out of the produced results. - -** Please adjust the `target_qps` value as per your system performance to get a valid submission - - -``` -cmr "run mlperf inference generate-run-cmds _submission" \ ---quiet --submitter="MLCommons" --hw_name=default --model=bert-99 --implementation=reference \ ---backend=onnxruntime --device=cpu --scenario=Offline --adr.compiler.tags=gcc --target_qps=1 \ ---category=edge --division=open -``` -* Use `--device=cuda` to run the inference on Nvidia GPU -* Use `--division=closed` to run all scenarios for the closed division -* Use `--category=datacenter` to run datacenter scenarios. Both `bert-99` and `bert-99.9` are applicable to datacenter category whereas only `bert-99` is applicable to edge category -* Use `--backend=tf` and `--backend=pytorch` to use tensorflow and pytorch backends respectively - - -More details and commands to run different implementations like NVIDIA implementation can be seen [here](https://github.com/mlcommons/ck/tree/master/docs/mlperf/inference/bert). diff --git a/language/gpt-j/README.md b/language/gpt-j/README.md index 7c5301482..17c72f7eb 100644 --- a/language/gpt-j/README.md +++ b/language/gpt-j/README.md @@ -1,6 +1,6 @@ # GPT-J Reference Implementation -Please see [this readme](README_cm.md) file for an automated way to run this benchmark out of the box and do an end-to-end submission with or without docker using the [MLCommons CM](https://github.com/mlcommons/ck/tree/master/cm) language. +Please see the [new docs site](https://docs.mlcommons.org/inference/benchmarks/language/gpt-j) for an automated way to run this benchmark across different available implementations and do an end-to-end submission with or without docker. ### Setup Instructions diff --git a/language/gpt-j/README_cm.md b/language/gpt-j/README_cm.md deleted file mode 100644 index 409fb4731..000000000 --- a/language/gpt-j/README_cm.md +++ /dev/null @@ -1,43 +0,0 @@ -## Setup - -Please follow the MLCommons CK [installation guide](https://github.com/mlcommons/ck/blob/master/docs/installation.md) to install CM. - -Install MLCommons CK repository with automation workflows for MLPerf. - -``` -cm pull repo mlcommons@ck -``` - -# GPT-J - -Bert has two variants - `gpt-j-99` and `gpt-j-99.9` where the `99` and `99.9` specifies the required accuracy constraint with respect to the reference floating point model. `bert-99.9` model is applicable only on a datacenter system. - -In the edge category, gpt-j-99 has Offline and SingleStream scenarios and in the datacenter category, both `gpt-j-99` and `gpt-j-99.9` have Offline and Server scenarios. - - -## Run Command - -### One liner to do an end-to-end submission using the reference implementation - -The below command will automatically preprocess the dataset for a given backend, builds the loadgen, runs the inference for all the required scenarios and modes, and generate a submission folder out of the produced results. - -** Please modify the `--adr.gptj-model.checkpoint` value to the path containing the shared GPT-J checkpoint.** - - -``` -cmr "run mlperf inference generate-run-cmds _submission" \ ---quiet --submitter="MLCommons" --hw_name=default --model=gpt-j-99 --implementation=reference \ ---backend=pytorch --device=cpu --scenario=Offline --adr.compiler.tags=gcc \ ---category=edge --division=open --adr.gptj-model.checkpoint=$HOME/checkpoint-final \ ---test_query_count=1 --execution_mode=test --precision=bfloat16 --offline_target_qps=1 -``` - -* Use `--device=cuda` to run the inference on Nvidia GPU -* Use `--division=closed` to run all scenarios for the closed division -* Use `--scenario=SingleStream` to run SingleStream scenario -* Use `--category=datacenter` to run datacenter scenarios. `--scenario=Server` and `--server_target_qps=` should be given for server scenario -* Can use `--adr.mlperf-inference-implementation.tags=_BEAM_SIZE.2` to change the beam size from the official value of 4 to 2 (can also be 1 or 3). This is allowed only for `open` submissions -* Skip `--test_query_count=1 --execution_mode=test` to do a full valid run for submission - - -More details and commands to run different implementations like NVIDIA implementation can be seen [here](https://github.com/mlcommons/ck/tree/master/docs/mlperf/inference/gpt-j). diff --git a/language/llama2-70b/README.md b/language/llama2-70b/README.md index 837f24ade..b5d4440e4 100644 --- a/language/llama2-70b/README.md +++ b/language/llama2-70b/README.md @@ -2,22 +2,13 @@ **Basic implementation for llama2-70b. Few noteworthy items:** -+ Processing of Validation dataset is not finalized yet. Decision on input token lengths is pending + Streamer for communicating with loadgen has quite some overhead. This is only meant to provide functional implementation + For custom/optimized implementations of this benchmark it is important to include the : - For server scenario, it is necessary to call `lg.FirstTokenComplete(response)` for each query. This way the first token will be reported and it's latency will be measured. - For all scenarios, when calling `lg.QuerySamplesComplete(response)`, it is necessary that each of the elements in response is a `lg.QuerySampleResponse` that contains the number of tokens (can be create this way: `lg.QuerySampleResponse(qitem.id, bi[0], bi[1], n_tokens)`). The number of tokens reported should match with the number of tokens on your answer and this will be checked in [TEST06](../../compliance/nvidia/TEST06/) -## Automated command to run the benchmark via MLCommons CM +Please see the [new docs site](https://docs.mlcommons.org/inference/benchmarks/language/llama2-70b) for an automated way to run this benchmark across different available implementations and do an end-to-end submission with or without docker. -``` -python3 -m pip install cmind -cm pull repo mlcommons@ck -cm run script --tags=run-mlperf,inference --model=llama2-70b-99 --implementation=reference --backend=pytorch --device=cpu --precision=float32 --scenario=Offline --quiet -``` -* `--device=cuda` can be used to run on Nvidia GPUs and `--device=rocm` can be used to run on AMD GPUs -* `--precision=float16` or `--precision=bfloat16` can be used to change the model precision -* `--model=llama2-7b` can be used to run the llama2-7b variant (this is not the official MLPerf model and so not valid for a closed division submission) ## Prepare environment diff --git a/language/mixtral-8x7b/README.md b/language/mixtral-8x7b/README.md index 34402a9c3..1ebce0446 100644 --- a/language/mixtral-8x7b/README.md +++ b/language/mixtral-8x7b/README.md @@ -8,9 +8,8 @@ - For server scenario, it is necessary to call `lg.FirstTokenComplete(response)` for each query. This way the first token will be reported and it's latency will be measured. - For all scenarios, when calling `lg.QuerySamplesComplete(response)`, it is necessary that each of the elements in response is a `lg.QuerySampleResponse` that contains the number of tokens (can be create this way: `lg.QuerySampleResponse(qitem.id, bi[0], bi[1], n_tokens)`). The number of tokens reported should match with the number of tokens on your answer and this will be checked in [TEST06](../../compliance/nvidia/TEST06/) -## Automated command to run the benchmark via MLCommons CM -TODO +Please see the [new docs site](https://docs.mlcommons.org/inference/benchmarks/language/mixtral-8x7b) for an automated way to run this benchmark across different available implementations and do an end-to-end submission with or without docker. ## Prepare environment diff --git a/recommendation/dlrm_v2/pytorch/README.md b/recommendation/dlrm_v2/pytorch/README.md index 12937c0c8..647e6830c 100755 --- a/recommendation/dlrm_v2/pytorch/README.md +++ b/recommendation/dlrm_v2/pytorch/README.md @@ -2,6 +2,8 @@ This is the reference implementation for MLCommons Inference benchmarks. +Please see the [new docs site](https://docs.mlcommons.org/inference/benchmarks/recommendation/dlrm-v2/) for an automated way to run this benchmark across different available implementations and do an end-to-end submission with or without docker. + ### Supported Models **TODO: Decide benchmark name** @@ -80,8 +82,7 @@ N/A | pytorch | <2GB | - The following MLCommons CM commands can be used to programmatically download the model checkpoint. ``` -pip install cmind -cm pull repo mlcommons@ck +pip install cm4mlops cm run script --tags=get,ml-model,dlrm,_pytorch,_weight_sharded,_rclone -j ``` diff --git a/vision/medical_imaging/3d-unet-brats19/.dockerignore b/retired_benchmarks/vision/medical_imaging/3d-unet-brats19/.dockerignore similarity index 100% rename from vision/medical_imaging/3d-unet-brats19/.dockerignore rename to retired_benchmarks/vision/medical_imaging/3d-unet-brats19/.dockerignore diff --git a/vision/medical_imaging/3d-unet-brats19/.gitignore b/retired_benchmarks/vision/medical_imaging/3d-unet-brats19/.gitignore similarity index 100% rename from vision/medical_imaging/3d-unet-brats19/.gitignore rename to retired_benchmarks/vision/medical_imaging/3d-unet-brats19/.gitignore diff --git a/vision/medical_imaging/3d-unet-brats19/Dockerfile b/retired_benchmarks/vision/medical_imaging/3d-unet-brats19/Dockerfile similarity index 100% rename from vision/medical_imaging/3d-unet-brats19/Dockerfile rename to retired_benchmarks/vision/medical_imaging/3d-unet-brats19/Dockerfile diff --git a/vision/medical_imaging/3d-unet-brats19/Makefile b/retired_benchmarks/vision/medical_imaging/3d-unet-brats19/Makefile similarity index 100% rename from vision/medical_imaging/3d-unet-brats19/Makefile rename to retired_benchmarks/vision/medical_imaging/3d-unet-brats19/Makefile diff --git a/vision/medical_imaging/3d-unet-brats19/README.md b/retired_benchmarks/vision/medical_imaging/3d-unet-brats19/README.md similarity index 100% rename from vision/medical_imaging/3d-unet-brats19/README.md rename to retired_benchmarks/vision/medical_imaging/3d-unet-brats19/README.md diff --git a/vision/medical_imaging/3d-unet-brats19/Task043_BraTS_2019.py b/retired_benchmarks/vision/medical_imaging/3d-unet-brats19/Task043_BraTS_2019.py similarity index 100% rename from vision/medical_imaging/3d-unet-brats19/Task043_BraTS_2019.py rename to retired_benchmarks/vision/medical_imaging/3d-unet-brats19/Task043_BraTS_2019.py diff --git a/vision/medical_imaging/3d-unet-brats19/accuracy-brats.py b/retired_benchmarks/vision/medical_imaging/3d-unet-brats19/accuracy-brats.py similarity index 100% rename from vision/medical_imaging/3d-unet-brats19/accuracy-brats.py rename to retired_benchmarks/vision/medical_imaging/3d-unet-brats19/accuracy-brats.py diff --git a/vision/medical_imaging/3d-unet-brats19/brats_QSL.py b/retired_benchmarks/vision/medical_imaging/3d-unet-brats19/brats_QSL.py similarity index 100% rename from vision/medical_imaging/3d-unet-brats19/brats_QSL.py rename to retired_benchmarks/vision/medical_imaging/3d-unet-brats19/brats_QSL.py diff --git a/vision/medical_imaging/3d-unet-brats19/folds/fold0_validation.txt b/retired_benchmarks/vision/medical_imaging/3d-unet-brats19/folds/fold0_validation.txt similarity index 100% rename from vision/medical_imaging/3d-unet-brats19/folds/fold0_validation.txt rename to retired_benchmarks/vision/medical_imaging/3d-unet-brats19/folds/fold0_validation.txt diff --git a/vision/medical_imaging/3d-unet-brats19/folds/fold1_validation.txt b/retired_benchmarks/vision/medical_imaging/3d-unet-brats19/folds/fold1_validation.txt similarity index 100% rename from vision/medical_imaging/3d-unet-brats19/folds/fold1_validation.txt rename to retired_benchmarks/vision/medical_imaging/3d-unet-brats19/folds/fold1_validation.txt diff --git a/vision/medical_imaging/3d-unet-brats19/folds/fold2_validation.txt b/retired_benchmarks/vision/medical_imaging/3d-unet-brats19/folds/fold2_validation.txt similarity index 100% rename from vision/medical_imaging/3d-unet-brats19/folds/fold2_validation.txt rename to retired_benchmarks/vision/medical_imaging/3d-unet-brats19/folds/fold2_validation.txt diff --git a/vision/medical_imaging/3d-unet-brats19/folds/fold3_validation.txt b/retired_benchmarks/vision/medical_imaging/3d-unet-brats19/folds/fold3_validation.txt similarity index 100% rename from vision/medical_imaging/3d-unet-brats19/folds/fold3_validation.txt rename to retired_benchmarks/vision/medical_imaging/3d-unet-brats19/folds/fold3_validation.txt diff --git a/vision/medical_imaging/3d-unet-brats19/folds/fold4_validation.txt b/retired_benchmarks/vision/medical_imaging/3d-unet-brats19/folds/fold4_validation.txt similarity index 100% rename from vision/medical_imaging/3d-unet-brats19/folds/fold4_validation.txt rename to retired_benchmarks/vision/medical_imaging/3d-unet-brats19/folds/fold4_validation.txt diff --git a/vision/medical_imaging/3d-unet-brats19/onnxruntime_SUT.py b/retired_benchmarks/vision/medical_imaging/3d-unet-brats19/onnxruntime_SUT.py similarity index 100% rename from vision/medical_imaging/3d-unet-brats19/onnxruntime_SUT.py rename to retired_benchmarks/vision/medical_imaging/3d-unet-brats19/onnxruntime_SUT.py diff --git a/vision/medical_imaging/3d-unet-brats19/ov_SUT.py b/retired_benchmarks/vision/medical_imaging/3d-unet-brats19/ov_SUT.py similarity index 100% rename from vision/medical_imaging/3d-unet-brats19/ov_SUT.py rename to retired_benchmarks/vision/medical_imaging/3d-unet-brats19/ov_SUT.py diff --git a/vision/medical_imaging/3d-unet-brats19/preprocess.py b/retired_benchmarks/vision/medical_imaging/3d-unet-brats19/preprocess.py similarity index 100% rename from vision/medical_imaging/3d-unet-brats19/preprocess.py rename to retired_benchmarks/vision/medical_imaging/3d-unet-brats19/preprocess.py diff --git a/vision/medical_imaging/3d-unet-brats19/pytorch_SUT.py b/retired_benchmarks/vision/medical_imaging/3d-unet-brats19/pytorch_SUT.py similarity index 100% rename from vision/medical_imaging/3d-unet-brats19/pytorch_SUT.py rename to retired_benchmarks/vision/medical_imaging/3d-unet-brats19/pytorch_SUT.py diff --git a/vision/medical_imaging/3d-unet-brats19/run.py b/retired_benchmarks/vision/medical_imaging/3d-unet-brats19/run.py similarity index 100% rename from vision/medical_imaging/3d-unet-brats19/run.py rename to retired_benchmarks/vision/medical_imaging/3d-unet-brats19/run.py diff --git a/vision/medical_imaging/3d-unet-brats19/tf_SUT.py b/retired_benchmarks/vision/medical_imaging/3d-unet-brats19/tf_SUT.py similarity index 100% rename from vision/medical_imaging/3d-unet-brats19/tf_SUT.py rename to retired_benchmarks/vision/medical_imaging/3d-unet-brats19/tf_SUT.py diff --git a/vision/medical_imaging/3d-unet-brats19/unet_onnx_to_tf.py b/retired_benchmarks/vision/medical_imaging/3d-unet-brats19/unet_onnx_to_tf.py similarity index 100% rename from vision/medical_imaging/3d-unet-brats19/unet_onnx_to_tf.py rename to retired_benchmarks/vision/medical_imaging/3d-unet-brats19/unet_onnx_to_tf.py diff --git a/vision/medical_imaging/3d-unet-brats19/unet_pytorch_to_onnx.py b/retired_benchmarks/vision/medical_imaging/3d-unet-brats19/unet_pytorch_to_onnx.py similarity index 100% rename from vision/medical_imaging/3d-unet-brats19/unet_pytorch_to_onnx.py rename to retired_benchmarks/vision/medical_imaging/3d-unet-brats19/unet_pytorch_to_onnx.py diff --git a/vision/medical_imaging/3d-unet-brats19/user.conf b/retired_benchmarks/vision/medical_imaging/3d-unet-brats19/user.conf similarity index 100% rename from vision/medical_imaging/3d-unet-brats19/user.conf rename to retired_benchmarks/vision/medical_imaging/3d-unet-brats19/user.conf diff --git a/text_to_image/README.md b/text_to_image/README.md index bb46406a8..bdd190129 100644 --- a/text_to_image/README.md +++ b/text_to_image/README.md @@ -1,6 +1,8 @@ # MLPerf™ Inference Benchmarks for Text to Image -This is the reference implementation for MLPerf Inference text to image +This is the reference implementation for MLPerf Inference text to image. + +Please see the [new docs site](https://docs.mlcommons.org/inference/benchmarks/text_to_image/sdxl) for an automated way to run this benchmark across different available implementations and do an end-to-end submission with or without docker. ## Supported Models @@ -16,15 +18,6 @@ This is the reference implementation for MLPerf Inference text to image | Coco-2014 | We use a subset of 5000 images and captions of the coco 2014 validation dataset, so that there is exaclty one caption per image. The model takes as input the caption of the image and generates an image from it. The original images and the generated images are used to compute FID score. The caption and the generated images are used to compute the CLIP score. We provide a [script](tools/coco.py) to automatically download the dataset | | Coco-2014 (calibration) | We use a subset of 500 captions and images of the coco 2014 training dataset, so that there is exaclty one caption per image. The subset was generated using this [script](tools/coco_generate_calibration.py). We provide the [caption ids](../calibration/COCO-2014/coco_cal_captions_list.txt) and a [script](tools/coco_calibration.py) to download them. | -## Automated command to run the benchmark via MLCommons CM - -``` -python3 -m pip install cmind -cm pull repo mlcommons@ck -cm run script --tags=run-mlperf,inference --model=sdxl --implementation=reference --backend=pytorch --device=cpu --precision=float32 --scenario=Offline --quiet -``` -* `--device=cuda` can be used to run on Nvidia GPUs and `--device=rocm` can be used to run on AMD GPUs -* `--precision=float16` or `--precision=bfloat16` can be used to change the model precision ## Setup Set the following helper variables diff --git a/vision/classification_and_detection/README.md b/vision/classification_and_detection/README.md index 8384c5aed..50b57acc5 100755 --- a/vision/classification_and_detection/README.md +++ b/vision/classification_and_detection/README.md @@ -2,11 +2,9 @@ This is the reference implementation for MLPerf Inference Classification and Object Detection benchmarks -Please see the below readme files for an automated way to run this benchmark out of the box and do an end-to-end submission with or without docker using the [MLCommons CM](https://github.com/mlcommons/ck/tree/master/cm) language. -1. [Resnet50](README_cm_resnet50.md) -2. [Retinanet](README_cm_retinanet.md) +Please see the [new docs site](https://docs.mlcommons.org/inference/benchmarks) for an automated way to run this benchmark across different available implementations and do an end-to-end submission with or without docker. -Alternatively, you can find a short tutorial on how to use this benchmark [here](https://github.com/mlperf/inference/blob/master/vision/classification_and_detection/GettingStarted.ipynb). +You can find a short tutorial on how to use this benchmark [here](https://github.com/mlperf/inference/blob/master/vision/classification_and_detection/GettingStarted.ipynb). ## Supported Models @@ -48,8 +46,7 @@ Alternatively, you can find a short tutorial on how to use this benchmark [here] ## Disclaimer This benchmark app is a reference implementation that is not meant to be the fastest implementation possible. It is written in python which might make it less suitable for lite models like mobilenet or large number of cpu's. -There is a [C++ implementation](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/app-mlperf-inference-cpp) in -[MLCommons CK repository](https://github.com/mlcommons/ck) which currently supports onnxruntime backend. +There is a [C++ implementation](https://github.com/mlcommons/cm4mlops/tree/mlperf-inference/script/app-mlperf-inference-mlcommons-cpp) available which currently supports onnxruntime backend for CPUs and Nvidia GPUs. ## Tools for preparing datasets and validating accuracy The reference implementation includes all required pre-processing of datasets. diff --git a/vision/classification_and_detection/README_cm_resnet50.md b/vision/classification_and_detection/README_cm_resnet50.md deleted file mode 100644 index 2f1c14c68..000000000 --- a/vision/classification_and_detection/README_cm_resnet50.md +++ /dev/null @@ -1,42 +0,0 @@ -## Setup - -Please follow the MLCommons CK [installation guide](https://github.com/mlcommons/ck/blob/master/docs/installation.md) to install CM. - -Install MLCommons CK repository with automation workflows for MLPerf. - -``` -cm pull repo mlcommons@ck -``` - -# Resnet50 - -## Get Imagenet Dataset - -We need to get imagenet full dataset to make image-classification submissions for MLPerf inference. Since this dataset is not publicly available via a URL please follow the instructions given [here](https://github.com/mlcommons/ck/blob/master/cm-mlops/script/get-dataset-imagenet-val/README-extra.md) to download the dataset and register in CM. - -In the edge category, ResNet50 has Offline, SingleStream, and MultiStream scenarios and in the datacenter category, it has Offline and Server scenarios. The below commands are assuming an edge category system. - -## Run Command - -### One liner to do an end-to-end submission using the reference implementation - -The below command will automatically preprocess the dataset for a given backend, builds the loadgen, runs the inference for all the required scenarios and modes, and generate a submission folder out of the produced results. - -** Please adjust the `target_qps` value as per your system performance to get a valid submission - - -``` -cmr "run mlperf inference generate-run-cmds _submission" \ ---quiet --submitter="MLCommons" --hw_name=default --model=resnet50 --implementation=reference \ ---backend=onnxruntime --device=cpu --scenario=Offline --adr.compiler.tags=gcc --target_qps=1 \ ---category=edge --division=open -``` -* Use `--device=cuda` to run the inference on Nvidia GPU -* Use `--division=closed` to run all scenarios for the closed division (compliance tests are skipped for `_find-performance` mode) -* Use `--category=datacenter` to run datacenter scenarios -* Use `--backend=tf` or `--backend=tvm-onnx` to use tensorflow and tvm-onnx backends respectively - - -More details and commands to run different implementations like NVIDIA implementation can be seen [here](https://github.com/mlcommons/ck/tree/master/docs/mlperf/inference/resnet50). - - diff --git a/vision/classification_and_detection/README_cm_retinanet.md b/vision/classification_and_detection/README_cm_retinanet.md deleted file mode 100644 index e782c539f..000000000 --- a/vision/classification_and_detection/README_cm_retinanet.md +++ /dev/null @@ -1,38 +0,0 @@ -## Setup - -Please follow the MLCommons CK [installation guide](https://github.com/mlcommons/ck/blob/master/docs/installation.md) to install CM. - -Install MLCommons CK repository with automation workflows for MLPerf - -``` -cm pull repo mlcommons@ck -``` - -# Retinanet - -In the edge category, Retinanet has Offline, SingleStream, and MultiStream scenarios and in the datacenter category, it has Offline and Server scenarios. The below commands are assuming an edge category system. - -## Run Command - -### One liner to do an end-to-end submission using the reference implementation - -The below command will automatically preprocess the dataset for a given backend, builds the loadgen, runs the inference for all the required scenarios and modes, and generate a submission folder out of the produced results. - -** Please adjust the `target_qps` value as per your system performance to get a valid submission - -``` -cmr "run mlperf inference generate-run-cmds _submission" \ ---quiet --submitter="MLCommons" --hw_name=default --model=retinanet --implementation=reference \ ---backend=onnxruntime --device=cpu --scenario=Offline --adr.compiler.tags=gcc --target_qps=1 \ ---category=edge --division=open -``` - -* Use `--device=cuda` to run the inference on Nvidia GPU -* Use `--division=closed` to run all scenarios for the closed division (compliance tests are skipped for `_find-performance` mode) -* Use `--category=datacenter` to run datacenter scenarios -* Use `--backend=pytorch` to use pytorch backend - - -More details and commands to run different implementations like NVIDIA implementation can be seen [here](https://github.com/mlcommons/ck/tree/master/docs/mlperf/inference/retinanet). - - diff --git a/vision/medical_imaging/3d-unet-brats19/nnUnet b/vision/medical_imaging/3d-unet-brats19/nnUnet deleted file mode 160000 index b38c69b34..000000000 --- a/vision/medical_imaging/3d-unet-brats19/nnUnet +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b38c69b345b2f60cd0d053039669e8f988b0c0af diff --git a/vision/medical_imaging/3d-unet-kits19/README.md b/vision/medical_imaging/3d-unet-kits19/README.md index 39363d853..ae2b81607 100644 --- a/vision/medical_imaging/3d-unet-kits19/README.md +++ b/vision/medical_imaging/3d-unet-kits19/README.md @@ -6,7 +6,8 @@ This model performing KiTS19 dataset for kidney tumor segmentation task is propo [3D-UNet BraTS19 model](https://github.com/mlcommons/inference/tree/master/vision/medical_imaging/3d-unet-brats19), which has been used for MLPerf-Inference v0.7 and v1.0, is the valid model for the submissions until the end of year 2021. Please use BraTS19 model for v.1.1 submission. -Please see [this readme](README_cm.md) file for an automated way to run this benchmark out of the box and do an end-to-end submission with or without docker using the [MLCommons CM](https://github.com/mlcommons/ck/tree/master/cm) language. +Please see the [new docs site](https://docs.mlcommons.org/inference/benchmarks/medical_imaging/3d-unet/) for an automated way to run this benchmark across different available implementations and do an end-to-end submission with or without docker. + ## Prerequisites diff --git a/vision/medical_imaging/3d-unet-kits19/README_cm.md b/vision/medical_imaging/3d-unet-kits19/README_cm.md deleted file mode 100644 index 772d0feb0..000000000 --- a/vision/medical_imaging/3d-unet-kits19/README_cm.md +++ /dev/null @@ -1,39 +0,0 @@ -## Setup - -Please follow the MLCommons CK [installation guide](https://github.com/mlcommons/ck/blob/master/docs/installation.md) to install CM. - -Install MLCommons CK repository with automation workflows for MLPerf. - -``` -cm pull repo mlcommons@ck -``` - -# 3d-unet - -3d-unet has two variants - `3d-unet-99` and `3d-unet-99.9` where the `99` and `99.9` specifies the required accuracy constraint with respect to the reference floating point model. Both the models are applicable to edge as well as datacenter category. - -In the edge category, Offline and SingleStream are the applicable scenarios and in the datacenter category, the applicable ones are Offline and Server scenarios. - - -## Run Command - -### One liner to do an end-to-end submission using the reference implementation - -The below command will automatically preprocess the dataset for a given backend, builds the loadgen, runs the inference for all the required scenarios and modes, and generate a submission folder out of the produced results. - -** Please adjust the `target_qps` value as per your system performance to get a valid submission - - -``` -cmr "run mlperf inference generate-run-cmds _submission" \ ---quiet --submitter="MLCommons" --hw_name=default --model=3d-unet-99 --implementation=reference \ ---backend=onnxruntime --device=cpu --scenario=Offline --adr.compiler.tags=gcc --target_qps=1 \ ---category=edge --division=open -``` -* Use `--device=cuda` to run the inference on Nvidia GPU -* Use `--division=closed` to run all scenarios for the closed division -* Use `--category=datacenter` to run datacenter scenarios. -* Use `--backend=tf` and `--backend=pytorch` to use tensorflow and pytorch backends respectively - - -More details and commands to run different implementations like NVIDIA implementation can be seen [here](https://github.com/mlcommons/ck/tree/master/docs/mlperf/inference/3d-unet). From 1f9bc3bee650b9d4adc4c201457e9869385a4309 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 8 Jul 2024 14:18:04 +0100 Subject: [PATCH 046/169] Update docs for MLPerf inference v4.1 --- .github/workflows/test-submission-checker.yml | 3 +-- language/bert/README.md | 4 ++++ language/gpt-j/README.md | 7 +++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-submission-checker.yml b/.github/workflows/test-submission-checker.yml index 2d1b12903..31b8e4aac 100644 --- a/.github/workflows/test-submission-checker.yml +++ b/.github/workflows/test-submission-checker.yml @@ -28,8 +28,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python3 -m pip install cmind - cm pull repo mlcommons@ck + python3 -m pip install cm4mlops git clone https://github.com/mlcommons/inference_results_v4.0 --depth 1 - name: Test MLPerf inference submission checker run: | diff --git a/language/bert/README.md b/language/bert/README.md index 0fa29e8e0..6e0468666 100644 --- a/language/bert/README.md +++ b/language/bert/README.md @@ -47,6 +47,10 @@ Please run the following commands: ## Loadgen over the Network +``` +pip install cm4mlops +``` + The below CM command will launch the SUT server ``` diff --git a/language/gpt-j/README.md b/language/gpt-j/README.md index 17c72f7eb..9dc024a8e 100644 --- a/language/gpt-j/README.md +++ b/language/gpt-j/README.md @@ -75,8 +75,7 @@ Please download the fine-tuned GPT-J checkpoint using the instructions below. Th The following MLCommons CM commands can be used to programmatically download the model checkpoint. ``` -pip install cmind -cm pull repo mlcommons@ck +pip install cm4mlops cm run script --tags=get,ml-model,gptj,_pytorch,_rclone -j ``` @@ -139,6 +138,10 @@ Intel expressly disclaims the accuracy, adequacy, or completeness of any data, d ## Loadgen over the Network +``` +pip install cm4mlops +``` + The below CM command will launch the SUT server ``` From 8ff59e19dce90a3529ba0b192028de90d59fbef9 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 8 Jul 2024 21:44:41 +0100 Subject: [PATCH 047/169] Fix typo --- Submission_Guidelines.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Submission_Guidelines.md b/Submission_Guidelines.md index 4e3ca2c1a..51ce3db74 100644 --- a/Submission_Guidelines.md +++ b/Submission_Guidelines.md @@ -21,7 +21,7 @@ The MLPerf inference submission rules are spread between the [MLCommons policies MLPerf inference submissions are expected to be run on various hardware and supported software stacks. Therefore, MLCommons provides only reference implementations to guide submitters in creating optimal implementations for their specific software and hardware configurations. Additionally, all implementations used for MLPerf inference submissions are available in the MLCommons Inference results repositories, offering further guidance for submitters developing their own implementations. ### Expected time to do benchmark runs -1. Closed submission under data enter needs offline and server scenario runs with a minimum of ten minutes needed for both. +1. Closed submission under datacenter needs offline and server scenario runs with a minimum of ten minutes needed for both. 2. Closed submission under the edge category needs single stream, multi-stream (only for R50 and retinanet), and offline scenarios. A minimum of ten minutes is needed for each scenario. 3. Further two (three for ResNet50) compliance runs are needed for closed division, each taking at least 10 minutes for each scenario. 4. SingleStream, MultiStream and Server scenarios use early stopping and so can always finish around 10 minutes From 7faa76244fff3cac938b36865f2d970634cbf689 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 9 Jul 2024 10:38:51 +0100 Subject: [PATCH 048/169] Gave direct link to implementation readmes --- vision/classification_and_detection/README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/vision/classification_and_detection/README.md b/vision/classification_and_detection/README.md index 50b57acc5..ede044f5b 100755 --- a/vision/classification_and_detection/README.md +++ b/vision/classification_and_detection/README.md @@ -2,7 +2,20 @@ This is the reference implementation for MLPerf Inference Classification and Object Detection benchmarks -Please see the [new docs site](https://docs.mlcommons.org/inference/benchmarks) for an automated way to run this benchmark across different available implementations and do an end-to-end submission with or without docker. +## Automated Run Commands +Please see the [new docs site](https://docs.mlcommons.org/inference/benchmarks) for an automated way to run this benchmark across different available implementations and do an end-to-end submission with or without docker. + +### ResNet50 +- [ResNet50 MLCommons Python Reference Implementation](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_1) (Onnxruntime, Tensorflow) +- [ResNet50 Nvidia Implementation](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_2) (TensorRT) +- [ResNet50 Intel Implementation](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_3) (PyTorch) +- [ResNet50 MLCommons C++ Implementation](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_5) (Onnxruntime) + +### Retinanet +- [Retinanet MLCommons Python Reference Implementation](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_1_1) (PyTorch, Onnxruntime) +- [Retinanet Nvidia Implementation](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_1_2) (TensorRT) +- [Retinanet Intel Implementation](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_1_3) (PyTorch) +- [Retinanet MLCommons C++ Implementation](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_1_5) (Onnxruntime) You can find a short tutorial on how to use this benchmark [here](https://github.com/mlperf/inference/blob/master/vision/classification_and_detection/GettingStarted.ipynb). From b8573fe808ad48cf8406429045cb23449dd898d4 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 9 Jul 2024 16:36:20 +0530 Subject: [PATCH 049/169] Added tables detailing implementations --- vision/classification_and_detection/README.md | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/vision/classification_and_detection/README.md b/vision/classification_and_detection/README.md index ede044f5b..572308ed6 100755 --- a/vision/classification_and_detection/README.md +++ b/vision/classification_and_detection/README.md @@ -6,16 +6,23 @@ This is the reference implementation for MLPerf Inference Classification and Obj Please see the [new docs site](https://docs.mlcommons.org/inference/benchmarks) for an automated way to run this benchmark across different available implementations and do an end-to-end submission with or without docker. ### ResNet50 -- [ResNet50 MLCommons Python Reference Implementation](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_1) (Onnxruntime, Tensorflow) -- [ResNet50 Nvidia Implementation](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_2) (TensorRT) -- [ResNet50 Intel Implementation](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_3) (PyTorch) -- [ResNet50 MLCommons C++ Implementation](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_5) (Onnxruntime) + +| Implementation | Framework | Supported Devices | Supported Precision | +| -------------- | --------- | ----------------- | ------------------- | +| [Reference](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_1) | Onnxruntime, Tensorflow, Deepsparse | CPU, CUDA, ROCm | fp32 | +| [Nvidia](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_2) | TensorRT | CUDA | int8 | +| [Intel](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_3) | PyTorch | CPU | int8 | +| [MLCommons C++](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_5) | Onnxruntime | CPU, CUDA | fp32 | ### Retinanet -- [Retinanet MLCommons Python Reference Implementation](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_1_1) (PyTorch, Onnxruntime) -- [Retinanet Nvidia Implementation](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_1_2) (TensorRT) -- [Retinanet Intel Implementation](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_1_3) (PyTorch) -- [Retinanet MLCommons C++ Implementation](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_1_5) (Onnxruntime) + +| Implementation | Framework | Supported Devices | Supported Precision | +| -------------- | --------- | ----------------- | ------------------- | +| [Reference](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_1_1) | Onnxruntime, PyTorch | CPU, CUDA, ROCm | fp32 | +| [Nvidia](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_1_2) | TensorRT | CUDA | int8 | +| [Intel](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_1_3) | PyTorch | CPU | int8 | +| [MLCommons C++](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_1_5) | Onnxruntime | CPU, CUDA | fp32 | + You can find a short tutorial on how to use this benchmark [here](https://github.com/mlperf/inference/blob/master/vision/classification_and_detection/GettingStarted.ipynb). From 35e3bbdb9859cdcbd3e0b0a0e4b6eb43b2b5d0d4 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 9 Jul 2024 15:16:37 +0100 Subject: [PATCH 050/169] Update vision README.md, split the frameworks into separate rows --- vision/classification_and_detection/README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vision/classification_and_detection/README.md b/vision/classification_and_detection/README.md index 572308ed6..c2c43a81c 100755 --- a/vision/classification_and_detection/README.md +++ b/vision/classification_and_detection/README.md @@ -9,7 +9,9 @@ Please see the [new docs site](https://docs.mlcommons.org/inference/benchmarks) | Implementation | Framework | Supported Devices | Supported Precision | | -------------- | --------- | ----------------- | ------------------- | -| [Reference](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_1) | Onnxruntime, Tensorflow, Deepsparse | CPU, CUDA, ROCm | fp32 | +| [Reference](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_1) | Onnxruntime | CPU, CUDA, ROCm | fp32 | +| [Reference](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_1) | Tensorflow | CPU, CUDA, ROCm | fp32 | +| [Reference](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_1) | Deepsparse | CPU | fp32, int8 | | [Nvidia](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_2) | TensorRT | CUDA | int8 | | [Intel](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_3) | PyTorch | CPU | int8 | | [MLCommons C++](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_5) | Onnxruntime | CPU, CUDA | fp32 | @@ -18,7 +20,8 @@ Please see the [new docs site](https://docs.mlcommons.org/inference/benchmarks) | Implementation | Framework | Supported Devices | Supported Precision | | -------------- | --------- | ----------------- | ------------------- | -| [Reference](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_1_1) | Onnxruntime, PyTorch | CPU, CUDA, ROCm | fp32 | +| [Reference](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_1_1) | Onnxruntime | CPU, CUDA, ROCm | fp32 | +| [Reference](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_1_1) | PyTorch | CPU, CUDA, ROCm | fp32 | | [Nvidia](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_1_2) | TensorRT | CUDA | int8 | | [Intel](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_1_3) | PyTorch | CPU | int8 | | [MLCommons C++](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_1_5) | Onnxruntime | CPU, CUDA | fp32 | From 3e7f86ca4a07098eda493471d88142192ae1c2d5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 9 Jul 2024 15:17:34 +0100 Subject: [PATCH 051/169] Update README.md --- vision/classification_and_detection/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vision/classification_and_detection/README.md b/vision/classification_and_detection/README.md index c2c43a81c..902535e56 100755 --- a/vision/classification_and_detection/README.md +++ b/vision/classification_and_detection/README.md @@ -27,7 +27,7 @@ Please see the [new docs site](https://docs.mlcommons.org/inference/benchmarks) | [MLCommons C++](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_1_5) | Onnxruntime | CPU, CUDA | fp32 | -You can find a short tutorial on how to use this benchmark [here](https://github.com/mlperf/inference/blob/master/vision/classification_and_detection/GettingStarted.ipynb). +Additionally, you can find a short tutorial on how to use this benchmark [here](https://github.com/mlperf/inference/blob/master/vision/classification_and_detection/GettingStarted.ipynb). ## Supported Models From d43165d65b91c44e541e32d26f4e49ba6263a3b4 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 9 Jul 2024 20:07:06 +0530 Subject: [PATCH 052/169] pointed links to specific frameworks --- vision/classification_and_detection/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/vision/classification_and_detection/README.md b/vision/classification_and_detection/README.md index 902535e56..12320b5fd 100755 --- a/vision/classification_and_detection/README.md +++ b/vision/classification_and_detection/README.md @@ -9,9 +9,9 @@ Please see the [new docs site](https://docs.mlcommons.org/inference/benchmarks) | Implementation | Framework | Supported Devices | Supported Precision | | -------------- | --------- | ----------------- | ------------------- | -| [Reference](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_1) | Onnxruntime | CPU, CUDA, ROCm | fp32 | -| [Reference](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_1) | Tensorflow | CPU, CUDA, ROCm | fp32 | -| [Reference](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_1) | Deepsparse | CPU | fp32, int8 | +| [Reference](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_3_1) | Onnxruntime | CPU, CUDA, ROCm | fp32 | +| [Reference](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_3_2) | Tensorflow | CPU, CUDA, ROCm | fp32 | +| [Reference](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_3_3) | Deepsparse | CPU | fp32, int8 | | [Nvidia](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_2) | TensorRT | CUDA | int8 | | [Intel](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_3) | PyTorch | CPU | int8 | | [MLCommons C++](https://docs.mlcommons.org/inference/benchmarks/image_classification/resnet50/#__tabbed_1_5) | Onnxruntime | CPU, CUDA | fp32 | @@ -20,8 +20,8 @@ Please see the [new docs site](https://docs.mlcommons.org/inference/benchmarks) | Implementation | Framework | Supported Devices | Supported Precision | | -------------- | --------- | ----------------- | ------------------- | -| [Reference](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_1_1) | Onnxruntime | CPU, CUDA, ROCm | fp32 | -| [Reference](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_1_1) | PyTorch | CPU, CUDA, ROCm | fp32 | +| [Reference](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_3_1) | Onnxruntime | CPU, CUDA, ROCm | fp32 | +| [Reference](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_3_2) | PyTorch | CPU, CUDA, ROCm | fp32 | | [Nvidia](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_1_2) | TensorRT | CUDA | int8 | | [Intel](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_1_3) | PyTorch | CPU | int8 | | [MLCommons C++](https://docs.mlcommons.org/inference/benchmarks/object_detection/retinanet/#__tabbed_1_5) | Onnxruntime | CPU, CUDA | fp32 | From 1e3cc6cced3e3ca69ffa058720e15e3743940753 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 9 Jul 2024 20:12:22 +0530 Subject: [PATCH 053/169] pointed links to specific frameworks --- Submission_Guidelines.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Submission_Guidelines.md b/Submission_Guidelines.md index 51ce3db74..898718c03 100644 --- a/Submission_Guidelines.md +++ b/Submission_Guidelines.md @@ -18,7 +18,7 @@ The MLPerf inference submission rules are spread between the [MLCommons policies ### Is there an automatic way to run the MLPerf inference benchmarks? -MLPerf inference submissions are expected to be run on various hardware and supported software stacks. Therefore, MLCommons provides only reference implementations to guide submitters in creating optimal implementations for their specific software and hardware configurations. Additionally, all implementations used for MLPerf inference submissions are available in the MLCommons Inference results repositories, offering further guidance for submitters developing their own implementations. +MLPerf inference submissions are expected to be run on various hardware and supported software stacks. Therefore, MLCommons provides only reference implementations to guide submitters in creating optimal implementations for their specific software and hardware configurations. Additionally, all implementations used for MLPerf inference submissions are available in the MLCommons [Inference_results](https://github.com/orgs/mlcommons/repositories?q=inference_results_v) repositories, offering further guidance for submitters developing their own implementations. ### Expected time to do benchmark runs 1. Closed submission under datacenter needs offline and server scenario runs with a minimum of ten minutes needed for both. From 291537e4929b82cda65e944e5beedecb5d444150 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 9 Jul 2024 15:50:45 +0100 Subject: [PATCH 054/169] Update Submission_Guidelines.md --- Submission_Guidelines.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Submission_Guidelines.md b/Submission_Guidelines.md index 898718c03..20fba08a4 100644 --- a/Submission_Guidelines.md +++ b/Submission_Guidelines.md @@ -18,7 +18,7 @@ The MLPerf inference submission rules are spread between the [MLCommons policies ### Is there an automatic way to run the MLPerf inference benchmarks? -MLPerf inference submissions are expected to be run on various hardware and supported software stacks. Therefore, MLCommons provides only reference implementations to guide submitters in creating optimal implementations for their specific software and hardware configurations. Additionally, all implementations used for MLPerf inference submissions are available in the MLCommons [Inference_results](https://github.com/orgs/mlcommons/repositories?q=inference_results_v) repositories, offering further guidance for submitters developing their own implementations. +MLPerf inference submissions are expected to be run on various hardware and supported software stacks. Therefore, MLCommons provides only reference implementations to guide submitters in creating optimal implementations for their specific software and hardware configurations. Additionally, all implementations used for MLPerf inference submissions are available in the MLCommons [Inference_results](https://github.com/orgs/mlcommons/repositories?q=inference_results_v+sort%3Aname) repositories (under `closed//code` directory), offering further guidance for submitters developing their own implementations. ### Expected time to do benchmark runs 1. Closed submission under datacenter needs offline and server scenario runs with a minimum of ten minutes needed for both. From 3ea811e85b8e3e7218e9bb2ee54620af0b3a6b91 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 9 Jul 2024 15:53:08 +0100 Subject: [PATCH 055/169] Update Submission_Guidelines.md --- Submission_Guidelines.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Submission_Guidelines.md b/Submission_Guidelines.md index 20fba08a4..443a71bc1 100644 --- a/Submission_Guidelines.md +++ b/Submission_Guidelines.md @@ -27,13 +27,13 @@ MLPerf inference submissions are expected to be run on various hardware and supp 4. SingleStream, MultiStream and Server scenarios use early stopping and so can always finish around 10 minutes 5. Offline scenario needs a minimum of 24756 input queries to be processed -- can take hours for low-performing models like 3dunet, LLMs, etc. 6. Open division has no accuracy constraints, no required compliance runs, and can be submitted for any single scenario. There is no constraint on the model used except that the model accuracy must be validated on the accuracy dataset used in the corresponding MLPerf inference task [or must be preapproved](https://github.com/mlcommons/inference_policies/blob/master/inference_rules.adoc#412-relaxed-constraints-for-the-open-division). -7. Power submission needs an extra ranging mode to determine the peak current usage and this often doubles the overall experiment run time. +7. Power submission needs an extra ranging mode to determine the peak current usage and this often doubles the overall experiment run time. If this overhead is too much, ranging run can be reduced to 5 minutes run using mechanisms like [this](https://github.com/mlcommons/cm4mlops/blob/main/script/benchmark-program-mlperf/customize.py#L18). ## Validity of the submission 1. [MLCommons Inference submission checker](https://github.com/mlcommons/inference/blob/master/tools/submission/submission_checker.py) is provided to ensure that all submissions are passing the required checks. -2. In the unlikely event that there is an error on the submission checker for your submission, please raise a Github issue [here](https://github.com/mlcommons/inference/issues) +2. In the unlikely event that there is an error on the submission checker for your submission, please raise a GitHub issue [here](https://github.com/mlcommons/inference/issues) 3. Any submission passing the submission checker is valid to go to the review discussions but submitters are still required to answer any queries and fix any issues being reported by other submitters. ### Reviewing other submissions From ef51ae3d8c17a34a46f46bab27ee09b09c2af4d9 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 9 Jul 2024 15:53:41 +0100 Subject: [PATCH 056/169] Update Submission_Guidelines.md --- Submission_Guidelines.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Submission_Guidelines.md b/Submission_Guidelines.md index 443a71bc1..5c8cfee84 100644 --- a/Submission_Guidelines.md +++ b/Submission_Guidelines.md @@ -18,7 +18,7 @@ The MLPerf inference submission rules are spread between the [MLCommons policies ### Is there an automatic way to run the MLPerf inference benchmarks? -MLPerf inference submissions are expected to be run on various hardware and supported software stacks. Therefore, MLCommons provides only reference implementations to guide submitters in creating optimal implementations for their specific software and hardware configurations. Additionally, all implementations used for MLPerf inference submissions are available in the MLCommons [Inference_results](https://github.com/orgs/mlcommons/repositories?q=inference_results_v+sort%3Aname) repositories (under `closed//code` directory), offering further guidance for submitters developing their own implementations. +MLPerf inference submissions are expected to be run on various hardware and supported software stacks. Therefore, MLCommons provides only reference implementations to guide submitters in creating optimal implementations for their specific software and hardware configurations. Additionally, all implementations used for MLPerf inference submissions are available in the MLCommons [Inference results](https://github.com/orgs/mlcommons/repositories?q=inference_results_v+sort%3Aname) repositories (under `closed//code` directory), offering further guidance for submitters developing their own implementations. ### Expected time to do benchmark runs 1. Closed submission under datacenter needs offline and server scenario runs with a minimum of ten minutes needed for both. From 4d36503c1a1100a1854346bd58d8ed5e73733669 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 16 Jul 2024 16:18:41 +0530 Subject: [PATCH 057/169] api support llama2 --- language/llama2-70b/SUT_API.py | 464 +++++++++++++++++++++++++++++++++ language/llama2-70b/main.py | 19 +- 2 files changed, 477 insertions(+), 6 deletions(-) create mode 100644 language/llama2-70b/SUT_API.py diff --git a/language/llama2-70b/SUT_API.py b/language/llama2-70b/SUT_API.py new file mode 100644 index 000000000..798c8d880 --- /dev/null +++ b/language/llama2-70b/SUT_API.py @@ -0,0 +1,464 @@ +import os +import time +import numpy as np +import array +import torch +from torch.nn.functional import pad +from torch.utils.data import DataLoader +from transformers import AutoModelForCausalLM, AutoTokenizer, LlamaForCausalLM +from transformers.generation.streamers import BaseStreamer + +import pickle +import time +import threading +import tqdm +import queue + +import logging +from typing import TYPE_CHECKING, Optional, List +from pathlib import Path + +import mlperf_loadgen as lg +from dataset import Dataset + +logging.basicConfig(level=logging.INFO) +log = logging.getLogger("Llama-70B-SUT") + +gen_kwargs = { + "early_stopping": True, + "max_new_tokens": 1024, + "min_new_tokens": 1, + "num_beams": 1, + "do_sample": False +} + + + +class FirstTokenStreamer(BaseStreamer): + """ Streams first tokens to a 'holder' """ + + def __init__(self, first_token, tokens_cache=[], is_first_token=True, response_ids=[] ): + """ Response ids added to 'sign' the first token""" + + self.first_token = first_token # Queue for first token + self.is_first_token = is_first_token + + # Cache for subsequent generated tokens + self.tokens_cache = tokens_cache + + self.response_ids = response_ids + + self.is_prompt = True # The first tokens sent to the streamer are actually the input prompts + + def put(self, value): + """ Caches the tokens as they're generated. Assumes bs=1 """ + + # Prompts are streamed first so we need to skip the first time value that arrives + if self.is_prompt: + self.is_prompt = False + return + + value = value.item() + if self.is_first_token: + + # Add generated first token together with its query response_id to first tokens queue + self.first_token.put((value, self.response_ids[0])) + + self.is_first_token = False + return + + self.tokens_cache.append(value) + + + def end(self): + pass + + def get_out_tokens(self): + return self.tokens_cache + + +class SUT(): + def __init__(self, + model_path=None, + api_server=None, + api_model_name=None, + dtype="bfloat16", + device="cpu", + batch_size=None, + total_sample_count=24576, + dataset_path=None, + use_cached_outputs=False, # Set this to True *only for test accuracy runs* in case your prior session was killed partway through + workers=1): + + self.model_path = model_path or "meta-llama/Llama-2-70b-chat-hf" + self.device = device + self.api_servers = [] + if api_server: + self.api_servers.append(api_server) + self.api_model_name = api_model_name + self.device = device + + if not batch_size: + if device == "cpu": + batch_size = 1 + else: + batch_size = 32 # Reduce to 8 if using 4 GPUs, 16 for 8. + self.batch_size = batch_size + + # dtype + if dtype == 'bfloat16': + self.amp_enabled = True + self.amp_dtype = torch.bfloat16 + elif dtype == 'float16': + self.amp_enabled = True + self.amp_dtype = torch.float16 + else: + self.amp_enabled = False + self.amp_dtype = torch.float32 + + if 'cuda' in self.device: + assert torch.cuda.is_available(), "torch gpu is not available, exiting..." + + self.dataset_path = dataset_path + self.data_object = Dataset(self.model_path, + dataset_path=self.dataset_path, + total_sample_count=total_sample_count, + device=self.device) + self.qsl = lg.ConstructQSL(self.data_object.total_sample_count, self.data_object.perf_count, + self.data_object.LoadSamplesToRam, self.data_object.UnloadSamplesFromRam) + + #self.load_model() + self.tokenizer = AutoTokenizer.from_pretrained( + self.model_path, + model_max_length=1024, + padding_side="left", + use_fast=True,) #changed from false + + self.tokenizer.pad_token = self.tokenizer.eos_token + + self.num_workers = workers + self.worker_threads = [None] * self.num_workers + self.query_queue = queue.Queue() + + self.use_cached_outputs = use_cached_outputs + self.sample_counter = 0 + self.sample_counter_lock = threading.Lock() + + + def start(self): + # Create worker threads + for j in range(self.num_workers): + worker = threading.Thread(target=self.process_queries) + worker.start() + self.worker_threads[j] = worker + + def stop(self): + for _ in range(self.num_workers): + self.query_queue.put(None) + + for worker in self.worker_threads: + worker.join() + + def query_api_vllm(self, inputs, idx): + headers = { + 'Content-Type': 'application/json', + } + json_data = { + 'model': self.api_model_name, + 'prompt': inputs, + 'max_tokens': 1024, + 'temperature': 0, + } + + response_code = 0 + print(f"Server path {{self.api_servers[idx]}/v1/completions}") + while response_code != 200: + try: + response = requests.post(f'{self.api_servers[idx]}/v1/completions', headers=headers, json=json_data, verify=False) + response_code = response.status_code + except: + print("connection failure") + exit() + return [resp["text"] for resp in json.loads(response.text)["choices"]] + + def api_action_handler(self, chunk, server_idx): + output = self.query_api_vllm(chunk, server_idx) + return output + + def process_queries(self): + """Processor of the queued queries. User may choose to add batching logic """ + + while True: + qitem = self.query_queue.get() + if qitem is None: + break + + query_ids = [q.index for q in qitem] + + fname = "q" + "_".join([str(i) for i in query_ids]) + fname = f"run_outputs/{fname}.pkl" + _p = Path(fname) + if self.use_cached_outputs and _p.exists(): + # Read cache + with _p.open(mode="rb") as f: + d = pickle.load(f) + processed_output = d["outputs"] + tik1 = None + tik2 = None + tik3 = None + tok = None + else: + # Construct / collate batch + max_seq_len = 1024 + + tik1 = time.time() + + input_ids_tensor = [] + input_masks_tensor = [] + input_len = [] + for q in qitem: + input_ids_tensor.append(pad(self.data_object.input_ids[q.index], + (max_seq_len - self.data_object.input_lens[q.index], 0, 0, 0), + value=self.tokenizer.pad_token_id)) + input_masks_tensor.append(pad(self.data_object.attention_masks[q.index], + (max_seq_len - self.data_object.input_lens[q.index], 0, 0, 0), + value=0)) + input_len.append(self.data_object.input_lens[q.index]) + input_ids_tensor = torch.cat(input_ids_tensor) + input_masks_tensor = torch.cat(input_masks_tensor) + + assert input_ids_tensor.shape == input_masks_tensor.shape + assert input_ids_tensor.shape[0] <= self.batch_size + + if self.api_servers: + decoded = self.tokenizer.batch_decode(input_ids_tensor) + cleaned = [entry.replace('','').replace('','') for entry in decoded] + cleaned_chunks = [list(c) for c in mit.divide(len(self.api_servers), cleaned)] + + tik2 = time.time() + + if self.api_servers: + with ThreadPoolExecutor(max_workers=len(self.api_servers)) as executor: + #needs to be tested + output_chunks = list(executor.map(self.api_action_handler,cleaned_chunks,range(len(self.api_servers)))) + output = [] + for row in output_chunks: + output += row + else: + print("Error: Specify at least one API to which the request is to be sent!") + exit() + + tik3 = time.time() + + processed_output = self.data_object.postProcess(pred_output_tokens, + input_seq_lens=input_len, + query_id_list=query_ids) + if self.api_servers: + processed_output = np.array(self.tokenizer(output, padding='longest')['input_ids']) + + for i in range(len(qitem)): + unpadded = np.delete(processed_output[i], np.where(processed_output[i] == 2)) + n_tokens = unpadded.shape[0] + response_array = array.array("B", unpadded.tobytes()) + bi = response_array.buffer_info() + response = [lg.QuerySampleResponse(qitem[i].id, bi[0], bi[1], n_tokens)] + lg.QuerySamplesComplete(response) + + tok = time.time() + + with self.sample_counter_lock: + self.sample_counter += len(qitem) + print(f"Samples run: {self.sample_counter}") + if tik1: + print(f"\tBatchMaker time: {tik2 - tik1}") + print(f"\tInference time: {tik3 - tik2}") + print(f"\tPostprocess time: {tok - tik3}") + print(f"\t==== Total time: {tok - tik1}") + else: + print(f"\tLoaded from cache: {_p}") + + def get_sut(self): + self.sut = lg.ConstructSUT(self.issue_queries, self.flush_queries) + return self.sut + + def get_qsl(self): + return self.qsl + + + def predict(self,**kwargs): + raise NotImplementedError + + + def issue_queries(self, query_samples): + """ Receives samples from loadgen and adds them to queue. Users may choose to batch here""" + + list_prompts_tokens = [] + list_prompts_attn_masks = [] + + print(f"IssueQuery started with {len(query_samples)} samples") + while len(query_samples) > 0: + self.query_queue.put(query_samples[:self.batch_size]) + query_samples = query_samples[self.batch_size:] + print(f"IssueQuery done") + + + def flush_queries(self): + pass + + def __del__(self): + pass + + +class SUTServer(SUT): + def __init__(self, model_path=None, api_server=None, api_model_name=None, dtype="bfloat16", device="cpu", total_sample_count=24576, dataset_path=None, batch_size=None, workers=1): + + super().__init__(model_path=model_path, api_server=None, api_model_name=None, dtype=dtype, device=device, total_sample_count=total_sample_count, dataset_path=dataset_path, workers=workers) + + with open(f"{self.model_path}/tokenizer.json", 'r') as token_file: + llama_tokenizer = json.load(token_file) + self.llama_vocab = llama_tokenizer["model"]["vocab"] + + self.first_token_queue = queue.Queue() + + def start(self): + + # Create worker threads + for j in range(self.num_workers): + worker = threading.Thread(target=self.process_queries) + worker.start() + self.worker_threads[j] = worker + + # Create first token response thread + self.ft_response_thread = threading.Thread(target=self.process_first_tokens) + self.ft_response_thread.start() + + + def process_first_tokens(self): + + while True: + first_token_item = self.first_token_queue.get() + + if first_token_item is None: + log.info("Exiting First token response thread") + break + + first_tokens, response_id = first_token_item + + response_data = array.array("B", np.array(first_tokens, np.float32).tobytes()) + bi = response_data.buffer_info() + response = [lg.QuerySampleResponse(response_id, bi[0], bi[1])] + lg.FirstTokenComplete(response) + + def stream_api_vllm(self, input, response_ids, idx): + headers = { + 'Content-Type': 'application/json', + } + + json_data = { + 'model': '/opt/app-root/share/models', + 'prompt': input, + 'max_tokens': 1024, + 'temperature': 0, + 'stream': True, + 'logprobs': 1 + } + + while True: + try: + token_cache = [] + s = requests.Session() + first = True + with s.post( + f'{self.api_servers[idx]}/v1/completions', + headers=headers, + json=json_data, + verify=False, + stream=True + ) as resp: + for line in resp.iter_lines(): + if line: + decoded = line.decode() + if decoded.startswith("data") and "[DONE]" not in decoded: + inter = json.loads(decoded[6:])["choices"][0]["logprobs"] + if "top_logprobs" in inter: + token_s = list(inter["top_logprobs"][0].keys())[0] + token = self.llama_vocab[token_s] + if first: + self.first_token_queue.put((token, response_ids[0])) + first = False + token_cache.append(token) + s.close() + if token_cache: + return token_cache + except Exception as e: + s.close() + print("Connection failure") + print(f"An exception occurred: {type(e).__name__}") + print(f"Exception details: {e}") + + def async_process_query(self, input_ids_tensor, qitem_id, idx): + decoded = self.tokenizer.decode(input_ids_tensor[0]) + response_ids = [qitem_id] + output_tokens = self.stream_api_vllm(decoded, response_ids, idx) + n_tokens = len(output_tokens) + if n_tokens <= 1: + print("WARNING: caught low token count") + print(input_ids_tensor) + print(output_tokens) + response_array = array.array("B", np.array(output_tokens, np.int32).tobytes()) + bi = response_array.buffer_info() + response = [lg.QuerySampleResponse( + qitem_id, bi[0], bi[1], n_tokens)] + lg.QuerySamplesComplete(response) + sys.exit() + + def process_queries(self): + """Processor of the queued queries. User may choose to add batching logic """ + server_idx = 0 + while True: + + qitem = self.query_queue.get() + if qitem is None: + break + + input_ids_tensor = self.data_object.input_ids[qitem.index] + input_masks_tensor = self.data_object.attention_masks[qitem.index] + + if self.api_servers: + threading.Thread(target=self.async_process_query, args=(input_ids_tensor, qitem.id, server_idx)).start() + server_idx = (server_idx + 1) % len(self.api_servers) + else: + #TODO: This PoC is super slow with significant overhead. Best to create a patch to `generate` + tokens_cache = [] + tokens_streamer = FirstTokenStreamer(self.first_token_queue, tokens_cache=tokens_cache, is_first_token=True, response_ids=[qitem.id]) + + _ = self.model.generate( input_ids=input_ids_tensor, + attention_mask=input_masks_tensor, + pad_token_id=self.tokenizer.pad_token_id, + streamer = tokens_streamer, + **gen_kwargs + ) + + output_tokens = tokens_streamer.get_out_tokens() + + n_tokens = len(output_tokens) + response_array = array.array("B", np.array(output_tokens, np.int32).tobytes()) + bi = response_array.buffer_info() + response = [lg.QuerySampleResponse( + qitem.id, bi[0], bi[1], n_tokens)] + lg.QuerySamplesComplete(response) + + def issue_queries(self, query_samples): + + self.query_queue.put(query_samples[0]) + + + def stop(self): + for _ in range(self.num_workers): + self.query_queue.put(None) + + for worker in self.worker_threads: + worker.join() + + self.first_token_queue.put(None) + self.ft_response_thread.join() diff --git a/language/llama2-70b/main.py b/language/llama2-70b/main.py index 15dbdd2ed..c33d4f0c6 100644 --- a/language/llama2-70b/main.py +++ b/language/llama2-70b/main.py @@ -4,7 +4,6 @@ import os import logging import sys -from SUT import SUT, SUTServer sys.path.insert(0, os.getcwd()) @@ -27,6 +26,9 @@ def get_args(): parser.add_argument("--output-log-dir", type=str, default="output-logs", help="Where logs are saved") parser.add_argument("--enable-log-trace", action="store_true", help="Enable log tracing. This file can become quite large") parser.add_argument("--num-workers", type=int, default=1, help="Number of workers to process queries") + parser.add_argument("--vllm", action="store_true", help="vllm mode") + parser.add_argument("--api-model-name", type=str, default="meta-llama/Llama-2-70b-chat-hf", help="Model name(specified in llm server)") + parser.add_argument("--api-server", type=str, default=None, help="Specify an api endpoint call to use api mode") args = parser.parse_args() return args @@ -37,11 +39,6 @@ def get_args(): "server": lg.TestScenario.Server, } -sut_map = { - "offline": SUT, - "server": SUTServer - } - def main(): args = get_args() @@ -64,6 +61,16 @@ def main(): log_settings.log_output = log_output_settings log_settings.enable_trace = args.enable_log_trace + if args.vllm: + from SUT_API import SUT, SUTServer + else: + from SUT import SUT, SUTServer + + sut_map = { + "offline": SUT, + "server": SUTServer + } + sut_cls = sut_map[args.scenario.lower()] sut = sut_cls( From 641df17ea9edfdba9f89844619afb483da32e639 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 16 Jul 2024 17:46:33 +0530 Subject: [PATCH 058/169] Added request module and reduced max token len --- language/llama2-70b/SUT_API.py | 19 +++++++++++++------ language/llama2-70b/main.py | 2 ++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/language/llama2-70b/SUT_API.py b/language/llama2-70b/SUT_API.py index 798c8d880..a0912461f 100644 --- a/language/llama2-70b/SUT_API.py +++ b/language/llama2-70b/SUT_API.py @@ -18,6 +18,12 @@ from typing import TYPE_CHECKING, Optional, List from pathlib import Path +import more_itertools as mit +from concurrent.futures.thread import ThreadPoolExecutor + +import requests +from urllib3.exceptions import InsecureRequestWarning + import mlperf_loadgen as lg from dataset import Dataset @@ -166,19 +172,20 @@ def query_api_vllm(self, inputs, idx): json_data = { 'model': self.api_model_name, 'prompt': inputs, - 'max_tokens': 1024, + 'max_tokens': 8, 'temperature': 0, } response_code = 0 - print(f"Server path {{self.api_servers[idx]}/v1/completions}") + print(f"Server path {self.api_servers[idx]}/v1/completions") while response_code != 200: try: - response = requests.post(f'{self.api_servers[idx]}/v1/completions', headers=headers, json=json_data, verify=False) + response = requests.post(f"{self.api_servers[idx]}/v1/completions", headers=headers, json=json_data, verify=False) response_code = response.status_code - except: + except Exception as e: + print(e) print("connection failure") - exit() + exit(1) return [resp["text"] for resp in json.loads(response.text)["choices"]] def api_action_handler(self, chunk, server_idx): @@ -246,7 +253,7 @@ def process_queries(self): output += row else: print("Error: Specify at least one API to which the request is to be sent!") - exit() + exit(1) tik3 = time.time() diff --git a/language/llama2-70b/main.py b/language/llama2-70b/main.py index c33d4f0c6..c48e02988 100644 --- a/language/llama2-70b/main.py +++ b/language/llama2-70b/main.py @@ -80,6 +80,8 @@ def main(): dataset_path=args.dataset_path, total_sample_count=args.total_sample_count, device=args.device, + api_server=args.api_server, + api_model_name=args.api_model_name, ) # Start sut before loadgen starts From 99ee8b6dc7a8d4f63ae54d6d5d4c0ff725ab63ff Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 16 Jul 2024 15:10:13 +0100 Subject: [PATCH 059/169] Fix for llama2 api server --- language/llama2-70b/main.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/language/llama2-70b/main.py b/language/llama2-70b/main.py index c48e02988..746f9872f 100644 --- a/language/llama2-70b/main.py +++ b/language/llama2-70b/main.py @@ -73,16 +73,26 @@ def main(): sut_cls = sut_map[args.scenario.lower()] - sut = sut_cls( - model_path=args.model_path, - dtype=args.dtype, - batch_size=args.batch_size, - dataset_path=args.dataset_path, - total_sample_count=args.total_sample_count, - device=args.device, - api_server=args.api_server, - api_model_name=args.api_model_name, - ) + if args.vllm: + sut = sut_cls( + model_path=args.model_path, + dtype=args.dtype, + batch_size=args.batch_size, + dataset_path=args.dataset_path, + total_sample_count=args.total_sample_count, + device=args.device, + api_server=args.api_server, + api_model_name=args.api_model_name, + ) + else: + sut = sut_cls( + model_path=args.model_path, + dtype=args.dtype, + batch_size=args.batch_size, + dataset_path=args.dataset_path, + total_sample_count=args.total_sample_count, + device=args.device + ) # Start sut before loadgen starts sut.start() From c736d334706c1cd75327cdf2ab3a58e119ed70e2 Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Tue, 16 Jul 2024 11:50:27 -0400 Subject: [PATCH 060/169] Update SUT_API offline to work for OpenAI --- language/llama2-70b/SUT_API.py | 37 +++++++++++----------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/language/llama2-70b/SUT_API.py b/language/llama2-70b/SUT_API.py index a0912461f..1abb77c37 100644 --- a/language/llama2-70b/SUT_API.py +++ b/language/llama2-70b/SUT_API.py @@ -220,30 +220,21 @@ def process_queries(self): tik1 = time.time() + # vLLM doesn't require padding and can take input tokens + # directly, so we build our input_ids_tensor as a jagged list input_ids_tensor = [] - input_masks_tensor = [] - input_len = [] for q in qitem: - input_ids_tensor.append(pad(self.data_object.input_ids[q.index], - (max_seq_len - self.data_object.input_lens[q.index], 0, 0, 0), - value=self.tokenizer.pad_token_id)) - input_masks_tensor.append(pad(self.data_object.attention_masks[q.index], - (max_seq_len - self.data_object.input_lens[q.index], 0, 0, 0), - value=0)) - input_len.append(self.data_object.input_lens[q.index]) - input_ids_tensor = torch.cat(input_ids_tensor) - input_masks_tensor = torch.cat(input_masks_tensor) - - assert input_ids_tensor.shape == input_masks_tensor.shape - assert input_ids_tensor.shape[0] <= self.batch_size + input_ids_tensor.append(self.data_object.input_ids[q.index]) + + # NOTE(mgoin): I don't think this has to be a torch tensor + # input_ids_tensor = torch.cat(input_ids_tensor) - if self.api_servers: - decoded = self.tokenizer.batch_decode(input_ids_tensor) - cleaned = [entry.replace('','').replace('','') for entry in decoded] - cleaned_chunks = [list(c) for c in mit.divide(len(self.api_servers), cleaned)] + assert len(input_ids_tensor) <= self.batch_size tik2 = time.time() + # NOTE(mgoin): I don't think threading is necessary since we are submitting all queries in one request + # vLLM takes care of mini-batches and scheduling if self.api_servers: with ThreadPoolExecutor(max_workers=len(self.api_servers)) as executor: #needs to be tested @@ -257,14 +248,10 @@ def process_queries(self): tik3 = time.time() - processed_output = self.data_object.postProcess(pred_output_tokens, - input_seq_lens=input_len, - query_id_list=query_ids) - if self.api_servers: - processed_output = np.array(self.tokenizer(output, padding='longest')['input_ids']) - + processed_output = self.tokenizer(output)['input_ids'] for i in range(len(qitem)): - unpadded = np.delete(processed_output[i], np.where(processed_output[i] == 2)) + # NOTE(mgoin): Not optimal to make numpy arrays just to serialize + unpadded = np.array(processed_output[i]) n_tokens = unpadded.shape[0] response_array = array.array("B", unpadded.tobytes()) bi = response_array.buffer_info() From 280a2949da6237ecdb55ab1aa7faf29b34a0fcff Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Tue, 16 Jul 2024 11:59:02 -0400 Subject: [PATCH 061/169] Update SUT_API.py --- language/llama2-70b/SUT_API.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/language/llama2-70b/SUT_API.py b/language/llama2-70b/SUT_API.py index 1abb77c37..140fb226c 100644 --- a/language/llama2-70b/SUT_API.py +++ b/language/llama2-70b/SUT_API.py @@ -220,7 +220,7 @@ def process_queries(self): tik1 = time.time() - # vLLM doesn't require padding and can take input tokens + # OpenAI-API servers don't require padding and can take input tokens # directly, so we build our input_ids_tensor as a jagged list input_ids_tensor = [] for q in qitem: @@ -234,7 +234,7 @@ def process_queries(self): tik2 = time.time() # NOTE(mgoin): I don't think threading is necessary since we are submitting all queries in one request - # vLLM takes care of mini-batches and scheduling + # The API server should take care of mini-batches and scheduling if self.api_servers: with ThreadPoolExecutor(max_workers=len(self.api_servers)) as executor: #needs to be tested From 8b4c88f2de2109514937a9a6988953c8e98ec037 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 17 Jul 2024 00:21:07 +0530 Subject: [PATCH 062/169] Minor fixes --- language/llama2-70b/SUT_API.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/language/llama2-70b/SUT_API.py b/language/llama2-70b/SUT_API.py index 140fb226c..6f44e133d 100644 --- a/language/llama2-70b/SUT_API.py +++ b/language/llama2-70b/SUT_API.py @@ -170,10 +170,9 @@ def query_api_vllm(self, inputs, idx): 'Content-Type': 'application/json', } json_data = { - 'model': self.api_model_name, - 'prompt': inputs, - 'max_tokens': 8, - 'temperature': 0, + "model": self.api_model_name, + "prompt": inputs, + "max_tokens": 8 } response_code = 0 @@ -227,7 +226,7 @@ def process_queries(self): input_ids_tensor.append(self.data_object.input_ids[q.index]) # NOTE(mgoin): I don't think this has to be a torch tensor - # input_ids_tensor = torch.cat(input_ids_tensor) + input_ids_tensor = torch.cat(input_ids_tensor) assert len(input_ids_tensor) <= self.batch_size @@ -236,6 +235,9 @@ def process_queries(self): # NOTE(mgoin): I don't think threading is necessary since we are submitting all queries in one request # The API server should take care of mini-batches and scheduling if self.api_servers: + decoded = self.tokenizer.batch_decode(input_ids_tensor) + cleaned = [entry.replace('','').replace('','') for entry in decoded] + cleaned_chunks = [list(c) for c in mit.divide(len(self.api_servers), cleaned)] with ThreadPoolExecutor(max_workers=len(self.api_servers)) as executor: #needs to be tested output_chunks = list(executor.map(self.api_action_handler,cleaned_chunks,range(len(self.api_servers)))) From 84649020a848ac8629bde89428879a4ee2225696 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 17 Jul 2024 00:37:47 +0530 Subject: [PATCH 063/169] Fix json import in SUT_API.py --- language/llama2-70b/SUT_API.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/language/llama2-70b/SUT_API.py b/language/llama2-70b/SUT_API.py index 6f44e133d..82af52bd5 100644 --- a/language/llama2-70b/SUT_API.py +++ b/language/llama2-70b/SUT_API.py @@ -8,6 +8,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizer, LlamaForCausalLM from transformers.generation.streamers import BaseStreamer +import json import pickle import time import threading @@ -184,7 +185,7 @@ def query_api_vllm(self, inputs, idx): except Exception as e: print(e) print("connection failure") - exit(1) + break return [resp["text"] for resp in json.loads(response.text)["choices"]] def api_action_handler(self, chunk, server_idx): From b00755d42daaf6cf1f1aede17b11feabfae47c8b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 16 Jul 2024 21:59:30 +0100 Subject: [PATCH 064/169] Fix llama2 token length --- language/llama2-70b/SUT_API.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/language/llama2-70b/SUT_API.py b/language/llama2-70b/SUT_API.py index 82af52bd5..882714849 100644 --- a/language/llama2-70b/SUT_API.py +++ b/language/llama2-70b/SUT_API.py @@ -173,7 +173,8 @@ def query_api_vllm(self, inputs, idx): json_data = { "model": self.api_model_name, "prompt": inputs, - "max_tokens": 8 + "min_tokens": 1, + "max_tokens": 1024 } response_code = 0 From 802374b6b748120a7e3af3700dcb46a77e17ccbe Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 17 Jul 2024 14:52:31 +0530 Subject: [PATCH 065/169] Added model name verification with server --- language/llama2-70b/exitstatus | 1 + language/llama2-70b/main.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 language/llama2-70b/exitstatus diff --git a/language/llama2-70b/exitstatus b/language/llama2-70b/exitstatus new file mode 100644 index 000000000..d00491fd7 --- /dev/null +++ b/language/llama2-70b/exitstatus @@ -0,0 +1 @@ +1 diff --git a/language/llama2-70b/main.py b/language/llama2-70b/main.py index 746f9872f..724a5341d 100644 --- a/language/llama2-70b/main.py +++ b/language/llama2-70b/main.py @@ -4,12 +4,27 @@ import os import logging import sys +import requests +import json sys.path.insert(0, os.getcwd()) logging.basicConfig(level=logging.INFO) log = logging.getLogger("Llama-70B-MAIN") +# function to check the model name in server matches the user specified one +def verify_model_name(user_specified_name, url): + response = requests.get(url) + if response.status_code == 200: + response_dict = response.json() + server_model_name = response_dict["data"][0]["id"] + if user_specified_name == server_model_name: + return {"matched":True, "error":False} + else: + return {"matched":False, "error":f"User specified {user_specified_name} and server model name {server_model_name} mismatch!"} + else: + return {"matched":False, "error":f"Failed to get a valid response. Status code: {response.status_code}"} + def get_args(): parser = argparse.ArgumentParser() parser.add_argument("--scenario", type=str, choices=["Offline", "Server"], default="Offline", help="Scenario") @@ -41,6 +56,13 @@ def get_args(): def main(): args = get_args() + + if args.vllm: + resp = verify_model_name(args.api_model_name, args.api_server+"/v1/models") + if resp["error"]: + print(f"\n\n\033[91mError:\033[0m", end=" ") + print(resp["error"]) + sys.exit(1) settings = lg.TestSettings() settings.scenario = scenario_map[args.scenario.lower()] From 44ae1d91a76a1968cc0cf7c6b3b2063b9f9a54aa Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 17 Jul 2024 14:52:51 +0530 Subject: [PATCH 066/169] clean temp files --- language/llama2-70b/exitstatus | 1 - 1 file changed, 1 deletion(-) delete mode 100644 language/llama2-70b/exitstatus diff --git a/language/llama2-70b/exitstatus b/language/llama2-70b/exitstatus deleted file mode 100644 index d00491fd7..000000000 --- a/language/llama2-70b/exitstatus +++ /dev/null @@ -1 +0,0 @@ -1 From fe3644e98cad710f5d9e029ab27b374ebf398031 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 17 Jul 2024 12:56:10 +0100 Subject: [PATCH 067/169] support num_workers in LLAMA2 SUTs --- language/llama2-70b/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/language/llama2-70b/main.py b/language/llama2-70b/main.py index 746f9872f..ba9d32bec 100644 --- a/language/llama2-70b/main.py +++ b/language/llama2-70b/main.py @@ -83,6 +83,7 @@ def main(): device=args.device, api_server=args.api_server, api_model_name=args.api_model_name, + workers=args.num_workers ) else: sut = sut_cls( @@ -91,7 +92,8 @@ def main(): batch_size=args.batch_size, dataset_path=args.dataset_path, total_sample_count=args.total_sample_count, - device=args.device + device=args.device, + workers=args.num_workers ) # Start sut before loadgen starts From 1ef8072548c840b52dcc53df37c65656b4d3e593 Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Wed, 17 Jul 2024 06:24:21 -0600 Subject: [PATCH 068/169] Remove batching from Offline SUT_API.py --- language/llama2-70b/SUT_API.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/language/llama2-70b/SUT_API.py b/language/llama2-70b/SUT_API.py index 882714849..9736ea7da 100644 --- a/language/llama2-70b/SUT_API.py +++ b/language/llama2-70b/SUT_API.py @@ -106,10 +106,8 @@ def __init__(self, self.device = device if not batch_size: - if device == "cpu": - batch_size = 1 - else: - batch_size = 32 # Reduce to 8 if using 4 GPUs, 16 for 8. + # API Servers can handle batching themselves + batch_size = total_sample_count self.batch_size = batch_size # dtype From c0dc52ea391d61941d76d9de52b09b7212dca109 Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Wed, 17 Jul 2024 06:41:25 -0600 Subject: [PATCH 069/169] Update SUT_API.py --- language/llama2-70b/SUT_API.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/language/llama2-70b/SUT_API.py b/language/llama2-70b/SUT_API.py index 9736ea7da..c2e4859ed 100644 --- a/language/llama2-70b/SUT_API.py +++ b/language/llama2-70b/SUT_API.py @@ -223,10 +223,7 @@ def process_queries(self): # directly, so we build our input_ids_tensor as a jagged list input_ids_tensor = [] for q in qitem: - input_ids_tensor.append(self.data_object.input_ids[q.index]) - - # NOTE(mgoin): I don't think this has to be a torch tensor - input_ids_tensor = torch.cat(input_ids_tensor) + input_ids_tensor.append(self.data_object.input_ids[q.index].tolist()) assert len(input_ids_tensor) <= self.batch_size From 7517a90a2305a4063fcadc1d77e2ecaa8856ea64 Mon Sep 17 00:00:00 2001 From: Arjun Date: Wed, 17 Jul 2024 12:52:43 +0000 Subject: [PATCH 070/169] Minor fixes for llama2 API --- language/llama2-70b/SUT_API.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/language/llama2-70b/SUT_API.py b/language/llama2-70b/SUT_API.py index c2e4859ed..9bb77927e 100644 --- a/language/llama2-70b/SUT_API.py +++ b/language/llama2-70b/SUT_API.py @@ -105,9 +105,7 @@ def __init__(self, self.api_model_name = api_model_name self.device = device - if not batch_size: - # API Servers can handle batching themselves - batch_size = total_sample_count + batch_size = total_sample_count self.batch_size = batch_size # dtype @@ -224,6 +222,10 @@ def process_queries(self): input_ids_tensor = [] for q in qitem: input_ids_tensor.append(self.data_object.input_ids[q.index].tolist()) + + # NOTE(mgoin): I don't think this has to be a torch tensor + #input_ids_tensor = torch.cat(input_ids_tensor) + assert len(input_ids_tensor) <= self.batch_size @@ -232,9 +234,12 @@ def process_queries(self): # NOTE(mgoin): I don't think threading is necessary since we are submitting all queries in one request # The API server should take care of mini-batches and scheduling if self.api_servers: + ''' decoded = self.tokenizer.batch_decode(input_ids_tensor) cleaned = [entry.replace('','').replace('','') for entry in decoded] cleaned_chunks = [list(c) for c in mit.divide(len(self.api_servers), cleaned)] + ''' + cleaned_chunks = input_ids_tensor with ThreadPoolExecutor(max_workers=len(self.api_servers)) as executor: #needs to be tested output_chunks = list(executor.map(self.api_action_handler,cleaned_chunks,range(len(self.api_servers)))) From d3db567e074c2551e0cd4b8144c859666a206862 Mon Sep 17 00:00:00 2001 From: Arjun Date: Wed, 17 Jul 2024 13:42:12 +0000 Subject: [PATCH 071/169] Fix for llama2 API --- language/llama2-70b/SUT_API.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/language/llama2-70b/SUT_API.py b/language/llama2-70b/SUT_API.py index 9bb77927e..da28d98cd 100644 --- a/language/llama2-70b/SUT_API.py +++ b/language/llama2-70b/SUT_API.py @@ -221,11 +221,13 @@ def process_queries(self): # directly, so we build our input_ids_tensor as a jagged list input_ids_tensor = [] for q in qitem: - input_ids_tensor.append(self.data_object.input_ids[q.index].tolist()) + #input_ids_tensor.append(self.data_object.input_ids[q.index].tolist()) + input_ids_tensor += self.data_object.input_ids[q.index].tolist() # NOTE(mgoin): I don't think this has to be a torch tensor #input_ids_tensor = torch.cat(input_ids_tensor) + #print(input_ids_tensor) assert len(input_ids_tensor) <= self.batch_size @@ -239,7 +241,7 @@ def process_queries(self): cleaned = [entry.replace('','').replace('','') for entry in decoded] cleaned_chunks = [list(c) for c in mit.divide(len(self.api_servers), cleaned)] ''' - cleaned_chunks = input_ids_tensor + cleaned_chunks = [input_ids_tensor] with ThreadPoolExecutor(max_workers=len(self.api_servers)) as executor: #needs to be tested output_chunks = list(executor.map(self.api_action_handler,cleaned_chunks,range(len(self.api_servers)))) @@ -253,7 +255,8 @@ def process_queries(self): tik3 = time.time() processed_output = self.tokenizer(output)['input_ids'] - for i in range(len(qitem)): + #for i in range(len(qitem)): + for i in range(len(processed_output)): # NOTE(mgoin): Not optimal to make numpy arrays just to serialize unpadded = np.array(processed_output[i]) n_tokens = unpadded.shape[0] From bd60060b5163035ec9661ac2eaeb5ec719c16e35 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 12 Aug 2024 11:46:51 +0530 Subject: [PATCH 072/169] removed table of contents --- .../image_classification/get-resnet50-data.md | 5 +++++ .../image_classification/mobilenets.md | 5 +++++ docs/benchmarks/image_classification/resnet50.md | 5 +++++ docs/benchmarks/index.md | 5 +++++ docs/benchmarks/language/bert.md | 5 +++++ docs/benchmarks/language/get-bert-data.md | 5 +++++ docs/benchmarks/language/get-gptj-data.md | 5 +++++ docs/benchmarks/language/get-llama2-70b-data.md | 5 +++++ .../benchmarks/language/get-mixtral-8x7b-data.md | 5 +++++ docs/benchmarks/language/gpt-j.md | 5 +++++ docs/benchmarks/language/llama2-70b.md | 16 +++++++++++----- docs/benchmarks/language/mixtral-8x7b.md | 4 ++++ docs/benchmarks/medical_imaging/3d-unet.md | 5 +++++ .../medical_imaging/get-3d-unet-data.md | 5 +++++ .../object_detection/get-retinanet-data.md | 5 +++++ docs/benchmarks/object_detection/retinanet.md | 5 +++++ docs/benchmarks/recommendation/dlrm-v2.md | 5 +++++ .../recommendation/get-dlrm_v2-data.md | 5 +++++ docs/benchmarks/text_to_image/get-sdxl-data.md | 5 +++++ docs/benchmarks/text_to_image/sdxl.md | 5 +++++ docs/changelog/index.md | 5 +++++ docs/demos/index.md | 5 +++++ docs/install/index.md | 5 +++++ docs/submission/index.md | 5 +++++ docs/usage/index.md | 5 +++++ 25 files changed, 130 insertions(+), 5 deletions(-) diff --git a/docs/benchmarks/image_classification/get-resnet50-data.md b/docs/benchmarks/image_classification/get-resnet50-data.md index d1f83ae9a..900379d5a 100644 --- a/docs/benchmarks/image_classification/get-resnet50-data.md +++ b/docs/benchmarks/image_classification/get-resnet50-data.md @@ -1,3 +1,8 @@ +--- +hide: + - toc +--- + # Image Classification using ResNet50 ## Dataset diff --git a/docs/benchmarks/image_classification/mobilenets.md b/docs/benchmarks/image_classification/mobilenets.md index d676300df..f276008ef 100644 --- a/docs/benchmarks/image_classification/mobilenets.md +++ b/docs/benchmarks/image_classification/mobilenets.md @@ -1,3 +1,8 @@ +--- +hide: + - toc +--- + # Image Classification using Mobilenet models Mobilenet models are not official MLPerf models and so cannot be used for a Closed division MLPerf inference submission. But since they can be run with Imagenet dataset, we are allowed to use them for Open division submission. Only CPU runs are supported now. diff --git a/docs/benchmarks/image_classification/resnet50.md b/docs/benchmarks/image_classification/resnet50.md index 26875258d..62b966e0d 100644 --- a/docs/benchmarks/image_classification/resnet50.md +++ b/docs/benchmarks/image_classification/resnet50.md @@ -1,3 +1,8 @@ +--- +hide: + - toc +--- + # Image Classification using ResNet50 === "MLCommons-Python" diff --git a/docs/benchmarks/index.md b/docs/benchmarks/index.md index 39e3e72ac..1b46ab803 100644 --- a/docs/benchmarks/index.md +++ b/docs/benchmarks/index.md @@ -1,3 +1,8 @@ +--- +hide: + - toc +--- + # MLPerf Inference Benchmarks Please visit the individual benchmark links to see the run commands using the unified CM interface. diff --git a/docs/benchmarks/language/bert.md b/docs/benchmarks/language/bert.md index 782340b3d..cb9a955f1 100644 --- a/docs/benchmarks/language/bert.md +++ b/docs/benchmarks/language/bert.md @@ -1,3 +1,8 @@ +--- +hide: + - toc +--- + # Question Answering using Bert-Large === "MLCommons-Python" diff --git a/docs/benchmarks/language/get-bert-data.md b/docs/benchmarks/language/get-bert-data.md index f5462b181..fed637572 100644 --- a/docs/benchmarks/language/get-bert-data.md +++ b/docs/benchmarks/language/get-bert-data.md @@ -1,3 +1,8 @@ +--- +hide: + - toc +--- + # Question Answering using Bert-Large ## Dataset diff --git a/docs/benchmarks/language/get-gptj-data.md b/docs/benchmarks/language/get-gptj-data.md index 9ea31feb4..90591fb76 100644 --- a/docs/benchmarks/language/get-gptj-data.md +++ b/docs/benchmarks/language/get-gptj-data.md @@ -1,3 +1,8 @@ +--- +hide: + - toc +--- + # Text Summarization using GPT-J ## Dataset diff --git a/docs/benchmarks/language/get-llama2-70b-data.md b/docs/benchmarks/language/get-llama2-70b-data.md index 4b04f7068..917f4705e 100644 --- a/docs/benchmarks/language/get-llama2-70b-data.md +++ b/docs/benchmarks/language/get-llama2-70b-data.md @@ -1,3 +1,8 @@ +--- +hide: + - toc +--- + # Text Summarization using LLAMA2-70b ## Dataset diff --git a/docs/benchmarks/language/get-mixtral-8x7b-data.md b/docs/benchmarks/language/get-mixtral-8x7b-data.md index 601e5c1dd..1b2df1b9e 100644 --- a/docs/benchmarks/language/get-mixtral-8x7b-data.md +++ b/docs/benchmarks/language/get-mixtral-8x7b-data.md @@ -1,3 +1,8 @@ +--- +hide: + - toc +--- + ## Dataset The benchmark implementation run command will automatically download the preprocessed validation and calibration datasets. In case you want to download only the datasets, you can use the below commands. diff --git a/docs/benchmarks/language/gpt-j.md b/docs/benchmarks/language/gpt-j.md index 2eefbbc79..125c87ef8 100644 --- a/docs/benchmarks/language/gpt-j.md +++ b/docs/benchmarks/language/gpt-j.md @@ -1,3 +1,8 @@ +--- +hide: + - toc +--- + # Text Summarization using GPT-J diff --git a/docs/benchmarks/language/llama2-70b.md b/docs/benchmarks/language/llama2-70b.md index f1785dcb3..c8df1e2fa 100644 --- a/docs/benchmarks/language/llama2-70b.md +++ b/docs/benchmarks/language/llama2-70b.md @@ -1,3 +1,8 @@ +--- +hide: + - toc +--- + # Text Summarization using LLAMA2-70b @@ -19,10 +24,11 @@ LLAMA2-70b-99.9 {{ mlperf_inference_implementation_readme (4, "llama2-70b-99.9", "nvidia") }} - -=== "Qualcomm" - ## Qualcomm AI100 MLPerf Implementation - +=== "Neural Magic" + ## Neural Magic MLPerf Implementation + LLAMA2-70b-99 -{{ mlperf_inference_implementation_readme (4, "llama2-70b-99", "qualcomm") }} +{{ mlperf_inference_implementation_readme (4, "llama2-70b-99", "NeuralMagic") }} + LLAMA2-70b-99.9 +{{ mlperf_inference_implementation_readme (4, "llama2-70b-99.9", "NeuralMagic") }} \ No newline at end of file diff --git a/docs/benchmarks/language/mixtral-8x7b.md b/docs/benchmarks/language/mixtral-8x7b.md index eb138dc46..f4e84e8cf 100644 --- a/docs/benchmarks/language/mixtral-8x7b.md +++ b/docs/benchmarks/language/mixtral-8x7b.md @@ -1,3 +1,7 @@ +--- +hide: + - toc +--- === "MLCommons-Python" ## MLPerf Reference Implementation in Python diff --git a/docs/benchmarks/medical_imaging/3d-unet.md b/docs/benchmarks/medical_imaging/3d-unet.md index b58ea7f2e..8d5017bf5 100644 --- a/docs/benchmarks/medical_imaging/3d-unet.md +++ b/docs/benchmarks/medical_imaging/3d-unet.md @@ -1,3 +1,8 @@ +--- +hide: + - toc +--- + # Medical Imaging using 3d-unet (KiTS 2019 kidney tumor segmentation task) diff --git a/docs/benchmarks/medical_imaging/get-3d-unet-data.md b/docs/benchmarks/medical_imaging/get-3d-unet-data.md index efc6ce6ed..e5bbfbab6 100644 --- a/docs/benchmarks/medical_imaging/get-3d-unet-data.md +++ b/docs/benchmarks/medical_imaging/get-3d-unet-data.md @@ -1,3 +1,8 @@ +--- +hide: + - toc +--- + # Medical Imaging using 3d-unet (KiTS 2019 kidney tumor segmentation task) ## Dataset diff --git a/docs/benchmarks/object_detection/get-retinanet-data.md b/docs/benchmarks/object_detection/get-retinanet-data.md index f2d432210..900fd572a 100644 --- a/docs/benchmarks/object_detection/get-retinanet-data.md +++ b/docs/benchmarks/object_detection/get-retinanet-data.md @@ -1,3 +1,8 @@ +--- +hide: + - toc +--- + # Object Detection using Retinanet ## Dataset diff --git a/docs/benchmarks/object_detection/retinanet.md b/docs/benchmarks/object_detection/retinanet.md index 383a2ec1b..699d92050 100644 --- a/docs/benchmarks/object_detection/retinanet.md +++ b/docs/benchmarks/object_detection/retinanet.md @@ -1,3 +1,8 @@ +--- +hide: + - toc +--- + # Object Detection using Retinanet === "MLCommons-Python" diff --git a/docs/benchmarks/recommendation/dlrm-v2.md b/docs/benchmarks/recommendation/dlrm-v2.md index 18266615f..aa93e981b 100644 --- a/docs/benchmarks/recommendation/dlrm-v2.md +++ b/docs/benchmarks/recommendation/dlrm-v2.md @@ -1,3 +1,8 @@ +--- +hide: + - toc +--- + # Recommendation using DLRM v2 diff --git a/docs/benchmarks/recommendation/get-dlrm_v2-data.md b/docs/benchmarks/recommendation/get-dlrm_v2-data.md index 97464a164..e70c500c5 100644 --- a/docs/benchmarks/recommendation/get-dlrm_v2-data.md +++ b/docs/benchmarks/recommendation/get-dlrm_v2-data.md @@ -1,3 +1,8 @@ +--- +hide: + - toc +--- + # Recommendation using DLRM v2 ## Dataset diff --git a/docs/benchmarks/text_to_image/get-sdxl-data.md b/docs/benchmarks/text_to_image/get-sdxl-data.md index 830465d44..f0d1376bd 100644 --- a/docs/benchmarks/text_to_image/get-sdxl-data.md +++ b/docs/benchmarks/text_to_image/get-sdxl-data.md @@ -1,3 +1,8 @@ +--- +hide: + - toc +--- + # Text to Image using Stable Diffusion ## Dataset diff --git a/docs/benchmarks/text_to_image/sdxl.md b/docs/benchmarks/text_to_image/sdxl.md index 2d84838d4..13c87611a 100644 --- a/docs/benchmarks/text_to_image/sdxl.md +++ b/docs/benchmarks/text_to_image/sdxl.md @@ -1,3 +1,8 @@ +--- +hide: + - toc +--- + # Text to Image using Stable Diffusion diff --git a/docs/changelog/index.md b/docs/changelog/index.md index f68abc5b1..6033a4826 100644 --- a/docs/changelog/index.md +++ b/docs/changelog/index.md @@ -1,2 +1,7 @@ +--- +hide: + - toc +--- + # What's New, What's Coming diff --git a/docs/demos/index.md b/docs/demos/index.md index 1c23a5f60..a9e6424ea 100644 --- a/docs/demos/index.md +++ b/docs/demos/index.md @@ -1,2 +1,7 @@ +--- +hide: + - toc +--- + # Demos diff --git a/docs/install/index.md b/docs/install/index.md index 9575e86c8..a8a37c950 100644 --- a/docs/install/index.md +++ b/docs/install/index.md @@ -1,3 +1,8 @@ +--- +hide: + - toc +--- + # Installation We use MLCommons CM Automation framework to run MLPerf inference benchmarks. diff --git a/docs/submission/index.md b/docs/submission/index.md index 94287f5d2..e294efb71 100644 --- a/docs/submission/index.md +++ b/docs/submission/index.md @@ -1,3 +1,8 @@ +--- +hide: + - toc +--- + If you follow the `cm run` commands under the individual model pages in the [benchmarks](../benchmarks/index.md) directory, all the valid results will get aggregated to the `cm cache` folder. Once all the results across all the modelsare ready you can use the following command to generate a valid submission tree compliant with the [MLPerf requirements](https://github.com/mlcommons/policies/blob/master/submission_rules.adoc#inference-1). ## Generate actual submission tree diff --git a/docs/usage/index.md b/docs/usage/index.md index 2e92a6e00..01d54c3aa 100644 --- a/docs/usage/index.md +++ b/docs/usage/index.md @@ -1 +1,6 @@ +--- +hide: + - toc +--- + # Using CM for MLPerf Inference From 32c4702220e65ceae31378d873fb125b56bd3b33 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 12 Aug 2024 11:48:21 +0530 Subject: [PATCH 073/169] enabled llama2-nvidia + vllm-NM : WIP --- main.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 2f1c036fe..b977ae1fb 100644 --- a/main.py +++ b/main.py @@ -29,10 +29,14 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): frameworks = [ "Pytorch" ] elif implementation == "nvidia": - if model in [ "sdxl", "llama2-70b-99", "llama2-70b-99.9", "mixtral-8x7b" ]: + if model in [ "sdxl", "mixtral-8x7b" ]: return pre_space+" WIP" devices = [ "CUDA" ] frameworks = [ "TensorRT" ] + + elif implementation == "NeuralMagic": + devices = [ "CUDA" ] + frameworks = [ "vLLM" ] elif implementation == "intel": if model not in [ "bert-99", "bert-99.9", "gptj-99", "gptj-99.9", "resnet50", "retinanet", "3d-unet-99", "3d-unet-99.9" ]: @@ -208,7 +212,7 @@ def get_docker_info(spaces, model, implementation, device): #pre_space = " " if implementation == "nvidia": info += f"\n{pre_space}!!! tip\n\n" - info+= f"{pre_space} All the Nvidia benchmarks, except GPT-J and LLAMA2-70B, use the same Docker container. Therefore, if you have already executed the Docker setup command for any benchmark, you can skip the Docker setup command below and run the commands inside the existing Docker container. The Docker container for GPT-J and LLAMA2-70B is the same and can be used for the other benchmarks, but not vice versa. This is because TensorRT-LLM is built specifically for the LLM benchmarks. If you are already inside a Docker container, execute the below Docker setup command without the --docker option for performance estimation.\n\n" + info+= f"{pre_space} If ran with `--all_models=yes`, all the benchmark models of NVIDIA implementation could be run within the same container.\n\n" return info def get_readme_suffix(spaces, model, implementation): @@ -263,6 +267,13 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ if docker: docker_cmd_suffix = f" \\\n{pre_space} --docker --quiet" docker_cmd_suffix += f" \\\n{pre_space} --test_query_count={test_query_count}" + + if "llama2-70b" in model: + if implementation != "NeuralMagic": + docker_cmd_suffix += f" \\\n{pre_space} --tp_size=" + docker_cmd_suffix += f" \\\n{pre_space} --nvidia_llama2_dataset_file_path=" + else: + docker_cmd_suffix += f" \\\n{pre_space} --api_server=" docker_setup_cmd = f"""\n {f_pre_space}```bash @@ -283,6 +294,13 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ if execution_mode == "test": cmd_suffix += f" \\\n {pre_space} --test_query_count={test_query_count}" + if "llama2-70b" in model: + if implementation != "NeuralMagic": + cmd_suffix += f" \\\n{pre_space} --tp_size=" + cmd_suffix += f" \\\n{pre_space} --nvidia_llama2_dataset_file_path=" + else: + cmd_suffix += f" \\\n{pre_space} --api_server=" + run_cmd = f"""\n {f_pre_space}```bash {f_pre_space}cm run script --tags=run-mlperf,inference,_{code_version}{scenario_variation_tag} \\ From cd823cc3d157848a80bd5d220d618e44d4d29add Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 12 Aug 2024 12:15:22 +0530 Subject: [PATCH 074/169] enabled dlrm for intel --- docs/benchmarks/recommendation/dlrm-v2.md | 8 ++++++++ main.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/benchmarks/recommendation/dlrm-v2.md b/docs/benchmarks/recommendation/dlrm-v2.md index aa93e981b..0657176c7 100644 --- a/docs/benchmarks/recommendation/dlrm-v2.md +++ b/docs/benchmarks/recommendation/dlrm-v2.md @@ -25,3 +25,11 @@ hide: DLRM-v2-99.9 {{ mlperf_inference_implementation_readme (4, "dlrm_v2-99.9", "nvidia") }} +=== "Intel" + ## Intel MLPerf Implementation + + DLRM-v2-99 +{{ mlperf_inference_implementation_readme (4, "dlrm_v2-99", "intel") }} + + DLRM-v2-99.9 +{{ mlperf_inference_implementation_readme (4, "dlrm_v2-99.9", "intel") }} \ No newline at end of file diff --git a/main.py b/main.py index b977ae1fb..92d5e9ccf 100644 --- a/main.py +++ b/main.py @@ -39,7 +39,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): frameworks = [ "vLLM" ] elif implementation == "intel": - if model not in [ "bert-99", "bert-99.9", "gptj-99", "gptj-99.9", "resnet50", "retinanet", "3d-unet-99", "3d-unet-99.9" ]: + if model not in [ "bert-99", "bert-99.9", "gptj-99", "gptj-99.9", "resnet50", "retinanet", "3d-unet-99", "3d-unet-99.9", "dlrm_v2-99", "dlrm_v2-99.9" ]: return pre_space+" WIP" if model in [ "bert-99", "bert-99.9", "retinanet", "3d-unet-99", "3d-unet-99.9" ]: code_version="r4.0" From 0c94ea99cf3b33d8e6391df8d80d80e13a82f03e Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 12 Aug 2024 15:29:23 +0530 Subject: [PATCH 075/169] lower cased implementation --- docs/benchmarks/language/llama2-70b.md | 4 ++-- main.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/benchmarks/language/llama2-70b.md b/docs/benchmarks/language/llama2-70b.md index c8df1e2fa..87b637b42 100644 --- a/docs/benchmarks/language/llama2-70b.md +++ b/docs/benchmarks/language/llama2-70b.md @@ -28,7 +28,7 @@ hide: ## Neural Magic MLPerf Implementation LLAMA2-70b-99 -{{ mlperf_inference_implementation_readme (4, "llama2-70b-99", "NeuralMagic") }} +{{ mlperf_inference_implementation_readme (4, "llama2-70b-99", "neuralmagic") }} LLAMA2-70b-99.9 -{{ mlperf_inference_implementation_readme (4, "llama2-70b-99.9", "NeuralMagic") }} \ No newline at end of file +{{ mlperf_inference_implementation_readme (4, "llama2-70b-99.9", "neuralmagic") }} \ No newline at end of file diff --git a/main.py b/main.py index 92d5e9ccf..d48d9f3d5 100644 --- a/main.py +++ b/main.py @@ -34,9 +34,9 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): devices = [ "CUDA" ] frameworks = [ "TensorRT" ] - elif implementation == "NeuralMagic": + elif implementation == "neuralmagic": devices = [ "CUDA" ] - frameworks = [ "vLLM" ] + frameworks = [ "pytorch" ] elif implementation == "intel": if model not in [ "bert-99", "bert-99.9", "gptj-99", "gptj-99.9", "resnet50", "retinanet", "3d-unet-99", "3d-unet-99.9", "dlrm_v2-99", "dlrm_v2-99.9" ]: @@ -269,7 +269,7 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ docker_cmd_suffix += f" \\\n{pre_space} --test_query_count={test_query_count}" if "llama2-70b" in model: - if implementation != "NeuralMagic": + if implementation != "neuralmagic": docker_cmd_suffix += f" \\\n{pre_space} --tp_size=" docker_cmd_suffix += f" \\\n{pre_space} --nvidia_llama2_dataset_file_path=" else: @@ -295,7 +295,7 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ cmd_suffix += f" \\\n {pre_space} --test_query_count={test_query_count}" if "llama2-70b" in model: - if implementation != "NeuralMagic": + if implementation != "neuralmagic": cmd_suffix += f" \\\n{pre_space} --tp_size=" cmd_suffix += f" \\\n{pre_space} --nvidia_llama2_dataset_file_path=" else: From 5b0df157df19e2a396f083e0e0ef1a141cfae7e1 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 12 Aug 2024 15:49:29 +0530 Subject: [PATCH 076/169] added raw data input --- docs/benchmarks/recommendation/dlrm-v2.md | 12 ++++++------ main.py | 8 +++++++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/benchmarks/recommendation/dlrm-v2.md b/docs/benchmarks/recommendation/dlrm-v2.md index 0657176c7..d1b41deb1 100644 --- a/docs/benchmarks/recommendation/dlrm-v2.md +++ b/docs/benchmarks/recommendation/dlrm-v2.md @@ -11,25 +11,25 @@ hide: ## MLPerf Reference Implementation in Python DLRM-v2-99 -{{ mlperf_inference_implementation_readme (4, "dlrm_v2-99", "reference") }} +{{ mlperf_inference_implementation_readme (4, "dlrm-v2-99", "reference") }} DLRM-v2-99.9 -{{ mlperf_inference_implementation_readme (4, "dlrm_v2-99.9", "reference") }} +{{ mlperf_inference_implementation_readme (4, "dlrm-v2-99.9", "reference") }} === "Nvidia" ## Nvidia MLPerf Implementation DLRM-v2-99 -{{ mlperf_inference_implementation_readme (4, "dlrm_v2-99", "nvidia") }} +{{ mlperf_inference_implementation_readme (4, "dlrm-v2-99", "nvidia") }} DLRM-v2-99.9 -{{ mlperf_inference_implementation_readme (4, "dlrm_v2-99.9", "nvidia") }} +{{ mlperf_inference_implementation_readme (4, "dlrm-v2-99.9", "nvidia") }} === "Intel" ## Intel MLPerf Implementation DLRM-v2-99 -{{ mlperf_inference_implementation_readme (4, "dlrm_v2-99", "intel") }} +{{ mlperf_inference_implementation_readme (4, "dlrm-v2-99", "intel") }} DLRM-v2-99.9 -{{ mlperf_inference_implementation_readme (4, "dlrm_v2-99.9", "intel") }} \ No newline at end of file +{{ mlperf_inference_implementation_readme (4, "dlrm-v2-99.9", "intel") }} \ No newline at end of file diff --git a/main.py b/main.py index d48d9f3d5..dc71f195a 100644 --- a/main.py +++ b/main.py @@ -39,7 +39,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): frameworks = [ "pytorch" ] elif implementation == "intel": - if model not in [ "bert-99", "bert-99.9", "gptj-99", "gptj-99.9", "resnet50", "retinanet", "3d-unet-99", "3d-unet-99.9", "dlrm_v2-99", "dlrm_v2-99.9" ]: + if model not in [ "bert-99", "bert-99.9", "gptj-99", "gptj-99.9", "resnet50", "retinanet", "3d-unet-99", "3d-unet-99.9", "dlrm-v2-99", "dlrm-v2-99.9" ]: return pre_space+" WIP" if model in [ "bert-99", "bert-99.9", "retinanet", "3d-unet-99", "3d-unet-99.9" ]: code_version="r4.0" @@ -274,6 +274,9 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ docker_cmd_suffix += f" \\\n{pre_space} --nvidia_llama2_dataset_file_path=" else: docker_cmd_suffix += f" \\\n{pre_space} --api_server=" + + if "dlrm-v2" in model and implementation == "nvidia": + docker_cmd_suffix += f" \\\n{pre_space} --criteo_day23_raw_data_path=" docker_setup_cmd = f"""\n {f_pre_space}```bash @@ -300,6 +303,9 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ cmd_suffix += f" \\\n{pre_space} --nvidia_llama2_dataset_file_path=" else: cmd_suffix += f" \\\n{pre_space} --api_server=" + + if "dlrm-v2" in model and implementation == "nvidia": + cmd_suffix += f" \\\n{pre_space} --criteo_day23_raw_data_path=" run_cmd = f"""\n {f_pre_space}```bash From f89295c357da3c2429a633225eb1672771bff336 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 12 Aug 2024 17:41:36 +0530 Subject: [PATCH 077/169] corrected data download commands --- .../recommendation/get-dlrm-v2-data.md | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 docs/benchmarks/recommendation/get-dlrm-v2-data.md diff --git a/docs/benchmarks/recommendation/get-dlrm-v2-data.md b/docs/benchmarks/recommendation/get-dlrm-v2-data.md new file mode 100644 index 000000000..1c44ec471 --- /dev/null +++ b/docs/benchmarks/recommendation/get-dlrm-v2-data.md @@ -0,0 +1,30 @@ +--- +hide: + - toc +--- + +# Recommendation using DLRM v2 + +## Dataset + +The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. + +=== "Validation" + DLRM validation run uses the Criteo dataset (Day 23). + + ### Get Validation Dataset + ``` + cm run script --tags=get,dataset,criteo,_validation -j + ``` +## Model +The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. + +Get the Official MLPerf DLRM v2 Model + +=== "Pytorch" + + ### Pytorch + ``` + cm run script --tags=get,ml-model,dlrm,_pytorch -j + ``` + From 8e1eb75b5413efb33c2994cc6f28d253ca3dd637 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 12 Aug 2024 17:42:20 +0530 Subject: [PATCH 078/169] renamed filename --- .../recommendation/get-dlrm_v2-data.md | 30 ------------------- 1 file changed, 30 deletions(-) delete mode 100644 docs/benchmarks/recommendation/get-dlrm_v2-data.md diff --git a/docs/benchmarks/recommendation/get-dlrm_v2-data.md b/docs/benchmarks/recommendation/get-dlrm_v2-data.md deleted file mode 100644 index e70c500c5..000000000 --- a/docs/benchmarks/recommendation/get-dlrm_v2-data.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -hide: - - toc ---- - -# Recommendation using DLRM v2 - -## Dataset - -The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands. - -=== "Validation" - DLRM validation run uses the Criteo dataset (Day 23). - - ### Get Validation Dataset - ``` - cm run script --tags=get,dataset,criteo,validation -j - ``` -## Model -The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands. - -Get the Official MLPerf DLRM v2 Model - -=== "Pytorch" - - ### Pytorch - ``` - cm run script --tags=get,ml-model,dlrm_v2,_pytorch -j - ``` - From 087dad94d2f65e3e522c6018a192570bcae6cdb6 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 13 Aug 2024 18:36:42 +0530 Subject: [PATCH 079/169] changes for bert and vllm --- main.py | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/main.py b/main.py index dc71f195a..b086bbad5 100644 --- a/main.py +++ b/main.py @@ -24,7 +24,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): elif model.lower() == "retinanet": frameworks = [ "Onnxruntime", "Pytorch" ] elif "bert" in model.lower(): - frameworks = [ "Onnxruntime", "Pytorch", "Tensorflow" ] + frameworks = [ "Pytorch" ] else: frameworks = [ "Pytorch" ] @@ -115,6 +115,13 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): test_query_count=get_test_query_count(model, implementation, device) if "99.9" not in model: #not showing docker command as it is already done for the 99% variant + if implementation == "neuralmagic": + content += f"{cur_space3}####### Run the Inference Server\n" + content += get_inference_server_run_cmd(spaces+16,implementation) + # tips regarding the running of nural magic server + content += f"\n{cur_space3}!!! tip\n\n" + content += f"{cur_space3} - Host and Port number of the server can be configured through `--host` and `--port`. Otherwise, server will run on default host `localhost` and port `8000`.\n\n" + if execution_env == "Native": # Native implementation steps through virtual environment content += f"{cur_space3}####### Setup a virtual environment for Python\n" content += get_venv_command(spaces+16) @@ -159,7 +166,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): #content += run_suffix content += f"{cur_space3}=== \"All Scenarios\"\n{cur_space4}###### All Scenarios\n\n" - run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), "valid", scenarios, code_version) + run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), "valid", 0, False, scenarios, code_version) content += run_cmd content += run_suffix @@ -195,6 +202,16 @@ def get_readme_prefix(spaces, model, implementation): return readme_prefix + def get_inference_server_run_cmd(spaces, implementation): + indent = " "*spaces + " " + if implementation == "neuralmagic": + pre_space = " "*spaces + return f"""\n +{pre_space}```bash +{pre_space}cm run script --tags=run,vllm-server \\ +{indent}--model=nm-testing/Llama-2-70b-chat-hf-FP8 +{pre_space}```\n""" + def get_venv_command(spaces): pre_space = " "*spaces return f"""\n @@ -260,7 +277,7 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ scenario_option = f"\\\n{pre_space} --scenario={scenario}" if scenario == "Server" or (scenario == "All Scenarios" and "Server" in scenarios): - scenario_option = f"\\\n{pre_space} --server_target_qps=" + scenario_option += f"\\\n{pre_space} --server_target_qps=" run_cmd_extra = get_run_cmd_extra(f_pre_space, model, implementation, device, scenario, scenarios) @@ -269,11 +286,12 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ docker_cmd_suffix += f" \\\n{pre_space} --test_query_count={test_query_count}" if "llama2-70b" in model: - if implementation != "neuralmagic": - docker_cmd_suffix += f" \\\n{pre_space} --tp_size=" + if implementation == "nvidia": + docker_cmd_suffix += f" \\\n{pre_space} --tp_size=2" docker_cmd_suffix += f" \\\n{pre_space} --nvidia_llama2_dataset_file_path=" - else: - docker_cmd_suffix += f" \\\n{pre_space} --api_server=" + elif implementation == "neuralmagic": + docker_cmd_suffix += f" \\\n{pre_space} --api_server=http://localhost:8000" + docker_cmd_suffix += f" \\\n{pre_space} --vllm_model_name=nm-testing/Llama-2-70b-chat-hf-FP8" if "dlrm-v2" in model and implementation == "nvidia": docker_cmd_suffix += f" \\\n{pre_space} --criteo_day23_raw_data_path=" @@ -298,11 +316,12 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ cmd_suffix += f" \\\n {pre_space} --test_query_count={test_query_count}" if "llama2-70b" in model: - if implementation != "neuralmagic": + if implementation == "nvidia": cmd_suffix += f" \\\n{pre_space} --tp_size=" cmd_suffix += f" \\\n{pre_space} --nvidia_llama2_dataset_file_path=" - else: - cmd_suffix += f" \\\n{pre_space} --api_server=" + elif implementation == "neuralmagic": + cmd_suffix += f" \\\n{pre_space} --api_server=http://localhost:8000" + cmd_suffix += f" \\\n{pre_space} --vllm_model_name=nm-testing/Llama-2-70b-chat-hf-FP8" if "dlrm-v2" in model and implementation == "nvidia": cmd_suffix += f" \\\n{pre_space} --criteo_day23_raw_data_path=" From c1032c230ec8bc4b65c07f45c9f24ddbe692b593 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 13 Aug 2024 18:40:23 +0530 Subject: [PATCH 080/169] documentation to work on custom repo and branch --- docs/install/index.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/install/index.md b/docs/install/index.md index a8a37c950..f365aade3 100644 --- a/docs/install/index.md +++ b/docs/install/index.md @@ -20,5 +20,12 @@ CM needs `git`, `python3-pip` and `python3-venv` installed on your system. If an pip install cm4mlops ``` +## To work on custom GitHub repo and branch + +```bash + pip install cmind && cm init --quiet --repo=mlcommons@cm4mlops --branch=mlperf-inference +``` + +Here, repo is in the format `githubUsername@githubRepo`. Now, you are ready to use the `cm` commands to run MLPerf inference as given in the [benchmarks](../benchmarks/index.md) page From e8cb2a89f32eb02b4bb735628eac801f17f4c0b5 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 13 Aug 2024 19:00:15 +0530 Subject: [PATCH 081/169] benchmark index page update --- docs/benchmarks/index.md | 126 +++++++++++++++++++++++++++++++++------ 1 file changed, 109 insertions(+), 17 deletions(-) diff --git a/docs/benchmarks/index.md b/docs/benchmarks/index.md index 1b46ab803..af8090615 100644 --- a/docs/benchmarks/index.md +++ b/docs/benchmarks/index.md @@ -1,32 +1,124 @@ +# MLPerf Inference Benchmarks + +## Overview +This document provides details on various MLPerf Inference Benchmarks categorized by tasks, models, and datasets. Each section lists the models performing similar tasks, with details on datasets, accuracy, and server latency constraints. + --- -hide: - - toc + +## 1. Image Classification +### [ResNet50-v1.5](image_classification/resnet50.md) +- **Dataset**: Imagenet-2012 (224x224) Validation + - **Size**: 50,000 + - **QSL Size**: 1,024 +- **Reference Model Accuracy**: 76.46% +- **Server Scenario Latency Constraint**: 15ms + --- -# MLPerf Inference Benchmarks +## 2. Text to Image +### [Stable Diffusion](text_to_image/sdxl.md) +- **Dataset**: Subset of Coco2014 + - **Size**: 5,000 + - **QSL Size**: 5,000 +- **Required Accuracy (Closed Division)**: + - FID: 23.01085758 ≤ FID ≤ 23.95007626 + - CLIP: 32.68631873 ≤ CLIP ≤ 31.81331801 -Please visit the individual benchmark links to see the run commands using the unified CM interface. +--- -1. [Image Classification](image_classification/resnet50.md) using ResNet50-v1.5 model and Imagenet-2012 (224x224) validation dataset. Dataset size is 50,000 and QSL size is 1024. Reference model accuracy is 76.46%. Server scenario latency constraint is 15ms. +## 3. Object Detection +### [Retinanet](object_detection/retinanet.md) +- **Dataset**: OpenImages + - **Size**: 24,781 + - **QSL Size**: 64 +- **Reference Model Accuracy**: 0.3755 mAP +- **Server Scenario Latency Constraint**: 100ms -2. [Text to Image](text_to_image/sdxl.md) using Stable Diffusion model and subset of Coco2014 dataset. Dataset size is 5000 amd QSL size is the same. Required accuracy for closed division is (23.01085758 <= FID <= 23.95007626, 32.68631873 <= CLIP <= 31.81331801). +--- -3. [Object Detection](object_detection/retinanet.md) using Retinanet model and OpenImages dataset.Dataset size is 24781 and QSL size is 64. Reference model accuracy is 0.3755 mAP. Server scenario latency constraint is 100ms. +## 4. Medical Image Segmentation +### [3d-unet](medical_imaging/3d-unet.md) +- **Dataset**: KiTS2019 + - **Size**: 42 + - **QSL Size**: 42 +- **Reference Model Accuracy**: 0.86330 Mean DICE Score +- **Server Scenario**: Not Applicable -4. [Medical Image Segmentation](medical_imaging/3d-unet.md) using 3d-unet model and KiTS2019 dataset. Dataset size is 42 and QSL size is the same. Reference model accuracy is 0.86330 mean DIXE score. Server scenario is not applicable. +--- -5. [Question Answering](language/bert.md) using Bert-Large model and Squad v1.1 dataset with 384 sequence length. Dataset size is 10833 and QSL size is the same. Reference model accuracy is f1 score = 90.874%. Server scenario latency constraint is 130ms. +## 5. Language Tasks -6. [Text Summarization](language/gpt-j.md) using GPT-J model and CNN Daily Mail v3.0.0 dataset. Dataset size is 13368 amd QSL size is the same. Reference model accuracy is (rouge1=42.9865, rouge2=20.1235, rougeL=29.9881, gen_len=4016878). Server scenario latency sconstraint is 20s. +### 5.1. Question Answering -7. [Question Answering](language/llama2-70b.md) using LLAMA2-70b model and OpenORCA (GPT-4 split, max_seq_len=1024) dataset. Dataset size is 24576 and QSL size is the same. Reference model accuracy is (rouge1=44.4312, rouge2=22.0352, rougeL=28.6162, tokens_per_sample=294.45). Server scenario latency constraint is TTFT=2000ms, TPOT=200ms. +### [Bert-Large](language/bert.md) +- **Dataset**: Squad v1.1 (384 Sequence Length) + - **Size**: 10,833 + - **QSL Size**: 10,833 +- **Reference Model Accuracy**: F1 Score = 90.874% +- **Server Scenario Latency Constraint**: 130ms -8. [Question Answering, Math and Code Generation](language/mixtral-8x7b.md) using Mixtral-8x7B model and OpenORCA (5k samples of GPT-4 split, max_seq_len=2048), GSM8K (5k samples of the validation split, max_seq_len=2048), MBXP (5k samples of the validation split, max_seq_len=2048) datasets. Dataset size is 15000 and QSL size is the same. Reference model accuracy is (rouge1=45.4911, rouge2=23.2829, rougeL=30.3615, gsm8k accuracy = 73.78, mbxp accuracy = 60.12, tokens_per_sample=294.45). Server scenario latency constraint is TTFT=2000ms, TPOT=200ms. +### [LLAMA2-70B](language/llama2-70b.md) +- **Dataset**: OpenORCA (GPT-4 split, max_seq_len=1024) + - **Size**: 24,576 + - **QSL Size**: 24,576 +- **Reference Model Accuracy**: + - Rouge1: 44.4312 + - Rouge2: 22.0352 + - RougeL: 28.6162 + - Tokens_per_sample: 294.45 +- **Server Scenario Latency Constraint**: + - TTFT: 2000ms + - TPOT: 200ms -9. [Recommendation](recommendation/dlrm-v2.md) using DLRMv2 model and Synthetic Multihot Criteo dataset. Dataset size is 204800 and QSL size is the same. Reference model accuracy is AUC=80.31%. Server scenario latency constraint is 60 ms. +### 5.2. Text Summarization + +### [GPT-J](language/gpt-j.md) +- **Dataset**: CNN Daily Mail v3.0.0 + - **Size**: 13,368 + - **QSL Size**: 13,368 +- **Reference Model Accuracy**: + - Rouge1: 42.9865 + - Rouge2: 20.1235 + - RougeL: 29.9881 + - Gen_len: 4,016,878 +- **Server Scenario Latency Constraint**: 20s + +### 5.3. Mixed Tasks (Question Answering, Math, and Code Generation) + +### [Mixtral-8x7B](language/mixtral-8x7b.md) +- **Datasets**: + - OpenORCA (5k samples of GPT-4 split, max_seq_len=2048) + - GSM8K (5k samples of the validation split, max_seq_len=2048) + - MBXP (5k samples of the validation split, max_seq_len=2048) + - **Size**: 15,000 + - **QSL Size**: 15,000 +- **Reference Model Accuracy**: + - Rouge1: 45.4911 + - Rouge2: 23.2829 + - RougeL: 30.3615 + - GSM8K Accuracy: 73.78% + - MBXP Accuracy: 60.12% + - Tokens_per_sample: 294.45 +- **Server Scenario Latency Constraint**: + - TTFT: 2000ms + - TPOT: 200ms + +--- + +## 6. Recommendation +### [DLRMv2](recommendation/dlrm-v2.md) +- **Dataset**: Synthetic Multihot Criteo + - **Size**: 204,800 + - **QSL Size**: 204,800 +- **Reference Model Accuracy**: AUC = 80.31% +- **Server Scenario Latency Constraint**: 60ms + +--- -All the nine benchmarks can participate in the datacenter category. -All the nine benchmarks except DLRMv2, LLAMA2 and Mixtral-8x7B and can participate in the edge category. +### Participation Categories +- **Datacenter Category**: All nine benchmarks can participate. +- **Edge Category**: All benchmarks except DLRMv2, LLAMA2, and Mixtral-8x7B can participate. -`bert`, `llama2-70b`, `dlrm_v2` and `3d-unet` has a high accuracy (99.9%) variant, where the benchmark run must achieve a higher accuracy of at least `99.9%` of the FP32 reference model -in comparison with the `99%` default accuracy requirement. +### High Accuracy Variants +- **Benchmarks**: `bert`, `llama2-70b`, `dlrm_v2`, and `3d-unet` +- **Requirement**: Must achieve at least 99.9% of the FP32 reference model accuracy, compared to the default 99% accuracy requirement. From 7e37072be082e242f8adba675403fe924b28cb80 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 13 Aug 2024 19:08:12 +0530 Subject: [PATCH 082/169] enabled sdxl for nvidia and intel --- docs/benchmarks/text_to_image/sdxl.md | 6 ------ main.py | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/docs/benchmarks/text_to_image/sdxl.md b/docs/benchmarks/text_to_image/sdxl.md index 13c87611a..575f4cabb 100644 --- a/docs/benchmarks/text_to_image/sdxl.md +++ b/docs/benchmarks/text_to_image/sdxl.md @@ -20,9 +20,3 @@ hide: ## Intel MLPerf Implementation {{ mlperf_inference_implementation_readme (4, "sdxl", "intel") }} - -=== "Qualcomm" - ## Qualcomm AI100 MLPerf Implementation - -{{ mlperf_inference_implementation_readme (4, "sdxl", "qualcomm") }} - diff --git a/main.py b/main.py index b086bbad5..da6a1d3e8 100644 --- a/main.py +++ b/main.py @@ -29,7 +29,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): frameworks = [ "Pytorch" ] elif implementation == "nvidia": - if model in [ "sdxl", "mixtral-8x7b" ]: + if model in [ "mixtral-8x7b" ]: return pre_space+" WIP" devices = [ "CUDA" ] frameworks = [ "TensorRT" ] @@ -39,7 +39,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): frameworks = [ "pytorch" ] elif implementation == "intel": - if model not in [ "bert-99", "bert-99.9", "gptj-99", "gptj-99.9", "resnet50", "retinanet", "3d-unet-99", "3d-unet-99.9", "dlrm-v2-99", "dlrm-v2-99.9" ]: + if model not in [ "bert-99", "bert-99.9", "gptj-99", "gptj-99.9", "resnet50", "retinanet", "3d-unet-99", "3d-unet-99.9", "dlrm-v2-99", "dlrm-v2-99.9", "sdxl" ]: return pre_space+" WIP" if model in [ "bert-99", "bert-99.9", "retinanet", "3d-unet-99", "3d-unet-99.9" ]: code_version="r4.0" From 283b39c760705b2cbaaf9f1ecd4f3820b9b13440 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 13 Aug 2024 21:13:25 +0530 Subject: [PATCH 083/169] updated vllm server run cmd --- main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index da6a1d3e8..4705ead83 100644 --- a/main.py +++ b/main.py @@ -209,7 +209,9 @@ def get_inference_server_run_cmd(spaces, implementation): return f"""\n {pre_space}```bash {pre_space}cm run script --tags=run,vllm-server \\ -{indent}--model=nm-testing/Llama-2-70b-chat-hf-FP8 +{indent}--model=nm-testing/Llama-2-70b-chat-hf-FP8 \\ +{indent}--vllm_model_name=nm-testing/Llama-2-70b-chat-hf-FP8 \\ +{indent}--quiet {pre_space}```\n""" def get_venv_command(spaces): From 4f5cbcdf549253cae5d202d1485ab86cedc03c6e Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 14 Aug 2024 13:04:31 +0530 Subject: [PATCH 084/169] benchmark page information addition --- docs/benchmarks/index.md | 124 ----------------------------- docs/index.md | 162 +++++++++++++++++++++++++++++++++++++- docs/install/index.md | 2 +- docs/official_gh_index.md | 1 + docs/submission/index.md | 2 +- mkdocs.yml | 2 +- 6 files changed, 165 insertions(+), 128 deletions(-) delete mode 100644 docs/benchmarks/index.md mode change 120000 => 100644 docs/index.md create mode 120000 docs/official_gh_index.md diff --git a/docs/benchmarks/index.md b/docs/benchmarks/index.md deleted file mode 100644 index af8090615..000000000 --- a/docs/benchmarks/index.md +++ /dev/null @@ -1,124 +0,0 @@ -# MLPerf Inference Benchmarks - -## Overview -This document provides details on various MLPerf Inference Benchmarks categorized by tasks, models, and datasets. Each section lists the models performing similar tasks, with details on datasets, accuracy, and server latency constraints. - ---- - -## 1. Image Classification -### [ResNet50-v1.5](image_classification/resnet50.md) -- **Dataset**: Imagenet-2012 (224x224) Validation - - **Size**: 50,000 - - **QSL Size**: 1,024 -- **Reference Model Accuracy**: 76.46% -- **Server Scenario Latency Constraint**: 15ms - ---- - -## 2. Text to Image -### [Stable Diffusion](text_to_image/sdxl.md) -- **Dataset**: Subset of Coco2014 - - **Size**: 5,000 - - **QSL Size**: 5,000 -- **Required Accuracy (Closed Division)**: - - FID: 23.01085758 ≤ FID ≤ 23.95007626 - - CLIP: 32.68631873 ≤ CLIP ≤ 31.81331801 - ---- - -## 3. Object Detection -### [Retinanet](object_detection/retinanet.md) -- **Dataset**: OpenImages - - **Size**: 24,781 - - **QSL Size**: 64 -- **Reference Model Accuracy**: 0.3755 mAP -- **Server Scenario Latency Constraint**: 100ms - ---- - -## 4. Medical Image Segmentation -### [3d-unet](medical_imaging/3d-unet.md) -- **Dataset**: KiTS2019 - - **Size**: 42 - - **QSL Size**: 42 -- **Reference Model Accuracy**: 0.86330 Mean DICE Score -- **Server Scenario**: Not Applicable - ---- - -## 5. Language Tasks - -### 5.1. Question Answering - -### [Bert-Large](language/bert.md) -- **Dataset**: Squad v1.1 (384 Sequence Length) - - **Size**: 10,833 - - **QSL Size**: 10,833 -- **Reference Model Accuracy**: F1 Score = 90.874% -- **Server Scenario Latency Constraint**: 130ms - -### [LLAMA2-70B](language/llama2-70b.md) -- **Dataset**: OpenORCA (GPT-4 split, max_seq_len=1024) - - **Size**: 24,576 - - **QSL Size**: 24,576 -- **Reference Model Accuracy**: - - Rouge1: 44.4312 - - Rouge2: 22.0352 - - RougeL: 28.6162 - - Tokens_per_sample: 294.45 -- **Server Scenario Latency Constraint**: - - TTFT: 2000ms - - TPOT: 200ms - -### 5.2. Text Summarization - -### [GPT-J](language/gpt-j.md) -- **Dataset**: CNN Daily Mail v3.0.0 - - **Size**: 13,368 - - **QSL Size**: 13,368 -- **Reference Model Accuracy**: - - Rouge1: 42.9865 - - Rouge2: 20.1235 - - RougeL: 29.9881 - - Gen_len: 4,016,878 -- **Server Scenario Latency Constraint**: 20s - -### 5.3. Mixed Tasks (Question Answering, Math, and Code Generation) - -### [Mixtral-8x7B](language/mixtral-8x7b.md) -- **Datasets**: - - OpenORCA (5k samples of GPT-4 split, max_seq_len=2048) - - GSM8K (5k samples of the validation split, max_seq_len=2048) - - MBXP (5k samples of the validation split, max_seq_len=2048) - - **Size**: 15,000 - - **QSL Size**: 15,000 -- **Reference Model Accuracy**: - - Rouge1: 45.4911 - - Rouge2: 23.2829 - - RougeL: 30.3615 - - GSM8K Accuracy: 73.78% - - MBXP Accuracy: 60.12% - - Tokens_per_sample: 294.45 -- **Server Scenario Latency Constraint**: - - TTFT: 2000ms - - TPOT: 200ms - ---- - -## 6. Recommendation -### [DLRMv2](recommendation/dlrm-v2.md) -- **Dataset**: Synthetic Multihot Criteo - - **Size**: 204,800 - - **QSL Size**: 204,800 -- **Reference Model Accuracy**: AUC = 80.31% -- **Server Scenario Latency Constraint**: 60ms - ---- - -### Participation Categories -- **Datacenter Category**: All nine benchmarks can participate. -- **Edge Category**: All benchmarks except DLRMv2, LLAMA2, and Mixtral-8x7B can participate. - -### High Accuracy Variants -- **Benchmarks**: `bert`, `llama2-70b`, `dlrm_v2`, and `3d-unet` -- **Requirement**: Must achieve at least 99.9% of the FP32 reference model accuracy, compared to the default 99% accuracy requirement. diff --git a/docs/index.md b/docs/index.md deleted file mode 120000 index 32d46ee88..000000000 --- a/docs/index.md +++ /dev/null @@ -1 +0,0 @@ -../README.md \ No newline at end of file diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 000000000..c77175e2f --- /dev/null +++ b/docs/index.md @@ -0,0 +1,161 @@ +# MLPerf Inference Benchmarks + +## Overview +This document provides details on various [MLPerf Inference Benchmarks](official_gh_index.md) categorized by tasks, models, and datasets. Each section lists the models performing similar tasks, with details on datasets, accuracy, and server latency constraints. + +--- + +## 1. Image Classification +### [ResNet50-v1.5](benchmarks/image_classification/resnet50.md) +- **Dataset**: Imagenet-2012 (224x224) Validation + - **Dataset Size**: 50,000 + - **QSL Size**: 1,024 +- **Number of Parameters**: 25,636,712 +- **Reference Model Accuracy**: 76.46% ACC +- **Server Scenario Latency Constraint**: 15ms +- **Equal Issue mode**: False +- **Accuracy Variants**: None +- **Precision**: fp32, int8 + +--- + +## 2. Text to Image +### [Stable Diffusion](benchmarks/text_to_image/sdxl.md) +- **Dataset**: Subset of Coco2014 + - **Dataset Size**: 5,000 + - **QSL Size**: 5,000 +- **Number of Parameters**: 890,000,000 +- **Required Accuracy (Closed Division)**: + - FID: 23.01085758 ≤ FID ≤ 23.95007626 + - CLIP: 32.68631873 ≤ CLIP ≤ 31.81331801 +- **Equal Issue mode**: True +- **Accuracy Variants**: None +- **Precision**: fp32, fp16 + +--- + +## 3. Object Detection +### [Retinanet](benchmarks/object_detection/retinanet.md) +- **Dataset**: OpenImages + - **Dataset Size**: 24,781 + - **QSL Size**: 64 +- **Number of Parameters**: 50,000,000 +- **Reference Model Accuracy**: 0.3755 mAP +- **Server Scenario Latency Constraint**: 100ms +- **Equal Issue mode**: False +- **Accuracy Variants**: None +- **Accuracy Variants**: 99% and 99.9% of reference model accuracy +- **Precision**: fp32 + +--- + +## 4. Medical Image Segmentation +### [3d-unet](benchmarks/medical_imaging/3d-unet.md) +- **Dataset**: KiTS2019 + - **Dataset Size**: 42 + - **QSL Size**: 42 +- **Number of Parameters**: 8,000,000 +- **Reference Model Accuracy**: 0.86330 Mean DICE Score +- **Server Scenario**: Not Applicable +- **Equal Issue mode**: True +- **Accuracy Variants**: 99% and 99.9% of reference model accuracy +- **Precision**: fp32 + +--- + +## 5. Language Tasks + +### 5.1. Question Answering + +### [Bert-Large](benchmarks/language/bert.md) +- **Dataset**: Squad v1.1 (384 Sequence Length) + - **Dataset Size**: 10,833 + - **QSL Size**: 10,833 +- **Number of Parameters**: 345,000,000 +- **Reference Model Accuracy**: F1 Score = 90.874% +- **Server Scenario Latency Constraint**: 130ms +- **Equal Issue mode**: True +- **Accuracy Variants**: 99% and 99.9% of reference model accuracy +- **Precision**: fp32, int8 + +### [LLAMA2-70B](benchmarks/language/llama2-70b.md) +- **Dataset**: OpenORCA (GPT-4 split, max_seq_len=1024) + - **Dataset Size**: 24,576 + - **QSL Size**: 24,576 +- **Number of Parameters**: 70,000,000,000 +- **Reference Model Accuracy**: + - Rouge1: 44.4312 + - Rouge2: 22.0352 + - RougeL: 28.6162 + - Tokens_per_sample: 294.45 +- **Server Scenario Latency Constraint**: + - TTFT: 2000ms + - TPOT: 200ms +- **Equal Issue mode**: True +- **Accuracy Variants**: 99% and 99.9% of reference model accuracy +- **Precision**: fp32 + +### 5.2. Text Summarization + +### [GPT-J](benchmarks/language/gpt-j.md) +- **Dataset**: CNN Daily Mail v3.0.0 + - **Dataset Size**: 13,368 + - **QSL Size**: 13,368 +- **Number of Parameters**: 6,000,000,000 +- **Reference Model Accuracy**: + - Rouge1: 42.9865 + - Rouge2: 20.1235 + - RougeL: 29.9881 + - Gen_len: 4,016,878 +- **Server Scenario Latency Constraint**: 20s +- **Equal Issue mode**: True +- **Accuracy Variants**: 99% and 99.9% of reference model accuracy +- **Precision**: fp32 + +### 5.3. Mixed Tasks (Question Answering, Math, and Code Generation) + +### [Mixtral-8x7B](benchmarks/language/mixtral-8x7b.md) +- **Datasets**: + - OpenORCA (5k samples of GPT-4 split, max_seq_len=2048) + - GSM8K (5k samples of the validation split, max_seq_len=2048) + - MBXP (5k samples of the validation split, max_seq_len=2048) + - **Dataset Size**: 15,000 + - **QSL Size**: 15,000 +- **Number of Parameters**: 56,000,000,000 +- **Reference Model Accuracy**: + - Rouge1: 45.4911 + - Rouge2: 23.2829 + - RougeL: 30.3615 + - GSM8K Accuracy: 73.78% + - MBXP Accuracy: 60.12% + - Tokens_per_sample: 294.45 +- **Server Scenario Latency Constraint**: + - TTFT: 2000ms + - TPOT: 200ms +- **Equal Issue mode**: True +- **Accuracy Variants**: None +- **Precision**: fp16 + +--- + +## 6. Recommendation +### [DLRMv2](benchmarks/recommendation/dlrm-v2.md) +- **Dataset**: Synthetic Multihot Criteo + - **Dataset Size**: 204,800 + - **QSL Size**: 204,800 +- **Number of Parameters**: 1,000,000,000 +- **Reference Model Accuracy**: AUC = 80.31% +- **Server Scenario Latency Constraint**: 60ms +- **Equal Issue mode**: False +- **Accuracy Variants**: 99% and 99.9% of reference model accuracy +- **Precision**: fp32 + +--- + +### Participation Categories +- **Datacenter Category**: All nine benchmarks can participate. +- **Edge Category**: All benchmarks except DLRMv2, LLAMA2, and Mixtral-8x7B can participate. + +### High Accuracy Variants +- **Benchmarks**: `bert`, `llama2-70b`, `dlrm_v2`, and `3d-unet` +- **Requirement**: Must achieve at least 99.9% of the reference model accuracy, compared to the default 99% accuracy requirement. diff --git a/docs/install/index.md b/docs/install/index.md index f365aade3..60377adee 100644 --- a/docs/install/index.md +++ b/docs/install/index.md @@ -28,4 +28,4 @@ CM needs `git`, `python3-pip` and `python3-venv` installed on your system. If an Here, repo is in the format `githubUsername@githubRepo`. -Now, you are ready to use the `cm` commands to run MLPerf inference as given in the [benchmarks](../benchmarks/index.md) page +Now, you are ready to use the `cm` commands to run MLPerf inference as given in the [benchmarks](../index.md) page diff --git a/docs/official_gh_index.md b/docs/official_gh_index.md new file mode 120000 index 000000000..32d46ee88 --- /dev/null +++ b/docs/official_gh_index.md @@ -0,0 +1 @@ +../README.md \ No newline at end of file diff --git a/docs/submission/index.md b/docs/submission/index.md index e294efb71..80088d444 100644 --- a/docs/submission/index.md +++ b/docs/submission/index.md @@ -3,7 +3,7 @@ hide: - toc --- -If you follow the `cm run` commands under the individual model pages in the [benchmarks](../benchmarks/index.md) directory, all the valid results will get aggregated to the `cm cache` folder. Once all the results across all the modelsare ready you can use the following command to generate a valid submission tree compliant with the [MLPerf requirements](https://github.com/mlcommons/policies/blob/master/submission_rules.adoc#inference-1). +If you follow the `cm run` commands under the individual model pages in the [benchmarks](../index.md) directory, all the valid results will get aggregated to the `cm cache` folder. Once all the results across all the modelsare ready you can use the following command to generate a valid submission tree compliant with the [MLPerf requirements](https://github.com/mlcommons/policies/blob/master/submission_rules.adoc#inference-1). ## Generate actual submission tree diff --git a/mkdocs.yml b/mkdocs.yml index 90a3ce9fd..8e59acf63 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -22,7 +22,7 @@ nav: - Install: - install/index.md - Benchmarks: - - benchmarks/index.md + - index.md - Image Classification: - ResNet50: benchmarks/image_classification/resnet50.md - Text to Image: From 8e71518726249bf51a60ea8565eff16f15e780b0 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 14 Aug 2024 13:07:10 +0530 Subject: [PATCH 085/169] fix indendation issue --- docs/submission/index.md | 80 ++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/docs/submission/index.md b/docs/submission/index.md index 80088d444..ffd9371e7 100644 --- a/docs/submission/index.md +++ b/docs/submission/index.md @@ -11,62 +11,62 @@ If you follow the `cm run` commands under the individual model pages in the [ben ### Closed Edge Submission ```bash cm run script -tags=generate,inference,submission \ - --clean \ - --preprocess_submission=yes \ - --run-checker \ - --submitter=MLCommons \ - --tar=yes \ - --env.CM_TAR_OUTFILE=submission.tar.gz \ - --division=closed \ - --category=edge \ - --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ - --quiet + --clean \ + --preprocess_submission=yes \ + --run-checker \ + --submitter=MLCommons \ + --tar=yes \ + --env.CM_TAR_OUTFILE=submission.tar.gz \ + --division=closed \ + --category=edge \ + --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ + --quiet ``` === "Closed Datacenter" ### Closed Datacenter Submission ```bash cm run script -tags=generate,inference,submission \ - --clean \ - --preprocess_submission=yes \ - --run-checker \ - --submitter=MLCommons \ - --tar=yes \ - --env.CM_TAR_OUTFILE=submission.tar.gz \ - --division=closed \ - --category=datacenter \ - --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ - --quiet + --clean \ + --preprocess_submission=yes \ + --run-checker \ + --submitter=MLCommons \ + --tar=yes \ + --env.CM_TAR_OUTFILE=submission.tar.gz \ + --division=closed \ + --category=datacenter \ + --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ + --quiet ``` === "Open Edge" ### Open Edge Submission ```bash cm run script -tags=generate,inference,submission \ - --clean \ - --preprocess_submission=yes \ - --run-checker \ - --submitter=MLCommons \ - --tar=yes \ - --env.CM_TAR_OUTFILE=submission.tar.gz \ - --division=open \ - --category=edge \ - --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ - --quiet + --clean \ + --preprocess_submission=yes \ + --run-checker \ + --submitter=MLCommons \ + --tar=yes \ + --env.CM_TAR_OUTFILE=submission.tar.gz \ + --division=open \ + --category=edge \ + --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ + --quiet ``` === "Open Datacenter" ### Closed Datacenter Submission ```bash cm run script -tags=generate,inference,submission \ - --clean \ - --preprocess_submission=yes \ - --run-checker \ - --submitter=MLCommons \ - --tar=yes \ - --env.CM_TAR_OUTFILE=submission.tar.gz \ - --division=open \ - --category=datacenter \ - --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ - --quiet + --clean \ + --preprocess_submission=yes \ + --run-checker \ + --submitter=MLCommons \ + --tar=yes \ + --env.CM_TAR_OUTFILE=submission.tar.gz \ + --division=open \ + --category=datacenter \ + --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ + --quiet ``` * Use `--hw_name="My system name"` to give a meaningful system name. Examples can be seen [here](https://github.com/mlcommons/inference_results_v3.0/tree/main/open/cTuning/systems) From e9dcf17a08939bba1f3d639c2616a532dfd71a16 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 14 Aug 2024 16:08:31 +0530 Subject: [PATCH 086/169] Added submission categories --- docs/index.md | 19 ++++++++++++++----- docs/{official_gh_index.md => index_gh.md} | 0 2 files changed, 14 insertions(+), 5 deletions(-) rename docs/{official_gh_index.md => index_gh.md} (100%) diff --git a/docs/index.md b/docs/index.md index c77175e2f..c49c5a355 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,7 +1,7 @@ # MLPerf Inference Benchmarks ## Overview -This document provides details on various [MLPerf Inference Benchmarks](official_gh_index.md) categorized by tasks, models, and datasets. Each section lists the models performing similar tasks, with details on datasets, accuracy, and server latency constraints. +This document provides details on various [MLPerf Inference Benchmarks](index_gh.md) categorized by tasks, models, and datasets. Each section lists the models performing similar tasks, with details on datasets, accuracy, and server latency constraints. --- @@ -16,6 +16,7 @@ This document provides details on various [MLPerf Inference Benchmarks](official - **Equal Issue mode**: False - **Accuracy Variants**: None - **Precision**: fp32, int8 +- **Submission Category**: Data Center, Edge --- @@ -30,7 +31,8 @@ This document provides details on various [MLPerf Inference Benchmarks](official - CLIP: 32.68631873 ≤ CLIP ≤ 31.81331801 - **Equal Issue mode**: True - **Accuracy Variants**: None -- **Precision**: fp32, fp16 +- **Precision**: fp32 +- **Submission Category**: Data Center, Edge --- @@ -46,6 +48,7 @@ This document provides details on various [MLPerf Inference Benchmarks](official - **Accuracy Variants**: None - **Accuracy Variants**: 99% and 99.9% of reference model accuracy - **Precision**: fp32 +- **Submission Category**: Data Center, Edge --- @@ -60,6 +63,7 @@ This document provides details on various [MLPerf Inference Benchmarks](official - **Equal Issue mode**: True - **Accuracy Variants**: 99% and 99.9% of reference model accuracy - **Precision**: fp32 +- **Submission Category**: Data Center, Edge --- @@ -77,6 +81,7 @@ This document provides details on various [MLPerf Inference Benchmarks](official - **Equal Issue mode**: True - **Accuracy Variants**: 99% and 99.9% of reference model accuracy - **Precision**: fp32, int8 +- **Submission Category**: Data Center, Edge ### [LLAMA2-70B](benchmarks/language/llama2-70b.md) - **Dataset**: OpenORCA (GPT-4 split, max_seq_len=1024) @@ -94,6 +99,7 @@ This document provides details on various [MLPerf Inference Benchmarks](official - **Equal Issue mode**: True - **Accuracy Variants**: 99% and 99.9% of reference model accuracy - **Precision**: fp32 +- **Submission Category**: Data Center ### 5.2. Text Summarization @@ -111,6 +117,7 @@ This document provides details on various [MLPerf Inference Benchmarks](official - **Equal Issue mode**: True - **Accuracy Variants**: 99% and 99.9% of reference model accuracy - **Precision**: fp32 +- **Submission Category**: Data Center, Edge ### 5.3. Mixed Tasks (Question Answering, Math, and Code Generation) @@ -133,8 +140,9 @@ This document provides details on various [MLPerf Inference Benchmarks](official - TTFT: 2000ms - TPOT: 200ms - **Equal Issue mode**: True -- **Accuracy Variants**: None +- **Accuracy Variants**: 99% and of reference model - **Precision**: fp16 +- **Submission Category**: Data Center --- @@ -149,11 +157,12 @@ This document provides details on various [MLPerf Inference Benchmarks](official - **Equal Issue mode**: False - **Accuracy Variants**: 99% and 99.9% of reference model accuracy - **Precision**: fp32 +- **Submission Category**: Data Center --- -### Participation Categories -- **Datacenter Category**: All nine benchmarks can participate. +### Submission Categories +- **Datacenter Category**: All the benchmarks can participate. - **Edge Category**: All benchmarks except DLRMv2, LLAMA2, and Mixtral-8x7B can participate. ### High Accuracy Variants diff --git a/docs/official_gh_index.md b/docs/index_gh.md similarity index 100% rename from docs/official_gh_index.md rename to docs/index_gh.md From 4f56494dd8bbbc9395900667f7066ce95fd91b82 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 14 Aug 2024 19:21:10 +0530 Subject: [PATCH 087/169] update submission page - generate submission with or w/o using CM for benchmarking --- docs/submission/index.md | 47 +++++++++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/docs/submission/index.md b/docs/submission/index.md index ffd9371e7..6f6d82a39 100644 --- a/docs/submission/index.md +++ b/docs/submission/index.md @@ -3,14 +3,51 @@ hide: - toc --- -If you follow the `cm run` commands under the individual model pages in the [benchmarks](../index.md) directory, all the valid results will get aggregated to the `cm cache` folder. Once all the results across all the modelsare ready you can use the following command to generate a valid submission tree compliant with the [MLPerf requirements](https://github.com/mlcommons/policies/blob/master/submission_rules.adoc#inference-1). +=== "CM based benchmark" + If you have followed the `cm run` commands under the individual model pages in the [benchmarks](../index.md) directory, all the valid results will get aggregated to the `cm cache` folder. The following command could be used to browse the structure of inference results folder generated by CM. + ### Get results folder structure + ```bash + cm find cache --tags=get,mlperf,inference,results,dir | xargs tree + ``` +=== "Non CM based benchmark" + If you have not followed the `cm run` commands under the individual model pages in the [benchmarks](../index.md) directory, make sure that the result directory is structured in the following way. + ``` + └── SUT_Name + ├── cm_sut_info.json + ├── model_mapping.json + └── model + └── Scenario + └── loadgen_Mode + ├── file-1 + ├── ... + └── file-n + ``` + + !!! tip + + - The `cm_sut_info.json` should contain the following keys + - `system_name` + - `implementation` + - `device` + - `framework` + - `run_config` + - The `model_mapping.json` is used to map the custom model full name to the official model name. The format of json file is: + ``` + { + "custom_model_name_for_model1":"official_model_name_for_model1", + "custom_model_name_for_model2":"official_model_name_for_model2", + + } + ``` + +Once all the results across all the models are ready you can use the following command to generate a valid submission tree compliant with the [MLPerf requirements](https://github.com/mlcommons/policies/blob/master/submission_rules.adoc#inference-1). ## Generate actual submission tree === "Closed Edge" ### Closed Edge Submission ```bash - cm run script -tags=generate,inference,submission \ + cm run script --tags=generate,inference,submission \ --clean \ --preprocess_submission=yes \ --run-checker \ @@ -26,7 +63,7 @@ If you follow the `cm run` commands under the individual model pages in the [ben === "Closed Datacenter" ### Closed Datacenter Submission ```bash - cm run script -tags=generate,inference,submission \ + cm run script --tags=generate,inference,submission \ --clean \ --preprocess_submission=yes \ --run-checker \ @@ -41,7 +78,7 @@ If you follow the `cm run` commands under the individual model pages in the [ben === "Open Edge" ### Open Edge Submission ```bash - cm run script -tags=generate,inference,submission \ + cm run script --tags=generate,inference,submission \ --clean \ --preprocess_submission=yes \ --run-checker \ @@ -56,7 +93,7 @@ If you follow the `cm run` commands under the individual model pages in the [ben === "Open Datacenter" ### Closed Datacenter Submission ```bash - cm run script -tags=generate,inference,submission \ + cm run script --tags=generate,inference,submission \ --clean \ --preprocess_submission=yes \ --run-checker \ From f1135ead5ec149cc2c7badaf07ebc1381143e18a Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 16 Aug 2024 14:52:45 +0530 Subject: [PATCH 088/169] Updated kits dataset documentation --- docs/benchmarks/medical_imaging/get-3d-unet-data.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/benchmarks/medical_imaging/get-3d-unet-data.md b/docs/benchmarks/medical_imaging/get-3d-unet-data.md index e5bbfbab6..6c361f6f1 100644 --- a/docs/benchmarks/medical_imaging/get-3d-unet-data.md +++ b/docs/benchmarks/medical_imaging/get-3d-unet-data.md @@ -12,9 +12,14 @@ The benchmark implementation run command will automatically download the validat === "Validation" 3d-unet validation run uses the KiTS19 dataset performing [KiTS 2019](https://kits19.grand-challenge.org/) kidney tumor segmentation task - ### Get Validation Dataset + ### Get Validation Dataset(Original) ``` - cm run script --tags=get,dataset,kits19,validation -j + cm run script --tags=get,dataset,kits19,_validation -j + ``` + + ### Get Validation Dataset(Preprocessed) + ``` + cm run script --tags=get,dataset,kits19,preprocessed -j ``` ## Model From cb71cd167d50f6c53fa0603452c7bf2b72432a5c Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 16 Aug 2024 21:28:36 +0530 Subject: [PATCH 089/169] Updated model parameters --- docs/index.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/index.md b/docs/index.md index c49c5a355..7b787441e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -10,12 +10,12 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Dataset**: Imagenet-2012 (224x224) Validation - **Dataset Size**: 50,000 - **QSL Size**: 1,024 -- **Number of Parameters**: 25,636,712 +- **Number of Parameters**: 25.6 Million - **Reference Model Accuracy**: 76.46% ACC - **Server Scenario Latency Constraint**: 15ms - **Equal Issue mode**: False - **Accuracy Variants**: None -- **Precision**: fp32, int8 +- **Precision**: fp32 - **Submission Category**: Data Center, Edge --- @@ -25,11 +25,11 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Dataset**: Subset of Coco2014 - **Dataset Size**: 5,000 - **QSL Size**: 5,000 -- **Number of Parameters**: 890,000,000 +- **Number of Parameters**: 3.5 Billion - **Required Accuracy (Closed Division)**: - FID: 23.01085758 ≤ FID ≤ 23.95007626 - CLIP: 32.68631873 ≤ CLIP ≤ 31.81331801 -- **Equal Issue mode**: True +- **Equal Issue mode**: False - **Accuracy Variants**: None - **Precision**: fp32 - **Submission Category**: Data Center, Edge @@ -41,7 +41,7 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Dataset**: OpenImages - **Dataset Size**: 24,781 - **QSL Size**: 64 -- **Number of Parameters**: 50,000,000 +- **Number of Parameters**: TBD - **Reference Model Accuracy**: 0.3755 mAP - **Server Scenario Latency Constraint**: 100ms - **Equal Issue mode**: False @@ -57,7 +57,7 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Dataset**: KiTS2019 - **Dataset Size**: 42 - **QSL Size**: 42 -- **Number of Parameters**: 8,000,000 +- **Number of Parameters**: 19 Million - **Reference Model Accuracy**: 0.86330 Mean DICE Score - **Server Scenario**: Not Applicable - **Equal Issue mode**: True @@ -75,10 +75,10 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Dataset**: Squad v1.1 (384 Sequence Length) - **Dataset Size**: 10,833 - **QSL Size**: 10,833 -- **Number of Parameters**: 345,000,000 +- **Number of Parameters**: 340 Million - **Reference Model Accuracy**: F1 Score = 90.874% - **Server Scenario Latency Constraint**: 130ms -- **Equal Issue mode**: True +- **Equal Issue mode**: False - **Accuracy Variants**: 99% and 99.9% of reference model accuracy - **Precision**: fp32, int8 - **Submission Category**: Data Center, Edge @@ -87,7 +87,7 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Dataset**: OpenORCA (GPT-4 split, max_seq_len=1024) - **Dataset Size**: 24,576 - **QSL Size**: 24,576 -- **Number of Parameters**: 70,000,000,000 +- **Number of Parameters**: 70 Billion - **Reference Model Accuracy**: - Rouge1: 44.4312 - Rouge2: 22.0352 @@ -107,7 +107,7 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Dataset**: CNN Daily Mail v3.0.0 - **Dataset Size**: 13,368 - **QSL Size**: 13,368 -- **Number of Parameters**: 6,000,000,000 +- **Number of Parameters**: 6 Billion - **Reference Model Accuracy**: - Rouge1: 42.9865 - Rouge2: 20.1235 @@ -128,7 +128,7 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - MBXP (5k samples of the validation split, max_seq_len=2048) - **Dataset Size**: 15,000 - **QSL Size**: 15,000 -- **Number of Parameters**: 56,000,000,000 +- **Number of Parameters**: 47 Billion - **Reference Model Accuracy**: - Rouge1: 45.4911 - Rouge2: 23.2829 @@ -151,7 +151,7 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Dataset**: Synthetic Multihot Criteo - **Dataset Size**: 204,800 - **QSL Size**: 204,800 -- **Number of Parameters**: 1,000,000,000 +- **Number of Parameters**: TBD - **Reference Model Accuracy**: AUC = 80.31% - **Server Scenario Latency Constraint**: 60ms - **Equal Issue mode**: False From 1579967ff341e3d69920150380abf4ba8070a144 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 19 Aug 2024 15:19:29 +0530 Subject: [PATCH 090/169] updation of information --- docs/index.md | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/docs/index.md b/docs/index.md index 7b787441e..907c38ab2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,11 +11,11 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Dataset Size**: 50,000 - **QSL Size**: 1,024 - **Number of Parameters**: 25.6 Million +- **FLOPs**: TBD - **Reference Model Accuracy**: 76.46% ACC - **Server Scenario Latency Constraint**: 15ms - **Equal Issue mode**: False -- **Accuracy Variants**: None -- **Precision**: fp32 +- **Accuracy Variants**: 99% of fp32 reference model accuracy - **Submission Category**: Data Center, Edge --- @@ -25,13 +25,13 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Dataset**: Subset of Coco2014 - **Dataset Size**: 5,000 - **QSL Size**: 5,000 -- **Number of Parameters**: 3.5 Billion +- **Number of Parameters**: 3.5 Billion +- **FLOPs**: TBD - **Required Accuracy (Closed Division)**: - FID: 23.01085758 ≤ FID ≤ 23.95007626 - CLIP: 32.68631873 ≤ CLIP ≤ 31.81331801 - **Equal Issue mode**: False -- **Accuracy Variants**: None -- **Precision**: fp32 +- **Accuracy Variants**: 98% of fp32 reference model accuracy - **Submission Category**: Data Center, Edge --- @@ -42,12 +42,11 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Dataset Size**: 24,781 - **QSL Size**: 64 - **Number of Parameters**: TBD +- **FLOPs**: TBD - **Reference Model Accuracy**: 0.3755 mAP - **Server Scenario Latency Constraint**: 100ms - **Equal Issue mode**: False -- **Accuracy Variants**: None -- **Accuracy Variants**: 99% and 99.9% of reference model accuracy -- **Precision**: fp32 +- **Accuracy Variants**: 99% of fp32 reference model accuracy - **Submission Category**: Data Center, Edge --- @@ -58,11 +57,11 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Dataset Size**: 42 - **QSL Size**: 42 - **Number of Parameters**: 19 Million +- **FLOPs**: TBD - **Reference Model Accuracy**: 0.86330 Mean DICE Score - **Server Scenario**: Not Applicable - **Equal Issue mode**: True -- **Accuracy Variants**: 99% and 99.9% of reference model accuracy -- **Precision**: fp32 +- **Accuracy Variants**: 99% and 99.9% of fp32 reference model accuracy - **Submission Category**: Data Center, Edge --- @@ -76,11 +75,11 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Dataset Size**: 10,833 - **QSL Size**: 10,833 - **Number of Parameters**: 340 Million +- **FLOPs**: TBD - **Reference Model Accuracy**: F1 Score = 90.874% - **Server Scenario Latency Constraint**: 130ms - **Equal Issue mode**: False -- **Accuracy Variants**: 99% and 99.9% of reference model accuracy -- **Precision**: fp32, int8 +- **Accuracy Variants**: 99% and 99.9% of fp32 reference model accuracy - **Submission Category**: Data Center, Edge ### [LLAMA2-70B](benchmarks/language/llama2-70b.md) @@ -88,6 +87,7 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Dataset Size**: 24,576 - **QSL Size**: 24,576 - **Number of Parameters**: 70 Billion +- **FLOPs**: TBD - **Reference Model Accuracy**: - Rouge1: 44.4312 - Rouge2: 22.0352 @@ -97,8 +97,7 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - TTFT: 2000ms - TPOT: 200ms - **Equal Issue mode**: True -- **Accuracy Variants**: 99% and 99.9% of reference model accuracy -- **Precision**: fp32 +- **Accuracy Variants**: 99% and 99.9% of fp32 reference model accuracy - **Submission Category**: Data Center ### 5.2. Text Summarization @@ -108,6 +107,7 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Dataset Size**: 13,368 - **QSL Size**: 13,368 - **Number of Parameters**: 6 Billion +- **FLOPs**: TBD - **Reference Model Accuracy**: - Rouge1: 42.9865 - Rouge2: 20.1235 @@ -115,8 +115,7 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - Gen_len: 4,016,878 - **Server Scenario Latency Constraint**: 20s - **Equal Issue mode**: True -- **Accuracy Variants**: 99% and 99.9% of reference model accuracy -- **Precision**: fp32 +- **Accuracy Variants**: 99% and 99.9% of fp32 reference model accuracy - **Submission Category**: Data Center, Edge ### 5.3. Mixed Tasks (Question Answering, Math, and Code Generation) @@ -129,6 +128,7 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Dataset Size**: 15,000 - **QSL Size**: 15,000 - **Number of Parameters**: 47 Billion +- **FLOPs**: TBD - **Reference Model Accuracy**: - Rouge1: 45.4911 - Rouge2: 23.2829 @@ -140,8 +140,7 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - TTFT: 2000ms - TPOT: 200ms - **Equal Issue mode**: True -- **Accuracy Variants**: 99% and of reference model -- **Precision**: fp16 +- **Accuracy Variants**: 99% and of fp16 reference model - **Submission Category**: Data Center --- @@ -152,11 +151,11 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Dataset Size**: 204,800 - **QSL Size**: 204,800 - **Number of Parameters**: TBD +- **FLOPs**: TBD - **Reference Model Accuracy**: AUC = 80.31% - **Server Scenario Latency Constraint**: 60ms - **Equal Issue mode**: False -- **Accuracy Variants**: 99% and 99.9% of reference model accuracy -- **Precision**: fp32 +- **Accuracy Variants**: 99% and 99.9% of fp32 reference model accuracy - **Submission Category**: Data Center --- From 5ba36a9a9ef232c3de92c7bcecab721261588c61 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 19 Aug 2024 15:20:12 +0530 Subject: [PATCH 091/169] updated non cm based benchmark --- docs/submission/index.md | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/docs/submission/index.md b/docs/submission/index.md index 6f6d82a39..9e6b66044 100644 --- a/docs/submission/index.md +++ b/docs/submission/index.md @@ -10,11 +10,10 @@ hide: cm find cache --tags=get,mlperf,inference,results,dir | xargs tree ``` === "Non CM based benchmark" - If you have not followed the `cm run` commands under the individual model pages in the [benchmarks](../index.md) directory, make sure that the result directory is structured in the following way. + If you have not followed the `cm run` commands under the individual model pages in the [benchmarks](../index.md) directory, please make sure that the result directory is structured in the following way. ``` └── SUT_Name ├── cm_sut_info.json - ├── model_mapping.json └── model └── Scenario └── loadgen_Mode @@ -31,14 +30,20 @@ hide: - `device` - `framework` - `run_config` - - The `model_mapping.json` is used to map the custom model full name to the official model name. The format of json file is: - ``` - { - "custom_model_name_for_model1":"official_model_name_for_model1", - "custom_model_name_for_model2":"official_model_name_for_model2", - - } - ``` + +
    + Click here if you are submitting custom model in open division + + * The `model_mapping.json` should be included inside the SUT folder which is used to map the custom model full name to the official model name. The format of json file is: + + ``` + { + "custom_model_name_for_model1":"official_model_name_for_model1", + "custom_model_name_for_model2":"official_model_name_for_model2", + + } + ``` +
    Once all the results across all the models are ready you can use the following command to generate a valid submission tree compliant with the [MLPerf requirements](https://github.com/mlcommons/policies/blob/master/submission_rules.adoc#inference-1). From a46164678667ca7ccd8764cd7c80fdd35751e596 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 20 Aug 2024 11:09:31 +0530 Subject: [PATCH 092/169] added info about hf password --- docs/benchmarks/language/get-llama2-70b-data.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/benchmarks/language/get-llama2-70b-data.md b/docs/benchmarks/language/get-llama2-70b-data.md index 917f4705e..bdbb97b87 100644 --- a/docs/benchmarks/language/get-llama2-70b-data.md +++ b/docs/benchmarks/language/get-llama2-70b-data.md @@ -28,4 +28,8 @@ Get the Official MLPerf LLAMA2-70b Model ``` cm run script --tags=get,ml-model,llama2-70b,_pytorch -j ``` + +!!! tip + + Downloading llama2-70B model from Hugging Face will prompt you to enter the Hugging Face username and password. Please note that the password required is the **access token** generated for your account. Additionally, ensure that your account has access to the llama2-70B model. From dd47e451e59005aaf90bdf6f4c0d56da614568fe Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 20 Aug 2024 14:14:13 +0530 Subject: [PATCH 093/169] added links to model and access tokens --- docs/benchmarks/language/get-llama2-70b-data.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/benchmarks/language/get-llama2-70b-data.md b/docs/benchmarks/language/get-llama2-70b-data.md index bdbb97b87..0214d95a5 100644 --- a/docs/benchmarks/language/get-llama2-70b-data.md +++ b/docs/benchmarks/language/get-llama2-70b-data.md @@ -31,5 +31,5 @@ Get the Official MLPerf LLAMA2-70b Model !!! tip - Downloading llama2-70B model from Hugging Face will prompt you to enter the Hugging Face username and password. Please note that the password required is the **access token** generated for your account. Additionally, ensure that your account has access to the llama2-70B model. + Downloading llama2-70B model from Hugging Face will prompt you to enter the Hugging Face username and password. Please note that the password required is the [**access token**](https://huggingface.co/settings/tokens) generated for your account. Additionally, ensure that your account has access to the [llama2-70B](https://huggingface.co/meta-llama/Llama-2-70b-chat-hf) model. From a1d66d4ef66fbf15de8ef0fc7270d2dd06d3e52f Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 20 Aug 2024 17:06:04 +0530 Subject: [PATCH 094/169] Updated reference results structuree tree --- docs/submission/index.md | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/docs/submission/index.md b/docs/submission/index.md index 9e6b66044..4c892379b 100644 --- a/docs/submission/index.md +++ b/docs/submission/index.md @@ -12,20 +12,37 @@ hide: === "Non CM based benchmark" If you have not followed the `cm run` commands under the individual model pages in the [benchmarks](../index.md) directory, please make sure that the result directory is structured in the following way. ``` - └── SUT_Name + └── System description ID ├── cm_sut_info.json - └── model + └── Benchmark └── Scenario - └── loadgen_Mode - ├── file-1 - ├── ... - └── file-n + ├── Performance + | └── run_x/#1 run for all scenarios + | ├── mlperf_log_summary.txt + | └── mlperf_log_detail.txt + ├── Accuracy + | ├── mlperf_log_summary.txt + | ├── mlperf_log_detail.txt + | ├── mlperf_log_accuracy.json + | └── accuracy.txt + └── Test_ID + ├── Performance + | └── run_x/#1 run for all scenarios + | ├── mlperf_log_summary.txt + | └── mlperf_log_detail.txt + ├── Accuracy + | ├── baseline_accuracy.txt + | ├── compliance_accuracy.txt + | ├── mlperf_log_accuracy.json + | └── accuracy.txt + ├── verify_performance.txt + └── verify_accuracy.txt #for TEST01 only ``` !!! tip - The `cm_sut_info.json` should contain the following keys - - `system_name` + - `hardware_name` - `implementation` - `device` - `framework` From c5ae6ed9ae6877a99efc657a64664e426ba9e6f2 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 20 Aug 2024 18:23:20 +0530 Subject: [PATCH 095/169] submission docs cleanup --- docs/submission/index.md | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/docs/submission/index.md b/docs/submission/index.md index 4c892379b..a75bc3259 100644 --- a/docs/submission/index.md +++ b/docs/submission/index.md @@ -12,8 +12,8 @@ hide: === "Non CM based benchmark" If you have not followed the `cm run` commands under the individual model pages in the [benchmarks](../index.md) directory, please make sure that the result directory is structured in the following way. ``` - └── System description ID - ├── cm_sut_info.json + └── System description ID(SUT Name) + ├── system_meta.json └── Benchmark └── Scenario ├── Performance @@ -25,7 +25,7 @@ hide: | ├── mlperf_log_detail.txt | ├── mlperf_log_accuracy.json | └── accuracy.txt - └── Test_ID + └── Compliance_Test_ID ├── Performance | └── run_x/#1 run for all scenarios | ├── mlperf_log_summary.txt @@ -38,18 +38,9 @@ hide: ├── verify_performance.txt └── verify_accuracy.txt #for TEST01 only ``` - - !!! tip - - - The `cm_sut_info.json` should contain the following keys - - `hardware_name` - - `implementation` - - `device` - - `framework` - - `run_config`
    - Click here if you are submitting custom model in open division + Click here if you are submitting in open division * The `model_mapping.json` should be included inside the SUT folder which is used to map the custom model full name to the official model name. The format of json file is: From bc80f65edf70fc12c8ac1b0da98ebccf8b0db1a8 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 22 Aug 2024 18:10:16 +0100 Subject: [PATCH 096/169] Some cleanups for benchmark info --- docs/index.md | 81 +++++++++++++++++++++++++-------------------------- 1 file changed, 39 insertions(+), 42 deletions(-) diff --git a/docs/index.md b/docs/index.md index 907c38ab2..b408e8212 100644 --- a/docs/index.md +++ b/docs/index.md @@ -10,13 +10,13 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Dataset**: Imagenet-2012 (224x224) Validation - **Dataset Size**: 50,000 - **QSL Size**: 1,024 -- **Number of Parameters**: 25.6 Million -- **FLOPs**: TBD +- **Number of Parameters**: 25.6 million +- **FLOPs**: 3.8 billion - **Reference Model Accuracy**: 76.46% ACC - **Server Scenario Latency Constraint**: 15ms - **Equal Issue mode**: False -- **Accuracy Variants**: 99% of fp32 reference model accuracy -- **Submission Category**: Data Center, Edge +- **High accuracy variant**: No +- **Submission Category**: Datacenter, Edge --- @@ -25,14 +25,14 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Dataset**: Subset of Coco2014 - **Dataset Size**: 5,000 - **QSL Size**: 5,000 -- **Number of Parameters**: 3.5 Billion -- **FLOPs**: TBD +- **Number of Parameters**: 3.5 billion +- **FLOPs**: 1.28 - 2.4 trillion - **Required Accuracy (Closed Division)**: - FID: 23.01085758 ≤ FID ≤ 23.95007626 - CLIP: 32.68631873 ≤ CLIP ≤ 31.81331801 - **Equal Issue mode**: False -- **Accuracy Variants**: 98% of fp32 reference model accuracy -- **Submission Category**: Data Center, Edge +- **High accuracy variant**: No +- **Submission Category**: Datacenter, Edge --- @@ -42,27 +42,26 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Dataset Size**: 24,781 - **QSL Size**: 64 - **Number of Parameters**: TBD -- **FLOPs**: TBD -- **Reference Model Accuracy**: 0.3755 mAP +- **Reference Model Accuracy (fp32) **: 0.3755 mAP - **Server Scenario Latency Constraint**: 100ms - **Equal Issue mode**: False -- **Accuracy Variants**: 99% of fp32 reference model accuracy -- **Submission Category**: Data Center, Edge +- **High accuracy variant**: Yes +- **Submission Category**: Datacenter, Edge --- ## 4. Medical Image Segmentation -### [3d-unet](benchmarks/medical_imaging/3d-unet.md) +### [3d-unet](benchmarks/medical_imaging/3d-unet.md) - **Dataset**: KiTS2019 - **Dataset Size**: 42 - **QSL Size**: 42 -- **Number of Parameters**: 19 Million -- **FLOPs**: TBD -- **Reference Model Accuracy**: 0.86330 Mean DICE Score +- **Number of Parameters**: 32.5 million +- **FLOPs**: 100-300 billion +- **Reference Model Accuracy (fp32) **: 0.86330 Mean DICE Score - **Server Scenario**: Not Applicable - **Equal Issue mode**: True -- **Accuracy Variants**: 99% and 99.9% of fp32 reference model accuracy -- **Submission Category**: Data Center, Edge +- **High accuracy variant**: Yes +- **Submission Category**: Datacenter, Edge --- @@ -74,21 +73,21 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Dataset**: Squad v1.1 (384 Sequence Length) - **Dataset Size**: 10,833 - **QSL Size**: 10,833 -- **Number of Parameters**: 340 Million -- **FLOPs**: TBD -- **Reference Model Accuracy**: F1 Score = 90.874% +- **Number of Parameters**: 340 million +- **FLOPs**: ~128 billion +- **Reference Model Accuracy (fp32) **: F1 Score = 90.874% - **Server Scenario Latency Constraint**: 130ms - **Equal Issue mode**: False -- **Accuracy Variants**: 99% and 99.9% of fp32 reference model accuracy -- **Submission Category**: Data Center, Edge +- **High accuracy variant**: yes +- **Submission Category**: Datacenter, Edge ### [LLAMA2-70B](benchmarks/language/llama2-70b.md) - **Dataset**: OpenORCA (GPT-4 split, max_seq_len=1024) - **Dataset Size**: 24,576 - **QSL Size**: 24,576 -- **Number of Parameters**: 70 Billion -- **FLOPs**: TBD -- **Reference Model Accuracy**: +- **Number of Parameters**: 70 billion +- **FLOPs**: ~500 trillion +- **Reference Model Accuracy (fp16) **: - Rouge1: 44.4312 - Rouge2: 22.0352 - RougeL: 28.6162 @@ -97,8 +96,8 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - TTFT: 2000ms - TPOT: 200ms - **Equal Issue mode**: True -- **Accuracy Variants**: 99% and 99.9% of fp32 reference model accuracy -- **Submission Category**: Data Center +- **High accuracy variant**: Yes +- **Submission Category**: Datacenter ### 5.2. Text Summarization @@ -106,17 +105,17 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Dataset**: CNN Daily Mail v3.0.0 - **Dataset Size**: 13,368 - **QSL Size**: 13,368 -- **Number of Parameters**: 6 Billion -- **FLOPs**: TBD -- **Reference Model Accuracy**: +- **Number of Parameters**: 6 billion +- **FLOPs**: ~148 billion +- **Reference Model Accuracy (fp32) **: - Rouge1: 42.9865 - Rouge2: 20.1235 - RougeL: 29.9881 - Gen_len: 4,016,878 - **Server Scenario Latency Constraint**: 20s - **Equal Issue mode**: True -- **Accuracy Variants**: 99% and 99.9% of fp32 reference model accuracy -- **Submission Category**: Data Center, Edge +- **High accuracy variant**: Yes +- **Submission Category**: Datacenter, Edge ### 5.3. Mixed Tasks (Question Answering, Math, and Code Generation) @@ -127,9 +126,8 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - MBXP (5k samples of the validation split, max_seq_len=2048) - **Dataset Size**: 15,000 - **QSL Size**: 15,000 -- **Number of Parameters**: 47 Billion -- **FLOPs**: TBD -- **Reference Model Accuracy**: +- **Number of Parameters**: 47 billion +- **Reference Model Accuracy (fp16) **: - Rouge1: 45.4911 - Rouge2: 23.2829 - RougeL: 30.3615 @@ -140,8 +138,8 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - TTFT: 2000ms - TPOT: 200ms - **Equal Issue mode**: True -- **Accuracy Variants**: 99% and of fp16 reference model -- **Submission Category**: Data Center +- **High accuracy variant**: Yes +- **Submission Category**: Datacenter --- @@ -150,13 +148,12 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Dataset**: Synthetic Multihot Criteo - **Dataset Size**: 204,800 - **QSL Size**: 204,800 -- **Number of Parameters**: TBD -- **FLOPs**: TBD +- **Number of Parameters**: ~23 billion - **Reference Model Accuracy**: AUC = 80.31% - **Server Scenario Latency Constraint**: 60ms - **Equal Issue mode**: False -- **Accuracy Variants**: 99% and 99.9% of fp32 reference model accuracy -- **Submission Category**: Data Center +- **High accuracy variant**: Yes +- **Submission Category**: Datacenter --- From 65e63db60ddb8c99cee7b41375a1f064cc5eb7e2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 22 Aug 2024 18:14:59 +0100 Subject: [PATCH 097/169] Some cleanups for benchmark info --- docs/index.md | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/docs/index.md b/docs/index.md index b408e8212..4c8530197 100644 --- a/docs/index.md +++ b/docs/index.md @@ -88,10 +88,7 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Number of Parameters**: 70 billion - **FLOPs**: ~500 trillion - **Reference Model Accuracy (fp16) **: - - Rouge1: 44.4312 - - Rouge2: 22.0352 - - RougeL: 28.6162 - - Tokens_per_sample: 294.45 + - Rouge1: 44.4312, Rouge2: 22.0352, RougeL: 28.6162, Tokens_per_sample: 294.45 - **Server Scenario Latency Constraint**: - TTFT: 2000ms - TPOT: 200ms @@ -108,10 +105,7 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Number of Parameters**: 6 billion - **FLOPs**: ~148 billion - **Reference Model Accuracy (fp32) **: - - Rouge1: 42.9865 - - Rouge2: 20.1235 - - RougeL: 29.9881 - - Gen_len: 4,016,878 + - Rouge1: 42.9865, Rouge2: 20.1235, RougeL: 29.9881, Gen_len: 4,016,878 - **Server Scenario Latency Constraint**: 20s - **Equal Issue mode**: True - **High accuracy variant**: Yes @@ -128,9 +122,7 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **QSL Size**: 15,000 - **Number of Parameters**: 47 billion - **Reference Model Accuracy (fp16) **: - - Rouge1: 45.4911 - - Rouge2: 23.2829 - - RougeL: 30.3615 + - Rouge1: 45.4911, Rouge2: 23.2829, RougeL: 30.3615 - GSM8K Accuracy: 73.78% - MBXP Accuracy: 60.12% - Tokens_per_sample: 294.45 From ba9820d4f1152b4ee0975b118539ec21a472616f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 22 Aug 2024 18:18:46 +0100 Subject: [PATCH 098/169] Some cleanups for benchmark info --- docs/index.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/index.md b/docs/index.md index 4c8530197..aed0438d7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -150,9 +150,9 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh --- ### Submission Categories -- **Datacenter Category**: All the benchmarks can participate. -- **Edge Category**: All benchmarks except DLRMv2, LLAMA2, and Mixtral-8x7B can participate. +- **Datacenter Category**: All the current inference benchmarks are applicable to the datacenter category. +- **Edge Category**: All benchmarks except DLRMv2, LLAMA2-70B, and Mixtral-8x7B are applicable to the edge category. ### High Accuracy Variants -- **Benchmarks**: `bert`, `llama2-70b`, `dlrm_v2`, and `3d-unet` +- **Benchmarks**: `bert`, `llama2-70b`, `gpt-j`, `dlrm_v2`, and `3d-unet` have a normal accuracy variant as well as a high accuracy variant. - **Requirement**: Must achieve at least 99.9% of the reference model accuracy, compared to the default 99% accuracy requirement. From 9ea1d14092b78a284494eaac025c63d94ee5ab96 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 23 Aug 2024 00:05:45 +0530 Subject: [PATCH 099/169] added generic stubs deepsparse --- main.py | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/main.py b/main.py index 4705ead83..73690b1ec 100644 --- a/main.py +++ b/main.py @@ -24,7 +24,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): elif model.lower() == "retinanet": frameworks = [ "Onnxruntime", "Pytorch" ] elif "bert" in model.lower(): - frameworks = [ "Pytorch" ] + frameworks = [ "Pytorch", "deepsparse" ] else: frameworks = [ "Pytorch" ] @@ -157,7 +157,28 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): run_suffix += f"{cur_space3} Please click here to see more options for the RUN command\n\n" run_suffix += f"{cur_space3}* Use `--division=closed` to do a closed division submission which includes compliance runs\n\n" run_suffix += f"{cur_space3}* Use `--rerun` to do a rerun even when a valid run exists\n" - run_suffix += f"{cur_space3}
    \n" + run_suffix += f"{cur_space3}
    \n\n" + + if "bert" in model.lower() and framework == "deepsparse": + run_suffix += f"{cur_space3}
    \n" + run_suffix += f"{cur_space3} Please click here for generic model stubs for bert deepsparse\n\n" + run_suffix += f"{cur_space3}* zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/pruned95_quant-none-vnni\n\n" + run_suffix += f"{cur_space3}* zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/14layer_pruned50_quant-none-vnni\n\n" + run_suffix += f"{cur_space3}* zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/base_quant-none\n\n" + run_suffix += f"{cur_space3}* zoo:nlp/question_answering/bert-base/pytorch/huggingface/squad/pruned95_obs_quant-none\n\n" + run_suffix += f"{cur_space3}* zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/14layer_pruned50-none-vnni\n\n" + run_suffix += f"{cur_space3}* zoo:nlp/question_answering/obert-base/pytorch/huggingface/squad/pruned90-none\n\n" + run_suffix += f"{cur_space3}* zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/pruned97_quant-none\n\n" + run_suffix += f"{cur_space3}* zoo:nlp/question_answering/bert-base/pytorch/huggingface/squad/pruned90-none\n\n" + run_suffix += f"{cur_space3}* zoo:nlp/question_answering/bert-large/pytorch/huggingface/squad/pruned80_quant-none-vnni\n\n" + run_suffix += f"{cur_space3}* zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/pruned95-none-vnni\n\n" + run_suffix += f"{cur_space3}* zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/pruned97-none\n\n" + run_suffix += f"{cur_space3}* zoo:nlp/question_answering/bert-large/pytorch/huggingface/squad/base-none\n\n" + run_suffix += f"{cur_space3}* zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/base-none\n\n" + run_suffix += f"{cur_space3}* zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/base-none\n" + run_suffix += f"{cur_space3}
    \n" + + for scenario in scenarios: content += f"{cur_space3}=== \"{scenario}\"\n{cur_space4}###### {scenario}\n\n" @@ -287,7 +308,9 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ docker_cmd_suffix = f" \\\n{pre_space} --docker --quiet" docker_cmd_suffix += f" \\\n{pre_space} --test_query_count={test_query_count}" - if "llama2-70b" in model: + if "bert" in model.lower() and framework == "deepsparse": + docker_cmd_suffix += f"\\\n{pre_space} --env.CM_MLPERF_NEURALMAGIC_MODEL_ZOO_STUB=zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/base_quant-none" + if "llama2-70b" in model.lower(): if implementation == "nvidia": docker_cmd_suffix += f" \\\n{pre_space} --tp_size=2" docker_cmd_suffix += f" \\\n{pre_space} --nvidia_llama2_dataset_file_path=" @@ -295,7 +318,7 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ docker_cmd_suffix += f" \\\n{pre_space} --api_server=http://localhost:8000" docker_cmd_suffix += f" \\\n{pre_space} --vllm_model_name=nm-testing/Llama-2-70b-chat-hf-FP8" - if "dlrm-v2" in model and implementation == "nvidia": + if "dlrm-v2" in model.lower() and implementation == "nvidia": docker_cmd_suffix += f" \\\n{pre_space} --criteo_day23_raw_data_path=" docker_setup_cmd = f"""\n @@ -317,7 +340,9 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ if execution_mode == "test": cmd_suffix += f" \\\n {pre_space} --test_query_count={test_query_count}" - if "llama2-70b" in model: + if "bert" in model.lower() and framework == "deepsparse": + cmd_suffix += f"\\\n{pre_space} --env.CM_MLPERF_NEURALMAGIC_MODEL_ZOO_STUB=zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/base_quant-none" + if "llama2-70b" in model.lower(): if implementation == "nvidia": cmd_suffix += f" \\\n{pre_space} --tp_size=" cmd_suffix += f" \\\n{pre_space} --nvidia_llama2_dataset_file_path=" From 63888bcf4f7398ee69a1a984e61fe2e26b7c6f95 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 22 Aug 2024 19:44:51 +0100 Subject: [PATCH 100/169] Some cleanups for benchmark info --- docs/index.md | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/docs/index.md b/docs/index.md index aed0438d7..289ea509a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,11 +1,11 @@ # MLPerf Inference Benchmarks ## Overview -This document provides details on various [MLPerf Inference Benchmarks](index_gh.md) categorized by tasks, models, and datasets. Each section lists the models performing similar tasks, with details on datasets, accuracy, and server latency constraints. +The currently valid [MLPerf Inference Benchmarks](index_gh.md) as of MLPerf inference v4.0 round are listed below, categorized by tasks. Under each model you can find its details like the dataset used, accuracy, server latency constraints etc. --- -## 1. Image Classification +## Image Classification ### [ResNet50-v1.5](benchmarks/image_classification/resnet50.md) - **Dataset**: Imagenet-2012 (224x224) Validation - **Dataset Size**: 50,000 @@ -20,7 +20,7 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh --- -## 2. Text to Image +## Text to Image ### [Stable Diffusion](benchmarks/text_to_image/sdxl.md) - **Dataset**: Subset of Coco2014 - **Dataset Size**: 5,000 @@ -28,15 +28,15 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **Number of Parameters**: 3.5 billion - **FLOPs**: 1.28 - 2.4 trillion - **Required Accuracy (Closed Division)**: - - FID: 23.01085758 ≤ FID ≤ 23.95007626 - - CLIP: 32.68631873 ≤ CLIP ≤ 31.81331801 + - FID: 23.01085758 ≤ FID ≤ 23.95007626 + - CLIP: 32.68631873 ≤ CLIP ≤ 31.81331801 - **Equal Issue mode**: False - **High accuracy variant**: No - **Submission Category**: Datacenter, Edge --- -## 3. Object Detection +## Object Detection ### [Retinanet](benchmarks/object_detection/retinanet.md) - **Dataset**: OpenImages - **Dataset Size**: 24,781 @@ -50,7 +50,7 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh --- -## 4. Medical Image Segmentation +## Medical Image Segmentation ### [3d-unet](benchmarks/medical_imaging/3d-unet.md) - **Dataset**: KiTS2019 - **Dataset Size**: 42 @@ -65,11 +65,11 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh --- -## 5. Language Tasks +## Language Tasks -### 5.1. Question Answering +### Question Answering -### [Bert-Large](benchmarks/language/bert.md) +#### [Bert-Large](benchmarks/language/bert.md) - **Dataset**: Squad v1.1 (384 Sequence Length) - **Dataset Size**: 10,833 - **QSL Size**: 10,833 @@ -81,7 +81,7 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **High accuracy variant**: yes - **Submission Category**: Datacenter, Edge -### [LLAMA2-70B](benchmarks/language/llama2-70b.md) +#### [LLAMA2-70B](benchmarks/language/llama2-70b.md) - **Dataset**: OpenORCA (GPT-4 split, max_seq_len=1024) - **Dataset Size**: 24,576 - **QSL Size**: 24,576 @@ -96,9 +96,9 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **High accuracy variant**: Yes - **Submission Category**: Datacenter -### 5.2. Text Summarization +### Text Summarization -### [GPT-J](benchmarks/language/gpt-j.md) +#### [GPT-J](benchmarks/language/gpt-j.md) - **Dataset**: CNN Daily Mail v3.0.0 - **Dataset Size**: 13,368 - **QSL Size**: 13,368 @@ -111,13 +111,13 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh - **High accuracy variant**: Yes - **Submission Category**: Datacenter, Edge -### 5.3. Mixed Tasks (Question Answering, Math, and Code Generation) +### Mixed Tasks (Question Answering, Math, and Code Generation) -### [Mixtral-8x7B](benchmarks/language/mixtral-8x7b.md) +#### [Mixtral-8x7B](benchmarks/language/mixtral-8x7b.md) - **Datasets**: - - OpenORCA (5k samples of GPT-4 split, max_seq_len=2048) - - GSM8K (5k samples of the validation split, max_seq_len=2048) - - MBXP (5k samples of the validation split, max_seq_len=2048) + - OpenORCA (5k samples of GPT-4 split, max_seq_len=2048) + - GSM8K (5k samples of the validation split, max_seq_len=2048) + - MBXP (5k samples of the validation split, max_seq_len=2048) - **Dataset Size**: 15,000 - **QSL Size**: 15,000 - **Number of Parameters**: 47 billion @@ -135,8 +135,8 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh --- -## 6. Recommendation -### [DLRMv2](benchmarks/recommendation/dlrm-v2.md) +## Recommendation +### [DLRM_v2](benchmarks/recommendation/dlrm-v2.md) - **Dataset**: Synthetic Multihot Criteo - **Dataset Size**: 204,800 - **QSL Size**: 204,800 @@ -149,10 +149,10 @@ This document provides details on various [MLPerf Inference Benchmarks](index_gh --- -### Submission Categories +## Submission Categories - **Datacenter Category**: All the current inference benchmarks are applicable to the datacenter category. - **Edge Category**: All benchmarks except DLRMv2, LLAMA2-70B, and Mixtral-8x7B are applicable to the edge category. -### High Accuracy Variants +## High Accuracy Variants - **Benchmarks**: `bert`, `llama2-70b`, `gpt-j`, `dlrm_v2`, and `3d-unet` have a normal accuracy variant as well as a high accuracy variant. - **Requirement**: Must achieve at least 99.9% of the reference model accuracy, compared to the default 99% accuracy requirement. From b956c6d09697ce8e4174219445b47bb6b9e493ee Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 22 Aug 2024 19:51:37 +0100 Subject: [PATCH 101/169] Some cleanups for benchmark info --- docs/index.md | 59 +++++++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/docs/index.md b/docs/index.md index 289ea509a..580a4336d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,8 +8,8 @@ The currently valid [MLPerf Inference Benchmarks](index_gh.md) as of MLPerf infe ## Image Classification ### [ResNet50-v1.5](benchmarks/image_classification/resnet50.md) - **Dataset**: Imagenet-2012 (224x224) Validation - - **Dataset Size**: 50,000 - - **QSL Size**: 1,024 + - **Dataset Size**: 50,000 + - **QSL Size**: 1,024 - **Number of Parameters**: 25.6 million - **FLOPs**: 3.8 billion - **Reference Model Accuracy**: 76.46% ACC @@ -23,8 +23,8 @@ The currently valid [MLPerf Inference Benchmarks](index_gh.md) as of MLPerf infe ## Text to Image ### [Stable Diffusion](benchmarks/text_to_image/sdxl.md) - **Dataset**: Subset of Coco2014 - - **Dataset Size**: 5,000 - - **QSL Size**: 5,000 + - **Dataset Size**: 5,000 + - **QSL Size**: 5,000 - **Number of Parameters**: 3.5 billion - **FLOPs**: 1.28 - 2.4 trillion - **Required Accuracy (Closed Division)**: @@ -39,8 +39,8 @@ The currently valid [MLPerf Inference Benchmarks](index_gh.md) as of MLPerf infe ## Object Detection ### [Retinanet](benchmarks/object_detection/retinanet.md) - **Dataset**: OpenImages - - **Dataset Size**: 24,781 - - **QSL Size**: 64 + - **Dataset Size**: 24,781 + - **QSL Size**: 64 - **Number of Parameters**: TBD - **Reference Model Accuracy (fp32) **: 0.3755 mAP - **Server Scenario Latency Constraint**: 100ms @@ -53,8 +53,8 @@ The currently valid [MLPerf Inference Benchmarks](index_gh.md) as of MLPerf infe ## Medical Image Segmentation ### [3d-unet](benchmarks/medical_imaging/3d-unet.md) - **Dataset**: KiTS2019 - - **Dataset Size**: 42 - - **QSL Size**: 42 + - **Dataset Size**: 42 + - **QSL Size**: 42 - **Number of Parameters**: 32.5 million - **FLOPs**: 100-300 billion - **Reference Model Accuracy (fp32) **: 0.86330 Mean DICE Score @@ -71,8 +71,8 @@ The currently valid [MLPerf Inference Benchmarks](index_gh.md) as of MLPerf infe #### [Bert-Large](benchmarks/language/bert.md) - **Dataset**: Squad v1.1 (384 Sequence Length) - - **Dataset Size**: 10,833 - - **QSL Size**: 10,833 + - **Dataset Size**: 10,833 + - **QSL Size**: 10,833 - **Number of Parameters**: 340 million - **FLOPs**: ~128 billion - **Reference Model Accuracy (fp32) **: F1 Score = 90.874% @@ -83,15 +83,18 @@ The currently valid [MLPerf Inference Benchmarks](index_gh.md) as of MLPerf infe #### [LLAMA2-70B](benchmarks/language/llama2-70b.md) - **Dataset**: OpenORCA (GPT-4 split, max_seq_len=1024) - - **Dataset Size**: 24,576 - - **QSL Size**: 24,576 + - **Dataset Size**: 24,576 + - **QSL Size**: 24,576 - **Number of Parameters**: 70 billion - **FLOPs**: ~500 trillion - **Reference Model Accuracy (fp16) **: - - Rouge1: 44.4312, Rouge2: 22.0352, RougeL: 28.6162, Tokens_per_sample: 294.45 + - Rouge1: 44.4312 + - Rouge2: 22.0352 + - RougeL: 28.6162 + - Tokens_per_sample: 294.45 - **Server Scenario Latency Constraint**: - - TTFT: 2000ms - - TPOT: 200ms + - TTFT: 2000ms + - TPOT: 200ms - **Equal Issue mode**: True - **High accuracy variant**: Yes - **Submission Category**: Datacenter @@ -100,12 +103,15 @@ The currently valid [MLPerf Inference Benchmarks](index_gh.md) as of MLPerf infe #### [GPT-J](benchmarks/language/gpt-j.md) - **Dataset**: CNN Daily Mail v3.0.0 - - **Dataset Size**: 13,368 - - **QSL Size**: 13,368 + - **Dataset Size**: 13,368 + - **QSL Size**: 13,368 - **Number of Parameters**: 6 billion - **FLOPs**: ~148 billion - **Reference Model Accuracy (fp32) **: - - Rouge1: 42.9865, Rouge2: 20.1235, RougeL: 29.9881, Gen_len: 4,016,878 + - Rouge1: 42.9865 + - Rouge2: 20.1235 + - RougeL: 29.9881 + - Gen_len: 4,016,878 - **Server Scenario Latency Constraint**: 20s - **Equal Issue mode**: True - **High accuracy variant**: Yes @@ -118,17 +124,20 @@ The currently valid [MLPerf Inference Benchmarks](index_gh.md) as of MLPerf infe - OpenORCA (5k samples of GPT-4 split, max_seq_len=2048) - GSM8K (5k samples of the validation split, max_seq_len=2048) - MBXP (5k samples of the validation split, max_seq_len=2048) - - **Dataset Size**: 15,000 - - **QSL Size**: 15,000 + - **Dataset Size**: 15,000 + - **QSL Size**: 15,000 - **Number of Parameters**: 47 billion - **Reference Model Accuracy (fp16) **: - - Rouge1: 45.4911, Rouge2: 23.2829, RougeL: 30.3615 - - GSM8K Accuracy: 73.78% - - MBXP Accuracy: 60.12% + - OpenORCA + - Rouge1: 45.4911 + - Rouge2: 23.2829 + - RougeL: 30.3615 + - GSM8K Accuracy: 73.78% + - MBXP Accuracy: 60.12% - Tokens_per_sample: 294.45 - **Server Scenario Latency Constraint**: - - TTFT: 2000ms - - TPOT: 200ms + - TTFT: 2000ms + - TPOT: 200ms - **Equal Issue mode**: True - **High accuracy variant**: Yes - **Submission Category**: Datacenter From 2c493340316b6f2cd30d4840bbd3093d5a674a79 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 22 Aug 2024 19:52:15 +0100 Subject: [PATCH 102/169] Some cleanups for benchmark info --- docs/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.md b/docs/index.md index 580a4336d..f5473afa7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -147,8 +147,8 @@ The currently valid [MLPerf Inference Benchmarks](index_gh.md) as of MLPerf infe ## Recommendation ### [DLRM_v2](benchmarks/recommendation/dlrm-v2.md) - **Dataset**: Synthetic Multihot Criteo - - **Dataset Size**: 204,800 - - **QSL Size**: 204,800 + - **Dataset Size**: 204,800 + - **QSL Size**: 204,800 - **Number of Parameters**: ~23 billion - **Reference Model Accuracy**: AUC = 80.31% - **Server Scenario Latency Constraint**: 60ms From 13db0f80be51bfca95dd5198773a16eae2ca1833 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 22 Aug 2024 20:47:34 +0100 Subject: [PATCH 103/169] Some cleanups for benchmark info (FID and CLIP data added) --- docs/index.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/index.md b/docs/index.md index f5473afa7..11f2a52c2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,7 +1,7 @@ # MLPerf Inference Benchmarks ## Overview -The currently valid [MLPerf Inference Benchmarks](index_gh.md) as of MLPerf inference v4.0 round are listed below, categorized by tasks. Under each model you can find its details like the dataset used, accuracy, server latency constraints etc. +The currently valid [MLPerf Inference Benchmarks](index_gh.md) as of MLPerf inference v4.0 round are listed below, categorized by tasks. Under each model you can find its details like the dataset used, reference accuracy, server latency constraints etc. --- @@ -27,9 +27,10 @@ The currently valid [MLPerf Inference Benchmarks](index_gh.md) as of MLPerf infe - **QSL Size**: 5,000 - **Number of Parameters**: 3.5 billion - **FLOPs**: 1.28 - 2.4 trillion +- **Reference Model Accuracy (fp32)**: CLIP: 31.74981837, FID: 23.48046692 - **Required Accuracy (Closed Division)**: - - FID: 23.01085758 ≤ FID ≤ 23.95007626 - - CLIP: 32.68631873 ≤ CLIP ≤ 31.81331801 + - CLIP: 31.68631873 ≤ CLIP ≤ 31.81331801 (within 0.2% of the reference model CLIP score) + - FID: 23.01085758 ≤ FID ≤ 23.95007626 (within 2% of the reference model FID score) - **Equal Issue mode**: False - **High accuracy variant**: No - **Submission Category**: Datacenter, Edge @@ -87,7 +88,7 @@ The currently valid [MLPerf Inference Benchmarks](index_gh.md) as of MLPerf infe - **QSL Size**: 24,576 - **Number of Parameters**: 70 billion - **FLOPs**: ~500 trillion -- **Reference Model Accuracy (fp16) **: +- **Reference Model Accuracy (fp32) **: - Rouge1: 44.4312 - Rouge2: 22.0352 - RougeL: 28.6162 From 4eefc940af31f2e4353448725d115a5829235f8b Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 23 Aug 2024 16:54:39 +0530 Subject: [PATCH 104/169] typo fix for bert deepsparse framework --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 73690b1ec..958f6d90b 100644 --- a/main.py +++ b/main.py @@ -24,7 +24,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): elif model.lower() == "retinanet": frameworks = [ "Onnxruntime", "Pytorch" ] elif "bert" in model.lower(): - frameworks = [ "Pytorch", "deepsparse" ] + frameworks = [ "Pytorch", "Deepsparse" ] else: frameworks = [ "Pytorch" ] From e6abadd56d29005a622cbd5428f97ff10aa8bbd4 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Fri, 23 Aug 2024 18:34:22 +0530 Subject: [PATCH 105/169] added min system requirements for models --- main.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/main.py b/main.py index 958f6d90b..d8f4ba847 100644 --- a/main.py +++ b/main.py @@ -104,6 +104,9 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): content += f"{cur_space1}=== \"{device}\"\n" content += f"{cur_space2}##### {device} device\n\n" + # minimum system requirements + content += get_min_system_requirements(cur_space2, model, implementation, device) + # to select the execution environments(currently Docker and Native) for execution_env in execution_envs: if (device == "ROCm" or implementation == "qualcomm") and execution_env == "Docker": @@ -213,6 +216,55 @@ def get_test_query_count(model, implementation, device, num_devices=1): p_range *= num_devices return p_range + + def get_min_system_requirements(spaces, model, implementation, device): + model = model.lower() + min_sys_req_content = "" + min_sys_req_content += f"{spaces}
    \n" + min_sys_req_content += f"{spaces}Please click here to see the minimum system requirements for running the benchmark\n\n" + # device memory + if device.lower() == "cuda" and (implementation.lower() == "nvidia" or implementation.lower() == "reference"): + if implementation.lower() == "nvidia": + if "dlrm" in model: + device_memory = "24GB" + elif "llama2-70b" in model or "mixtral" in model: + device_memory = "80GB" + elif "sdxl" in model or "gptj" in model: + device_memory = "16GB" + else: + device_memory = "8GB" + elif implementation.lower() == "reference": + if "dlrm" in model: + device_memory = "2x80GB" + elif "llama2-70b" in model: + device_memory = "8x80GB" + elif "mixtral" in model: + device_memory = "4x80GB" + elif "sdxl" in model: + device_memory = "24GB(fp32), 16GB(fp16)" + elif "gptj" in model: + device_memory = "80GB(fp32). 40GB(fp16)" + else: + device_memory = "8GB" + min_sys_req_content += f"{spaces}* **Device Memory**: {device_memory}\n\n" + # disk space + if "dlrm" in model: + disk_space = "500GB" + elif "llama2-70b" in model: + disk_space = "700GB" + elif "mixtral" in model: + disk_space = "100GB" + elif "retinanet" in model: + disk_space = "200GB" + else: + disk_space = "50GB" + min_sys_req_content += f"{spaces}* **Disk Space**: {disk_space}\n\n" + # System memory + if "dlrm" in model: + system_memory = "512GB" + min_sys_req_content += f"{spaces}* **System Memory(RAM+SWAP)**: {system_memory}\n\n" + min_sys_req_content += f"{spaces}
    \n" + return min_sys_req_content def get_readme_prefix(spaces, model, implementation): readme_prefix = "" From d994a863d793114d2018082d4208319eabe5e39f Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 3 Sep 2024 15:44:18 +0530 Subject: [PATCH 106/169] fixed code version --- main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) mode change 100644 => 100755 main.py diff --git a/main.py b/main.py old mode 100644 new mode 100755 index d8f4ba847..367bf7178 --- a/main.py +++ b/main.py @@ -12,7 +12,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): content="" scenarios = [] execution_envs = ["Docker","Native"] - code_version="r4.1" + code_version="r4.1-dev" if model == "rnnt": code_version="r4.0" @@ -140,7 +140,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): content += f"{cur_space3}The above command should get you to an interactive shell inside the docker container and do a quick test run for the Offline scenario. Once inside the docker container please do the below commands to do the accuracy + performance runs for each scenario.\n\n" content += f"{cur_space3}
    \n" content += f"{cur_space3} Please click here to see more options for the docker launch \n\n" - content += f"{cur_space3}* `--docker_cm_repo `: to use a custom fork of cm4mlops repository inside the docker image\n\n" + content += f"{cur_space3}* `--docker_cm_repo=`: to use a custom fork of cm4mlops repository inside the docker image\n\n" content += f"{cur_space3}* `--docker_cache=no`: to not use docker cache during the image build\n" if device.lower() not in [ "cuda" ]: @@ -337,7 +337,7 @@ def get_run_cmd_extra(f_pre_space, model, implementation, device, scenario, scen return extra_content @env.macro - def mlperf_inference_run_command(spaces, model, implementation, framework, category, scenario, device="cpu", execution_mode="test", test_query_count="20", docker=False, scenarios = [], code_version="r4.1"): + def mlperf_inference_run_command(spaces, model, implementation, framework, category, scenario, device="cpu", execution_mode="test", test_query_count="20", docker=False, scenarios = [], code_version="r4.1-dev"): pre_space = "" for i in range(1,spaces): pre_space = pre_space + " " From fd8945ce66f69b15291e39f63d0ba5feb1550380 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 3 Sep 2024 16:31:49 +0530 Subject: [PATCH 107/169] changes for displaying reference and intel implementation tip --- docs/benchmarks/language/bert.md | 9 +-------- docs/benchmarks/language/gpt-j.md | 8 +------- docs/benchmarks/language/llama2-70b.md | 6 ------ docs/benchmarks/language/mixtral-8x7b.md | 1 - docs/benchmarks/medical_imaging/3d-unet.md | 6 ------ docs/benchmarks/recommendation/dlrm-v2.md | 8 +------- main.py | 12 ++++++++++++ 7 files changed, 15 insertions(+), 35 deletions(-) diff --git a/docs/benchmarks/language/bert.md b/docs/benchmarks/language/bert.md index cb9a955f1..57b1cf224 100644 --- a/docs/benchmarks/language/bert.md +++ b/docs/benchmarks/language/bert.md @@ -8,34 +8,27 @@ hide: === "MLCommons-Python" ## MLPerf Reference Implementation in Python - BERT-99 {{ mlperf_inference_implementation_readme (4, "bert-99", "reference") }} - BERT-99.9 {{ mlperf_inference_implementation_readme (4, "bert-99.9", "reference") }} === "Nvidia" ## Nvidia MLPerf Implementation - BERT-99 {{ mlperf_inference_implementation_readme (4, "bert-99", "nvidia") }} - BERT-99.9 {{ mlperf_inference_implementation_readme (4, "bert-99.9", "nvidia") }} === "Intel" ## Intel MLPerf Implementation - BERT-99 + {{ mlperf_inference_implementation_readme (4, "bert-99", "intel") }} - BERT-99.9 {{ mlperf_inference_implementation_readme (4, "bert-99.9", "intel") }} === "Qualcomm" ## Qualcomm AI100 MLPerf Implementation - BERT-99 {{ mlperf_inference_implementation_readme (4, "bert-99", "qualcomm") }} - BERT-99.9 {{ mlperf_inference_implementation_readme (4, "bert-99.9", "qualcomm") }} diff --git a/docs/benchmarks/language/gpt-j.md b/docs/benchmarks/language/gpt-j.md index 125c87ef8..4dcb3d70e 100644 --- a/docs/benchmarks/language/gpt-j.md +++ b/docs/benchmarks/language/gpt-j.md @@ -8,29 +8,24 @@ hide: === "MLCommons-Python" ## MLPerf Reference Implementation in Python - - GPT-J-99 + {{ mlperf_inference_implementation_readme (4, "gptj-99", "reference") }} - GPTJ-99.9 {{ mlperf_inference_implementation_readme (4, "gptj-99.9", "reference") }} === "Nvidia" ## Nvidia MLPerf Implementation - GPTJ-99 {{ mlperf_inference_implementation_readme (4, "gptj-99", "nvidia") }} - GPTJ-99.9 {{ mlperf_inference_implementation_readme (4, "gptj-99.9", "nvidia") }} === "Intel" ## Intel MLPerf Implementation - GPTJ-99 {{ mlperf_inference_implementation_readme (4, "gptj-99", "intel") }} @@ -38,7 +33,6 @@ hide: === "Qualcomm" ## Qualcomm AI100 MLPerf Implementation - GPTJ-99 {{ mlperf_inference_implementation_readme (4, "gptj-99", "qualcomm") }} diff --git a/docs/benchmarks/language/llama2-70b.md b/docs/benchmarks/language/llama2-70b.md index 87b637b42..0d9a0504d 100644 --- a/docs/benchmarks/language/llama2-70b.md +++ b/docs/benchmarks/language/llama2-70b.md @@ -9,26 +9,20 @@ hide: === "MLCommons-Python" ## MLPerf Reference Implementation in Python - LLAMA2-70b-99 {{ mlperf_inference_implementation_readme (4, "llama2-70b-99", "reference") }} - LLAMA2-70b-99.9 {{ mlperf_inference_implementation_readme (4, "llama2-70b-99.9", "reference") }} === "Nvidia" ## Nvidia MLPerf Implementation - LLAMA2-70b-99 {{ mlperf_inference_implementation_readme (4, "llama2-70b-99", "nvidia") }} - LLAMA2-70b-99.9 {{ mlperf_inference_implementation_readme (4, "llama2-70b-99.9", "nvidia") }} === "Neural Magic" ## Neural Magic MLPerf Implementation - LLAMA2-70b-99 {{ mlperf_inference_implementation_readme (4, "llama2-70b-99", "neuralmagic") }} - LLAMA2-70b-99.9 {{ mlperf_inference_implementation_readme (4, "llama2-70b-99.9", "neuralmagic") }} \ No newline at end of file diff --git a/docs/benchmarks/language/mixtral-8x7b.md b/docs/benchmarks/language/mixtral-8x7b.md index f4e84e8cf..9f3bf2992 100644 --- a/docs/benchmarks/language/mixtral-8x7b.md +++ b/docs/benchmarks/language/mixtral-8x7b.md @@ -6,5 +6,4 @@ hide: === "MLCommons-Python" ## MLPerf Reference Implementation in Python - MIXTRAL-8x7b {{ mlperf_inference_implementation_readme (4, "mixtral-8x7b", "reference") }} \ No newline at end of file diff --git a/docs/benchmarks/medical_imaging/3d-unet.md b/docs/benchmarks/medical_imaging/3d-unet.md index 8d5017bf5..01a54c63e 100644 --- a/docs/benchmarks/medical_imaging/3d-unet.md +++ b/docs/benchmarks/medical_imaging/3d-unet.md @@ -9,30 +9,24 @@ hide: === "MLCommons-Python" ## MLPerf Reference Implementation in Python - 3d-unet-99 {{ mlperf_inference_implementation_readme (4, "3d-unet-99", "reference") }} - 3d-unet-99.9 {{ mlperf_inference_implementation_readme (4, "3d-unet-99.9", "reference") }} === "Nvidia" ## Nvidia MLPerf Implementation - 3d-unet-99 {{ mlperf_inference_implementation_readme (4, "3d-unet-99", "nvidia") }} - 3d-unet-99.9 {{ mlperf_inference_implementation_readme (4, "3d-unet-99.9", "nvidia") }} === "Intel" ## Intel MLPerf Implementation - 3d-unet-99 {{ mlperf_inference_implementation_readme (4, "3d-unet-99", "intel") }} - 3d-unet-99.9 {{ mlperf_inference_implementation_readme (4, "3d-unet-99.9", "intel") }} diff --git a/docs/benchmarks/recommendation/dlrm-v2.md b/docs/benchmarks/recommendation/dlrm-v2.md index d1b41deb1..ce3081077 100644 --- a/docs/benchmarks/recommendation/dlrm-v2.md +++ b/docs/benchmarks/recommendation/dlrm-v2.md @@ -10,26 +10,20 @@ hide: === "MLCommons-Python" ## MLPerf Reference Implementation in Python - DLRM-v2-99 {{ mlperf_inference_implementation_readme (4, "dlrm-v2-99", "reference") }} - DLRM-v2-99.9 {{ mlperf_inference_implementation_readme (4, "dlrm-v2-99.9", "reference") }} === "Nvidia" ## Nvidia MLPerf Implementation - - DLRM-v2-99 + {{ mlperf_inference_implementation_readme (4, "dlrm-v2-99", "nvidia") }} - DLRM-v2-99.9 {{ mlperf_inference_implementation_readme (4, "dlrm-v2-99.9", "nvidia") }} === "Intel" ## Intel MLPerf Implementation - DLRM-v2-99 {{ mlperf_inference_implementation_readme (4, "dlrm-v2-99", "intel") }} - DLRM-v2-99.9 {{ mlperf_inference_implementation_readme (4, "dlrm-v2-99.9", "intel") }} \ No newline at end of file diff --git a/main.py b/main.py index 367bf7178..19f2793d9 100755 --- a/main.py +++ b/main.py @@ -18,6 +18,11 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): code_version="r4.0" if implementation == "reference": + # Tip + if "99.9" not in model: + content += f"\n{pre_space}!!! tip\n\n" + content += f"{pre_space} - MLCommons reference implementations are only meant to provide a rules compliant reference implementation for the submitters and in most cases are not best performing. If you want to benchmark any system, it is advisable to use the vendor MLPerf implementation for that system like Nvidia, Intel etc.\n\n" + devices = [ "CPU", "CUDA", "ROCm" ] if model.lower() == "resnet50": frameworks = [ "Onnxruntime", "Tensorflow", "Deepsparse" ] @@ -39,6 +44,11 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): frameworks = [ "pytorch" ] elif implementation == "intel": + # Tip + if "99.9" not in model: + content += f"\n{pre_space}!!! tip\n\n" + content += f"{pre_space} - Intel MLPerf inference implementation is available only for datacenter category and has been tested only on a limited number of systems. Most of the benchmarks using Intel implementation require at least Intel Sapphire Rapids or higher CPU generation.\n\n" + if model not in [ "bert-99", "bert-99.9", "gptj-99", "gptj-99.9", "resnet50", "retinanet", "3d-unet-99", "3d-unet-99.9", "dlrm-v2-99", "dlrm-v2-99.9", "sdxl" ]: return pre_space+" WIP" if model in [ "bert-99", "bert-99.9", "retinanet", "3d-unet-99", "3d-unet-99.9" ]: @@ -72,6 +82,8 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): else: categories = [ "Edge", "Datacenter" ] + # model name + content += f"{pre_space}{model.upper()}\n\n" for category in categories: if category == "Edge" and not scenarios: scenarios = [ "Offline", "SingleStream" ] From 8815065200982f4786ab957bcae53bfd0108a950 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 3 Sep 2024 16:46:36 +0530 Subject: [PATCH 108/169] added reference to installation page --- main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main.py b/main.py index 19f2793d9..77f10560d 100755 --- a/main.py +++ b/main.py @@ -127,6 +127,8 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): continue # Nvidia implementation only supports execution through docker content += f"{cur_space2}=== \"{execution_env}\"\n" content += f"{cur_space3}###### {execution_env} Environment\n\n" + # ref to cm installation + content += f"{cur_space3}Please refer to the [installation page](../../install/index.md) to install CM for running the automated benchmark commands.\n\n" test_query_count=get_test_query_count(model, implementation, device) if "99.9" not in model: #not showing docker command as it is already done for the 99% variant From 8085e8b150829c73516545f901330fc7a3019b24 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 3 Sep 2024 18:33:06 +0530 Subject: [PATCH 109/169] updated neural magic documentation --- main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main.py b/main.py index 77f10560d..c9e3e1b56 100755 --- a/main.py +++ b/main.py @@ -383,6 +383,7 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ elif implementation == "neuralmagic": docker_cmd_suffix += f" \\\n{pre_space} --api_server=http://localhost:8000" docker_cmd_suffix += f" \\\n{pre_space} --vllm_model_name=nm-testing/Llama-2-70b-chat-hf-FP8" + docker_cmd_suffix += f" \\\n{pre_space} --adr.mlperf-implementation.tags=_repo.https://github.com/neuralmagic/inference,_branch.vllm" if "dlrm-v2" in model.lower() and implementation == "nvidia": docker_cmd_suffix += f" \\\n{pre_space} --criteo_day23_raw_data_path=" @@ -415,6 +416,7 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ elif implementation == "neuralmagic": cmd_suffix += f" \\\n{pre_space} --api_server=http://localhost:8000" cmd_suffix += f" \\\n{pre_space} --vllm_model_name=nm-testing/Llama-2-70b-chat-hf-FP8" + cmd_suffix += f" \\\n{pre_space} --adr.mlperf-implementation.tags=_repo.https://github.com/neuralmagic/inference,_branch.vllm" if "dlrm-v2" in model and implementation == "nvidia": cmd_suffix += f" \\\n{pre_space} --criteo_day23_raw_data_path=" From 9a271059cdbdf44f9a1941b1f419dba4bf464af2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 4 Sep 2024 15:14:54 +0100 Subject: [PATCH 110/169] Added links to the install page, redirect benchmarks page --- .../image_classification/mobilenets.md | 2 ++ .../image_classification/resnet50.md | 3 +++ docs/benchmarks/language/bert.md | 2 ++ docs/benchmarks/language/gpt-j.md | 1 + docs/benchmarks/language/llama2-70b.md | 3 ++- docs/benchmarks/language/mixtral-8x7b.md | 6 ++++- docs/benchmarks/medical_imaging/3d-unet.md | 1 + docs/benchmarks/object_detection/retinanet.md | 2 ++ docs/benchmarks/recommendation/dlrm-v2.md | 4 +-- docs/install/index.md | 26 +++++++++---------- docs/requirements.txt | 1 + mkdocs.yml | 9 ++++--- 12 files changed, 40 insertions(+), 20 deletions(-) diff --git a/docs/benchmarks/image_classification/mobilenets.md b/docs/benchmarks/image_classification/mobilenets.md index f276008ef..5b96a78e2 100644 --- a/docs/benchmarks/image_classification/mobilenets.md +++ b/docs/benchmarks/image_classification/mobilenets.md @@ -5,6 +5,8 @@ hide: # Image Classification using Mobilenet models +Install CM following the [installation page](/install). + Mobilenet models are not official MLPerf models and so cannot be used for a Closed division MLPerf inference submission. But since they can be run with Imagenet dataset, we are allowed to use them for Open division submission. Only CPU runs are supported now. ## TFLite Backend diff --git a/docs/benchmarks/image_classification/resnet50.md b/docs/benchmarks/image_classification/resnet50.md index 62b966e0d..8fec36a64 100644 --- a/docs/benchmarks/image_classification/resnet50.md +++ b/docs/benchmarks/image_classification/resnet50.md @@ -3,8 +3,11 @@ hide: - toc --- + # Image Classification using ResNet50 +Install CM following the [installation page](/install). + === "MLCommons-Python" ## MLPerf Reference Implementation in Python diff --git a/docs/benchmarks/language/bert.md b/docs/benchmarks/language/bert.md index 57b1cf224..7c76543e8 100644 --- a/docs/benchmarks/language/bert.md +++ b/docs/benchmarks/language/bert.md @@ -5,6 +5,8 @@ hide: # Question Answering using Bert-Large +Install CM following the [installation page](/install). + === "MLCommons-Python" ## MLPerf Reference Implementation in Python diff --git a/docs/benchmarks/language/gpt-j.md b/docs/benchmarks/language/gpt-j.md index 4dcb3d70e..3ca1157a6 100644 --- a/docs/benchmarks/language/gpt-j.md +++ b/docs/benchmarks/language/gpt-j.md @@ -5,6 +5,7 @@ hide: # Text Summarization using GPT-J +Install CM following the [installation page](/install). === "MLCommons-Python" ## MLPerf Reference Implementation in Python diff --git a/docs/benchmarks/language/llama2-70b.md b/docs/benchmarks/language/llama2-70b.md index 0d9a0504d..726a6dd8a 100644 --- a/docs/benchmarks/language/llama2-70b.md +++ b/docs/benchmarks/language/llama2-70b.md @@ -5,6 +5,7 @@ hide: # Text Summarization using LLAMA2-70b +Install CM following the [installation page](/install). === "MLCommons-Python" ## MLPerf Reference Implementation in Python @@ -25,4 +26,4 @@ hide: {{ mlperf_inference_implementation_readme (4, "llama2-70b-99", "neuralmagic") }} -{{ mlperf_inference_implementation_readme (4, "llama2-70b-99.9", "neuralmagic") }} \ No newline at end of file +{{ mlperf_inference_implementation_readme (4, "llama2-70b-99.9", "neuralmagic") }} diff --git a/docs/benchmarks/language/mixtral-8x7b.md b/docs/benchmarks/language/mixtral-8x7b.md index 9f3bf2992..c66dfdff0 100644 --- a/docs/benchmarks/language/mixtral-8x7b.md +++ b/docs/benchmarks/language/mixtral-8x7b.md @@ -3,7 +3,11 @@ hide: - toc --- +# Question Answering, Math, and Code Generation using Mixtral-8x7B + +Install CM following the [installation page](/install). + === "MLCommons-Python" ## MLPerf Reference Implementation in Python -{{ mlperf_inference_implementation_readme (4, "mixtral-8x7b", "reference") }} \ No newline at end of file +{{ mlperf_inference_implementation_readme (4, "mixtral-8x7b", "reference") }} diff --git a/docs/benchmarks/medical_imaging/3d-unet.md b/docs/benchmarks/medical_imaging/3d-unet.md index 01a54c63e..43495bc47 100644 --- a/docs/benchmarks/medical_imaging/3d-unet.md +++ b/docs/benchmarks/medical_imaging/3d-unet.md @@ -5,6 +5,7 @@ hide: # Medical Imaging using 3d-unet (KiTS 2019 kidney tumor segmentation task) +Install CM following the [installation page](/install). === "MLCommons-Python" ## MLPerf Reference Implementation in Python diff --git a/docs/benchmarks/object_detection/retinanet.md b/docs/benchmarks/object_detection/retinanet.md index 699d92050..47504481b 100644 --- a/docs/benchmarks/object_detection/retinanet.md +++ b/docs/benchmarks/object_detection/retinanet.md @@ -5,6 +5,8 @@ hide: # Object Detection using Retinanet +Install CM following the [installation page](/install). + === "MLCommons-Python" ## MLPerf Reference Implementation in Python diff --git a/docs/benchmarks/recommendation/dlrm-v2.md b/docs/benchmarks/recommendation/dlrm-v2.md index ce3081077..c50c69417 100644 --- a/docs/benchmarks/recommendation/dlrm-v2.md +++ b/docs/benchmarks/recommendation/dlrm-v2.md @@ -5,8 +5,8 @@ hide: # Recommendation using DLRM v2 +Install CM following the [installation page](/install). -## Benchmark Implementations === "MLCommons-Python" ## MLPerf Reference Implementation in Python @@ -26,4 +26,4 @@ hide: {{ mlperf_inference_implementation_readme (4, "dlrm-v2-99", "intel") }} -{{ mlperf_inference_implementation_readme (4, "dlrm-v2-99.9", "intel") }} \ No newline at end of file +{{ mlperf_inference_implementation_readme (4, "dlrm-v2-99.9", "intel") }} diff --git a/docs/install/index.md b/docs/install/index.md index 60377adee..195521c7e 100644 --- a/docs/install/index.md +++ b/docs/install/index.md @@ -8,24 +8,24 @@ We use MLCommons CM Automation framework to run MLPerf inference benchmarks. CM needs `git`, `python3-pip` and `python3-venv` installed on your system. If any of these are absent, please follow the [official CM installation page](https://docs.mlcommons.org/ck/install) to install them. Once the dependencies are installed, do the following -## Activate a VENV for CM +## Activate a Virtual ENV for CM +This step is not mandatory as CM can use separate virtual environment for MLPerf inference. But the latest `pip` install requires this or else will need the `--break-system-packages` flag while installing `cm4mlops`. + ```bash python3 -m venv cm source cm/bin/activate ``` ## Install CM and pulls any needed repositories - -```bash - pip install cm4mlops -``` - -## To work on custom GitHub repo and branch - -```bash - pip install cmind && cm init --quiet --repo=mlcommons@cm4mlops --branch=mlperf-inference -``` - -Here, repo is in the format `githubUsername@githubRepo`. +=== "Use the default fork of CM MLOps repository" + ```bash + pip install cm4mlops + ``` + +=== "Use custom fork/branch of the CM MLOps repository" + ```bash + pip install cmind && cm init --quiet --repo=mlcommons@cm4mlops --branch=mlperf-inference + ``` + Here, `repo` is in the format `githubUsername@githubRepo`. Now, you are ready to use the `cm` commands to run MLPerf inference as given in the [benchmarks](../index.md) page diff --git a/docs/requirements.txt b/docs/requirements.txt index 39fab4e1f..35176cae0 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,3 +2,4 @@ mkdocs-material swagger-markdown mkdocs-macros-plugin ruamel.yaml +mkdocs-redirects diff --git a/mkdocs.yml b/mkdocs.yml index 8e59acf63..ceebf74cb 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -19,9 +19,7 @@ theme: - navigation.top - toc.follow nav: - - Install: - - install/index.md - - Benchmarks: + - Home: - index.md - Image Classification: - ResNet50: benchmarks/image_classification/resnet50.md @@ -38,6 +36,8 @@ nav: - MIXTRAL-8x7B: benchmarks/language/mixtral-8x7b.md - Recommendation: - DLRM-v2: benchmarks/recommendation/dlrm-v2.md + - Install CM: + - install/index.md - Submission: - Submission Generation: submission/index.md - Release Notes: @@ -62,3 +62,6 @@ markdown_extensions: plugins: - search - macros + - redirects: + redirect_maps: + 'benchmarks/index.md': 'index.md' From 8d76337815f33303fbcb52942fe8a7c164579bec Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 10 Sep 2024 16:55:33 +0530 Subject: [PATCH 111/169] added tips about batch size and dataset for nvidia llama2 --- main.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index c9e3e1b56..9759f1b4c 100755 --- a/main.py +++ b/main.py @@ -316,9 +316,15 @@ def get_docker_info(spaces, model, implementation, device): pre_space = pre_space + " " pre_space += " " #pre_space = " " + info += f"\n{pre_space}!!! tip\n\n" + info+= f"{pre_space} - Batch size could be adjusted using `--batch_size=#`, where `#` is the desired batch size.\n\n" + info+= f"{pre_space} - If batch size is not specified, CM scans for accelerator name and sets a predetermined batch size for known accelerators. For example, the batch size for BERT on an RTX 4090 can be found [here](https://github.com/anandhu-eng/cm4mlops/blob/54af88fb64d898758e2f0edfd79bb6e5aed9c38a/script/app-mlperf-inference-nvidia/_cm.yaml#L1381).\n\n" + info+= f"{pre_space} - If batch size is not specifieid and the accelerator name is unknown, CM sets default value to 1.\n\n" if implementation == "nvidia": - info += f"\n{pre_space}!!! tip\n\n" - info+= f"{pre_space} If ran with `--all_models=yes`, all the benchmark models of NVIDIA implementation could be run within the same container.\n\n" + info+= f"{pre_space} - when run with `--all_models=yes`, all the benchmark models of NVIDIA implementation can be executed within the same container.\n\n" + if "llama2" in model.lower(): + info+= f"{pre_space} - The dataset for NVIDIA's implementation of Llama2 is not publicly available. The user must fill [this](https://docs.google.com/forms/d/e/1FAIpQLSc_8VIvRmXM3I8KQaYnKf7gy27Z63BBoI_I1u02f4lw6rBp3g/viewform?pli=1&fbzx=-8842630989397184967) form and be verified as a MLCommons member to access the dataset.\n\n" + info+= f"{pre_space} - The path to the downloaded pickle file should be placed in tag `nvidia_llama2_dataset_file_path`.\n\n" return info def get_readme_suffix(spaces, model, implementation): From e1992210aa9be74b4d2a781380e73ae2841b1074 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 10 Sep 2024 17:40:15 +0530 Subject: [PATCH 112/169] fix conditions logic --- main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/main.py b/main.py index 9759f1b4c..d3ec7d106 100755 --- a/main.py +++ b/main.py @@ -320,11 +320,11 @@ def get_docker_info(spaces, model, implementation, device): info+= f"{pre_space} - Batch size could be adjusted using `--batch_size=#`, where `#` is the desired batch size.\n\n" info+= f"{pre_space} - If batch size is not specified, CM scans for accelerator name and sets a predetermined batch size for known accelerators. For example, the batch size for BERT on an RTX 4090 can be found [here](https://github.com/anandhu-eng/cm4mlops/blob/54af88fb64d898758e2f0edfd79bb6e5aed9c38a/script/app-mlperf-inference-nvidia/_cm.yaml#L1381).\n\n" info+= f"{pre_space} - If batch size is not specifieid and the accelerator name is unknown, CM sets default value to 1.\n\n" - if implementation == "nvidia": + if implementation.lower() == "nvidia": info+= f"{pre_space} - when run with `--all_models=yes`, all the benchmark models of NVIDIA implementation can be executed within the same container.\n\n" - if "llama2" in model.lower(): - info+= f"{pre_space} - The dataset for NVIDIA's implementation of Llama2 is not publicly available. The user must fill [this](https://docs.google.com/forms/d/e/1FAIpQLSc_8VIvRmXM3I8KQaYnKf7gy27Z63BBoI_I1u02f4lw6rBp3g/viewform?pli=1&fbzx=-8842630989397184967) form and be verified as a MLCommons member to access the dataset.\n\n" - info+= f"{pre_space} - The path to the downloaded pickle file should be placed in tag `nvidia_llama2_dataset_file_path`.\n\n" + if "llama2" in model.lower(): + info+= f"{pre_space} - The dataset for NVIDIA's implementation of Llama2 is not publicly available. The user must fill [this](https://docs.google.com/forms/d/e/1FAIpQLSc_8VIvRmXM3I8KQaYnKf7gy27Z63BBoI_I1u02f4lw6rBp3g/viewform?pli=1&fbzx=-8842630989397184967) form and be verified as a MLCommons member to access the dataset.\n\n" + info+= f"{pre_space} - `PATH_TO_PICKE_FILE` should be replaced with path to the downloaded pickle file.\n\n" return info def get_readme_suffix(spaces, model, implementation): From 439b150e154f0888c8d870a73bcc45b34b2f84ab Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 10 Sep 2024 20:17:10 +0530 Subject: [PATCH 113/169] modified tips and additional run cmds --- main.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/main.py b/main.py index d3ec7d106..96ee8bc3f 100755 --- a/main.py +++ b/main.py @@ -157,6 +157,9 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): content += f"{cur_space3}* `--docker_cm_repo=`: to use a custom fork of cm4mlops repository inside the docker image\n\n" content += f"{cur_space3}* `--docker_cache=no`: to not use docker cache during the image build\n" + if implementation.lower() == "nvidia": + content += f"{cur_space3}* `--gpu_name=` : The GPU's with configured batch sizes in CM are `orin`, `rtx_4090`, `rtx_a6000`, `rtx_6000_ada`, `l4`, `t4`and `a100`.\n" + if device.lower() not in [ "cuda" ]: content += f"{cur_space3}* `--docker_os=ubuntu`: ubuntu and rhel are supported. \n" content += f"{cur_space3}* `--docker_os_version=20.04`: [20.04, 22.04] are supported for Ubuntu and [8, 9] for RHEL\n" @@ -174,6 +177,8 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): run_suffix += f"{cur_space3} Please click here to see more options for the RUN command\n\n" run_suffix += f"{cur_space3}* Use `--division=closed` to do a closed division submission which includes compliance runs\n\n" run_suffix += f"{cur_space3}* Use `--rerun` to do a rerun even when a valid run exists\n" + if implementation.lower() == "nvidia": + run_suffix += f"{cur_space3}* `--gpu_name=` : The GPU's with configured batch sizes in CM are `orin`, `rtx_4090`, `rtx_a6000`, `rtx_6000_ada`, `l4`, `t4`and `a100`.\n" run_suffix += f"{cur_space3}
    \n\n" if "bert" in model.lower() and framework == "deepsparse": @@ -318,10 +323,9 @@ def get_docker_info(spaces, model, implementation, device): #pre_space = " " info += f"\n{pre_space}!!! tip\n\n" info+= f"{pre_space} - Batch size could be adjusted using `--batch_size=#`, where `#` is the desired batch size.\n\n" - info+= f"{pre_space} - If batch size is not specified, CM scans for accelerator name and sets a predetermined batch size for known accelerators. For example, the batch size for BERT on an RTX 4090 can be found [here](https://github.com/anandhu-eng/cm4mlops/blob/54af88fb64d898758e2f0edfd79bb6e5aed9c38a/script/app-mlperf-inference-nvidia/_cm.yaml#L1381).\n\n" - info+= f"{pre_space} - If batch size is not specifieid and the accelerator name is unknown, CM sets default value to 1.\n\n" if implementation.lower() == "nvidia": - info+= f"{pre_space} - when run with `--all_models=yes`, all the benchmark models of NVIDIA implementation can be executed within the same container.\n\n" + info+= f"{pre_space} - Default batch size [is assigned](https://github.com/mlcommons/cm4mlops/blob/dd0c35856969c68945524d5c80414c615f5fe42c/script/app-mlperf-inference-nvidia/_cm.yaml#L1129) as per GPU memory or [specified GPU](https://github.com/mlcommons/cm4mlops/blob/dd0c35856969c68945524d5c80414c615f5fe42c/script/app-mlperf-inference-nvidia/_cm.yaml#L1370). Please click more option for *docker launch* or *run command* to see how to specify GPU name.\n\n" + info+= f"{pre_space} - When run with `--all_models=yes`, all the benchmark models of NVIDIA implementation can be executed within the same container.\n\n" if "llama2" in model.lower(): info+= f"{pre_space} - The dataset for NVIDIA's implementation of Llama2 is not publicly available. The user must fill [this](https://docs.google.com/forms/d/e/1FAIpQLSc_8VIvRmXM3I8KQaYnKf7gy27Z63BBoI_I1u02f4lw6rBp3g/viewform?pli=1&fbzx=-8842630989397184967) form and be verified as a MLCommons member to access the dataset.\n\n" info+= f"{pre_space} - `PATH_TO_PICKE_FILE` should be replaced with path to the downloaded pickle file.\n\n" @@ -347,7 +351,6 @@ def get_run_cmd_extra(f_pre_space, model, implementation, device, scenario, scen f_pre_space += "" if scenario == "Server" or (scenario == "All Scenarios" and "Server" in scenarios): extra_content += f"{f_pre_space} * `` must be determined manually. It is usually around 80% of the Offline QPS, but on some systems, it can drop below 50%. If a higher value is specified, the latency constraint will not be met, and the run will be considered invalid.\n" - if "gptj" in model and device == "cuda" and implementation == "reference": extra_content += f"{f_pre_space} * `--precision=[float16|bfloat16]` can help run on GPUs with less RAM \n" extra_content += f"{f_pre_space} * `--beam-size=1` Beam size of 4 is mandatory for a closed division submission but reducing the beam size can help in running the model on GPUs with lower device memory\n" From b8188c25602c54b6abb62cd7292958ff0951fc69 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 10 Sep 2024 20:30:54 +0530 Subject: [PATCH 114/169] sentence corrections --- main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 96ee8bc3f..7ba058750 100755 --- a/main.py +++ b/main.py @@ -158,7 +158,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): content += f"{cur_space3}* `--docker_cache=no`: to not use docker cache during the image build\n" if implementation.lower() == "nvidia": - content += f"{cur_space3}* `--gpu_name=` : The GPU's with configured batch sizes in CM are `orin`, `rtx_4090`, `rtx_a6000`, `rtx_6000_ada`, `l4`, `t4`and `a100`.\n" + content += f"{cur_space3}* `--gpu_name=` : The GPUs with configured batch sizes in CM are `orin`, `rtx_4090`, `rtx_a6000`, `rtx_6000_ada`, `l4`, `t4`and `a100`.\n" if device.lower() not in [ "cuda" ]: content += f"{cur_space3}* `--docker_os=ubuntu`: ubuntu and rhel are supported. \n" @@ -324,7 +324,7 @@ def get_docker_info(spaces, model, implementation, device): info += f"\n{pre_space}!!! tip\n\n" info+= f"{pre_space} - Batch size could be adjusted using `--batch_size=#`, where `#` is the desired batch size.\n\n" if implementation.lower() == "nvidia": - info+= f"{pre_space} - Default batch size [is assigned](https://github.com/mlcommons/cm4mlops/blob/dd0c35856969c68945524d5c80414c615f5fe42c/script/app-mlperf-inference-nvidia/_cm.yaml#L1129) as per GPU memory or [specified GPU](https://github.com/mlcommons/cm4mlops/blob/dd0c35856969c68945524d5c80414c615f5fe42c/script/app-mlperf-inference-nvidia/_cm.yaml#L1370). Please click more option for *docker launch* or *run command* to see how to specify GPU name.\n\n" + info+= f"{pre_space} - Default batch size is assigned based on either [GPU memory](https://github.com/mlcommons/cm4mlops/blob/dd0c35856969c68945524d5c80414c615f5fe42c/script/app-mlperf-inference-nvidia/_cm.yaml#L1129) or the [specified GPU](https://github.com/mlcommons/cm4mlops/blob/dd0c35856969c68945524d5c80414c615f5fe42c/script/app-mlperf-inference-nvidia/_cm.yaml#L1370). Please click more option for *docker launch* or *run command* to see how to specify the GPU name.\n\n" info+= f"{pre_space} - When run with `--all_models=yes`, all the benchmark models of NVIDIA implementation can be executed within the same container.\n\n" if "llama2" in model.lower(): info+= f"{pre_space} - The dataset for NVIDIA's implementation of Llama2 is not publicly available. The user must fill [this](https://docs.google.com/forms/d/e/1FAIpQLSc_8VIvRmXM3I8KQaYnKf7gy27Z63BBoI_I1u02f4lw6rBp3g/viewform?pli=1&fbzx=-8842630989397184967) form and be verified as a MLCommons member to access the dataset.\n\n" From f0b9e7fc49fd7c73b28988b61fdb621e9927f1d4 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 10 Sep 2024 16:25:34 +0100 Subject: [PATCH 115/169] Minor fix for the documentation --- main.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/main.py b/main.py index 7ba058750..01fdf382e 100755 --- a/main.py +++ b/main.py @@ -13,6 +13,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): scenarios = [] execution_envs = ["Docker","Native"] code_version="r4.1-dev" + implementation_run_options = [] if model == "rnnt": code_version="r4.0" @@ -137,7 +138,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): content += get_inference_server_run_cmd(spaces+16,implementation) # tips regarding the running of nural magic server content += f"\n{cur_space3}!!! tip\n\n" - content += f"{cur_space3} - Host and Port number of the server can be configured through `--host` and `--port`. Otherwise, server will run on default host `localhost` and port `8000`.\n\n" + content += f"{cur_space3} - Host and Port number of the server can be configured through `--host` and `--port` options. Otherwise, server will run on the default host `localhost` and port `8000`.\n\n" if execution_env == "Native": # Native implementation steps through virtual environment content += f"{cur_space3}####### Setup a virtual environment for Python\n" @@ -158,7 +159,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): content += f"{cur_space3}* `--docker_cache=no`: to not use docker cache during the image build\n" if implementation.lower() == "nvidia": - content += f"{cur_space3}* `--gpu_name=` : The GPUs with configured batch sizes in CM are `orin`, `rtx_4090`, `rtx_a6000`, `rtx_6000_ada`, `l4`, `t4`and `a100`.\n" + content += f"{cur_space3}* `--gpu_name=` : The GPUs with supported configs in CM are `orin`, `rtx_4090`, `rtx_a6000`, `rtx_6000_ada`, `l4`, `t4`and `a100`. For other GPUs, default configuration as per the GPU memory will be used.\n" if device.lower() not in [ "cuda" ]: content += f"{cur_space3}* `--docker_os=ubuntu`: ubuntu and rhel are supported. \n" @@ -178,7 +179,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): run_suffix += f"{cur_space3}* Use `--division=closed` to do a closed division submission which includes compliance runs\n\n" run_suffix += f"{cur_space3}* Use `--rerun` to do a rerun even when a valid run exists\n" if implementation.lower() == "nvidia": - run_suffix += f"{cur_space3}* `--gpu_name=` : The GPU's with configured batch sizes in CM are `orin`, `rtx_4090`, `rtx_a6000`, `rtx_6000_ada`, `l4`, `t4`and `a100`.\n" + run_suffix += f"{cur_space3}* `--gpu_name=` : The GPUs with supported configs in CM are `orin`, `rtx_4090`, `rtx_a6000`, `rtx_6000_ada`, `l4`, `t4`and `a100`. For other GPUs, default configuration as per the GPU memory will be used.\n" run_suffix += f"{cur_space3}\n\n" if "bert" in model.lower() and framework == "deepsparse": @@ -322,9 +323,9 @@ def get_docker_info(spaces, model, implementation, device): pre_space += " " #pre_space = " " info += f"\n{pre_space}!!! tip\n\n" - info+= f"{pre_space} - Batch size could be adjusted using `--batch_size=#`, where `#` is the desired batch size.\n\n" + info+= f"{pre_space} - Batch size could be adjusted using `--batch_size=#`, where `#` is the desired batch size. This option works only if the implementation in use is supporting the given batch size.\n\n" if implementation.lower() == "nvidia": - info+= f"{pre_space} - Default batch size is assigned based on either [GPU memory](https://github.com/mlcommons/cm4mlops/blob/dd0c35856969c68945524d5c80414c615f5fe42c/script/app-mlperf-inference-nvidia/_cm.yaml#L1129) or the [specified GPU](https://github.com/mlcommons/cm4mlops/blob/dd0c35856969c68945524d5c80414c615f5fe42c/script/app-mlperf-inference-nvidia/_cm.yaml#L1370). Please click more option for *docker launch* or *run command* to see how to specify the GPU name.\n\n" + info+= f"{pre_space} - Default batch size is assigned based on [GPU memory](https://github.com/mlcommons/cm4mlops/blob/dd0c35856969c68945524d5c80414c615f5fe42c/script/app-mlperf-inference-nvidia/_cm.yaml#L1129) or the [specified GPU](https://github.com/mlcommons/cm4mlops/blob/dd0c35856969c68945524d5c80414c615f5fe42c/script/app-mlperf-inference-nvidia/_cm.yaml#L1370). Please click more option for *docker launch* or *run command* to see how to specify the GPU name.\n\n" info+= f"{pre_space} - When run with `--all_models=yes`, all the benchmark models of NVIDIA implementation can be executed within the same container.\n\n" if "llama2" in model.lower(): info+= f"{pre_space} - The dataset for NVIDIA's implementation of Llama2 is not publicly available. The user must fill [this](https://docs.google.com/forms/d/e/1FAIpQLSc_8VIvRmXM3I8KQaYnKf7gy27Z63BBoI_I1u02f4lw6rBp3g/viewform?pli=1&fbzx=-8842630989397184967) form and be verified as a MLCommons member to access the dataset.\n\n" From aba2ce8d080a5d734516959daac5f12577d776b7 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 17 Sep 2024 06:40:57 +0530 Subject: [PATCH 116/169] fixed bug in deepsparse generic model stubs + styling --- main.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/main.py b/main.py index 7ba058750..5294c03e0 100755 --- a/main.py +++ b/main.py @@ -181,23 +181,23 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): run_suffix += f"{cur_space3}* `--gpu_name=` : The GPU's with configured batch sizes in CM are `orin`, `rtx_4090`, `rtx_a6000`, `rtx_6000_ada`, `l4`, `t4`and `a100`.\n" run_suffix += f"{cur_space3}\n\n" - if "bert" in model.lower() and framework == "deepsparse": + if "bert" in model.lower() and framework.lower() == "deepsparse": run_suffix += f"{cur_space3}
    \n" - run_suffix += f"{cur_space3} Please click here for generic model stubs for bert deepsparse\n\n" - run_suffix += f"{cur_space3}* zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/pruned95_quant-none-vnni\n\n" - run_suffix += f"{cur_space3}* zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/14layer_pruned50_quant-none-vnni\n\n" - run_suffix += f"{cur_space3}* zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/base_quant-none\n\n" - run_suffix += f"{cur_space3}* zoo:nlp/question_answering/bert-base/pytorch/huggingface/squad/pruned95_obs_quant-none\n\n" - run_suffix += f"{cur_space3}* zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/14layer_pruned50-none-vnni\n\n" - run_suffix += f"{cur_space3}* zoo:nlp/question_answering/obert-base/pytorch/huggingface/squad/pruned90-none\n\n" - run_suffix += f"{cur_space3}* zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/pruned97_quant-none\n\n" - run_suffix += f"{cur_space3}* zoo:nlp/question_answering/bert-base/pytorch/huggingface/squad/pruned90-none\n\n" - run_suffix += f"{cur_space3}* zoo:nlp/question_answering/bert-large/pytorch/huggingface/squad/pruned80_quant-none-vnni\n\n" - run_suffix += f"{cur_space3}* zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/pruned95-none-vnni\n\n" - run_suffix += f"{cur_space3}* zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/pruned97-none\n\n" - run_suffix += f"{cur_space3}* zoo:nlp/question_answering/bert-large/pytorch/huggingface/squad/base-none\n\n" - run_suffix += f"{cur_space3}* zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/base-none\n\n" - run_suffix += f"{cur_space3}* zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/base-none\n" + run_suffix += f"{cur_space3} Please click here to view available generic model stubs for bert deepsparse\n\n" + run_suffix += f"{cur_space3}* **pruned95_quant-none-vnni:** zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/pruned95_quant-none-vnni\n\n" + run_suffix += f"{cur_space3}* **14layer_pruned50_quant-none-vnni:** zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/14layer_pruned50_quant-none-vnni\n\n" + run_suffix += f"{cur_space3}* **base_quant-none:** zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/base_quant-none\n\n" + run_suffix += f"{cur_space3}* **pruned95_obs_quant-none:** zoo:nlp/question_answering/bert-base/pytorch/huggingface/squad/pruned95_obs_quant-none\n\n" + run_suffix += f"{cur_space3}* **14layer_pruned50-none-vnni:** zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/14layer_pruned50-none-vnni\n\n" + run_suffix += f"{cur_space3}* **pruned90-none:** zoo:nlp/question_answering/obert-base/pytorch/huggingface/squad/pruned90-none\n\n" + run_suffix += f"{cur_space3}* **pruned97_quant-none:** zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/pruned97_quant-none\n\n" + run_suffix += f"{cur_space3}* **pruned90-none:** zoo:nlp/question_answering/bert-base/pytorch/huggingface/squad/pruned90-none\n\n" + run_suffix += f"{cur_space3}* **pruned80_quant-none-vnni:** zoo:nlp/question_answering/bert-large/pytorch/huggingface/squad/pruned80_quant-none-vnni\n\n" + run_suffix += f"{cur_space3}* **pruned95-none-vnni:** zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/pruned95-none-vnni\n\n" + run_suffix += f"{cur_space3}* **pruned97-none:** zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/pruned97-none\n\n" + run_suffix += f"{cur_space3}* **bert-large-base-none:** zoo:nlp/question_answering/bert-large/pytorch/huggingface/squad/base-none\n\n" + run_suffix += f"{cur_space3}* **obert-large-base-none:** zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/base-none\n\n" + run_suffix += f"{cur_space3}* **mobilebert-none-base-none:** zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/base-none\n" run_suffix += f"{cur_space3}
    \n" From 33cad4491c16607d4bf0bd449ffb2553a7eecb49 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 17 Sep 2024 06:54:54 +0530 Subject: [PATCH 117/169] added more information to stubs --- main.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/main.py b/main.py index 2dde297a9..28a4800d4 100755 --- a/main.py +++ b/main.py @@ -185,17 +185,17 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): if "bert" in model.lower() and framework.lower() == "deepsparse": run_suffix += f"{cur_space3}
    \n" run_suffix += f"{cur_space3} Please click here to view available generic model stubs for bert deepsparse\n\n" - run_suffix += f"{cur_space3}* **pruned95_quant-none-vnni:** zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/pruned95_quant-none-vnni\n\n" - run_suffix += f"{cur_space3}* **14layer_pruned50_quant-none-vnni:** zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/14layer_pruned50_quant-none-vnni\n\n" - run_suffix += f"{cur_space3}* **base_quant-none:** zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/base_quant-none\n\n" - run_suffix += f"{cur_space3}* **pruned95_obs_quant-none:** zoo:nlp/question_answering/bert-base/pytorch/huggingface/squad/pruned95_obs_quant-none\n\n" - run_suffix += f"{cur_space3}* **14layer_pruned50-none-vnni:** zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/14layer_pruned50-none-vnni\n\n" - run_suffix += f"{cur_space3}* **pruned90-none:** zoo:nlp/question_answering/obert-base/pytorch/huggingface/squad/pruned90-none\n\n" - run_suffix += f"{cur_space3}* **pruned97_quant-none:** zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/pruned97_quant-none\n\n" - run_suffix += f"{cur_space3}* **pruned90-none:** zoo:nlp/question_answering/bert-base/pytorch/huggingface/squad/pruned90-none\n\n" - run_suffix += f"{cur_space3}* **pruned80_quant-none-vnni:** zoo:nlp/question_answering/bert-large/pytorch/huggingface/squad/pruned80_quant-none-vnni\n\n" - run_suffix += f"{cur_space3}* **pruned95-none-vnni:** zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/pruned95-none-vnni\n\n" - run_suffix += f"{cur_space3}* **pruned97-none:** zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/pruned97-none\n\n" + run_suffix += f"{cur_space3}* **obert-large-pruned95_quant-none-vnni:** zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/pruned95_quant-none-vnni\n\n" + run_suffix += f"{cur_space3}* **mobilebert-none-14layer_pruned50_quant-none-vnni:** zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/14layer_pruned50_quant-none-vnni\n\n" + run_suffix += f"{cur_space3}* **mobilebert-none-base_quant-none:** zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/base_quant-none\n\n" + run_suffix += f"{cur_space3}* **bert-base-pruned95_obs_quant-none:** zoo:nlp/question_answering/bert-base/pytorch/huggingface/squad/pruned95_obs_quant-none\n\n" + run_suffix += f"{cur_space3}* **mobilebert-none-14layer_pruned50-none-vnni:** zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/14layer_pruned50-none-vnni\n\n" + run_suffix += f"{cur_space3}* **obert-base-pruned90-none:** zoo:nlp/question_answering/obert-base/pytorch/huggingface/squad/pruned90-none\n\n" + run_suffix += f"{cur_space3}* **obert-large-pruned97_quant-none:** zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/pruned97_quant-none\n\n" + run_suffix += f"{cur_space3}* **bert-base-pruned90-none:** zoo:nlp/question_answering/bert-base/pytorch/huggingface/squad/pruned90-none\n\n" + run_suffix += f"{cur_space3}* **bert-large-pruned80_quant-none-vnni:** zoo:nlp/question_answering/bert-large/pytorch/huggingface/squad/pruned80_quant-none-vnni\n\n" + run_suffix += f"{cur_space3}* **obert-large-pruned95-none-vnni:** zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/pruned95-none-vnni\n\n" + run_suffix += f"{cur_space3}* **obert-large-pruned97-none:** zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/pruned97-none\n\n" run_suffix += f"{cur_space3}* **bert-large-base-none:** zoo:nlp/question_answering/bert-large/pytorch/huggingface/squad/base-none\n\n" run_suffix += f"{cur_space3}* **obert-large-base-none:** zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/base-none\n\n" run_suffix += f"{cur_space3}* **mobilebert-none-base-none:** zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/base-none\n" From 8cea28a1c3baef5d4edb3bbb68bcd5862e81299a Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 18 Sep 2024 19:25:53 +0100 Subject: [PATCH 118/169] Added SCC24 readme, support reproducibility in the docs --- .../image_classification/mobilenets.md | 2 +- .../image_classification/resnet50.md | 1 - docs/benchmarks/language/bert.md | 2 - docs/benchmarks/language/gpt-j.md | 2 - docs/benchmarks/language/llama2-70b.md | 2 - docs/benchmarks/language/mixtral-8x7b.md | 2 - docs/benchmarks/medical_imaging/3d-unet.md | 2 - docs/benchmarks/object_detection/retinanet.md | 2 - docs/benchmarks/recommendation/dlrm-v2.md | 2 - .../text_to_image/reproducibility/scc24.md | 18 +++ docs/requirements.txt | 1 + main.py | 117 ++++++++++-------- mkdocs.yml | 6 +- 13 files changed, 91 insertions(+), 68 deletions(-) create mode 100644 docs/benchmarks/text_to_image/reproducibility/scc24.md diff --git a/docs/benchmarks/image_classification/mobilenets.md b/docs/benchmarks/image_classification/mobilenets.md index 5b96a78e2..09840ef1d 100644 --- a/docs/benchmarks/image_classification/mobilenets.md +++ b/docs/benchmarks/image_classification/mobilenets.md @@ -5,7 +5,7 @@ hide: # Image Classification using Mobilenet models -Install CM following the [installation page](/install). +Install CM following the [installation page](site:install). Mobilenet models are not official MLPerf models and so cannot be used for a Closed division MLPerf inference submission. But since they can be run with Imagenet dataset, we are allowed to use them for Open division submission. Only CPU runs are supported now. diff --git a/docs/benchmarks/image_classification/resnet50.md b/docs/benchmarks/image_classification/resnet50.md index 8fec36a64..4172158dc 100644 --- a/docs/benchmarks/image_classification/resnet50.md +++ b/docs/benchmarks/image_classification/resnet50.md @@ -6,7 +6,6 @@ hide: # Image Classification using ResNet50 -Install CM following the [installation page](/install). === "MLCommons-Python" ## MLPerf Reference Implementation in Python diff --git a/docs/benchmarks/language/bert.md b/docs/benchmarks/language/bert.md index 7c76543e8..57b1cf224 100644 --- a/docs/benchmarks/language/bert.md +++ b/docs/benchmarks/language/bert.md @@ -5,8 +5,6 @@ hide: # Question Answering using Bert-Large -Install CM following the [installation page](/install). - === "MLCommons-Python" ## MLPerf Reference Implementation in Python diff --git a/docs/benchmarks/language/gpt-j.md b/docs/benchmarks/language/gpt-j.md index 3ca1157a6..d2f545815 100644 --- a/docs/benchmarks/language/gpt-j.md +++ b/docs/benchmarks/language/gpt-j.md @@ -5,8 +5,6 @@ hide: # Text Summarization using GPT-J -Install CM following the [installation page](/install). - === "MLCommons-Python" ## MLPerf Reference Implementation in Python diff --git a/docs/benchmarks/language/llama2-70b.md b/docs/benchmarks/language/llama2-70b.md index 726a6dd8a..e68693716 100644 --- a/docs/benchmarks/language/llama2-70b.md +++ b/docs/benchmarks/language/llama2-70b.md @@ -5,8 +5,6 @@ hide: # Text Summarization using LLAMA2-70b -Install CM following the [installation page](/install). - === "MLCommons-Python" ## MLPerf Reference Implementation in Python diff --git a/docs/benchmarks/language/mixtral-8x7b.md b/docs/benchmarks/language/mixtral-8x7b.md index c66dfdff0..bdb26ae77 100644 --- a/docs/benchmarks/language/mixtral-8x7b.md +++ b/docs/benchmarks/language/mixtral-8x7b.md @@ -5,8 +5,6 @@ hide: # Question Answering, Math, and Code Generation using Mixtral-8x7B -Install CM following the [installation page](/install). - === "MLCommons-Python" ## MLPerf Reference Implementation in Python diff --git a/docs/benchmarks/medical_imaging/3d-unet.md b/docs/benchmarks/medical_imaging/3d-unet.md index 43495bc47..72d5eed49 100644 --- a/docs/benchmarks/medical_imaging/3d-unet.md +++ b/docs/benchmarks/medical_imaging/3d-unet.md @@ -5,8 +5,6 @@ hide: # Medical Imaging using 3d-unet (KiTS 2019 kidney tumor segmentation task) -Install CM following the [installation page](/install). - === "MLCommons-Python" ## MLPerf Reference Implementation in Python diff --git a/docs/benchmarks/object_detection/retinanet.md b/docs/benchmarks/object_detection/retinanet.md index 47504481b..699d92050 100644 --- a/docs/benchmarks/object_detection/retinanet.md +++ b/docs/benchmarks/object_detection/retinanet.md @@ -5,8 +5,6 @@ hide: # Object Detection using Retinanet -Install CM following the [installation page](/install). - === "MLCommons-Python" ## MLPerf Reference Implementation in Python diff --git a/docs/benchmarks/recommendation/dlrm-v2.md b/docs/benchmarks/recommendation/dlrm-v2.md index c50c69417..b539c1607 100644 --- a/docs/benchmarks/recommendation/dlrm-v2.md +++ b/docs/benchmarks/recommendation/dlrm-v2.md @@ -5,8 +5,6 @@ hide: # Recommendation using DLRM v2 -Install CM following the [installation page](/install). - === "MLCommons-Python" ## MLPerf Reference Implementation in Python diff --git a/docs/benchmarks/text_to_image/reproducibility/scc24.md b/docs/benchmarks/text_to_image/reproducibility/scc24.md new file mode 100644 index 000000000..9c48a969f --- /dev/null +++ b/docs/benchmarks/text_to_image/reproducibility/scc24.md @@ -0,0 +1,18 @@ +--- +hide: + - toc +--- + +# Text to Image using Stable Diffusion for Student Cluster Competition 2024 + + +=== "MLCommons-Python" + ## MLPerf Reference Implementation in Python + +{{ mlperf_inference_implementation_readme (4, "sdxl", "reference", extra_variation_tags=",_short", scenarios=["Offline"],categories=["Edge"]) }} + +=== "Nvidia" + ## Nvidia MLPerf Implementation +{{ mlperf_inference_implementation_readme (4, "sdxl", "nvidia", extra_variation_tags=",_short", scenarios=["Offline"],categories=["Edge"]) }} + + diff --git a/docs/requirements.txt b/docs/requirements.txt index 35176cae0..293abf164 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -3,3 +3,4 @@ swagger-markdown mkdocs-macros-plugin ruamel.yaml mkdocs-redirects +mkdocs-site-urls diff --git a/main.py b/main.py index 01fdf382e..daa3e3677 100755 --- a/main.py +++ b/main.py @@ -1,7 +1,7 @@ def define_env(env): @env.macro - def mlperf_inference_implementation_readme(spaces, model, implementation): + def mlperf_inference_implementation_readme(spaces, model, implementation, *, scenarios = [], devices=[], frameworks=[], categories=[], extra_variation_tags="", extra_input_string="", extra_docker_input_string=""): pre_space = "" for i in range(1,spaces): @@ -10,7 +10,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): pre_space += " " content="" - scenarios = [] + execution_envs = ["Docker","Native"] code_version="r4.1-dev" implementation_run_options = [] @@ -24,15 +24,18 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): content += f"\n{pre_space}!!! tip\n\n" content += f"{pre_space} - MLCommons reference implementations are only meant to provide a rules compliant reference implementation for the submitters and in most cases are not best performing. If you want to benchmark any system, it is advisable to use the vendor MLPerf implementation for that system like Nvidia, Intel etc.\n\n" - devices = [ "CPU", "CUDA", "ROCm" ] - if model.lower() == "resnet50": - frameworks = [ "Onnxruntime", "Tensorflow", "Deepsparse" ] - elif model.lower() == "retinanet": - frameworks = [ "Onnxruntime", "Pytorch" ] - elif "bert" in model.lower(): - frameworks = [ "Pytorch", "Deepsparse" ] - else: - frameworks = [ "Pytorch" ] + if not devices: + devices = [ "CPU", "CUDA", "ROCm" ] + + if not frameworks: + if model.lower() == "resnet50": + frameworks = [ "Onnxruntime", "Tensorflow", "Deepsparse" ] + elif model.lower() == "retinanet": + frameworks = [ "Onnxruntime", "Pytorch" ] + elif "bert" in model.lower(): + frameworks = [ "Pytorch", "Deepsparse" ] + else: + frameworks = [ "Pytorch" ] elif implementation == "nvidia": if model in [ "mixtral-8x7b" ]: @@ -65,7 +68,8 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): frameworks = [ "Glow" ] elif implementation == "cpp": - devices = [ "CPU", "CUDA" ] + if not devices: + devices = [ "CPU", "CUDA" ] frameworks = [ "Onnxruntime" ] elif implementation == "ctuning-cpp": @@ -76,22 +80,25 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): else: frameworks = [] - if model.lower() == "bert-99.9": - categories = [ "Datacenter" ] - elif "dlrm" in model.lower() or "llama2" in model.lower() or "mixtral" in model.lower(): - categories = [ "Datacenter" ] - else: - categories = [ "Edge", "Datacenter" ] + if not categories: + if model.lower() == "bert-99.9": + categories = [ "Datacenter" ] + elif "dlrm" in model.lower() or "llama2" in model.lower() or "mixtral" in model.lower(): + categories = [ "Datacenter" ] + else: + categories = [ "Edge", "Datacenter" ] # model name content += f"{pre_space}{model.upper()}\n\n" + for category in categories: - if category == "Edge" and not scenarios: - scenarios = [ "Offline", "SingleStream" ] - if model.lower() in [ "resnet50", "retinanet" ] and not "MultiStream" in scenarios:#MultiStream was duplicating - scenarios.append("MultiStream") - elif category == "Datacenter": - scenarios = [ "Offline", "Server" ] + if not scenarios: + if category == "Edge" and not scenarios: + scenarios = [ "Offline", "SingleStream" ] + if model.lower() in [ "resnet50", "retinanet" ] and not "MultiStream" in scenarios:#MultiStream was duplicating + scenarios.append("MultiStream") + elif category == "Datacenter": + scenarios = [ "Offline", "Server" ] content += f"{pre_space}=== \"{category.lower()}\"\n\n" @@ -129,7 +136,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): content += f"{cur_space2}=== \"{execution_env}\"\n" content += f"{cur_space3}###### {execution_env} Environment\n\n" # ref to cm installation - content += f"{cur_space3}Please refer to the [installation page](../../install/index.md) to install CM for running the automated benchmark commands.\n\n" + content += f"{cur_space3}Please refer to the [installation page](site:install/) to install CM for running the automated benchmark commands.\n\n" test_query_count=get_test_query_count(model, implementation, device) if "99.9" not in model: #not showing docker command as it is already done for the 99% variant @@ -140,18 +147,24 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): content += f"\n{cur_space3}!!! tip\n\n" content += f"{cur_space3} - Host and Port number of the server can be configured through `--host` and `--port` options. Otherwise, server will run on the default host `localhost` and port `8000`.\n\n" + setup_run_cmd = mlperf_inference_run_command(spaces+17, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True, scenarios, code_version, extra_variation_tags, extra_input_string, extra_docker_input_string) + if execution_env == "Native": # Native implementation steps through virtual environment content += f"{cur_space3}####### Setup a virtual environment for Python\n" content += get_venv_command(spaces+16) content += f"{cur_space3}####### Performance Estimation for Offline Scenario\n" - content += mlperf_inference_run_command(spaces+17, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True, scenarios, code_version).replace("--docker ","") + + content += setup_run_cmd.replace("--docker ", "") + content += f"{cur_space3}The above command should do a test run of Offline scenario and record the estimated offline_target_qps.\n\n" else: # Docker implementation steps content += f"{cur_space3}####### Docker Container Build and Performance Estimation for Offline Scenario\n" docker_info = get_docker_info(spaces+16, model, implementation, device) content += docker_info - content += mlperf_inference_run_command(spaces+17, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True, scenarios, code_version) + + content += setup_run_cmd + content += f"{cur_space3}The above command should get you to an interactive shell inside the docker container and do a quick test run for the Offline scenario. Once inside the docker container please do the below commands to do the accuracy + performance runs for each scenario.\n\n" content += f"{cur_space3}
    \n" content += f"{cur_space3} Please click here to see more options for the docker launch \n\n" @@ -169,6 +182,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): else: content += f"{cur_space3} You can reuse the same environment as described for {model.split('.')[0]}.\n" content += f"{cur_space3}###### Performance Estimation for Offline Scenario\n" + content += mlperf_inference_run_command(spaces+17, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True, scenarios, code_version).replace("--docker ","") content += f"{cur_space3}The above command should do a test run of Offline scenario and record the estimated offline_target_qps.\n\n" @@ -205,20 +219,21 @@ def mlperf_inference_implementation_readme(spaces, model, implementation): for scenario in scenarios: content += f"{cur_space3}=== \"{scenario}\"\n{cur_space4}###### {scenario}\n\n" - run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), "valid", 0, False, scenarios, code_version) + run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), "valid", 0, False, scenarios, code_version, extra_variation_tags, extra_input_string) content += run_cmd #content += run_suffix - - content += f"{cur_space3}=== \"All Scenarios\"\n{cur_space4}###### All Scenarios\n\n" - run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), "valid", 0, False, scenarios, code_version) - content += run_cmd - content += run_suffix + + if len(scenarios) > 1: + content += f"{cur_space3}=== \"All Scenarios\"\n{cur_space4}###### All Scenarios\n\n" + run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), "valid", 0, False, scenarios, code_version, extra_variation_tags, extra_input_string) + content += run_cmd + content += run_suffix - readme_prefix = get_readme_prefix(spaces, model, implementation) + readme_prefix = get_readme_prefix(spaces, model, implementation, extra_variation_tags) - readme_suffix = get_readme_suffix(spaces, model, implementation) + readme_suffix = get_readme_suffix(spaces, model, implementation, extra_variation_tags) return readme_prefix + content + readme_suffix @@ -286,15 +301,6 @@ def get_min_system_requirements(spaces, model, implementation, device): min_sys_req_content += f"{spaces}
    \n" return min_sys_req_content - def get_readme_prefix(spaces, model, implementation): - readme_prefix = "" - pre_space=" " - #for i in range(1,spaces): - # pre_space = pre_space + " " - #pre_space += " " - - return readme_prefix - def get_inference_server_run_cmd(spaces, implementation): indent = " "*spaces + " " if implementation == "neuralmagic": @@ -332,14 +338,23 @@ def get_docker_info(spaces, model, implementation, device): info+= f"{pre_space} - `PATH_TO_PICKE_FILE` should be replaced with path to the downloaded pickle file.\n\n" return info - def get_readme_suffix(spaces, model, implementation): + def get_readme_prefix(spaces, model, implementation, extra_variation_tags): + readme_prefix = "" + pre_space=" " + #for i in range(1,spaces): + # pre_space = pre_space + " " + #pre_space += " " + + return readme_prefix + + def get_readme_suffix(spaces, model, implementation, extra_variation_tags): readme_suffix = "" pre_space="" for i in range(1,spaces): pre_space = pre_space + " " pre_space += " " - if implementation == "reference": + if implementation == "reference" and not extra_variation_tags: if not model.endswith("-99"): model_base_name = model.replace("-99.9","").replace("-99","") readme_suffix+= f"{pre_space}* If you want to download the official MLPerf model and dataset for {model} you can follow [this README](get-{model_base_name}-data.md).\n" @@ -361,7 +376,7 @@ def get_run_cmd_extra(f_pre_space, model, implementation, device, scenario, scen return extra_content @env.macro - def mlperf_inference_run_command(spaces, model, implementation, framework, category, scenario, device="cpu", execution_mode="test", test_query_count="20", docker=False, scenarios = [], code_version="r4.1-dev"): + def mlperf_inference_run_command(spaces, model, implementation, framework, category, scenario, device="cpu", execution_mode="test", test_query_count="20", docker=False, scenarios = [], code_version="r4.1-dev", extra_variation_tags="", extra_input_string="", extra_docker_input_string=""): pre_space = "" for i in range(1,spaces): pre_space = pre_space + " " @@ -382,14 +397,14 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ if docker: docker_cmd_suffix = f" \\\n{pre_space} --docker --quiet" - docker_cmd_suffix += f" \\\n{pre_space} --test_query_count={test_query_count}" + docker_cmd_suffix += f" \\\n{pre_space} --test_query_count={test_query_count} {extra_docker_input_string} {extra_input_string}" if "bert" in model.lower() and framework == "deepsparse": docker_cmd_suffix += f"\\\n{pre_space} --env.CM_MLPERF_NEURALMAGIC_MODEL_ZOO_STUB=zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/base_quant-none" if "llama2-70b" in model.lower(): if implementation == "nvidia": docker_cmd_suffix += f" \\\n{pre_space} --tp_size=2" - docker_cmd_suffix += f" \\\n{pre_space} --nvidia_llama2_dataset_file_path=" + docker_cmd_suffix += f" \\\n{pre_space} --nvidia_llama2_dataset_file_path=" elif implementation == "neuralmagic": docker_cmd_suffix += f" \\\n{pre_space} --api_server=http://localhost:8000" docker_cmd_suffix += f" \\\n{pre_space} --vllm_model_name=nm-testing/Llama-2-70b-chat-hf-FP8" @@ -400,7 +415,7 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ docker_setup_cmd = f"""\n {f_pre_space}```bash -{f_pre_space}cm run script --tags=run-mlperf,inference,_find-performance,_full,_{code_version}{scenario_variation_tag} \\ +{f_pre_space}cm run script --tags=run-mlperf,inference,_find-performance,_full,_{code_version}{scenario_variation_tag}{extra_variation_tags} \\ {pre_space} --model={model} \\ {pre_space} --implementation={implementation} \\ {pre_space} --framework={framework} \\ @@ -412,7 +427,7 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ return docker_setup_cmd + run_cmd_extra else: - cmd_suffix = f"\\\n{pre_space} --quiet" + cmd_suffix = f"\\\n{pre_space} --quiet {extra_input_string}" if execution_mode == "test": cmd_suffix += f" \\\n {pre_space} --test_query_count={test_query_count}" diff --git a/mkdocs.yml b/mkdocs.yml index ceebf74cb..75556e05d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -24,7 +24,10 @@ nav: - Image Classification: - ResNet50: benchmarks/image_classification/resnet50.md - Text to Image: - - Stable Diffusion: benchmarks/text_to_image/sdxl.md + - Stable Diffusion: + - Run Commands: benchmarks/text_to_image/sdxl.md + - Reproducibility: + - SCC24: benchmarks/text_to_image/reproducibility/scc24.md - Object Detection: - RetinaNet: benchmarks/object_detection/retinanet.md - Medical Imaging: @@ -62,6 +65,7 @@ markdown_extensions: plugins: - search - macros + - site-urls - redirects: redirect_maps: 'benchmarks/index.md': 'index.md' From ae8f9e69dd81ff0d9183de05bdaa98bac2aeafbe Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 18 Sep 2024 19:34:32 +0100 Subject: [PATCH 119/169] Made clear the custom CM repo URL format --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index daa3e3677..0a37cf542 100755 --- a/main.py +++ b/main.py @@ -168,7 +168,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, sce content += f"{cur_space3}The above command should get you to an interactive shell inside the docker container and do a quick test run for the Offline scenario. Once inside the docker container please do the below commands to do the accuracy + performance runs for each scenario.\n\n" content += f"{cur_space3}
    \n" content += f"{cur_space3} Please click here to see more options for the docker launch \n\n" - content += f"{cur_space3}* `--docker_cm_repo=`: to use a custom fork of cm4mlops repository inside the docker image\n\n" + content += f"{cur_space3}* `--docker_cm_repo=`: to use a custom fork of cm4mlops repository inside the docker image\n\n" content += f"{cur_space3}* `--docker_cache=no`: to not use docker cache during the image build\n" if implementation.lower() == "nvidia": From a5c16275bab68c4693c6c14ef07558d2bfc673aa Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 18 Sep 2024 21:26:43 +0100 Subject: [PATCH 120/169] Support conditional implementation, setup and run tips --- .../text_to_image/reproducibility/scc24.md | 4 +- main.py | 42 +++++++++++-------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/docs/benchmarks/text_to_image/reproducibility/scc24.md b/docs/benchmarks/text_to_image/reproducibility/scc24.md index 9c48a969f..b55008bbc 100644 --- a/docs/benchmarks/text_to_image/reproducibility/scc24.md +++ b/docs/benchmarks/text_to_image/reproducibility/scc24.md @@ -9,10 +9,10 @@ hide: === "MLCommons-Python" ## MLPerf Reference Implementation in Python -{{ mlperf_inference_implementation_readme (4, "sdxl", "reference", extra_variation_tags=",_short", scenarios=["Offline"],categories=["Edge"]) }} +{{ mlperf_inference_implementation_readme (4, "sdxl", "reference", extra_variation_tags=",_short", scenarios=["Offline"],categories=["Edge"], setup_tips=False) }} === "Nvidia" ## Nvidia MLPerf Implementation -{{ mlperf_inference_implementation_readme (4, "sdxl", "nvidia", extra_variation_tags=",_short", scenarios=["Offline"],categories=["Edge"]) }} +{{ mlperf_inference_implementation_readme (4, "sdxl", "nvidia", extra_variation_tags=",_short", scenarios=["Offline"],categories=["Edge"], setup_tips=False, implementation_tips=False) }} diff --git a/main.py b/main.py index 0a37cf542..49c9249ea 100755 --- a/main.py +++ b/main.py @@ -1,7 +1,7 @@ def define_env(env): @env.macro - def mlperf_inference_implementation_readme(spaces, model, implementation, *, scenarios = [], devices=[], frameworks=[], categories=[], extra_variation_tags="", extra_input_string="", extra_docker_input_string=""): + def mlperf_inference_implementation_readme(spaces, model, implementation, *, implementation_tips=True, setup_tips=True, run_tips=True, scenarios = [], devices=[], frameworks=[], categories=[], extra_variation_tags="", extra_input_string="", extra_docker_input_string=""): pre_space = "" for i in range(1,spaces): @@ -20,7 +20,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, sce if implementation == "reference": # Tip - if "99.9" not in model: + if "99.9" not in model and implementation_tips: content += f"\n{pre_space}!!! tip\n\n" content += f"{pre_space} - MLCommons reference implementations are only meant to provide a rules compliant reference implementation for the submitters and in most cases are not best performing. If you want to benchmark any system, it is advisable to use the vendor MLPerf implementation for that system like Nvidia, Intel etc.\n\n" @@ -49,7 +49,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, sce elif implementation == "intel": # Tip - if "99.9" not in model: + if "99.9" not in model and implementation_tips: content += f"\n{pre_space}!!! tip\n\n" content += f"{pre_space} - Intel MLPerf inference implementation is available only for datacenter category and has been tested only on a limited number of systems. Most of the benchmarks using Intel implementation require at least Intel Sapphire Rapids or higher CPU generation.\n\n" @@ -143,9 +143,10 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, sce if implementation == "neuralmagic": content += f"{cur_space3}####### Run the Inference Server\n" content += get_inference_server_run_cmd(spaces+16,implementation) - # tips regarding the running of nural magic server - content += f"\n{cur_space3}!!! tip\n\n" - content += f"{cur_space3} - Host and Port number of the server can be configured through `--host` and `--port` options. Otherwise, server will run on the default host `localhost` and port `8000`.\n\n" + if run_tips: + # tips regarding the running of nural magic server + content += f"\n{cur_space3}!!! tip\n\n" + content += f"{cur_space3} - Host and Port number of the server can be configured through `--host` and `--port` options. Otherwise, server will run on the default host `localhost` and port `8000`.\n\n" setup_run_cmd = mlperf_inference_run_command(spaces+17, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True, scenarios, code_version, extra_variation_tags, extra_input_string, extra_docker_input_string) @@ -160,7 +161,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, sce else: # Docker implementation steps content += f"{cur_space3}####### Docker Container Build and Performance Estimation for Offline Scenario\n" - docker_info = get_docker_info(spaces+16, model, implementation, device) + docker_info = get_docker_info(spaces+16, model, implementation, device, setup_tips) content += docker_info content += setup_run_cmd @@ -321,21 +322,22 @@ def get_venv_command(spaces): {pre_space}export CM_SCRIPT_EXTRA_CMD=\"--adr.python.name=mlperf\" {pre_space}```\n""" - def get_docker_info(spaces, model, implementation, device): + def get_docker_info(spaces, model, implementation, device, setup_tips=True): info = "" pre_space="" for i in range(1,spaces): pre_space = pre_space + " " pre_space += " " #pre_space = " " - info += f"\n{pre_space}!!! tip\n\n" - info+= f"{pre_space} - Batch size could be adjusted using `--batch_size=#`, where `#` is the desired batch size. This option works only if the implementation in use is supporting the given batch size.\n\n" - if implementation.lower() == "nvidia": - info+= f"{pre_space} - Default batch size is assigned based on [GPU memory](https://github.com/mlcommons/cm4mlops/blob/dd0c35856969c68945524d5c80414c615f5fe42c/script/app-mlperf-inference-nvidia/_cm.yaml#L1129) or the [specified GPU](https://github.com/mlcommons/cm4mlops/blob/dd0c35856969c68945524d5c80414c615f5fe42c/script/app-mlperf-inference-nvidia/_cm.yaml#L1370). Please click more option for *docker launch* or *run command* to see how to specify the GPU name.\n\n" - info+= f"{pre_space} - When run with `--all_models=yes`, all the benchmark models of NVIDIA implementation can be executed within the same container.\n\n" - if "llama2" in model.lower(): - info+= f"{pre_space} - The dataset for NVIDIA's implementation of Llama2 is not publicly available. The user must fill [this](https://docs.google.com/forms/d/e/1FAIpQLSc_8VIvRmXM3I8KQaYnKf7gy27Z63BBoI_I1u02f4lw6rBp3g/viewform?pli=1&fbzx=-8842630989397184967) form and be verified as a MLCommons member to access the dataset.\n\n" - info+= f"{pre_space} - `PATH_TO_PICKE_FILE` should be replaced with path to the downloaded pickle file.\n\n" + if setup_tips: + info += f"\n{pre_space}!!! tip\n\n" + info+= f"{pre_space} - Batch size could be adjusted using `--batch_size=#`, where `#` is the desired batch size. This option works only if the implementation in use is supporting the given batch size.\n\n" + if implementation.lower() == "nvidia": + info+= f"{pre_space} - Default batch size is assigned based on [GPU memory](https://github.com/mlcommons/cm4mlops/blob/dd0c35856969c68945524d5c80414c615f5fe42c/script/app-mlperf-inference-nvidia/_cm.yaml#L1129) or the [specified GPU](https://github.com/mlcommons/cm4mlops/blob/dd0c35856969c68945524d5c80414c615f5fe42c/script/app-mlperf-inference-nvidia/_cm.yaml#L1370). Please click more option for *docker launch* or *run command* to see how to specify the GPU name.\n\n" + info+= f"{pre_space} - When run with `--all_models=yes`, all the benchmark models of NVIDIA implementation can be executed within the same container.\n\n" + if "llama2" in model.lower(): + info+= f"{pre_space} - The dataset for NVIDIA's implementation of Llama2 is not publicly available. The user must fill [this](https://docs.google.com/forms/d/e/1FAIpQLSc_8VIvRmXM3I8KQaYnKf7gy27Z63BBoI_I1u02f4lw6rBp3g/viewform?pli=1&fbzx=-8842630989397184967) form and be verified as a MLCommons member to access the dataset.\n\n" + info+= f"{pre_space} - `PATH_TO_PICKE_FILE` should be replaced with path to the downloaded pickle file.\n\n" return info def get_readme_prefix(spaces, model, implementation, extra_variation_tags): @@ -362,7 +364,7 @@ def get_readme_suffix(spaces, model, implementation, extra_variation_tags): readme_suffix += f"{pre_space}* Please see [mobilenets.md](mobilenets.md) for running mobilenet models for Image Classification." return readme_suffix - def get_run_cmd_extra(f_pre_space, model, implementation, device, scenario, scenarios = []): + def get_run_cmd_extra(f_pre_space, model, implementation, device, scenario, scenarios = [], run_tips=True): extra_content = "" f_pre_space += "" if scenario == "Server" or (scenario == "All Scenarios" and "Server" in scenarios): @@ -373,7 +375,11 @@ def get_run_cmd_extra(f_pre_space, model, implementation, device, scenario, scen if extra_content: extra_content = f"{f_pre_space}!!! tip\n\n" + extra_content - return extra_content + if run_tips: + return extra_content + else: + return "" + @env.macro def mlperf_inference_run_command(spaces, model, implementation, framework, category, scenario, device="cpu", execution_mode="test", test_query_count="20", docker=False, scenarios = [], code_version="r4.1-dev", extra_variation_tags="", extra_input_string="", extra_docker_input_string=""): From 3e24bb9bc4b5b3e81d2a633b349b2ce7b20f0ea3 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 20 Sep 2024 04:23:40 +0530 Subject: [PATCH 121/169] Support rocm for sdxl --- text_to_image/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/text_to_image/main.py b/text_to_image/main.py index 3d81b3fb0..07cf17472 100644 --- a/text_to_image/main.py +++ b/text_to_image/main.py @@ -111,7 +111,7 @@ def get_args(): parser.add_argument( "--device", default="cuda", - choices=["cuda", "cpu"], + choices=["cuda", "cpu", "rocm"], help="device to run the benchmark", ) parser.add_argument( From 8d6392d870cdae855ad59d661f856b6606ef8df1 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 19 Sep 2024 23:55:12 +0100 Subject: [PATCH 122/169] Fix _short tag support --- main.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 49c9249ea..a26c00ba1 100755 --- a/main.py +++ b/main.py @@ -419,9 +419,14 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ if "dlrm-v2" in model.lower() and implementation == "nvidia": docker_cmd_suffix += f" \\\n{pre_space} --criteo_day23_raw_data_path=" + if "short" in extra_variation_tags: + full_ds_needed_tag = "" + else: + full_ds_needed_tag = ",_full" + docker_setup_cmd = f"""\n {f_pre_space}```bash -{f_pre_space}cm run script --tags=run-mlperf,inference,_find-performance,_full,_{code_version}{scenario_variation_tag}{extra_variation_tags} \\ +{f_pre_space}cm run script --tags=run-mlperf,inference,_find-performance,{full_ds_needed_tag}_{code_version}{scenario_variation_tag}{extra_variation_tags} \\ {pre_space} --model={model} \\ {pre_space} --implementation={implementation} \\ {pre_space} --framework={framework} \\ From 0511c951dad0dabfd125099dcf60578fa98543b3 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 19 Sep 2024 23:59:38 +0100 Subject: [PATCH 123/169] Fix install URL --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index a26c00ba1..7f202e3c8 100755 --- a/main.py +++ b/main.py @@ -136,7 +136,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, imp content += f"{cur_space2}=== \"{execution_env}\"\n" content += f"{cur_space3}###### {execution_env} Environment\n\n" # ref to cm installation - content += f"{cur_space3}Please refer to the [installation page](site:install/) to install CM for running the automated benchmark commands.\n\n" + content += f"{cur_space3}Please refer to the [installation page](site:inference/install/) to install CM for running the automated benchmark commands.\n\n" test_query_count=get_test_query_count(model, implementation, device) if "99.9" not in model: #not showing docker command as it is already done for the 99% variant From e8b2adcd494e3ea36bc960b6f8f73c0ee7a6bafa Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 20 Sep 2024 00:25:50 +0100 Subject: [PATCH 124/169] Expose bfloat16 and float16 options for sdxl --- main.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 7f202e3c8..9faa8ba7b 100755 --- a/main.py +++ b/main.py @@ -369,8 +369,11 @@ def get_run_cmd_extra(f_pre_space, model, implementation, device, scenario, scen f_pre_space += "" if scenario == "Server" or (scenario == "All Scenarios" and "Server" in scenarios): extra_content += f"{f_pre_space} * `` must be determined manually. It is usually around 80% of the Offline QPS, but on some systems, it can drop below 50%. If a higher value is specified, the latency constraint will not be met, and the run will be considered invalid.\n" - if "gptj" in model and device == "cuda" and implementation == "reference": - extra_content += f"{f_pre_space} * `--precision=[float16|bfloat16]` can help run on GPUs with less RAM \n" + if implementation == "reference" and model in [ "sdxl", "gptj-99", "gptj-99.9" ] and device in ["cuda", "rocm"]: + extra_content += f"{f_pre_space} * `--precision=float16` can help run on GPUs with less RAM \n" + if implementation == "reference" and model in [ "sdxl", "gptj-99", "gptj-99.9" ] and device in ["cpu"]: + extra_content += f"{f_pre_space} * `--precision=bfloat16` can help run on GPUs with less RAM \n" + if "gptj" in model and implementation == "reference": extra_content += f"{f_pre_space} * `--beam-size=1` Beam size of 4 is mandatory for a closed division submission but reducing the beam size can help in running the model on GPUs with lower device memory\n" if extra_content: extra_content = f"{f_pre_space}!!! tip\n\n" + extra_content From d7080cdceffe28a54cbaedf0772a2bd9e35dcba3 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 20 Sep 2024 00:37:41 +0100 Subject: [PATCH 125/169] Expose download model to host option for sdxl --- main.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/main.py b/main.py index 9faa8ba7b..afc634b2d 100755 --- a/main.py +++ b/main.py @@ -331,6 +331,10 @@ def get_docker_info(spaces, model, implementation, device, setup_tips=True): #pre_space = " " if setup_tips: info += f"\n{pre_space}!!! tip\n\n" + + if model == "sdxl": + info+= f"{pre_space} - `--env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes` option can be used to download the model on the host so that it can be reused across different container lanuches. \n\n" + info+= f"{pre_space} - Batch size could be adjusted using `--batch_size=#`, where `#` is the desired batch size. This option works only if the implementation in use is supporting the given batch size.\n\n" if implementation.lower() == "nvidia": info+= f"{pre_space} - Default batch size is assigned based on [GPU memory](https://github.com/mlcommons/cm4mlops/blob/dd0c35856969c68945524d5c80414c615f5fe42c/script/app-mlperf-inference-nvidia/_cm.yaml#L1129) or the [specified GPU](https://github.com/mlcommons/cm4mlops/blob/dd0c35856969c68945524d5c80414c615f5fe42c/script/app-mlperf-inference-nvidia/_cm.yaml#L1370). Please click more option for *docker launch* or *run command* to see how to specify the GPU name.\n\n" @@ -338,6 +342,11 @@ def get_docker_info(spaces, model, implementation, device, setup_tips=True): if "llama2" in model.lower(): info+= f"{pre_space} - The dataset for NVIDIA's implementation of Llama2 is not publicly available. The user must fill [this](https://docs.google.com/forms/d/e/1FAIpQLSc_8VIvRmXM3I8KQaYnKf7gy27Z63BBoI_I1u02f4lw6rBp3g/viewform?pli=1&fbzx=-8842630989397184967) form and be verified as a MLCommons member to access the dataset.\n\n" info+= f"{pre_space} - `PATH_TO_PICKE_FILE` should be replaced with path to the downloaded pickle file.\n\n" + else: + if model == "sdxl": + info += f"\n{pre_space}!!! tip\n\n" + info+= f"{pre_space} - `--env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes` option can be used to download the model on the host so that it can be reused across different container lanuches. \n\n" + return info def get_readme_prefix(spaces, model, implementation, extra_variation_tags): From 900938295bfbe501a32b31ab8da3719fb94cdaf0 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 20 Sep 2024 16:24:47 +0100 Subject: [PATCH 126/169] IndySCC24 documentation added --- .../language/reproducibility/scc24-bert.md | 18 ++++++++++++++++++ mkdocs.yml | 5 ++++- 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 docs/benchmarks/language/reproducibility/scc24-bert.md diff --git a/docs/benchmarks/language/reproducibility/scc24-bert.md b/docs/benchmarks/language/reproducibility/scc24-bert.md new file mode 100644 index 000000000..4bd48135a --- /dev/null +++ b/docs/benchmarks/language/reproducibility/scc24-bert.md @@ -0,0 +1,18 @@ +--- +hide: + - toc +--- + +# Text to Image using Bert Large for IndySCC 2024 + + +=== "MLCommons-Python" + ## MLPerf Reference Implementation in Python + +{{ mlperf_inference_implementation_readme (4, "bert-99", "reference", extra_variation_tags=",_short", scenarios=["Offline"],categories=["Edge"], setup_tips=False) }} + +=== "Nvidia" + ## Nvidia MLPerf Implementation +{{ mlperf_inference_implementation_readme (4, "bert-99", "nvidia", extra_variation_tags=",_short", scenarios=["Offline"],categories=["Edge"], setup_tips=False, implementation_tips=False) }} + + diff --git a/mkdocs.yml b/mkdocs.yml index 75556e05d..14b49d0d4 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -33,7 +33,10 @@ nav: - Medical Imaging: - 3d-unet: benchmarks/medical_imaging/3d-unet.md - Language Processing: - - Bert-Large: benchmarks/language/bert.md + - Bert-Large: + - Run Commands: benchmarks/language/bert.md + - Reproducibility: + - IndySCC24: benchmarks/language/reproducibility/scc24-bert.md - GPT-J: benchmarks/language/gpt-j.md - LLAMA2-70B: benchmarks/language/llama2-70b.md - MIXTRAL-8x7B: benchmarks/language/mixtral-8x7b.md From 00c650fdd6fe1e47ed66885ff27d960eb0129b77 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 20 Sep 2024 17:26:29 +0100 Subject: [PATCH 127/169] Improve the SCC24 docs --- .../text_to_image/reproducibility/scc24.md | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/docs/benchmarks/text_to_image/reproducibility/scc24.md b/docs/benchmarks/text_to_image/reproducibility/scc24.md index b55008bbc..4f66097a6 100644 --- a/docs/benchmarks/text_to_image/reproducibility/scc24.md +++ b/docs/benchmarks/text_to_image/reproducibility/scc24.md @@ -3,8 +3,47 @@ hide: - toc --- -# Text to Image using Stable Diffusion for Student Cluster Competition 2024 +# Text-to-Image with Stable Diffusion for Student Cluster Competition 2024 +## Introduction + +This guide is designed for the [Student Cluster Competition 2024](https://studentclustercompetition.us/2024/index.html) to walk participants through running and optimizing the [MLPerf Inference Benchmark](https://arxiv.org/abs/1911.02549) using [Stable Diffusion XL 1.0](https://github.com/mlcommons/inference/tree/master/text_to_image#supported-models) across various software and hardware configurations. The goal is to maximize system throughput (measured in samples per second) without compromising accuracy. Since the model performs poorly on CPUs, it is essential to run it on GPUs. + +For a valid MLPerf inference submission, two types of runs are required: a performance run and an accuracy run. In this competition, we focus on the `Offline` scenario, where throughput is the key metric—higher values are better. The official MLPerf inference benchmark for Stable Diffusion XL requires processing a minimum of 5,000 samples in both performance and accuracy modes using the COCO 2014 dataset. However, for SCC, the dataset size has been reduced to 50 samples, making it possible to complete both runs in approximately 5-10 minutes. Setting up for Nvidia GPUs may take 2-3 hours but can be done offline. Your final output will be a tarball (`mlperf_submission.tar.gz`) containing MLPerf-compatible results, which you will submit to the SCC organizers for scoring. + +## Scoring + +In the SCC, your first objective will be to run a reference (unoptimized) Python implementation or a vendor-provided version (such as Nvidia's) of the MLPerf inference benchmark to secure a baseline score. + +Once the initial run is successful, you'll have the opportunity to optimize the benchmark further by maximizing system utilization, applying quantization techniques, adjusting ML frameworks, experimenting with batch sizes, and more, all of which can earn you additional points. + +Since vendor implementations of the MLPerf inference benchmark vary and are often limited to single-node benchmarking, teams will compete within their respective hardware categories (e.g., Nvidia GPUs, AMD GPUs). Points will be awarded based on the throughput achieved on your system. + +Additionally, significant bonus points will be awarded if your team enhances an existing implementation, adds support for new hardware (such as Intel GPUs), enables multi-node execution, or improves the MLPerf SDXL model's performance while maintaining accuracy. All improvements must be made publicly available under the Apache 2.0 license and submitted alongside your results to the SCC committee to earn these bonus points, contributing to the MLPerf community. + + +!!! info + Both MLPerf and CM automation are evolving projects. + If you encounter issues or have questions, please submit them [here](https://github.com/mlcommons/cm4mlops/issues) + +## Artifacts to submit to the SCC committee + +You will need to submit the following files: + +* `mlperf_submission_short.tar.gz` - automatically generated file with validated MLPerf results. +* `mlperf_submission_short_summary.json` - automatically generated summary of MLPerf results. +* `mlperf_submission_short.run` - CM commands to run MLPerf BERT inference benchmark saved to this file. +* `mlperf_submission_short.tstamps` - execution timestamps before and after CM command saved to this file. +* `mlperf_submission_short.md` - description of your platform and some highlights of the MLPerf benchmark execution. + + +## SCC interview + +You are encouraged to highlight and explain the obtained MLPerf inference throughput on your system +and describe any improvements and extensions to this benchmark (such as adding new hardware backend +or supporting multi-node execution) useful for the community and [MLCommons](https://mlcommons.org). + +## Run Commands === "MLCommons-Python" ## MLPerf Reference Implementation in Python From 4c92e2adbdfda9187649bb29ea631f7564f6d8f9 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 20 Sep 2024 17:39:26 +0100 Subject: [PATCH 128/169] Improve the support of short variation --- main.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/main.py b/main.py index d725a0374..4e9a93d9a 100755 --- a/main.py +++ b/main.py @@ -91,6 +91,8 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, imp # model name content += f"{pre_space}{model.upper()}\n\n" + final_run_mode = "valid" if "short" not in extra_variation_tags else "test" + for category in categories: if not scenarios: if category == "Edge" and not scenarios: @@ -137,7 +139,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, imp content += f"{cur_space3}###### {execution_env} Environment\n\n" # ref to cm installation content += f"{cur_space3}Please refer to the [installation page](site:inference/install/) to install CM for running the automated benchmark commands.\n\n" - test_query_count=get_test_query_count(model, implementation, device) + test_query_count=get_test_query_count(model, implementation, device.lower()) if "99.9" not in model: #not showing docker command as it is already done for the 99% variant if implementation == "neuralmagic": @@ -220,13 +222,13 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, imp for scenario in scenarios: content += f"{cur_space3}=== \"{scenario}\"\n{cur_space4}###### {scenario}\n\n" - run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), "valid", 0, False, scenarios, code_version, extra_variation_tags, extra_input_string) + run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), final_run_mode, -1, False, scenarios, code_version, extra_variation_tags, extra_input_string) content += run_cmd #content += run_suffix if len(scenarios) > 1: content += f"{cur_space3}=== \"All Scenarios\"\n{cur_space4}###### All Scenarios\n\n" - run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), "valid", 0, False, scenarios, code_version, extra_variation_tags, extra_input_string) + run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), final_run_mode, -1, False, scenarios, code_version, extra_variation_tags, extra_input_string) content += run_cmd content += run_suffix @@ -245,10 +247,10 @@ def get_test_query_count(model, implementation, device, num_devices=1): elif model in [ "retinanet", "bert-99", "bert-99.9" ]: p_range = 100 else: - p_range = 50 + p_range = 10 if device == "cuda": - p_range *= 40 + p_range *= 5 p_range *= num_devices return p_range @@ -452,7 +454,7 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ else: cmd_suffix = f"\\\n{pre_space} --quiet {extra_input_string}" - if execution_mode == "test": + if execution_mode == "test" and test_query_count > 0: cmd_suffix += f" \\\n {pre_space} --test_query_count={test_query_count}" if "bert" in model.lower() and framework == "deepsparse": @@ -471,7 +473,7 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ run_cmd = f"""\n {f_pre_space}```bash -{f_pre_space}cm run script --tags=run-mlperf,inference,_{code_version}{scenario_variation_tag} \\ +{f_pre_space}cm run script --tags=run-mlperf,inference,_{code_version}{scenario_variation_tag}{extra_variation_tags} \\ {pre_space} --model={model} \\ {pre_space} --implementation={implementation} \\ {pre_space} --framework={framework} \\ From 60d3a8a7a283f1d74989b027e5b85d0d1cf9eb5e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 20 Sep 2024 17:52:17 +0100 Subject: [PATCH 129/169] Improved the indyscc24 documentation --- .../reproducibility/indyscc24-bert.md | 48 +++++++++++++++++++ .../language/reproducibility/scc24-bert.md | 18 ------- mkdocs.yml | 2 +- 3 files changed, 49 insertions(+), 19 deletions(-) create mode 100644 docs/benchmarks/language/reproducibility/indyscc24-bert.md delete mode 100644 docs/benchmarks/language/reproducibility/scc24-bert.md diff --git a/docs/benchmarks/language/reproducibility/indyscc24-bert.md b/docs/benchmarks/language/reproducibility/indyscc24-bert.md new file mode 100644 index 000000000..68215c5e1 --- /dev/null +++ b/docs/benchmarks/language/reproducibility/indyscc24-bert.md @@ -0,0 +1,48 @@ +--- +hide: + - toc +--- + +# Question and Answering using Bert Large for IndySCC 2024 + +## Introduction + +This guide is designed for the [IndySCC 2024](https://sc24.supercomputing.org/students/indyscc/) to walk participants through running and optimizing the [MLPerf Inference Benchmark](https://arxiv.org/abs/1911.02549) using [Bert Large](https://github.com/mlcommons/inference/tree/master/language/bert#supported-models) across various software and hardware configurations. The goal is to maximize system throughput (measured in samples per second) without compromising accuracy. + +For a valid MLPerf inference submission, two types of runs are required: a performance run and an accuracy run. In this competition, we focus on the `Offline` scenario, where throughput is the key metric—higher values are better. The official MLPerf inference benchmark for Bert Large requires processing a minimum of 10833 samples in both performance and accuracy modes using the Squad v1.1 dataset. Setting up for Nvidia GPUs may take 2-3 hours but can be done offline. Your final output will be a tarball (`mlperf_submission.tar.gz`) containing MLPerf-compatible results, which you will submit to the SCC organizers for scoring. + +## Scoring + +In the SCC, your first objective will be to run a reference (unoptimized) Python implementation or a vendor-provided version (such as Nvidia's) of the MLPerf inference benchmark to secure a baseline score. + +Once the initial run is successful, you'll have the opportunity to optimize the benchmark further by maximizing system utilization, applying quantization techniques, adjusting ML frameworks, experimenting with batch sizes, and more, all of which can earn you additional points. + +Since vendor implementations of the MLPerf inference benchmark vary and are often limited to single-node benchmarking, teams will compete within their respective hardware categories (e.g., Nvidia GPUs, AMD GPUs). Points will be awarded based on the throughput achieved on your system. + + +!!! info + Both MLPerf and CM automation are evolving projects. + If you encounter issues or have questions, please submit them [here](https://github.com/mlcommons/cm4mlops/issues) + +## Artifacts to submit to the SCC committee + +You will need to submit the following files: + +* `mlperf_submission_short.tar.gz` - automatically generated file with validated MLPerf results. +* `mlperf_submission_short_summary.json` - automatically generated summary of MLPerf results. +* `mlperf_submission_short.run` - CM commands to run MLPerf BERT inference benchmark saved to this file. +* `mlperf_submission_short.tstamps` - execution timestamps before and after CM command saved to this file. +* `mlperf_submission_short.md` - description of your platform and some highlights of the MLPerf benchmark execution. + + + +=== "MLCommons-Python" + ## MLPerf Reference Implementation in Python + +{{ mlperf_inference_implementation_readme (4, "bert-99", "reference", extra_variation_tags=",_short", scenarios=["Offline"],categories=["Edge"], setup_tips=False) }} + +=== "Nvidia" + ## Nvidia MLPerf Implementation +{{ mlperf_inference_implementation_readme (4, "bert-99", "nvidia", extra_variation_tags=",_short", scenarios=["Offline"],categories=["Edge"], setup_tips=False, implementation_tips=False) }} + + diff --git a/docs/benchmarks/language/reproducibility/scc24-bert.md b/docs/benchmarks/language/reproducibility/scc24-bert.md deleted file mode 100644 index 4bd48135a..000000000 --- a/docs/benchmarks/language/reproducibility/scc24-bert.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -hide: - - toc ---- - -# Text to Image using Bert Large for IndySCC 2024 - - -=== "MLCommons-Python" - ## MLPerf Reference Implementation in Python - -{{ mlperf_inference_implementation_readme (4, "bert-99", "reference", extra_variation_tags=",_short", scenarios=["Offline"],categories=["Edge"], setup_tips=False) }} - -=== "Nvidia" - ## Nvidia MLPerf Implementation -{{ mlperf_inference_implementation_readme (4, "bert-99", "nvidia", extra_variation_tags=",_short", scenarios=["Offline"],categories=["Edge"], setup_tips=False, implementation_tips=False) }} - - diff --git a/mkdocs.yml b/mkdocs.yml index 14b49d0d4..95dfb6e86 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -36,7 +36,7 @@ nav: - Bert-Large: - Run Commands: benchmarks/language/bert.md - Reproducibility: - - IndySCC24: benchmarks/language/reproducibility/scc24-bert.md + - IndySCC24: benchmarks/language/reproducibility/indyscc24-bert.md - GPT-J: benchmarks/language/gpt-j.md - LLAMA2-70B: benchmarks/language/llama2-70b.md - MIXTRAL-8x7B: benchmarks/language/mixtral-8x7b.md From b2f95b2affa0613b0ecdaeee5eff4f0ddef40ab6 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 23 Sep 2024 23:25:36 +0530 Subject: [PATCH 130/169] Updated scc run commands --- .../text_to_image/reproducibility/scc24.md | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/docs/benchmarks/text_to_image/reproducibility/scc24.md b/docs/benchmarks/text_to_image/reproducibility/scc24.md index 4f66097a6..7f7617678 100644 --- a/docs/benchmarks/text_to_image/reproducibility/scc24.md +++ b/docs/benchmarks/text_to_image/reproducibility/scc24.md @@ -48,10 +48,45 @@ or supporting multi-node execution) useful for the community and [MLCommons](htt === "MLCommons-Python" ## MLPerf Reference Implementation in Python -{{ mlperf_inference_implementation_readme (4, "sdxl", "reference", extra_variation_tags=",_short", scenarios=["Offline"],categories=["Edge"], setup_tips=False) }} +{{ mlperf_inference_implementation_readme (4, "sdxl", "reference", extra_variation_tags=",_short,_scc24-base", devices=["ROCm", "CUDA"],scenarios=["Offline"],categories=["Edge"], setup_tips=False) }} === "Nvidia" ## Nvidia MLPerf Implementation {{ mlperf_inference_implementation_readme (4, "sdxl", "nvidia", extra_variation_tags=",_short", scenarios=["Offline"],categories=["Edge"], setup_tips=False, implementation_tips=False) }} +## Submission Commands +### Generate actual submission tree + +```bash + cm run script --tags=generate,inference,submission \ + --clean \ + --preprocess_submission=yes \ + --run-checker \ + --submitter= \ + --tar=yes \ + --env.CM_TAR_OUTFILE=submission.tar.gz \ + --division=open \ + --category=datacenter \ + --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ + --quiet +``` + +* Use `--hw_name="My system name"` to give a meaningful system name. Examples can be seen [here](https://github.com/mlcommons/inference_results_v3.0/tree/main/open/cTuning/systems) + +* Use `--hw_notes_extra` option to add additional notes like `--hw_notes_extra="Result taken by NAME" ` + + From 84ba6508e1d2bc0b8399242f3a3e90bfa8ec7ff2 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 23 Sep 2024 23:44:35 +0530 Subject: [PATCH 131/169] removed test_query_count option for scc --- main.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 4e9a93d9a..be6ae49ee 100755 --- a/main.py +++ b/main.py @@ -417,8 +417,10 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ if docker: docker_cmd_suffix = f" \\\n{pre_space} --docker --quiet" - docker_cmd_suffix += f" \\\n{pre_space} --test_query_count={test_query_count} {extra_docker_input_string} {extra_input_string}" - + if "scc24" not in extra_variation_tags: + docker_cmd_suffix += f" \\\n{pre_space} --test_query_count={test_query_count} {extra_docker_input_string} {extra_input_string}" + if extra_docker_input_string != "" or extra_input_string != "": + docker_cmd_suffix += f" \\\n{pre_space} {extra_docker_input_string} {extra_input_string}" if "bert" in model.lower() and framework == "deepsparse": docker_cmd_suffix += f"\\\n{pre_space} --env.CM_MLPERF_NEURALMAGIC_MODEL_ZOO_STUB=zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/base_quant-none" if "llama2-70b" in model.lower(): @@ -454,7 +456,7 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ else: cmd_suffix = f"\\\n{pre_space} --quiet {extra_input_string}" - if execution_mode == "test" and test_query_count > 0: + if execution_mode == "test" and test_query_count > 0 and "scc24" not in extra_variation_tags: cmd_suffix += f" \\\n {pre_space} --test_query_count={test_query_count}" if "bert" in model.lower() and framework == "deepsparse": From 6c2381603e1d106c9402181d8cdf0bf2c64d59f1 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 23 Sep 2024 22:55:27 +0100 Subject: [PATCH 132/169] Remove scc24 in the main docs --- .../text_to_image/reproducibility/scc24.md | 40 +++++++++---------- main.py | 6 +-- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/docs/benchmarks/text_to_image/reproducibility/scc24.md b/docs/benchmarks/text_to_image/reproducibility/scc24.md index 7f7617678..029e56116 100644 --- a/docs/benchmarks/text_to_image/reproducibility/scc24.md +++ b/docs/benchmarks/text_to_image/reproducibility/scc24.md @@ -48,45 +48,45 @@ or supporting multi-node execution) useful for the community and [MLCommons](htt === "MLCommons-Python" ## MLPerf Reference Implementation in Python -{{ mlperf_inference_implementation_readme (4, "sdxl", "reference", extra_variation_tags=",_short,_scc24-base", devices=["ROCm", "CUDA"],scenarios=["Offline"],categories=["Edge"], setup_tips=False) }} +{{ mlperf_inference_implementation_readme (4, "sdxl", "reference", extra_variation_tags=",_short,_scc24-base", devices=["ROCm", "CUDA"],scenarios=["Offline"],categories=["Datacenter"], setup_tips=False) }} === "Nvidia" ## Nvidia MLPerf Implementation -{{ mlperf_inference_implementation_readme (4, "sdxl", "nvidia", extra_variation_tags=",_short", scenarios=["Offline"],categories=["Edge"], setup_tips=False, implementation_tips=False) }} +{{ mlperf_inference_implementation_readme (4, "sdxl", "nvidia", extra_variation_tags=",_short", scenarios=["Offline"],categories=["Datacenter"], setup_tips=False, implementation_tips=False) }} ## Submission Commands ### Generate actual submission tree ```bash - cm run script --tags=generate,inference,submission \ - --clean \ - --preprocess_submission=yes \ - --run-checker \ - --submitter= \ - --tar=yes \ - --env.CM_TAR_OUTFILE=submission.tar.gz \ - --division=open \ - --category=datacenter \ - --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ - --quiet +cm run script --tags=generate,inference,submission \ + --clean \ + --preprocess_submission=yes \ + --run-checker \ + --tar=yes \ + --env.CM_TAR_OUTFILE=submission.tar.gz \ + --division=open \ + --category=datacenter \ + --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ + --run_style=test --adr.submission-checker.tags=_short-run \ + --quiet \ + --submitter= ``` -* Use `--hw_name="My system name"` to give a meaningful system name. Examples can be seen [here](https://github.com/mlcommons/inference_results_v3.0/tree/main/open/cTuning/systems) +* Use `--hw_name="My system name"` to give a meaningful system name. -* Use `--hw_notes_extra` option to add additional notes like `--hw_notes_extra="Result taken by NAME" ` - diff --git a/main.py b/main.py index be6ae49ee..b778c86e9 100755 --- a/main.py +++ b/main.py @@ -417,8 +417,8 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ if docker: docker_cmd_suffix = f" \\\n{pre_space} --docker --quiet" - if "scc24" not in extra_variation_tags: - docker_cmd_suffix += f" \\\n{pre_space} --test_query_count={test_query_count} {extra_docker_input_string} {extra_input_string}" + if test_query_count > 0: + docker_cmd_suffix += f" \\\n{pre_space} --test_query_count={test_query_count}"" if extra_docker_input_string != "" or extra_input_string != "": docker_cmd_suffix += f" \\\n{pre_space} {extra_docker_input_string} {extra_input_string}" if "bert" in model.lower() and framework == "deepsparse": @@ -456,7 +456,7 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ else: cmd_suffix = f"\\\n{pre_space} --quiet {extra_input_string}" - if execution_mode == "test" and test_query_count > 0 and "scc24" not in extra_variation_tags: + if execution_mode == "test" and test_query_count > 0: cmd_suffix += f" \\\n {pre_space} --test_query_count={test_query_count}" if "bert" in model.lower() and framework == "deepsparse": From 469b091ca904dbf72450c85e9c30b924df2e3e1d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 23 Sep 2024 22:56:48 +0100 Subject: [PATCH 133/169] Remove scc24 in the main docs --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index b778c86e9..e0f85fe2f 100755 --- a/main.py +++ b/main.py @@ -418,7 +418,7 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ if docker: docker_cmd_suffix = f" \\\n{pre_space} --docker --quiet" if test_query_count > 0: - docker_cmd_suffix += f" \\\n{pre_space} --test_query_count={test_query_count}"" + docker_cmd_suffix += f" \\\n{pre_space} --test_query_count={test_query_count}" if extra_docker_input_string != "" or extra_input_string != "": docker_cmd_suffix += f" \\\n{pre_space} {extra_docker_input_string} {extra_input_string}" if "bert" in model.lower() and framework == "deepsparse": From 21d16ed317326bf2e228d6f71deb66107b2889dc Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Mon, 23 Sep 2024 23:02:32 +0100 Subject: [PATCH 134/169] Fix docs: indendation issue on the submission page --- docs/submission/index.md | 90 ++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/docs/submission/index.md b/docs/submission/index.md index a75bc3259..4f6e05c25 100644 --- a/docs/submission/index.md +++ b/docs/submission/index.md @@ -60,63 +60,63 @@ Once all the results across all the models are ready you can use the following c === "Closed Edge" ### Closed Edge Submission ```bash - cm run script --tags=generate,inference,submission \ - --clean \ - --preprocess_submission=yes \ - --run-checker \ - --submitter=MLCommons \ - --tar=yes \ - --env.CM_TAR_OUTFILE=submission.tar.gz \ - --division=closed \ - --category=edge \ - --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ - --quiet + cm run script --tags=generate,inference,submission \ + --clean \ + --preprocess_submission=yes \ + --run-checker \ + --submitter=MLCommons \ + --tar=yes \ + --env.CM_TAR_OUTFILE=submission.tar.gz \ + --division=closed \ + --category=edge \ + --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ + --quiet ``` === "Closed Datacenter" ### Closed Datacenter Submission ```bash - cm run script --tags=generate,inference,submission \ - --clean \ - --preprocess_submission=yes \ - --run-checker \ - --submitter=MLCommons \ - --tar=yes \ - --env.CM_TAR_OUTFILE=submission.tar.gz \ - --division=closed \ - --category=datacenter \ - --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ - --quiet + cm run script --tags=generate,inference,submission \ + --clean \ + --preprocess_submission=yes \ + --run-checker \ + --submitter=MLCommons \ + --tar=yes \ + --env.CM_TAR_OUTFILE=submission.tar.gz \ + --division=closed \ + --category=datacenter \ + --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ + --quiet ``` === "Open Edge" ### Open Edge Submission ```bash - cm run script --tags=generate,inference,submission \ - --clean \ - --preprocess_submission=yes \ - --run-checker \ - --submitter=MLCommons \ - --tar=yes \ - --env.CM_TAR_OUTFILE=submission.tar.gz \ - --division=open \ - --category=edge \ - --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ - --quiet + cm run script --tags=generate,inference,submission \ + --clean \ + --preprocess_submission=yes \ + --run-checker \ + --submitter=MLCommons \ + --tar=yes \ + --env.CM_TAR_OUTFILE=submission.tar.gz \ + --division=open \ + --category=edge \ + --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ + --quiet ``` === "Open Datacenter" ### Closed Datacenter Submission ```bash - cm run script --tags=generate,inference,submission \ - --clean \ - --preprocess_submission=yes \ - --run-checker \ - --submitter=MLCommons \ - --tar=yes \ - --env.CM_TAR_OUTFILE=submission.tar.gz \ - --division=open \ - --category=datacenter \ - --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ - --quiet + cm run script --tags=generate,inference,submission \ + --clean \ + --preprocess_submission=yes \ + --run-checker \ + --submitter=MLCommons \ + --tar=yes \ + --env.CM_TAR_OUTFILE=submission.tar.gz \ + --division=open \ + --category=datacenter \ + --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ + --quiet ``` * Use `--hw_name="My system name"` to give a meaningful system name. Examples can be seen [here](https://github.com/mlcommons/inference_results_v3.0/tree/main/open/cTuning/systems) @@ -134,7 +134,7 @@ If you are collecting results across multiple systems you can generate different Run the following command after **replacing `--repo_url` with your GitHub repository URL**. ```bash - cm run script --tags=push,github,mlperf,inference,submission \ +cm run script --tags=push,github,mlperf,inference,submission \ --repo_url=https://github.com/GATEOverflow/mlperf_inference_submissions_v4.1 \ --commit_message="Results on added by " \ --quiet From 5d4a3026f19b7c10f5396e93da35334a4ceaf86f Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 24 Sep 2024 10:51:44 +0530 Subject: [PATCH 135/169] generalised code for skipping test query count --- .../text_to_image/reproducibility/scc24.md | 4 ++-- main.py | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/benchmarks/text_to_image/reproducibility/scc24.md b/docs/benchmarks/text_to_image/reproducibility/scc24.md index 029e56116..4dd638b88 100644 --- a/docs/benchmarks/text_to_image/reproducibility/scc24.md +++ b/docs/benchmarks/text_to_image/reproducibility/scc24.md @@ -48,11 +48,11 @@ or supporting multi-node execution) useful for the community and [MLCommons](htt === "MLCommons-Python" ## MLPerf Reference Implementation in Python -{{ mlperf_inference_implementation_readme (4, "sdxl", "reference", extra_variation_tags=",_short,_scc24-base", devices=["ROCm", "CUDA"],scenarios=["Offline"],categories=["Datacenter"], setup_tips=False) }} +{{ mlperf_inference_implementation_readme (4, "sdxl", "reference", extra_variation_tags=",_short,_scc24-base", devices=["ROCm", "CUDA"],scenarios=["Offline"],categories=["Datacenter"], setup_tips=False, skip_test_query_count=True) }} === "Nvidia" ## Nvidia MLPerf Implementation -{{ mlperf_inference_implementation_readme (4, "sdxl", "nvidia", extra_variation_tags=",_short", scenarios=["Offline"],categories=["Datacenter"], setup_tips=False, implementation_tips=False) }} +{{ mlperf_inference_implementation_readme (4, "sdxl", "nvidia", extra_variation_tags=",_short", scenarios=["Offline"],categories=["Datacenter"], setup_tips=False, implementation_tips=False, skip_test_query_count=True) }} ## Submission Commands diff --git a/main.py b/main.py index e0f85fe2f..81f5c355d 100755 --- a/main.py +++ b/main.py @@ -1,7 +1,7 @@ def define_env(env): @env.macro - def mlperf_inference_implementation_readme(spaces, model, implementation, *, implementation_tips=True, setup_tips=True, run_tips=True, scenarios = [], devices=[], frameworks=[], categories=[], extra_variation_tags="", extra_input_string="", extra_docker_input_string=""): + def mlperf_inference_implementation_readme(spaces, model, implementation, *, implementation_tips=True, setup_tips=True, run_tips=True, skip_test_query_count=False, scenarios = [], devices=[], frameworks=[], categories=[], extra_variation_tags="", extra_input_string="", extra_docker_input_string=""): pre_space = "" for i in range(1,spaces): @@ -150,7 +150,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, imp content += f"\n{cur_space3}!!! tip\n\n" content += f"{cur_space3} - Host and Port number of the server can be configured through `--host` and `--port` options. Otherwise, server will run on the default host `localhost` and port `8000`.\n\n" - setup_run_cmd = mlperf_inference_run_command(spaces+17, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True, scenarios, code_version, extra_variation_tags, extra_input_string, extra_docker_input_string) + setup_run_cmd = mlperf_inference_run_command(spaces+17, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True, skip_test_query_count, scenarios, code_version, extra_variation_tags, extra_input_string, extra_docker_input_string) if execution_env == "Native": # Native implementation steps through virtual environment content += f"{cur_space3}####### Setup a virtual environment for Python\n" @@ -186,7 +186,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, imp content += f"{cur_space3} You can reuse the same environment as described for {model.split('.')[0]}.\n" content += f"{cur_space3}###### Performance Estimation for Offline Scenario\n" - content += mlperf_inference_run_command(spaces+17, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True, scenarios, code_version).replace("--docker ","") + content += mlperf_inference_run_command(spaces+17, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True, skip_test_query_count, scenarios, code_version).replace("--docker ","") content += f"{cur_space3}The above command should do a test run of Offline scenario and record the estimated offline_target_qps.\n\n" @@ -222,13 +222,13 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, imp for scenario in scenarios: content += f"{cur_space3}=== \"{scenario}\"\n{cur_space4}###### {scenario}\n\n" - run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), final_run_mode, -1, False, scenarios, code_version, extra_variation_tags, extra_input_string) + run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), final_run_mode, -1, False, skip_test_query_count, scenarios, code_version, extra_variation_tags, extra_input_string) content += run_cmd #content += run_suffix if len(scenarios) > 1: content += f"{cur_space3}=== \"All Scenarios\"\n{cur_space4}###### All Scenarios\n\n" - run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), final_run_mode, -1, False, scenarios, code_version, extra_variation_tags, extra_input_string) + run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), final_run_mode, -1, False, skip_test_query_count, scenarios, code_version, extra_variation_tags, extra_input_string) content += run_cmd content += run_suffix @@ -396,7 +396,7 @@ def get_run_cmd_extra(f_pre_space, model, implementation, device, scenario, scen @env.macro - def mlperf_inference_run_command(spaces, model, implementation, framework, category, scenario, device="cpu", execution_mode="test", test_query_count="20", docker=False, scenarios = [], code_version="r4.1-dev", extra_variation_tags="", extra_input_string="", extra_docker_input_string=""): + def mlperf_inference_run_command(spaces, model, implementation, framework, category, scenario, device="cpu", execution_mode="test", test_query_count="20", docker=False, skip_test_query_count=False, scenarios = [], code_version="r4.1-dev", extra_variation_tags="", extra_input_string="", extra_docker_input_string=""): pre_space = "" for i in range(1,spaces): pre_space = pre_space + " " @@ -417,7 +417,7 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ if docker: docker_cmd_suffix = f" \\\n{pre_space} --docker --quiet" - if test_query_count > 0: + if not skip_test_query_count: docker_cmd_suffix += f" \\\n{pre_space} --test_query_count={test_query_count}" if extra_docker_input_string != "" or extra_input_string != "": docker_cmd_suffix += f" \\\n{pre_space} {extra_docker_input_string} {extra_input_string}" @@ -456,7 +456,7 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ else: cmd_suffix = f"\\\n{pre_space} --quiet {extra_input_string}" - if execution_mode == "test" and test_query_count > 0: + if execution_mode == "test" and not skip_test_query_count: cmd_suffix += f" \\\n {pre_space} --test_query_count={test_query_count}" if "bert" in model.lower() and framework == "deepsparse": From 21e72596149f818fe210b0409f58773be9ef9a16 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 24 Sep 2024 14:33:28 +0100 Subject: [PATCH 136/169] Fixes for SCC24 docs --- .../text_to_image/reproducibility/scc24.md | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/benchmarks/text_to_image/reproducibility/scc24.md b/docs/benchmarks/text_to_image/reproducibility/scc24.md index 4dd638b88..f4e0cbea9 100644 --- a/docs/benchmarks/text_to_image/reproducibility/scc24.md +++ b/docs/benchmarks/text_to_image/reproducibility/scc24.md @@ -9,17 +9,17 @@ hide: This guide is designed for the [Student Cluster Competition 2024](https://studentclustercompetition.us/2024/index.html) to walk participants through running and optimizing the [MLPerf Inference Benchmark](https://arxiv.org/abs/1911.02549) using [Stable Diffusion XL 1.0](https://github.com/mlcommons/inference/tree/master/text_to_image#supported-models) across various software and hardware configurations. The goal is to maximize system throughput (measured in samples per second) without compromising accuracy. Since the model performs poorly on CPUs, it is essential to run it on GPUs. -For a valid MLPerf inference submission, two types of runs are required: a performance run and an accuracy run. In this competition, we focus on the `Offline` scenario, where throughput is the key metric—higher values are better. The official MLPerf inference benchmark for Stable Diffusion XL requires processing a minimum of 5,000 samples in both performance and accuracy modes using the COCO 2014 dataset. However, for SCC, the dataset size has been reduced to 50 samples, making it possible to complete both runs in approximately 5-10 minutes. Setting up for Nvidia GPUs may take 2-3 hours but can be done offline. Your final output will be a tarball (`mlperf_submission.tar.gz`) containing MLPerf-compatible results, which you will submit to the SCC organizers for scoring. +For a valid MLPerf inference submission, two types of runs are required: a performance run and an accuracy run. In this competition, we focus on the `Offline` scenario, where throughput is the key metric—higher values are better. The official MLPerf inference benchmark for Stable Diffusion XL requires processing a minimum of 5,000 samples in both performance and accuracy modes using the COCO 2014 dataset. However, for SCC, we have reduced this and we also have two variants. `scc-base` variant has dataset size reduced to 50 samples, making it possible to complete both performance and accuracy runs in approximately 5-10 minutes. `scc-main` variant has dataset size of 500 and running it will fetch extra points as compared to running just the base variant. Setting up for Nvidia GPUs may take 2-3 hours but can be done offline. Your final output will be a tarball (`mlperf_submission.tar.gz`) containing MLPerf-compatible results, which you will submit to the SCC organizers for scoring. ## Scoring -In the SCC, your first objective will be to run a reference (unoptimized) Python implementation or a vendor-provided version (such as Nvidia's) of the MLPerf inference benchmark to secure a baseline score. +In the SCC, your first objective will be to run `scc-base` variant for reference (unoptimized) Python implementation or a vendor-provided version (such as Nvidia's) of the MLPerf inference benchmark to secure a baseline score. Once the initial run is successful, you'll have the opportunity to optimize the benchmark further by maximizing system utilization, applying quantization techniques, adjusting ML frameworks, experimenting with batch sizes, and more, all of which can earn you additional points. Since vendor implementations of the MLPerf inference benchmark vary and are often limited to single-node benchmarking, teams will compete within their respective hardware categories (e.g., Nvidia GPUs, AMD GPUs). Points will be awarded based on the throughput achieved on your system. -Additionally, significant bonus points will be awarded if your team enhances an existing implementation, adds support for new hardware (such as Intel GPUs), enables multi-node execution, or improves the MLPerf SDXL model's performance while maintaining accuracy. All improvements must be made publicly available under the Apache 2.0 license and submitted alongside your results to the SCC committee to earn these bonus points, contributing to the MLPerf community. +Additionally, significant bonus points will be awarded if your team enhances an existing implementation, adds support for new hardware (such as an unsupported GPU), enables multi-node execution, or adds new scripts to cm4mlops repository supporting new devices, frameworks, implementations etc. All improvements must be made publicly available under the Apache 2.0 license and submitted alongside your results to the SCC committee to earn these bonus points, contributing to the MLPerf community. !!! info @@ -30,11 +30,9 @@ Additionally, significant bonus points will be awarded if your team enhances an You will need to submit the following files: -* `mlperf_submission_short.tar.gz` - automatically generated file with validated MLPerf results. -* `mlperf_submission_short_summary.json` - automatically generated summary of MLPerf results. -* `mlperf_submission_short.run` - CM commands to run MLPerf BERT inference benchmark saved to this file. -* `mlperf_submission_short.tstamps` - execution timestamps before and after CM command saved to this file. -* `mlperf_submission_short.md` - description of your platform and some highlights of the MLPerf benchmark execution. +* `mlperf_submission.run` - CM commands to run MLPerf inference benchmark saved to this file. +* `mlperf_submission.md` - description of your platform and some highlights of the MLPerf benchmark execution. +* under which results are pushed to the github repository. ## SCC interview @@ -78,15 +76,18 @@ cm run script --tags=generate,inference,submission \ ### Aggregate Results in GitHub -If you are collecting results across multiple systems you can generate different submissions and aggregate all of them to a GitHub repository (can be private) and use it to generate a single tar ball which can be uploaded to the [MLCommons Submission UI](https://submissions-ui.mlcommons.org/submission). +Fork the repository URL at https://github.com/gateoverflow/cm4mlperf-inference. -Run the following command after **replacing `--repo_url` with your GitHub repository URL**. +Run the following command after **replacing `--repo_url` with your GitHub fork URL**. ```bash cm run script --tags=push,github,mlperf,inference,submission \ - --repo_url=https://github.com/mlcommons/cm4mlperf-inference \ + --repo_url=https://github.com/gateoverflow/cm4mlperf-inference \ --repo_branch=mlperf-inference-results-scc24 \ --commit_message="Results on system " \ --quiet ``` +Once uploaded give a Pull Request to the origin repository. Github action will be running there and once +finished you can see your submitted results at https://gateoverflow.github.io/cm4mlperf-inference. + From 93649dd7fad38ba2bf19bc4d6224feb40906ee02 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 24 Sep 2024 14:39:43 +0100 Subject: [PATCH 137/169] Fix scenario text in main.py --- main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 81f5c355d..85155a9be 100755 --- a/main.py +++ b/main.py @@ -168,7 +168,11 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, imp content += setup_run_cmd - content += f"{cur_space3}The above command should get you to an interactive shell inside the docker container and do a quick test run for the Offline scenario. Once inside the docker container please do the below commands to do the accuracy + performance runs for each scenario.\n\n" + if len(scenarios) == 1: + scenario_text = f"""the {scenarios[0]} scenario""" + else: + scenario_text = "each scenario""" + content += f"{cur_space3}The above command should get you to an interactive shell inside the docker container and do a quick test run for the Offline scenario. Once inside the docker container please do the below commands to do the accuracy + performance runs for {scenario_text}.\n\n" content += f"{cur_space3}
    \n" content += f"{cur_space3} Please click here to see more options for the docker launch \n\n" content += f"{cur_space3}* `--docker_cm_repo=`: to use a custom fork of cm4mlops repository inside the docker image\n\n" From 0cc5d7b02fe53a5cb3b6e1813d8945f4a090e1c9 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 24 Sep 2024 14:47:16 +0100 Subject: [PATCH 138/169] Fix links for scc24 --- docs/benchmarks/text_to_image/reproducibility/scc24.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/benchmarks/text_to_image/reproducibility/scc24.md b/docs/benchmarks/text_to_image/reproducibility/scc24.md index f4e0cbea9..93c9544db 100644 --- a/docs/benchmarks/text_to_image/reproducibility/scc24.md +++ b/docs/benchmarks/text_to_image/reproducibility/scc24.md @@ -7,7 +7,7 @@ hide: ## Introduction -This guide is designed for the [Student Cluster Competition 2024](https://studentclustercompetition.us/2024/index.html) to walk participants through running and optimizing the [MLPerf Inference Benchmark](https://arxiv.org/abs/1911.02549) using [Stable Diffusion XL 1.0](https://github.com/mlcommons/inference/tree/master/text_to_image#supported-models) across various software and hardware configurations. The goal is to maximize system throughput (measured in samples per second) without compromising accuracy. Since the model performs poorly on CPUs, it is essential to run it on GPUs. +This guide is designed for the [Student Cluster Competition 2024](https://sc24.supercomputing.org/students/student-cluster-competition/) to walk participants through running and optimizing the [MLPerf Inference Benchmark](https://arxiv.org/abs/1911.02549) using [Stable Diffusion XL 1.0](https://github.com/mlcommons/inference/tree/master/text_to_image#supported-models) across various software and hardware configurations. The goal is to maximize system throughput (measured in samples per second) without compromising accuracy. Since the model performs poorly on CPUs, it is essential to run it on GPUs. For a valid MLPerf inference submission, two types of runs are required: a performance run and an accuracy run. In this competition, we focus on the `Offline` scenario, where throughput is the key metric—higher values are better. The official MLPerf inference benchmark for Stable Diffusion XL requires processing a minimum of 5,000 samples in both performance and accuracy modes using the COCO 2014 dataset. However, for SCC, we have reduced this and we also have two variants. `scc-base` variant has dataset size reduced to 50 samples, making it possible to complete both performance and accuracy runs in approximately 5-10 minutes. `scc-main` variant has dataset size of 500 and running it will fetch extra points as compared to running just the base variant. Setting up for Nvidia GPUs may take 2-3 hours but can be done offline. Your final output will be a tarball (`mlperf_submission.tar.gz`) containing MLPerf-compatible results, which you will submit to the SCC organizers for scoring. @@ -19,7 +19,7 @@ Once the initial run is successful, you'll have the opportunity to optimize the Since vendor implementations of the MLPerf inference benchmark vary and are often limited to single-node benchmarking, teams will compete within their respective hardware categories (e.g., Nvidia GPUs, AMD GPUs). Points will be awarded based on the throughput achieved on your system. -Additionally, significant bonus points will be awarded if your team enhances an existing implementation, adds support for new hardware (such as an unsupported GPU), enables multi-node execution, or adds new scripts to cm4mlops repository supporting new devices, frameworks, implementations etc. All improvements must be made publicly available under the Apache 2.0 license and submitted alongside your results to the SCC committee to earn these bonus points, contributing to the MLPerf community. +Additionally, significant bonus points will be awarded if your team enhances an existing implementation, adds support for new hardware (such as an unsupported GPU), enables multi-node execution, or adds/extends scripts to [cm4mlops repository](https://github.com/mlcommons/cm4mlops/script) supporting new devices, frameworks, implementations etc. All improvements must be made publicly available under the Apache 2.0 license and submitted alongside your results to the SCC committee to earn these bonus points, contributing to the MLPerf community. !!! info @@ -51,7 +51,10 @@ or supporting multi-node execution) useful for the community and [MLCommons](htt === "Nvidia" ## Nvidia MLPerf Implementation {{ mlperf_inference_implementation_readme (4, "sdxl", "nvidia", extra_variation_tags=",_short", scenarios=["Offline"],categories=["Datacenter"], setup_tips=False, implementation_tips=False, skip_test_query_count=True) }} - + +!!! info + Once the above run is successful, you can change `_scc24-base` to `_scc24-main` to run the main variant. + ## Submission Commands ### Generate actual submission tree From 70f9a81e58b54fe6d7adca8cdcadf54fe376f669 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 24 Sep 2024 14:50:19 +0100 Subject: [PATCH 139/169] Fix links for scc24 --- docs/benchmarks/text_to_image/reproducibility/scc24.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/benchmarks/text_to_image/reproducibility/scc24.md b/docs/benchmarks/text_to_image/reproducibility/scc24.md index 93c9544db..a0d19ae82 100644 --- a/docs/benchmarks/text_to_image/reproducibility/scc24.md +++ b/docs/benchmarks/text_to_image/reproducibility/scc24.md @@ -79,7 +79,7 @@ cm run script --tags=generate,inference,submission \ ### Aggregate Results in GitHub -Fork the repository URL at https://github.com/gateoverflow/cm4mlperf-inference. +Fork the repository URL at [https://github.com/gateoverflow/cm4mlperf-inference](https://github.com/gateoverflow/cm4mlperf-inference). Run the following command after **replacing `--repo_url` with your GitHub fork URL**. @@ -92,5 +92,5 @@ cm run script --tags=push,github,mlperf,inference,submission \ ``` Once uploaded give a Pull Request to the origin repository. Github action will be running there and once -finished you can see your submitted results at https://gateoverflow.github.io/cm4mlperf-inference. +finished you can see your submitted results at [https://gateoverflow.github.io/cm4mlperf-inference](https://gateoverflow.github.io/cm4mlperf-inference). From 6f56438f81401558c96eb2df5a97e3f1e6385ece Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 24 Sep 2024 14:54:56 +0100 Subject: [PATCH 140/169] Improve the general docs --- docs/benchmarks/text_to_image/reproducibility/scc24.md | 2 +- main.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/benchmarks/text_to_image/reproducibility/scc24.md b/docs/benchmarks/text_to_image/reproducibility/scc24.md index a0d19ae82..567d4905a 100644 --- a/docs/benchmarks/text_to_image/reproducibility/scc24.md +++ b/docs/benchmarks/text_to_image/reproducibility/scc24.md @@ -50,7 +50,7 @@ or supporting multi-node execution) useful for the community and [MLCommons](htt === "Nvidia" ## Nvidia MLPerf Implementation -{{ mlperf_inference_implementation_readme (4, "sdxl", "nvidia", extra_variation_tags=",_short", scenarios=["Offline"],categories=["Datacenter"], setup_tips=False, implementation_tips=False, skip_test_query_count=True) }} +{{ mlperf_inference_implementation_readme (4, "sdxl", "nvidia", extra_variation_tags=",_short,_scc24-base", scenarios=["Offline"],categories=["Datacenter"], setup_tips=False, implementation_tips=False, skip_test_query_count=True) }} !!! info Once the above run is successful, you can change `_scc24-base` to `_scc24-main` to run the main variant. diff --git a/main.py b/main.py index 85155a9be..00e1bccf3 100755 --- a/main.py +++ b/main.py @@ -385,9 +385,9 @@ def get_run_cmd_extra(f_pre_space, model, implementation, device, scenario, scen if scenario == "Server" or (scenario == "All Scenarios" and "Server" in scenarios): extra_content += f"{f_pre_space} * `` must be determined manually. It is usually around 80% of the Offline QPS, but on some systems, it can drop below 50%. If a higher value is specified, the latency constraint will not be met, and the run will be considered invalid.\n" if implementation == "reference" and model in [ "sdxl", "gptj-99", "gptj-99.9" ] and device in ["cuda", "rocm"]: - extra_content += f"{f_pre_space} * `--precision=float16` can help run on GPUs with less RAM \n" + extra_content += f"{f_pre_space} * `--precision=float16` can help run on GPUs with less RAM / gives better performance \n" if implementation == "reference" and model in [ "sdxl", "gptj-99", "gptj-99.9" ] and device in ["cpu"]: - extra_content += f"{f_pre_space} * `--precision=bfloat16` can help run on GPUs with less RAM \n" + extra_content += f"{f_pre_space} * `--precision=bfloat16` can give better performance \n" if "gptj" in model and implementation == "reference": extra_content += f"{f_pre_space} * `--beam-size=1` Beam size of 4 is mandatory for a closed division submission but reducing the beam size can help in running the model on GPUs with lower device memory\n" if extra_content: From a46ebee121764950aba6835763329ec8b5b454f8 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 24 Sep 2024 14:56:38 +0100 Subject: [PATCH 141/169] Fix links for scc24 --- docs/benchmarks/text_to_image/reproducibility/scc24.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/benchmarks/text_to_image/reproducibility/scc24.md b/docs/benchmarks/text_to_image/reproducibility/scc24.md index 567d4905a..577c2ecde 100644 --- a/docs/benchmarks/text_to_image/reproducibility/scc24.md +++ b/docs/benchmarks/text_to_image/reproducibility/scc24.md @@ -19,7 +19,7 @@ Once the initial run is successful, you'll have the opportunity to optimize the Since vendor implementations of the MLPerf inference benchmark vary and are often limited to single-node benchmarking, teams will compete within their respective hardware categories (e.g., Nvidia GPUs, AMD GPUs). Points will be awarded based on the throughput achieved on your system. -Additionally, significant bonus points will be awarded if your team enhances an existing implementation, adds support for new hardware (such as an unsupported GPU), enables multi-node execution, or adds/extends scripts to [cm4mlops repository](https://github.com/mlcommons/cm4mlops/script) supporting new devices, frameworks, implementations etc. All improvements must be made publicly available under the Apache 2.0 license and submitted alongside your results to the SCC committee to earn these bonus points, contributing to the MLPerf community. +Additionally, significant bonus points will be awarded if your team enhances an existing implementation, adds support for new hardware (such as an unsupported GPU), enables multi-node execution, or adds/extends scripts to [cm4mlops repository](https://github.com/mlcommons/cm4mlops/tree/main/script) supporting new devices, frameworks, implementations etc. All improvements must be made publicly available under the Apache 2.0 license and submitted alongside your results to the SCC committee to earn these bonus points, contributing to the MLPerf community. !!! info From 913ffd436e69fd2b1e4e4e79307eb594b95c0383 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 24 Sep 2024 15:07:39 +0100 Subject: [PATCH 142/169] Use float16 in scc24 doc --- docs/benchmarks/text_to_image/reproducibility/scc24.md | 2 +- main.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/benchmarks/text_to_image/reproducibility/scc24.md b/docs/benchmarks/text_to_image/reproducibility/scc24.md index 577c2ecde..8d2f8525a 100644 --- a/docs/benchmarks/text_to_image/reproducibility/scc24.md +++ b/docs/benchmarks/text_to_image/reproducibility/scc24.md @@ -46,7 +46,7 @@ or supporting multi-node execution) useful for the community and [MLCommons](htt === "MLCommons-Python" ## MLPerf Reference Implementation in Python -{{ mlperf_inference_implementation_readme (4, "sdxl", "reference", extra_variation_tags=",_short,_scc24-base", devices=["ROCm", "CUDA"],scenarios=["Offline"],categories=["Datacenter"], setup_tips=False, skip_test_query_count=True) }} +{{ mlperf_inference_implementation_readme (4, "sdxl", "reference", extra_variation_tags=",_short,_scc24-base", devices=["ROCm", "CUDA"],scenarios=["Offline"],categories=["Datacenter"], setup_tips=False, skip_test_query_count=True, extra_input_string=" --precision=float16") }} === "Nvidia" ## Nvidia MLPerf Implementation diff --git a/main.py b/main.py index 00e1bccf3..7ab17f16a 100755 --- a/main.py +++ b/main.py @@ -379,14 +379,14 @@ def get_readme_suffix(spaces, model, implementation, extra_variation_tags): readme_suffix += f"{pre_space}* Please see [mobilenets.md](mobilenets.md) for running mobilenet models for Image Classification." return readme_suffix - def get_run_cmd_extra(f_pre_space, model, implementation, device, scenario, scenarios = [], run_tips=True): + def get_run_cmd_extra(f_pre_space, model, implementation, device, scenario, scenarios = [], run_tips=True, extra_input_string=""): extra_content = "" f_pre_space += "" if scenario == "Server" or (scenario == "All Scenarios" and "Server" in scenarios): extra_content += f"{f_pre_space} * `` must be determined manually. It is usually around 80% of the Offline QPS, but on some systems, it can drop below 50%. If a higher value is specified, the latency constraint will not be met, and the run will be considered invalid.\n" - if implementation == "reference" and model in [ "sdxl", "gptj-99", "gptj-99.9" ] and device in ["cuda", "rocm"]: + if implementation == "reference" and model in [ "sdxl", "gptj-99", "gptj-99.9" ] and device in ["cuda", "rocm"] and "precision" not in extra_input_string: extra_content += f"{f_pre_space} * `--precision=float16` can help run on GPUs with less RAM / gives better performance \n" - if implementation == "reference" and model in [ "sdxl", "gptj-99", "gptj-99.9" ] and device in ["cpu"]: + if implementation == "reference" and model in [ "sdxl", "gptj-99", "gptj-99.9" ] and device in ["cpu"] and "precision" not in extra_input_string: extra_content += f"{f_pre_space} * `--precision=bfloat16` can give better performance \n" if "gptj" in model and implementation == "reference": extra_content += f"{f_pre_space} * `--beam-size=1` Beam size of 4 is mandatory for a closed division submission but reducing the beam size can help in running the model on GPUs with lower device memory\n" @@ -417,7 +417,7 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ if scenario == "Server" or (scenario == "All Scenarios" and "Server" in scenarios): scenario_option += f"\\\n{pre_space} --server_target_qps=" - run_cmd_extra = get_run_cmd_extra(f_pre_space, model, implementation, device, scenario, scenarios) + run_cmd_extra = get_run_cmd_extra(f_pre_space, model, implementation, device, scenario, scenarios, extra_input_string) if docker: docker_cmd_suffix = f" \\\n{pre_space} --docker --quiet" From b21cf397b9871e8076964259317314f807e8d1b0 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 24 Sep 2024 15:09:14 +0100 Subject: [PATCH 143/169] Improve scc24 docs --- docs/benchmarks/text_to_image/reproducibility/scc24.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/benchmarks/text_to_image/reproducibility/scc24.md b/docs/benchmarks/text_to_image/reproducibility/scc24.md index 8d2f8525a..105c01505 100644 --- a/docs/benchmarks/text_to_image/reproducibility/scc24.md +++ b/docs/benchmarks/text_to_image/reproducibility/scc24.md @@ -77,7 +77,7 @@ cm run script --tags=generate,inference,submission \ * Use `--hw_name="My system name"` to give a meaningful system name. -### Aggregate Results in GitHub +### Push Results to GitHub Fork the repository URL at [https://github.com/gateoverflow/cm4mlperf-inference](https://github.com/gateoverflow/cm4mlperf-inference). @@ -93,4 +93,3 @@ cm run script --tags=push,github,mlperf,inference,submission \ Once uploaded give a Pull Request to the origin repository. Github action will be running there and once finished you can see your submitted results at [https://gateoverflow.github.io/cm4mlperf-inference](https://gateoverflow.github.io/cm4mlperf-inference). - From 227186604332fe8d359b4b3c70756e8d50891f4b Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 24 Sep 2024 15:19:34 +0100 Subject: [PATCH 144/169] Improve scc24 docs --- docs/benchmarks/text_to_image/reproducibility/scc24.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/benchmarks/text_to_image/reproducibility/scc24.md b/docs/benchmarks/text_to_image/reproducibility/scc24.md index 105c01505..b4b3c32f4 100644 --- a/docs/benchmarks/text_to_image/reproducibility/scc24.md +++ b/docs/benchmarks/text_to_image/reproducibility/scc24.md @@ -32,7 +32,7 @@ You will need to submit the following files: * `mlperf_submission.run` - CM commands to run MLPerf inference benchmark saved to this file. * `mlperf_submission.md` - description of your platform and some highlights of the MLPerf benchmark execution. -* under which results are pushed to the github repository. +* `` under which results are pushed to the github repository. ## SCC interview @@ -69,7 +69,8 @@ cm run script --tags=generate,inference,submission \ --division=open \ --category=datacenter \ --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ - --run_style=test --adr.submission-checker.tags=_short-run \ + --run_style=test \ + --adr.submission-checker.tags=_short-run \ --quiet \ --submitter= ``` From 3c072e06fecd1ac6b8cf283ba18de639a4a8883f Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 24 Sep 2024 15:34:24 +0100 Subject: [PATCH 145/169] Use float16 in scc24 doc --- docs/benchmarks/text_to_image/reproducibility/scc24.md | 2 +- main.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/benchmarks/text_to_image/reproducibility/scc24.md b/docs/benchmarks/text_to_image/reproducibility/scc24.md index b4b3c32f4..bae4eceb3 100644 --- a/docs/benchmarks/text_to_image/reproducibility/scc24.md +++ b/docs/benchmarks/text_to_image/reproducibility/scc24.md @@ -46,7 +46,7 @@ or supporting multi-node execution) useful for the community and [MLCommons](htt === "MLCommons-Python" ## MLPerf Reference Implementation in Python -{{ mlperf_inference_implementation_readme (4, "sdxl", "reference", extra_variation_tags=",_short,_scc24-base", devices=["ROCm", "CUDA"],scenarios=["Offline"],categories=["Datacenter"], setup_tips=False, skip_test_query_count=True, extra_input_string=" --precision=float16") }} +{{ mlperf_inference_implementation_readme (4, "sdxl", "reference", extra_variation_tags=",_short,_scc24-base", devices=["ROCm", "CUDA"],scenarios=["Offline"],categories=["Datacenter"], setup_tips=False, skip_test_query_count=True, extra_input_string="--precision=float16") }} === "Nvidia" ## Nvidia MLPerf Implementation diff --git a/main.py b/main.py index 7ab17f16a..968c71b16 100755 --- a/main.py +++ b/main.py @@ -417,7 +417,7 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ if scenario == "Server" or (scenario == "All Scenarios" and "Server" in scenarios): scenario_option += f"\\\n{pre_space} --server_target_qps=" - run_cmd_extra = get_run_cmd_extra(f_pre_space, model, implementation, device, scenario, scenarios, extra_input_string) + run_cmd_extra = get_run_cmd_extra(f_pre_space, model, implementation, device, scenario, scenarios, True, extra_input_string) if docker: docker_cmd_suffix = f" \\\n{pre_space} --docker --quiet" From 7b776b78ae21dfb3313ca49966e7a61f27fda84e Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Tue, 24 Sep 2024 20:53:27 +0530 Subject: [PATCH 146/169] fixed command bug --- main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 968c71b16..aa8dd769e 100755 --- a/main.py +++ b/main.py @@ -226,13 +226,13 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, imp for scenario in scenarios: content += f"{cur_space3}=== \"{scenario}\"\n{cur_space4}###### {scenario}\n\n" - run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), final_run_mode, -1, False, skip_test_query_count, scenarios, code_version, extra_variation_tags, extra_input_string) + run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), final_run_mode, test_query_count, False, skip_test_query_count, scenarios, code_version, extra_variation_tags, extra_input_string) content += run_cmd #content += run_suffix if len(scenarios) > 1: content += f"{cur_space3}=== \"All Scenarios\"\n{cur_space4}###### All Scenarios\n\n" - run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), final_run_mode, -1, False, skip_test_query_count, scenarios, code_version, extra_variation_tags, extra_input_string) + run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), final_run_mode, test_query_count, False, skip_test_query_count, scenarios, code_version, extra_variation_tags, extra_input_string) content += run_cmd content += run_suffix From c6078702b47953f0ec443c60ea807bee93dc32df Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 25 Sep 2024 22:58:20 +0100 Subject: [PATCH 147/169] Fix typo in docs --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index aa8dd769e..e475c25d0 100755 --- a/main.py +++ b/main.py @@ -442,7 +442,7 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ if "short" in extra_variation_tags: full_ds_needed_tag = "" else: - full_ds_needed_tag = ",_full" + full_ds_needed_tag = "_full" docker_setup_cmd = f"""\n {f_pre_space}```bash From 04a67dc6c83cfcfc58e083ea69490bafaa55db47 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 26 Sep 2024 11:40:47 +0100 Subject: [PATCH 148/169] Fix typo in docs --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 7b403ecfd..ab08233ec 100755 --- a/main.py +++ b/main.py @@ -442,7 +442,7 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ if "short" in extra_variation_tags: full_ds_needed_tag = "" else: - full_ds_needed_tag = "_full" + full_ds_needed_tag = "_full," docker_setup_cmd = f"""\n From a6e873a513c3c568433fa4908202ba096e6dfbdf Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 26 Sep 2024 21:35:44 +0100 Subject: [PATCH 149/169] Remove unnecessary indendation in docs --- docs/install/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/install/index.md b/docs/install/index.md index 195521c7e..1750d86e4 100644 --- a/docs/install/index.md +++ b/docs/install/index.md @@ -12,8 +12,8 @@ CM needs `git`, `python3-pip` and `python3-venv` installed on your system. If an This step is not mandatory as CM can use separate virtual environment for MLPerf inference. But the latest `pip` install requires this or else will need the `--break-system-packages` flag while installing `cm4mlops`. ```bash - python3 -m venv cm - source cm/bin/activate +python3 -m venv cm +source cm/bin/activate ``` ## Install CM and pulls any needed repositories From 8ab622534141d2e3184e17e72e74deb64d811778 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Mon, 30 Sep 2024 11:49:29 +0530 Subject: [PATCH 150/169] initial commit for tip - native run CUDA --- main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/main.py b/main.py index ab08233ec..cb2316ad2 100755 --- a/main.py +++ b/main.py @@ -140,6 +140,9 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, imp # ref to cm installation content += f"{cur_space3}Please refer to the [installation page](site:inference/install/) to install CM for running the automated benchmark commands.\n\n" test_query_count=get_test_query_count(model, implementation, device.lower()) + if device.lower() == "cuda" and execution_env.lower() == "native": + content += f"\n{cur_space3}!!! tip\n\n" + content += f"{cur_space3} - Unless CUDA, cuDNN and TensorRT are available in the environment it is recommended to use the Docker option.\n\n" if "99.9" not in model: #not showing docker command as it is already done for the 99% variant if implementation == "neuralmagic": From 54d8585a175362923f748271213f003c5e24e652 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 30 Sep 2024 16:28:23 +0530 Subject: [PATCH 151/169] Updated tip --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index cb2316ad2..6a607cc10 100755 --- a/main.py +++ b/main.py @@ -142,7 +142,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, imp test_query_count=get_test_query_count(model, implementation, device.lower()) if device.lower() == "cuda" and execution_env.lower() == "native": content += f"\n{cur_space3}!!! tip\n\n" - content += f"{cur_space3} - Unless CUDA, cuDNN and TensorRT are available in the environment it is recommended to use the Docker option.\n\n" + content += f"{cur_space3} - It is advisable to use the commands in the Docker tab for CUDA. Run the below native command only if you are already on a CUDA setup with cuDNN and TensorRT installed.\n\n" if "99.9" not in model: #not showing docker command as it is already done for the 99% variant if implementation == "neuralmagic": From ed593ba05adc080190d581f379a2cf73542f1e0c Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Sun, 6 Oct 2024 21:34:07 +0530 Subject: [PATCH 152/169] added docker_cm_repo_branch to more run option - docker --- main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/main.py b/main.py index 6a607cc10..38e6e1650 100755 --- a/main.py +++ b/main.py @@ -179,6 +179,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, imp content += f"{cur_space3}
    \n" content += f"{cur_space3} Please click here to see more options for the docker launch \n\n" content += f"{cur_space3}* `--docker_cm_repo=`: to use a custom fork of cm4mlops repository inside the docker image\n\n" + content += f"{cur_space3}* `--docker_cm_repo_branch=`: to checkout a custom branch of the cloned cm4mlops repository inside the docker image\n\n" content += f"{cur_space3}* `--docker_cache=no`: to not use docker cache during the image build\n" if implementation.lower() == "nvidia": From 6918b85b3aaf186147d247c3cd9cd623eb5faa9d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 9 Oct 2024 14:29:24 +0100 Subject: [PATCH 153/169] Update docs for IndySCC24 --- .../reproducibility/indyscc24-bert.md | 62 +++++++++++++------ .../text_to_image/reproducibility/scc24.md | 6 +- 2 files changed, 45 insertions(+), 23 deletions(-) diff --git a/docs/benchmarks/language/reproducibility/indyscc24-bert.md b/docs/benchmarks/language/reproducibility/indyscc24-bert.md index 68215c5e1..c82175a60 100644 --- a/docs/benchmarks/language/reproducibility/indyscc24-bert.md +++ b/docs/benchmarks/language/reproducibility/indyscc24-bert.md @@ -9,15 +9,11 @@ hide: This guide is designed for the [IndySCC 2024](https://sc24.supercomputing.org/students/indyscc/) to walk participants through running and optimizing the [MLPerf Inference Benchmark](https://arxiv.org/abs/1911.02549) using [Bert Large](https://github.com/mlcommons/inference/tree/master/language/bert#supported-models) across various software and hardware configurations. The goal is to maximize system throughput (measured in samples per second) without compromising accuracy. -For a valid MLPerf inference submission, two types of runs are required: a performance run and an accuracy run. In this competition, we focus on the `Offline` scenario, where throughput is the key metric—higher values are better. The official MLPerf inference benchmark for Bert Large requires processing a minimum of 10833 samples in both performance and accuracy modes using the Squad v1.1 dataset. Setting up for Nvidia GPUs may take 2-3 hours but can be done offline. Your final output will be a tarball (`mlperf_submission.tar.gz`) containing MLPerf-compatible results, which you will submit to the SCC organizers for scoring. +For a valid MLPerf inference submission, two types of runs are required: a performance run and an accuracy run. In this competition, we focus on the `Offline` scenario, where throughput is the key metric—higher values are better. The official MLPerf inference benchmark for Bert Large requires processing a minimum of 10833 samples in both performance and accuracy modes using the Squad v1.1 dataset. ## Scoring -In the SCC, your first objective will be to run a reference (unoptimized) Python implementation or a vendor-provided version (such as Nvidia's) of the MLPerf inference benchmark to secure a baseline score. - -Once the initial run is successful, you'll have the opportunity to optimize the benchmark further by maximizing system utilization, applying quantization techniques, adjusting ML frameworks, experimenting with batch sizes, and more, all of which can earn you additional points. - -Since vendor implementations of the MLPerf inference benchmark vary and are often limited to single-node benchmarking, teams will compete within their respective hardware categories (e.g., Nvidia GPUs, AMD GPUs). Points will be awarded based on the throughput achieved on your system. +In the IndySCC 2024, your objective will be to run a reference (unoptimized) Python implementation of the MLPerf inference benchmark to complete a successful submission passing the submission checker. Only one of the available framework needs to be submitted. !!! info @@ -25,24 +21,50 @@ Since vendor implementations of the MLPerf inference benchmark vary and are ofte If you encounter issues or have questions, please submit them [here](https://github.com/mlcommons/cm4mlops/issues) ## Artifacts to submit to the SCC committee - -You will need to submit the following files: - -* `mlperf_submission_short.tar.gz` - automatically generated file with validated MLPerf results. -* `mlperf_submission_short_summary.json` - automatically generated summary of MLPerf results. -* `mlperf_submission_short.run` - CM commands to run MLPerf BERT inference benchmark saved to this file. -* `mlperf_submission_short.tstamps` - execution timestamps before and after CM command saved to this file. -* `mlperf_submission_short.md` - description of your platform and some highlights of the MLPerf benchmark execution. - +All the needed files are automatically pushed to the GitHub repository if you manage to complete the given commands. No additional files need to be submitted. === "MLCommons-Python" ## MLPerf Reference Implementation in Python -{{ mlperf_inference_implementation_readme (4, "bert-99", "reference", extra_variation_tags=",_short", scenarios=["Offline"],categories=["Edge"], setup_tips=False) }} +{{ mlperf_inference_implementation_readme (4, "bert-99", "reference", extra_variation_tags="", scenarios=["Offline"],categories=["Edge"], setup_tips=False) }} -=== "Nvidia" - ## Nvidia MLPerf Implementation -{{ mlperf_inference_implementation_readme (4, "bert-99", "nvidia", extra_variation_tags=",_short", scenarios=["Offline"],categories=["Edge"], setup_tips=False, implementation_tips=False) }} - +## Submission Commands + +### Generate actual submission tree + +```bash +cm run script --tags=generate,inference,submission \ + --clean \ + --preprocess_submission=yes \ + --run-checker \ + --tar=yes \ + --env.CM_TAR_OUTFILE=submission.tar.gz \ + --division=open \ + --category=edge \ + --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes \ + --run_style=test \ + --quiet \ + --submitter= +``` + +* Use `--hw_name="My system name"` to give a meaningful system name. + + +### Push Results to GitHub + +Fork the `mlperf-inference-results-scc24` branch of the repository URL at [https://github.com/mlcommons/cm4mlperf-inference](https://github.com/mlcommons/cm4mlperf-inference). + +Run the following command after **replacing `--repo_url` with your GitHub fork URL**. + +```bash +cm run script --tags=push,github,mlperf,inference,submission \ + --repo_url=https://github.com//cm4mlperf-inference \ + --repo_branch=mlperf-inference-results-scc24 \ + --commit_message="Results on system " \ + --quiet +``` + +Once uploaded give a Pull Request to the origin repository. Github action will be running there and once +finished you can see your submitted results at [https://docs.mlcommons.org/cm4mlperf-inference](https://docs.mlcommons.org/cm4mlperf-inference). diff --git a/docs/benchmarks/text_to_image/reproducibility/scc24.md b/docs/benchmarks/text_to_image/reproducibility/scc24.md index bae4eceb3..d17079973 100644 --- a/docs/benchmarks/text_to_image/reproducibility/scc24.md +++ b/docs/benchmarks/text_to_image/reproducibility/scc24.md @@ -80,17 +80,17 @@ cm run script --tags=generate,inference,submission \ ### Push Results to GitHub -Fork the repository URL at [https://github.com/gateoverflow/cm4mlperf-inference](https://github.com/gateoverflow/cm4mlperf-inference). +Fork the `mlperf-inference-results-scc24` branch of the repository URL at [https://github.com/mlcommons/cm4mlperf-inference](https://github.com/mlcommons/cm4mlperf-inference). Run the following command after **replacing `--repo_url` with your GitHub fork URL**. ```bash cm run script --tags=push,github,mlperf,inference,submission \ - --repo_url=https://github.com/gateoverflow/cm4mlperf-inference \ + --repo_url=https://github.com//cm4mlperf-inference \ --repo_branch=mlperf-inference-results-scc24 \ --commit_message="Results on system " \ --quiet ``` Once uploaded give a Pull Request to the origin repository. Github action will be running there and once -finished you can see your submitted results at [https://gateoverflow.github.io/cm4mlperf-inference](https://gateoverflow.github.io/cm4mlperf-inference). +finished you can see your submitted results at [https://docs.mlcommons.org/cm4mlperf-inference](https://docs.mlcommons.org/cm4mlperf-inference). From fa0eeca68dfe1fd072494d41f6c082a6222d03d5 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 11 Oct 2024 16:13:57 +0100 Subject: [PATCH 154/169] Support custom repo branch and owner for final report generation --- tools/submission/generate_final_report.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/submission/generate_final_report.py b/tools/submission/generate_final_report.py index 160d27424..c55097dd4 100644 --- a/tools/submission/generate_final_report.py +++ b/tools/submission/generate_final_report.py @@ -14,8 +14,10 @@ def get_args(): """Parse commandline.""" parser = argparse.ArgumentParser() parser.add_argument('--input', required=True, help='results csv from checker') - parser.add_argument('--version', default='4.0', help='mlperf version') - parser.add_argument('--repository', default='submissions_inference_4.0', help='mlperf repository') + parser.add_argument('--version', default='4.1', help='mlperf version') + parser.add_argument('--repository', default='submissions_inference_4.1', help='mlperf repository') + parser.add_argument('--repository-owner', default='mlcommons', help='mlperf repository owner in GitHub') + parser.add_argument('--repository-branch', default='main', help='mlperf repository branch') args = parser.parse_args() return args @@ -59,7 +61,7 @@ def main(): df['p#'] = df.apply(lambda x: int(x['host_processors_per_node']), axis=1) # details url - base_url = f'https://github.com/mlcommons/{args.repository}/tree/main' + base_url = f'https://github.com/{args.repository_owner}/{args.repository}/tree/{args.repository_branch}' df['Details'] = df.apply( lambda x: '=HYPERLINK("{}","details")'.format('/'.join( [base_url, x['Category'], x['Submitter'], 'results', x['Platform']])), From 148650d157b8a6fa993ab01c352529a80db4e718 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sat, 19 Oct 2024 15:50:17 +0530 Subject: [PATCH 155/169] enabled amd implementation for llama2 --- docs/benchmarks/language/llama2-70b.md | 7 +++++++ main.py | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/docs/benchmarks/language/llama2-70b.md b/docs/benchmarks/language/llama2-70b.md index e68693716..40c62cf71 100644 --- a/docs/benchmarks/language/llama2-70b.md +++ b/docs/benchmarks/language/llama2-70b.md @@ -25,3 +25,10 @@ hide: {{ mlperf_inference_implementation_readme (4, "llama2-70b-99", "neuralmagic") }} {{ mlperf_inference_implementation_readme (4, "llama2-70b-99.9", "neuralmagic") }} + +=== "AMD" + ## AMD MLPerf Implementation + +{{ mlperf_inference_implementation_readme (4, "llama2-70b-99", "amd") }} + +{{ mlperf_inference_implementation_readme (4, "llama2-70b-99.9", "amd") }} diff --git a/main.py b/main.py index 38e6e1650..a4f09d559 100755 --- a/main.py +++ b/main.py @@ -43,6 +43,10 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, imp devices = [ "CUDA" ] frameworks = [ "TensorRT" ] + elif implementation == "amd": + devices = [ "ROCm" ] + frameworks = [ "pytorch" ] + elif implementation == "neuralmagic": devices = [ "CUDA" ] frameworks = [ "pytorch" ] From 7eee0725a5496b0e7991ce6f04fff608c7a7301d Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Mon, 21 Oct 2024 09:01:26 +0530 Subject: [PATCH 156/169] updations for amd - docs --- main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index a4f09d559..8b6ce3ea5 100755 --- a/main.py +++ b/main.py @@ -44,8 +44,9 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, imp frameworks = [ "TensorRT" ] elif implementation == "amd": - devices = [ "ROCm" ] + devices = [ "cuda" ] frameworks = [ "pytorch" ] + execution_envs.remove("Docker") elif implementation == "neuralmagic": devices = [ "CUDA" ] From 30c0c0bf6c111f38a6ad9b951edf85cb1f1832cf Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 22 Oct 2024 17:33:08 +0530 Subject: [PATCH 157/169] Fix scenarios in docs page --- main.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/main.py b/main.py index 8b6ce3ea5..847f95bec 100755 --- a/main.py +++ b/main.py @@ -1,7 +1,7 @@ def define_env(env): @env.macro - def mlperf_inference_implementation_readme(spaces, model, implementation, *, implementation_tips=True, setup_tips=True, run_tips=True, skip_test_query_count=False, scenarios = [], devices=[], frameworks=[], categories=[], extra_variation_tags="", extra_input_string="", extra_docker_input_string=""): + def mlperf_inference_implementation_readme(spaces, model, implementation, *, implementation_tips=True, setup_tips=True, run_tips=True, skip_test_query_count=False, fixed_scenarios = [], devices=[], frameworks=[], categories=[], extra_variation_tags="", extra_input_string="", extra_docker_input_string=""): pre_space = "" for i in range(1,spaces): @@ -78,7 +78,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, imp frameworks = [ "Onnxruntime" ] elif implementation == "ctuning-cpp": - scenarios = [ "SingleStream" ] + fixed_scenarios = [ "SingleStream" ] devices = [ "CPU" ] if model.lower() == "resnet50": frameworks = [ "TFLite" ] @@ -99,13 +99,14 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, imp final_run_mode = "valid" if "short" not in extra_variation_tags else "test" for category in categories: - if not scenarios: - if category == "Edge" and not scenarios: - scenarios = [ "Offline", "SingleStream" ] - if model.lower() in [ "resnet50", "retinanet" ] and not "MultiStream" in scenarios:#MultiStream was duplicating - scenarios.append("MultiStream") - elif category == "Datacenter": - scenarios = [ "Offline", "Server" ] + if category == "Edge": + scenarios = [ "Offline", "SingleStream" ] + if model.lower() in [ "resnet50", "retinanet" ] and not "MultiStream" in scenarios:#MultiStream was duplicating + scenarios.append("MultiStream") + elif category == "Datacenter": + scenarios = [ "Offline", "Server" ] + if fixed_scenarios: + scenarios = [ scenario for scenario in scenarios if scenario in fixed_scenarios ] content += f"{pre_space}=== \"{category.lower()}\"\n\n" From 8967ecf403f4ce9e4448cac74a97b63214b065e3 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 22 Oct 2024 17:38:15 +0530 Subject: [PATCH 158/169] formatted the files to pass the gh action --- main.py | 342 +++++++----- tools/submission/generate_final_report.py | 631 +++++++++++----------- 2 files changed, 537 insertions(+), 436 deletions(-) diff --git a/main.py b/main.py index 847f95bec..15b76c78c 100755 --- a/main.py +++ b/main.py @@ -1,97 +1,100 @@ def define_env(env): @env.macro - def mlperf_inference_implementation_readme(spaces, model, implementation, *, implementation_tips=True, setup_tips=True, run_tips=True, skip_test_query_count=False, fixed_scenarios = [], devices=[], frameworks=[], categories=[], extra_variation_tags="", extra_input_string="", extra_docker_input_string=""): + def mlperf_inference_implementation_readme(spaces, model, implementation, *, implementation_tips=True, setup_tips=True, run_tips=True, skip_test_query_count=False, fixed_scenarios=[ + ], devices=[], frameworks=[], categories=[], extra_variation_tags="", extra_input_string="", extra_docker_input_string=""): pre_space = "" - for i in range(1,spaces): - pre_space = pre_space + " " + for i in range(1, spaces): + pre_space = pre_space + " " f_pre_space = pre_space pre_space += " " - content="" + content = "" - execution_envs = ["Docker","Native"] - code_version="r4.1-dev" + execution_envs = ["Docker", "Native"] + code_version = "r4.1-dev" implementation_run_options = [] if model == "rnnt": - code_version="r4.0" + code_version = "r4.0" if implementation == "reference": # Tip if "99.9" not in model and implementation_tips: content += f"\n{pre_space}!!! tip\n\n" content += f"{pre_space} - MLCommons reference implementations are only meant to provide a rules compliant reference implementation for the submitters and in most cases are not best performing. If you want to benchmark any system, it is advisable to use the vendor MLPerf implementation for that system like Nvidia, Intel etc.\n\n" - + if not devices: - devices = [ "CPU", "CUDA", "ROCm" ] + devices = ["CPU", "CUDA", "ROCm"] if not frameworks: if model.lower() == "resnet50": - frameworks = [ "Onnxruntime", "Tensorflow", "Deepsparse" ] + frameworks = ["Onnxruntime", "Tensorflow", "Deepsparse"] elif model.lower() == "retinanet": - frameworks = [ "Onnxruntime", "Pytorch" ] + frameworks = ["Onnxruntime", "Pytorch"] elif "bert" in model.lower(): - frameworks = [ "Pytorch", "Deepsparse" ] + frameworks = ["Pytorch", "Deepsparse"] else: - frameworks = [ "Pytorch" ] + frameworks = ["Pytorch"] elif implementation == "nvidia": - if model in [ "mixtral-8x7b" ]: - return pre_space+" WIP" - devices = [ "CUDA" ] - frameworks = [ "TensorRT" ] - + if model in ["mixtral-8x7b"]: + return pre_space + " WIP" + devices = ["CUDA"] + frameworks = ["TensorRT"] + elif implementation == "amd": - devices = [ "cuda" ] - frameworks = [ "pytorch" ] + devices = ["cuda"] + frameworks = ["pytorch"] execution_envs.remove("Docker") - + elif implementation == "neuralmagic": - devices = [ "CUDA" ] - frameworks = [ "pytorch" ] + devices = ["CUDA"] + frameworks = ["pytorch"] elif implementation == "intel": # Tip if "99.9" not in model and implementation_tips: content += f"\n{pre_space}!!! tip\n\n" content += f"{pre_space} - Intel MLPerf inference implementation is available only for datacenter category and has been tested only on a limited number of systems. Most of the benchmarks using Intel implementation require at least Intel Sapphire Rapids or higher CPU generation.\n\n" - - if model not in [ "bert-99", "bert-99.9", "gptj-99", "gptj-99.9", "resnet50", "retinanet", "3d-unet-99", "3d-unet-99.9", "dlrm-v2-99", "dlrm-v2-99.9", "sdxl" ]: - return pre_space+" WIP" - if model in [ "bert-99", "bert-99.9", "retinanet", "3d-unet-99", "3d-unet-99.9" ]: - code_version="r4.0" - devices = [ "CPU" ] - frameworks = [ "Pytorch" ] + + if model not in ["bert-99", "bert-99.9", "gptj-99", "gptj-99.9", "resnet50", + "retinanet", "3d-unet-99", "3d-unet-99.9", "dlrm-v2-99", "dlrm-v2-99.9", "sdxl"]: + return pre_space + " WIP" + if model in ["bert-99", "bert-99.9", + "retinanet", "3d-unet-99", "3d-unet-99.9"]: + code_version = "r4.0" + devices = ["CPU"] + frameworks = ["Pytorch"] elif implementation == "qualcomm": - if model not in [ "resnet50", "retinanet", "bert-99", "bert-99.9" ]: - return pre_space+" WIP" + if model not in ["resnet50", "retinanet", "bert-99", "bert-99.9"]: + return pre_space + " WIP" - devices = [ "QAIC" ] - frameworks = [ "Glow" ] + devices = ["QAIC"] + frameworks = ["Glow"] elif implementation == "cpp": if not devices: - devices = [ "CPU", "CUDA" ] - frameworks = [ "Onnxruntime" ] + devices = ["CPU", "CUDA"] + frameworks = ["Onnxruntime"] elif implementation == "ctuning-cpp": - fixed_scenarios = [ "SingleStream" ] - devices = [ "CPU" ] + fixed_scenarios = ["SingleStream"] + devices = ["CPU"] if model.lower() == "resnet50": - frameworks = [ "TFLite" ] + frameworks = ["TFLite"] else: - frameworks = [] + frameworks = [] if not categories: if model.lower() == "bert-99.9": - categories = [ "Datacenter" ] + categories = ["Datacenter"] elif "dlrm" in model.lower() or "llama2" in model.lower() or "mixtral" in model.lower(): - categories = [ "Datacenter" ] + categories = ["Datacenter"] else: - categories = [ "Edge", "Datacenter" ] + categories = ["Edge", "Datacenter"] # model name content += f"{pre_space}{model.upper()}\n\n" @@ -100,13 +103,15 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, imp for category in categories: if category == "Edge": - scenarios = [ "Offline", "SingleStream" ] - if model.lower() in [ "resnet50", "retinanet" ] and not "MultiStream" in scenarios:#MultiStream was duplicating + scenarios = ["Offline", "SingleStream"] + if model.lower() in [ + "resnet50", "retinanet"] and not "MultiStream" in scenarios: # MultiStream was duplicating scenarios.append("MultiStream") elif category == "Datacenter": - scenarios = [ "Offline", "Server" ] + scenarios = ["Offline", "Server"] if fixed_scenarios: - scenarios = [ scenario for scenario in scenarios if scenario in fixed_scenarios ] + scenarios = [ + scenario for scenario in scenarios if scenario in fixed_scenarios] content += f"{pre_space}=== \"{category.lower()}\"\n\n" @@ -115,7 +120,6 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, imp content += f"{cur_space}### {category} category \n\n{cur_space} In the {category.lower()} category, {model} has {scenarios_string} scenarios and all the scenarios are mandatory for a closed division submission.\n\n" - for framework in frameworks: cur_space1 = cur_space + " " content += f"{cur_space}=== \"{framework}\"\n" @@ -124,20 +128,23 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, imp for device in devices: if framework.lower() == "deepsparse": if device.lower() != "cpu": - continue + continue cur_space2 = cur_space1 + " " cur_space3 = cur_space2 + " " cur_space4 = cur_space3 + " " - + content += f"{cur_space1}=== \"{device}\"\n" content += f"{cur_space2}##### {device} device\n\n" # minimum system requirements - content += get_min_system_requirements(cur_space2, model, implementation, device) + content += get_min_system_requirements( + cur_space2, model, implementation, device) - # to select the execution environments(currently Docker and Native) + # to select the execution environments(currently Docker and + # Native) for execution_env in execution_envs: - if (device == "ROCm" or implementation == "qualcomm") and execution_env == "Docker": + if (device == "ROCm" or implementation == + "qualcomm") and execution_env == "Docker": continue # docker not currently supported for Qualcomm implementation and ROCm device if implementation == "nvidia" and execution_env == "Native": continue # Nvidia implementation only supports execution through docker @@ -145,39 +152,60 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, imp content += f"{cur_space3}###### {execution_env} Environment\n\n" # ref to cm installation content += f"{cur_space3}Please refer to the [installation page](site:inference/install/) to install CM for running the automated benchmark commands.\n\n" - test_query_count=get_test_query_count(model, implementation, device.lower()) + test_query_count = get_test_query_count( + model, implementation, device.lower()) if device.lower() == "cuda" and execution_env.lower() == "native": content += f"\n{cur_space3}!!! tip\n\n" content += f"{cur_space3} - It is advisable to use the commands in the Docker tab for CUDA. Run the below native command only if you are already on a CUDA setup with cuDNN and TensorRT installed.\n\n" - if "99.9" not in model: #not showing docker command as it is already done for the 99% variant + if "99.9" not in model: # not showing docker command as it is already done for the 99% variant if implementation == "neuralmagic": content += f"{cur_space3}####### Run the Inference Server\n" - content += get_inference_server_run_cmd(spaces+16,implementation) + content += get_inference_server_run_cmd( + spaces + 16, implementation) if run_tips: - # tips regarding the running of nural magic server + # tips regarding the running of nural magic + # server content += f"\n{cur_space3}!!! tip\n\n" content += f"{cur_space3} - Host and Port number of the server can be configured through `--host` and `--port` options. Otherwise, server will run on the default host `localhost` and port `8000`.\n\n" - - setup_run_cmd = mlperf_inference_run_command(spaces+17, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True, skip_test_query_count, scenarios, code_version, extra_variation_tags, extra_input_string, extra_docker_input_string) - if execution_env == "Native": # Native implementation steps through virtual environment + setup_run_cmd = mlperf_inference_run_command( + spaces + 17, + model, + implementation, + framework.lower(), + category.lower(), + "Offline", + device.lower(), + "test", + test_query_count, + True, + skip_test_query_count, + scenarios, + code_version, + extra_variation_tags, + extra_input_string, + extra_docker_input_string) + + if execution_env == "Native": # Native implementation steps through virtual environment content += f"{cur_space3}####### Setup a virtual environment for Python\n" - content += get_venv_command(spaces+16) + content += get_venv_command(spaces + 16) content += f"{cur_space3}####### Performance Estimation for Offline Scenario\n" - content += setup_run_cmd.replace("--docker ", "") + content += setup_run_cmd.replace( + "--docker ", "") content += f"{cur_space3}The above command should do a test run of Offline scenario and record the estimated offline_target_qps.\n\n" - else: # Docker implementation steps + else: # Docker implementation steps content += f"{cur_space3}####### Docker Container Build and Performance Estimation for Offline Scenario\n" - docker_info = get_docker_info(spaces+16, model, implementation, device, setup_tips) + docker_info = get_docker_info( + spaces + 16, model, implementation, device, setup_tips) content += docker_info content += setup_run_cmd - if len(scenarios) == 1: + if len(scenarios) == 1: scenario_text = f"""the {scenarios[0]} scenario""" else: scenario_text = "each scenario""" @@ -191,7 +219,7 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, imp if implementation.lower() == "nvidia": content += f"{cur_space3}* `--gpu_name=` : The GPUs with supported configs in CM are `orin`, `rtx_4090`, `rtx_a6000`, `rtx_6000_ada`, `l4`, `t4`and `a100`. For other GPUs, default configuration as per the GPU memory will be used.\n" - if device.lower() not in [ "cuda" ]: + if device.lower() not in ["cuda"]: content += f"{cur_space3}* `--docker_os=ubuntu`: ubuntu and rhel are supported. \n" content += f"{cur_space3}* `--docker_os_version=20.04`: [20.04, 22.04] are supported for Ubuntu and [8, 9] for RHEL\n" @@ -200,15 +228,29 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, imp content += f"{cur_space3} You can reuse the same environment as described for {model.split('.')[0]}.\n" content += f"{cur_space3}###### Performance Estimation for Offline Scenario\n" - content += mlperf_inference_run_command(spaces+17, model, implementation, framework.lower(), category.lower(), "Offline", device.lower(), "test", test_query_count, True, skip_test_query_count, scenarios, code_version).replace("--docker ","") + content += mlperf_inference_run_command( + spaces + 17, + model, + implementation, + framework.lower(), + category.lower(), + "Offline", + device.lower(), + "test", + test_query_count, + True, + skip_test_query_count, + scenarios, + code_version).replace( + "--docker ", + "") content += f"{cur_space3}The above command should do a test run of Offline scenario and record the estimated offline_target_qps.\n\n" - run_suffix = "" run_suffix += f"{cur_space3}
    \n" run_suffix += f"{cur_space3} Please click here to see more options for the RUN command\n\n" run_suffix += f"{cur_space3}* Use `--division=closed` to do a closed division submission which includes compliance runs\n\n" - run_suffix += f"{cur_space3}* Use `--rerun` to do a rerun even when a valid run exists\n" + run_suffix += f"{cur_space3}* Use `--rerun` to do a rerun even when a valid run exists\n" if implementation.lower() == "nvidia": run_suffix += f"{cur_space3}* `--gpu_name=` : The GPUs with supported configs in CM are `orin`, `rtx_4090`, `rtx_a6000`, `rtx_6000_ada`, `l4`, `t4`and `a100`. For other GPUs, default configuration as per the GPU memory will be used.\n" run_suffix += f"{cur_space3}
    \n\n" @@ -216,8 +258,8 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, imp if "bert" in model.lower() and framework.lower() == "deepsparse": run_suffix += f"{cur_space3}
    \n" run_suffix += f"{cur_space3} Please click here to view available generic model stubs for bert deepsparse\n\n" - run_suffix += f"{cur_space3}* **obert-large-pruned95_quant-none-vnni:** zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/pruned95_quant-none-vnni\n\n" - run_suffix += f"{cur_space3}* **mobilebert-none-14layer_pruned50_quant-none-vnni:** zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/14layer_pruned50_quant-none-vnni\n\n" + run_suffix += f"{cur_space3}* **obert-large-pruned95_quant-none-vnni:** zoo:nlp/question_answering/obert-large/pytorch/huggingface/squad/pruned95_quant-none-vnni\n\n" + run_suffix += f"{cur_space3}* **mobilebert-none-14layer_pruned50_quant-none-vnni:** zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/14layer_pruned50_quant-none-vnni\n\n" run_suffix += f"{cur_space3}* **mobilebert-none-base_quant-none:** zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/base_quant-none\n\n" run_suffix += f"{cur_space3}* **bert-base-pruned95_obs_quant-none:** zoo:nlp/question_answering/bert-base/pytorch/huggingface/squad/pruned95_obs_quant-none\n\n" run_suffix += f"{cur_space3}* **mobilebert-none-14layer_pruned50-none-vnni:** zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/14layer_pruned50-none-vnni\n\n" @@ -232,50 +274,79 @@ def mlperf_inference_implementation_readme(spaces, model, implementation, *, imp run_suffix += f"{cur_space3}* **mobilebert-none-base-none:** zoo:nlp/question_answering/mobilebert-none/pytorch/huggingface/squad/base-none\n" run_suffix += f"{cur_space3}
    \n" - - for scenario in scenarios: content += f"{cur_space3}=== \"{scenario}\"\n{cur_space4}###### {scenario}\n\n" - run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), scenario, device.lower(), final_run_mode, test_query_count, False, skip_test_query_count, scenarios, code_version, extra_variation_tags, extra_input_string) + run_cmd = mlperf_inference_run_command( + spaces + 21, + model, + implementation, + framework.lower(), + category.lower(), + scenario, + device.lower(), + final_run_mode, + test_query_count, + False, + skip_test_query_count, + scenarios, + code_version, + extra_variation_tags, + extra_input_string) content += run_cmd - #content += run_suffix + # content += run_suffix - if len(scenarios) > 1: + if len(scenarios) > 1: content += f"{cur_space3}=== \"All Scenarios\"\n{cur_space4}###### All Scenarios\n\n" - run_cmd = mlperf_inference_run_command(spaces+21, model, implementation, framework.lower(), category.lower(), "All Scenarios", device.lower(), final_run_mode, test_query_count, False, skip_test_query_count, scenarios, code_version, extra_variation_tags, extra_input_string) + run_cmd = mlperf_inference_run_command( + spaces + 21, + model, + implementation, + framework.lower(), + category.lower(), + "All Scenarios", + device.lower(), + final_run_mode, + test_query_count, + False, + skip_test_query_count, + scenarios, + code_version, + extra_variation_tags, + extra_input_string) content += run_cmd content += run_suffix - - - readme_prefix = get_readme_prefix(spaces, model, implementation, extra_variation_tags) + readme_prefix = get_readme_prefix( + spaces, model, implementation, extra_variation_tags) - readme_suffix = get_readme_suffix(spaces, model, implementation, extra_variation_tags) + readme_suffix = get_readme_suffix( + spaces, model, implementation, extra_variation_tags) return readme_prefix + content + readme_suffix def get_test_query_count(model, implementation, device, num_devices=1): if model == "resnet50": - p_range = 1000 - elif model in [ "retinanet", "bert-99", "bert-99.9" ]: - p_range = 100 + p_range = 1000 + elif model in ["retinanet", "bert-99", "bert-99.9"]: + p_range = 100 else: - p_range = 10 + p_range = 10 if device == "cuda": p_range *= 5 p_range *= num_devices return p_range - + def get_min_system_requirements(spaces, model, implementation, device): model = model.lower() min_sys_req_content = "" min_sys_req_content += f"{spaces}
    \n" min_sys_req_content += f"{spaces}Please click here to see the minimum system requirements for running the benchmark\n\n" # device memory - if device.lower() == "cuda" and (implementation.lower() == "nvidia" or implementation.lower() == "reference"): + if device.lower() == "cuda" and (implementation.lower() == + "nvidia" or implementation.lower() == "reference"): if implementation.lower() == "nvidia": if "dlrm" in model: device_memory = "24GB" @@ -319,9 +390,9 @@ def get_min_system_requirements(spaces, model, implementation, device): return min_sys_req_content def get_inference_server_run_cmd(spaces, implementation): - indent = " "*spaces + " " + indent = " " * spaces + " " if implementation == "neuralmagic": - pre_space = " "*spaces + pre_space = " " * spaces return f"""\n {pre_space}```bash {pre_space}cm run script --tags=run,vllm-server \\ @@ -331,72 +402,77 @@ def get_inference_server_run_cmd(spaces, implementation): {pre_space}```\n""" def get_venv_command(spaces): - pre_space = " "*spaces - return f"""\n + pre_space = " " * spaces + return f"""\n {pre_space}```bash {pre_space}cm run script --tags=install,python-venv --name=mlperf {pre_space}export CM_SCRIPT_EXTRA_CMD=\"--adr.python.name=mlperf\" -{pre_space}```\n""" +{pre_space}```\n""" - def get_docker_info(spaces, model, implementation, device, setup_tips=True): + def get_docker_info(spaces, model, implementation, + device, setup_tips=True): info = "" - pre_space="" - for i in range(1,spaces): - pre_space = pre_space + " " + pre_space = "" + for i in range(1, spaces): + pre_space = pre_space + " " pre_space += " " - #pre_space = " " + # pre_space = " " if setup_tips: info += f"\n{pre_space}!!! tip\n\n" if model == "sdxl": - info+= f"{pre_space} - `--env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes` option can be used to download the model on the host so that it can be reused across different container lanuches. \n\n" + info += f"{pre_space} - `--env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes` option can be used to download the model on the host so that it can be reused across different container lanuches. \n\n" - info+= f"{pre_space} - Batch size could be adjusted using `--batch_size=#`, where `#` is the desired batch size. This option works only if the implementation in use is supporting the given batch size.\n\n" + info += f"{pre_space} - Batch size could be adjusted using `--batch_size=#`, where `#` is the desired batch size. This option works only if the implementation in use is supporting the given batch size.\n\n" if implementation.lower() == "nvidia": - info+= f"{pre_space} - Default batch size is assigned based on [GPU memory](https://github.com/mlcommons/cm4mlops/blob/dd0c35856969c68945524d5c80414c615f5fe42c/script/app-mlperf-inference-nvidia/_cm.yaml#L1129) or the [specified GPU](https://github.com/mlcommons/cm4mlops/blob/dd0c35856969c68945524d5c80414c615f5fe42c/script/app-mlperf-inference-nvidia/_cm.yaml#L1370). Please click more option for *docker launch* or *run command* to see how to specify the GPU name.\n\n" - info+= f"{pre_space} - When run with `--all_models=yes`, all the benchmark models of NVIDIA implementation can be executed within the same container.\n\n" + info += f"{pre_space} - Default batch size is assigned based on [GPU memory](https://github.com/mlcommons/cm4mlops/blob/dd0c35856969c68945524d5c80414c615f5fe42c/script/app-mlperf-inference-nvidia/_cm.yaml#L1129) or the [specified GPU](https://github.com/mlcommons/cm4mlops/blob/dd0c35856969c68945524d5c80414c615f5fe42c/script/app-mlperf-inference-nvidia/_cm.yaml#L1370). Please click more option for *docker launch* or *run command* to see how to specify the GPU name.\n\n" + info += f"{pre_space} - When run with `--all_models=yes`, all the benchmark models of NVIDIA implementation can be executed within the same container.\n\n" if "llama2" in model.lower(): - info+= f"{pre_space} - The dataset for NVIDIA's implementation of Llama2 is not publicly available. The user must fill [this](https://docs.google.com/forms/d/e/1FAIpQLSc_8VIvRmXM3I8KQaYnKf7gy27Z63BBoI_I1u02f4lw6rBp3g/viewform?pli=1&fbzx=-8842630989397184967) form and be verified as a MLCommons member to access the dataset.\n\n" - info+= f"{pre_space} - `PATH_TO_PICKE_FILE` should be replaced with path to the downloaded pickle file.\n\n" + info += f"{pre_space} - The dataset for NVIDIA's implementation of Llama2 is not publicly available. The user must fill [this](https://docs.google.com/forms/d/e/1FAIpQLSc_8VIvRmXM3I8KQaYnKf7gy27Z63BBoI_I1u02f4lw6rBp3g/viewform?pli=1&fbzx=-8842630989397184967) form and be verified as a MLCommons member to access the dataset.\n\n" + info += f"{pre_space} - `PATH_TO_PICKE_FILE` should be replaced with path to the downloaded pickle file.\n\n" else: if model == "sdxl": info += f"\n{pre_space}!!! tip\n\n" - info+= f"{pre_space} - `--env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes` option can be used to download the model on the host so that it can be reused across different container lanuches. \n\n" + info += f"{pre_space} - `--env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes` option can be used to download the model on the host so that it can be reused across different container lanuches. \n\n" return info def get_readme_prefix(spaces, model, implementation, extra_variation_tags): readme_prefix = "" - pre_space=" " - #for i in range(1,spaces): + pre_space = " " + # for i in range(1,spaces): # pre_space = pre_space + " " - #pre_space += " " + # pre_space += " " return readme_prefix - + def get_readme_suffix(spaces, model, implementation, extra_variation_tags): readme_suffix = "" - pre_space="" - for i in range(1,spaces): - pre_space = pre_space + " " + pre_space = "" + for i in range(1, spaces): + pre_space = pre_space + " " pre_space += " " if implementation == "reference" and not extra_variation_tags: if not model.endswith("-99"): - model_base_name = model.replace("-99.9","").replace("-99","") - readme_suffix+= f"{pre_space}* If you want to download the official MLPerf model and dataset for {model} you can follow [this README](get-{model_base_name}-data.md).\n" + model_base_name = model.replace("-99.9", "").replace("-99", "") + readme_suffix += f"{pre_space}* If you want to download the official MLPerf model and dataset for {model} you can follow [this README](get-{model_base_name}-data.md).\n" if model == "resnet50": - readme_suffix += f"{pre_space}* Please see [mobilenets.md](mobilenets.md) for running mobilenet models for Image Classification." + readme_suffix += f"{pre_space}* Please see [mobilenets.md](mobilenets.md) for running mobilenet models for Image Classification." return readme_suffix - def get_run_cmd_extra(f_pre_space, model, implementation, device, scenario, scenarios = [], run_tips=True, extra_input_string=""): + def get_run_cmd_extra(f_pre_space, model, implementation, device, + scenario, scenarios=[], run_tips=True, extra_input_string=""): extra_content = "" f_pre_space += "" - if scenario == "Server" or (scenario == "All Scenarios" and "Server" in scenarios): + if scenario == "Server" or ( + scenario == "All Scenarios" and "Server" in scenarios): extra_content += f"{f_pre_space} * `` must be determined manually. It is usually around 80% of the Offline QPS, but on some systems, it can drop below 50%. If a higher value is specified, the latency constraint will not be met, and the run will be considered invalid.\n" - if implementation == "reference" and model in [ "sdxl", "gptj-99", "gptj-99.9" ] and device in ["cuda", "rocm"] and "precision" not in extra_input_string: + if implementation == "reference" and model in [ + "sdxl", "gptj-99", "gptj-99.9"] and device in ["cuda", "rocm"] and "precision" not in extra_input_string: extra_content += f"{f_pre_space} * `--precision=float16` can help run on GPUs with less RAM / gives better performance \n" - if implementation == "reference" and model in [ "sdxl", "gptj-99", "gptj-99.9" ] and device in ["cpu"] and "precision" not in extra_input_string: + if implementation == "reference" and model in [ + "sdxl", "gptj-99", "gptj-99.9"] and device in ["cpu"] and "precision" not in extra_input_string: extra_content += f"{f_pre_space} * `--precision=bfloat16` can give better performance \n" if "gptj" in model and implementation == "reference": extra_content += f"{f_pre_space} * `--beam-size=1` Beam size of 4 is mandatory for a closed division submission but reducing the beam size can help in running the model on GPUs with lower device memory\n" @@ -407,27 +483,36 @@ def get_run_cmd_extra(f_pre_space, model, implementation, device, scenario, scen return extra_content else: return "" - @env.macro - def mlperf_inference_run_command(spaces, model, implementation, framework, category, scenario, device="cpu", execution_mode="test", test_query_count="20", docker=False, skip_test_query_count=False, scenarios = [], code_version="r4.1-dev", extra_variation_tags="", extra_input_string="", extra_docker_input_string=""): + def mlperf_inference_run_command(spaces, model, implementation, framework, category, scenario, device="cpu", execution_mode="test", test_query_count="20", + docker=False, skip_test_query_count=False, scenarios=[], code_version="r4.1-dev", extra_variation_tags="", extra_input_string="", extra_docker_input_string=""): pre_space = "" - for i in range(1,spaces): - pre_space = pre_space + " " + for i in range(1, spaces): + pre_space = pre_space + " " f_pre_space = pre_space pre_space += " " if scenario == "All Scenarios": - scenario_variation_tag = ",_all-scenarios" - scenario_option = "" + scenario_variation_tag = ",_all-scenarios" + scenario_option = "" else: scenario_variation_tag = "" scenario_option = f"\\\n{pre_space} --scenario={scenario}" - if scenario == "Server" or (scenario == "All Scenarios" and "Server" in scenarios): + if scenario == "Server" or ( + scenario == "All Scenarios" and "Server" in scenarios): scenario_option += f"\\\n{pre_space} --server_target_qps=" - run_cmd_extra = get_run_cmd_extra(f_pre_space, model, implementation, device, scenario, scenarios, True, extra_input_string) + run_cmd_extra = get_run_cmd_extra( + f_pre_space, + model, + implementation, + device, + scenario, + scenarios, + True, + extra_input_string) if docker: docker_cmd_suffix = f" \\\n{pre_space} --docker --quiet" @@ -445,7 +530,7 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ docker_cmd_suffix += f" \\\n{pre_space} --api_server=http://localhost:8000" docker_cmd_suffix += f" \\\n{pre_space} --vllm_model_name=nm-testing/Llama-2-70b-chat-hf-FP8" docker_cmd_suffix += f" \\\n{pre_space} --adr.mlperf-implementation.tags=_repo.https://github.com/neuralmagic/inference,_branch.vllm" - + if "dlrm-v2" in model.lower() and implementation == "nvidia": docker_cmd_suffix += f" \\\n{pre_space} --criteo_day23_raw_data_path=" @@ -454,7 +539,6 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ else: full_ds_needed_tag = "_full," - docker_setup_cmd = f"""\n {f_pre_space}```bash {f_pre_space}cm run script --tags=run-mlperf,inference,_find-performance,{full_ds_needed_tag}_{code_version}{scenario_variation_tag}{extra_variation_tags} \\ @@ -484,7 +568,7 @@ def mlperf_inference_run_command(spaces, model, implementation, framework, categ cmd_suffix += f" \\\n{pre_space} --api_server=http://localhost:8000" cmd_suffix += f" \\\n{pre_space} --vllm_model_name=nm-testing/Llama-2-70b-chat-hf-FP8" cmd_suffix += f" \\\n{pre_space} --adr.mlperf-implementation.tags=_repo.https://github.com/neuralmagic/inference,_branch.vllm" - + if "dlrm-v2" in model and implementation == "nvidia": cmd_suffix += f" \\\n{pre_space} --criteo_day23_raw_data_path=" diff --git a/tools/submission/generate_final_report.py b/tools/submission/generate_final_report.py index c55097dd4..45e3f45d9 100644 --- a/tools/submission/generate_final_report.py +++ b/tools/submission/generate_final_report.py @@ -10,317 +10,334 @@ import pandas as pd import json + def get_args(): - """Parse commandline.""" - parser = argparse.ArgumentParser() - parser.add_argument('--input', required=True, help='results csv from checker') - parser.add_argument('--version', default='4.1', help='mlperf version') - parser.add_argument('--repository', default='submissions_inference_4.1', help='mlperf repository') - parser.add_argument('--repository-owner', default='mlcommons', help='mlperf repository owner in GitHub') - parser.add_argument('--repository-branch', default='main', help='mlperf repository branch') - args = parser.parse_args() - return args + """Parse commandline.""" + parser = argparse.ArgumentParser() + parser.add_argument( + '--input', + required=True, + help='results csv from checker') + parser.add_argument('--version', default='4.1', help='mlperf version') + parser.add_argument( + '--repository', + default='submissions_inference_4.1', + help='mlperf repository') + parser.add_argument( + '--repository-owner', + default='mlcommons', + help='mlperf repository owner in GitHub') + parser.add_argument( + '--repository-branch', + default='main', + help='mlperf repository branch') + args = parser.parse_args() + return args def main(): - args = get_args() - - df = pd.read_csv(args.input).fillna('') - - if df.empty: - return - - # rename some fields - df.rename( - columns={ - 'Organization': 'Submitter', - 'Division': 'Category', - 'SystemType': 'Suite', - 'SystemName': 'System', - 'number_of_nodes': 'Nodes', - 'host_processor_model_name': 'Processor', - 'accelerator_model_name': 'Accelerator', - 'accelerators_per_node': 'a#', - 'notes': 'Notes', - 'framework': 'Software', - }, - inplace=True) - df.rename(columns={'Model': 'UsedModel'}, inplace=True) - df.rename(columns={'MlperfModel': 'Model'}, inplace=True) - - # fix issues with raw data - df['host_processor_core_count'] = df['host_processor_core_count'].apply( - lambda x: 2 if x == '2 (big); 4 (LITTLE)' else x) - df['Availability'] = df['Availability'].apply(lambda x: 'available' - if x == 'on-premise' else x) - - # cleanup counts - df['Accelerator'] = df['Accelerator'].apply(lambda x: x if x != '-' else '') - df['a#'] = df['a#'].apply(lambda x: int(x) if str(x).isnumeric() else x) - df['a#'] = df['a#'].apply(lambda x: x if x != 0 else '') - df['p#'] = df.apply(lambda x: int(x['host_processors_per_node']), axis=1) - - # details url - base_url = f'https://github.com/{args.repository_owner}/{args.repository}/tree/{args.repository_branch}' - df['Details'] = df.apply( - lambda x: '=HYPERLINK("{}","details")'.format('/'.join( - [base_url, x['Category'], x['Submitter'], 'results', x['Platform']])), - axis=1) - - # code url - df['Code'] = df.apply( - lambda x: '=HYPERLINK("{}","code")'.format('/'.join( - [base_url, x['Category'], x['Submitter'], 'code'])), - axis=1) - - output = args.input[:-4] - writer = pd.ExcelWriter(output + '.xlsx', engine='xlsxwriter') - outjsondata = [] - - indices = {} - indices['closed'] = [ - 'ID', - 'Unique ID (e.g. for Audit)', - 'ColorKey', - 'Submitter', - 'Availability', - 'System', - 'Nodes', - 'Processor', - 'p#', - 'Accelerator', - 'a#', - 'Software', - 'Notes', - ] - indices['open'] = indices['closed'].copy() - indices['closed'].append('Details') - indices['closed'].append('Code') - indices['network'] = indices['closed'].copy() - indices['open'].append('UsedModel') - indices['open'].append('Accuracy') - indices['open'].append('Details') - indices['open'].append('Code') - columns = [ - 'Model', - 'Scenario', - 'Units', - ] - columns_order = [['Result'], - [ - 'resnet', 'retinanet', '3d-unet-99', '3d-unet-99.9', - 'rnnt', 'bert-99', 'bert-99.9', 'dlrm-v2-99', 'dlrm-v2-99.9', - 'gptj-99', 'gptj-99.9', 'stable-diffusion-xl', 'llama2-70b-99', 'llama2-70b-99.9', - 'mixtral-8x7b', - ], ['SingleStream', 'MultiStream', 'Server', 'Offline'], - [ - 'Latency (ms)', - 'Samples/s', - 'Queries/s', - 'Tokens/s', - 'millijoules', - 'Watts', - ]] - - filter_scenarios = { - 'datacenter': { - 'resnet': ['Server', 'Offline'], - 'retinanet': ['Server', 'Offline'], - 'rnnt': ['Server', 'Offline'], - 'bert-99': ['Server', 'Offline'], - 'bert-99.9': ['Server', 'Offline'], - 'dlrm-v2-99': ['Server', 'Offline'], - 'dlrm-v2-99.9': ['Server', 'Offline'], - '3d-unet-99': ['Offline'], - '3d-unet-99.9': ['Offline'], - 'gptj-99': ['Server', 'Offline'], - 'gptj-99.9': ['Server', 'Offline'], - 'stable-diffusion-xl': ['Server', 'Offline'], - 'llama2-70b-99': ['Server', 'Offline'], - 'llama2-70b-99.9': ['Server', 'Offline'], - 'mixtral-8x7b': ['Server', 'Offline'], - }, - 'edge': { - 'resnet': ['SingleStream', 'MultiStream', 'Offline'], - 'retinanet': ['SingleStream', 'MultiStream', 'Offline'], - 'rnnt': ['SingleStream', 'Offline'], - 'bert-99': ['SingleStream', 'Offline'], - 'bert-99.9': [], - 'dlrm-v2-99': [], - 'dlrm-v2-99.9': [], - '3d-unet-99': ['SingleStream', 'Offline'], - '3d-unet-99.9': ['SingleStream', 'Offline'], - 'gptj-99': ['SingleStream', 'Offline'], - 'gptj-99.9': ['SingleStream', 'Offline'], - 'stable-diffusion-xl': ['SingleStream', 'Offline'], - } - } - - def MakeWorksheet(df, index, filter_dict, sheet_name, outjsondata=[]): - for key, value in filter_dict.items(): - if type(key) == tuple: - key = list(key) - df = df[value(df[key])] - if df.size == 0: - return - json_df = df.to_json(orient='records') - outjsondata += json.loads(json_df) - - df = df.pivot_table(index=index, columns=columns, values=['Result']) - df = df.fillna('') - for i, order in enumerate(columns_order): - df = df.reindex(columns=order, level=i) - df.to_excel(writer, sheet_name=sheet_name) - - def Equal(x): - return lambda y: y == x - - def NotEqual(x): - return lambda y: y != x - - def Contain(x): - return lambda y: y.str.find(x) != -1 - - def And(x, y): - return lambda z: x(z) & y(z) - - def Apply(f, *args): - return lambda x: f(x, *args) - - def FilterScenario(x, suite): - return x.apply( - lambda y: y['Scenario'] in filter_scenarios[suite][y['Model']], axis=1) - - def MakeUniqueID(x): - key_list = ['Suite', 'Category', 'Submitter', 'Platform'] - if x['Category'] == 'open': - key_list.append('UsedModel') - return '/'.join(x[key_list]) - - df['Unique ID (e.g. for Audit)'] = df.apply(MakeUniqueID, axis=1) - df['ColorKey'] = df.apply( - lambda x: ''.join(x[['Availability', 'Submitter']]), axis=1) - df.sort_values( - by=[ - 'Category', 'Availability', 'Submitter', 'Unique ID (e.g. for Audit)' - ], - inplace=True) - id_dict = { - key: 1 + value - for (value, - key) in enumerate(pd.unique(df['Unique ID (e.g. for Audit)'])) - } - df['ID'] = df.apply( - lambda x: '{}-{:04}'.format(args.version, id_dict[x['Unique ID (e.g. for Audit)']]), - axis=1) - - for category in ['closed', 'open', 'network']: - for suite in ['datacenter', 'edge']: - MakeWorksheet( - df, indices[category], { - 'Category': - Equal(category), - 'Suite': - Contain(suite), - 'Units': - And( - And(NotEqual('Watts'), NotEqual('millijoules')), - NotEqual('millijoules/Stream')), - ('Scenario', 'Model'): - Apply(FilterScenario, suite) - }, suite + ' - ' + category, outjsondata) - - MakeWorksheet( - df, indices[category], { - 'Category': Equal(category), - 'Suite': Contain(suite), - 'has_power': Equal(True), - ('Scenario', 'Model'): Apply(FilterScenario, suite) - }, suite + ' - ' + category + ' - power', outjsondata) - - def reformatlink(data, key): - details = data[key] - details = details[details.find("(")+2:details.find(",")-1] - return details - - for i,result in enumerate(outjsondata): - result['Details'] = reformatlink(result, "Details") - result['Code'] = reformatlink(result, "Code") - result_id = result.pop('ID') - outjsondata[i] = {'ID': result_id, **result} - - outjsondata.sort(key=lambda x:x["Units"]) - outjsondata.sort(key=lambda x:x["Scenario"]) - outjsondata.sort(key=lambda x:x["UsedModel"]) - outjsondata.sort(key=lambda x:x["ID"]) - - #remove duplicate perf results - keystomatch = ['ID', 'UsedModel', 'Scenario', 'Units'] - i = 0 - n = len(outjsondata) - while i < n: - result = outjsondata[i] - while i < n - 1 and all(result[key] == outjsondata[i+1][key] for key in keystomatch): - del(outjsondata[i+1]) - n -= 1 - i += 1 - - #merge perf and power results - keystomatch.pop() - - for i in range(len(outjsondata)): - result = outjsondata[i] - if not result: - continue - if i < len(outjsondata) - 1: - if all(result[key] == outjsondata[i+1][key] for key in keystomatch): - #print(result) - #print(outjsondata[i+1]) - if "Watts" in result['Units'] or "joules" in result['Units']: - result['Performance_Result'] = outjsondata[i+1]['Result'] - result['Performance_Units'] = outjsondata[i+1]['Units'] - result['Power_Result'] = result['Result'] - result['Power_Units'] = result['Units'] - else: - result['Power_Result'] = outjsondata[i+1]['Result'] - result['Power_Units'] = outjsondata[i+1]['Units'] - result['Performance_Result'] = result['Result'] - result['Performance_Units'] = result['Units'] - outjsondata[i+1] = {} - del(result['Result']) - del(result['Units']) - - for i,result in enumerate(outjsondata): - if result.get('Result'): - result['Performance_Result'] = result['Result'] - result['Performance_Units'] = result['Units'] - del(result['Result']) - del(result['Units']) - - outjsondata = [ i for i in outjsondata if i != {}] - with open(f"{output}_results.json", "w") as f: - f.write(json.dumps(outjsondata, indent=2)) - score_format = writer.book.add_format({'num_format': '#,##0.00'}) - bg_format = writer.book.add_format({'bg_color': '#efefef'}) - for ws in writer.book.worksheets(): - ws.set_column(1, 1, None, None, {'hidden': 1}) - ws.set_column(2, 2, None, None, {'hidden': 1}) - ws.set_column(len(indices['closed']), 100, None, score_format) - ws.conditional_format( - 2 + len(columns), 0, 200, 100, { - 'type': - 'formula', - 'criteria': - '=mod(countunique($c$' + str(len(columns) + 3) + ':$c' + - str(len(columns) + 3) + '), 2) = 0', - 'format': - bg_format, - }) - - writer.close() - - return 0 + args = get_args() + + df = pd.read_csv(args.input).fillna('') + + if df.empty: + return + + # rename some fields + df.rename( + columns={ + 'Organization': 'Submitter', + 'Division': 'Category', + 'SystemType': 'Suite', + 'SystemName': 'System', + 'number_of_nodes': 'Nodes', + 'host_processor_model_name': 'Processor', + 'accelerator_model_name': 'Accelerator', + 'accelerators_per_node': 'a#', + 'notes': 'Notes', + 'framework': 'Software', + }, + inplace=True) + df.rename(columns={'Model': 'UsedModel'}, inplace=True) + df.rename(columns={'MlperfModel': 'Model'}, inplace=True) + + # fix issues with raw data + df['host_processor_core_count'] = df['host_processor_core_count'].apply( + lambda x: 2 if x == '2 (big); 4 (LITTLE)' else x) + df['Availability'] = df['Availability'].apply(lambda x: 'available' + if x == 'on-premise' else x) + + # cleanup counts + df['Accelerator'] = df['Accelerator'].apply( + lambda x: x if x != '-' else '') + df['a#'] = df['a#'].apply(lambda x: int(x) if str(x).isnumeric() else x) + df['a#'] = df['a#'].apply(lambda x: x if x != 0 else '') + df['p#'] = df.apply(lambda x: int(x['host_processors_per_node']), axis=1) + + # details url + base_url = f'https://github.com/{args.repository_owner}/{args.repository}/tree/{args.repository_branch}' + df['Details'] = df.apply( + lambda x: '=HYPERLINK("{}","details")'.format('/'.join( + [base_url, x['Category'], x['Submitter'], 'results', x['Platform']])), + axis=1) + + # code url + df['Code'] = df.apply( + lambda x: '=HYPERLINK("{}","code")'.format('/'.join( + [base_url, x['Category'], x['Submitter'], 'code'])), + axis=1) + + output = args.input[:-4] + writer = pd.ExcelWriter(output + '.xlsx', engine='xlsxwriter') + outjsondata = [] + + indices = {} + indices['closed'] = [ + 'ID', + 'Unique ID (e.g. for Audit)', + 'ColorKey', + 'Submitter', + 'Availability', + 'System', + 'Nodes', + 'Processor', + 'p#', + 'Accelerator', + 'a#', + 'Software', + 'Notes', + ] + indices['open'] = indices['closed'].copy() + indices['closed'].append('Details') + indices['closed'].append('Code') + indices['network'] = indices['closed'].copy() + indices['open'].append('UsedModel') + indices['open'].append('Accuracy') + indices['open'].append('Details') + indices['open'].append('Code') + columns = [ + 'Model', + 'Scenario', + 'Units', + ] + columns_order = [['Result'], + [ + 'resnet', 'retinanet', '3d-unet-99', '3d-unet-99.9', + 'rnnt', 'bert-99', 'bert-99.9', 'dlrm-v2-99', 'dlrm-v2-99.9', + 'gptj-99', 'gptj-99.9', 'stable-diffusion-xl', 'llama2-70b-99', 'llama2-70b-99.9', + 'mixtral-8x7b', + ], ['SingleStream', 'MultiStream', 'Server', 'Offline'], + [ + 'Latency (ms)', + 'Samples/s', + 'Queries/s', + 'Tokens/s', + 'millijoules', + 'Watts', + ]] + + filter_scenarios = { + 'datacenter': { + 'resnet': ['Server', 'Offline'], + 'retinanet': ['Server', 'Offline'], + 'rnnt': ['Server', 'Offline'], + 'bert-99': ['Server', 'Offline'], + 'bert-99.9': ['Server', 'Offline'], + 'dlrm-v2-99': ['Server', 'Offline'], + 'dlrm-v2-99.9': ['Server', 'Offline'], + '3d-unet-99': ['Offline'], + '3d-unet-99.9': ['Offline'], + 'gptj-99': ['Server', 'Offline'], + 'gptj-99.9': ['Server', 'Offline'], + 'stable-diffusion-xl': ['Server', 'Offline'], + 'llama2-70b-99': ['Server', 'Offline'], + 'llama2-70b-99.9': ['Server', 'Offline'], + 'mixtral-8x7b': ['Server', 'Offline'], + }, + 'edge': { + 'resnet': ['SingleStream', 'MultiStream', 'Offline'], + 'retinanet': ['SingleStream', 'MultiStream', 'Offline'], + 'rnnt': ['SingleStream', 'Offline'], + 'bert-99': ['SingleStream', 'Offline'], + 'bert-99.9': [], + 'dlrm-v2-99': [], + 'dlrm-v2-99.9': [], + '3d-unet-99': ['SingleStream', 'Offline'], + '3d-unet-99.9': ['SingleStream', 'Offline'], + 'gptj-99': ['SingleStream', 'Offline'], + 'gptj-99.9': ['SingleStream', 'Offline'], + 'stable-diffusion-xl': ['SingleStream', 'Offline'], + } + } + + def MakeWorksheet(df, index, filter_dict, sheet_name, outjsondata=[]): + for key, value in filter_dict.items(): + if isinstance(key, tuple): + key = list(key) + df = df[value(df[key])] + if df.size == 0: + return + json_df = df.to_json(orient='records') + outjsondata += json.loads(json_df) + + df = df.pivot_table(index=index, columns=columns, values=['Result']) + df = df.fillna('') + for i, order in enumerate(columns_order): + df = df.reindex(columns=order, level=i) + df.to_excel(writer, sheet_name=sheet_name) + + def Equal(x): + return lambda y: y == x + + def NotEqual(x): + return lambda y: y != x + + def Contain(x): + return lambda y: y.str.find(x) != -1 + + def And(x, y): + return lambda z: x(z) & y(z) + + def Apply(f, *args): + return lambda x: f(x, *args) + + def FilterScenario(x, suite): + return x.apply( + lambda y: y['Scenario'] in filter_scenarios[suite][y['Model']], axis=1) + + def MakeUniqueID(x): + key_list = ['Suite', 'Category', 'Submitter', 'Platform'] + if x['Category'] == 'open': + key_list.append('UsedModel') + return '/'.join(x[key_list]) + + df['Unique ID (e.g. for Audit)'] = df.apply(MakeUniqueID, axis=1) + df['ColorKey'] = df.apply( + lambda x: ''.join(x[['Availability', 'Submitter']]), axis=1) + df.sort_values( + by=[ + 'Category', 'Availability', 'Submitter', 'Unique ID (e.g. for Audit)' + ], + inplace=True) + id_dict = { + key: 1 + value + for (value, + key) in enumerate(pd.unique(df['Unique ID (e.g. for Audit)'])) + } + df['ID'] = df.apply( + lambda x: '{}-{:04}'.format(args.version, + id_dict[x['Unique ID (e.g. for Audit)']]), + axis=1) + + for category in ['closed', 'open', 'network']: + for suite in ['datacenter', 'edge']: + MakeWorksheet( + df, indices[category], { + 'Category': + Equal(category), + 'Suite': + Contain(suite), + 'Units': + And( + And(NotEqual('Watts'), NotEqual('millijoules')), + NotEqual('millijoules/Stream')), + ('Scenario', 'Model'): + Apply(FilterScenario, suite) + }, suite + ' - ' + category, outjsondata) + + MakeWorksheet( + df, indices[category], { + 'Category': Equal(category), + 'Suite': Contain(suite), + 'has_power': Equal(True), + ('Scenario', 'Model'): Apply(FilterScenario, suite) + }, suite + ' - ' + category + ' - power', outjsondata) + + def reformatlink(data, key): + details = data[key] + details = details[details.find("(") + 2:details.find(",") - 1] + return details + + for i, result in enumerate(outjsondata): + result['Details'] = reformatlink(result, "Details") + result['Code'] = reformatlink(result, "Code") + result_id = result.pop('ID') + outjsondata[i] = {'ID': result_id, **result} + + outjsondata.sort(key=lambda x: x["Units"]) + outjsondata.sort(key=lambda x: x["Scenario"]) + outjsondata.sort(key=lambda x: x["UsedModel"]) + outjsondata.sort(key=lambda x: x["ID"]) + + # remove duplicate perf results + keystomatch = ['ID', 'UsedModel', 'Scenario', 'Units'] + i = 0 + n = len(outjsondata) + while i < n: + result = outjsondata[i] + while i < n - \ + 1 and all(result[key] == outjsondata[i + 1][key] for key in keystomatch): + del (outjsondata[i + 1]) + n -= 1 + i += 1 + + # merge perf and power results + keystomatch.pop() + + for i in range(len(outjsondata)): + result = outjsondata[i] + if not result: + continue + if i < len(outjsondata) - 1: + if all(result[key] == outjsondata[i + 1][key] + for key in keystomatch): + # print(result) + # print(outjsondata[i+1]) + if "Watts" in result['Units'] or "joules" in result['Units']: + result['Performance_Result'] = outjsondata[i + 1]['Result'] + result['Performance_Units'] = outjsondata[i + 1]['Units'] + result['Power_Result'] = result['Result'] + result['Power_Units'] = result['Units'] + else: + result['Power_Result'] = outjsondata[i + 1]['Result'] + result['Power_Units'] = outjsondata[i + 1]['Units'] + result['Performance_Result'] = result['Result'] + result['Performance_Units'] = result['Units'] + outjsondata[i + 1] = {} + del (result['Result']) + del (result['Units']) + + for i, result in enumerate(outjsondata): + if result.get('Result'): + result['Performance_Result'] = result['Result'] + result['Performance_Units'] = result['Units'] + del (result['Result']) + del (result['Units']) + + outjsondata = [i for i in outjsondata if i != {}] + with open(f"{output}_results.json", "w") as f: + f.write(json.dumps(outjsondata, indent=2)) + score_format = writer.book.add_format({'num_format': '#,##0.00'}) + bg_format = writer.book.add_format({'bg_color': '#efefef'}) + for ws in writer.book.worksheets(): + ws.set_column(1, 1, None, None, {'hidden': 1}) + ws.set_column(2, 2, None, None, {'hidden': 1}) + ws.set_column(len(indices['closed']), 100, None, score_format) + ws.conditional_format( + 2 + len(columns), 0, 200, 100, { + 'type': + 'formula', + 'criteria': + '=mod(countunique($c$' + str(len(columns) + 3) + ':$c' + + str(len(columns) + 3) + '), 2) = 0', + 'format': + bg_format, + }) + + writer.close() + + return 0 if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) From fe9976860a3f11259ed1c5b4f828bd1102cf1a33 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Tue, 22 Oct 2024 18:55:41 +0530 Subject: [PATCH 159/169] scenarios -> fixed_scenarios in docs --- docs/benchmarks/language/reproducibility/indyscc24-bert.md | 2 +- docs/benchmarks/text_to_image/reproducibility/scc24.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/benchmarks/language/reproducibility/indyscc24-bert.md b/docs/benchmarks/language/reproducibility/indyscc24-bert.md index c82175a60..86f4f285d 100644 --- a/docs/benchmarks/language/reproducibility/indyscc24-bert.md +++ b/docs/benchmarks/language/reproducibility/indyscc24-bert.md @@ -27,7 +27,7 @@ All the needed files are automatically pushed to the GitHub repository if you ma === "MLCommons-Python" ## MLPerf Reference Implementation in Python -{{ mlperf_inference_implementation_readme (4, "bert-99", "reference", extra_variation_tags="", scenarios=["Offline"],categories=["Edge"], setup_tips=False) }} +{{ mlperf_inference_implementation_readme (4, "bert-99", "reference", extra_variation_tags="", fixed_scenarios=["Offline"],categories=["Edge"], setup_tips=False) }} ## Submission Commands diff --git a/docs/benchmarks/text_to_image/reproducibility/scc24.md b/docs/benchmarks/text_to_image/reproducibility/scc24.md index d17079973..6fe119b53 100644 --- a/docs/benchmarks/text_to_image/reproducibility/scc24.md +++ b/docs/benchmarks/text_to_image/reproducibility/scc24.md @@ -46,11 +46,11 @@ or supporting multi-node execution) useful for the community and [MLCommons](htt === "MLCommons-Python" ## MLPerf Reference Implementation in Python -{{ mlperf_inference_implementation_readme (4, "sdxl", "reference", extra_variation_tags=",_short,_scc24-base", devices=["ROCm", "CUDA"],scenarios=["Offline"],categories=["Datacenter"], setup_tips=False, skip_test_query_count=True, extra_input_string="--precision=float16") }} +{{ mlperf_inference_implementation_readme (4, "sdxl", "reference", extra_variation_tags=",_short,_scc24-base", devices=["ROCm", "CUDA"],fixed_scenarios=["Offline"],categories=["Datacenter"], setup_tips=False, skip_test_query_count=True, extra_input_string="--precision=float16") }} === "Nvidia" ## Nvidia MLPerf Implementation -{{ mlperf_inference_implementation_readme (4, "sdxl", "nvidia", extra_variation_tags=",_short,_scc24-base", scenarios=["Offline"],categories=["Datacenter"], setup_tips=False, implementation_tips=False, skip_test_query_count=True) }} +{{ mlperf_inference_implementation_readme (4, "sdxl", "nvidia", extra_variation_tags=",_short,_scc24-base", fixed_scenarios=["Offline"],categories=["Datacenter"], setup_tips=False, implementation_tips=False, skip_test_query_count=True) }} !!! info Once the above run is successful, you can change `_scc24-base` to `_scc24-main` to run the main variant. From 2f603e59122899ec351f0dea66192f1d466d88ce Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Thu, 7 Nov 2024 11:26:05 +0000 Subject: [PATCH 160/169] [Automated Commit] Format Codebase --- main.py | 2 -- tools/submission/generate_final_report.py | 5 ++--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/main.py b/main.py index fd03171da..6038131da 100755 --- a/main.py +++ b/main.py @@ -74,7 +74,6 @@ def mlperf_inference_implementation_readme( content += f"\n{pre_space}!!! tip\n\n" content += f"{pre_space} - Intel MLPerf inference implementation is available only for datacenter category and has been tested only on a limited number of systems. Most of the benchmarks using Intel implementation require at least Intel Sapphire Rapids or higher CPU generation.\n\n" - if model not in [ "bert-99", "bert-99.9", @@ -569,7 +568,6 @@ def get_run_cmd_extra( return "" @env.macro - def mlperf_inference_run_command( spaces, model, diff --git a/tools/submission/generate_final_report.py b/tools/submission/generate_final_report.py index 2a036c77a..f9065a89b 100644 --- a/tools/submission/generate_final_report.py +++ b/tools/submission/generate_final_report.py @@ -329,7 +329,7 @@ def reformatlink(data, key): for key in keystomatch): # print(result) # print(outjsondata[i+1]) - + if "Watts" in result["Units"] or "joules" in result["Units"]: result["Performance_Result"] = outjsondata[i + 1]["Result"] result["Performance_Units"] = outjsondata[i + 1]["Units"] @@ -355,7 +355,6 @@ def reformatlink(data, key): with open(f"{output}_results.json", "w") as f: f.write(json.dumps(outjsondata, indent=2)) - score_format = writer.book.add_format({"num_format": "#,##0.00"}) bg_format = writer.book.add_format({"bg_color": "#efefef"}) for ws in writer.book.worksheets(): @@ -384,4 +383,4 @@ def reformatlink(data, key): if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file + sys.exit(main()) From 8f906be73c12cc073fef302733e304befdc1f7fd Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 11:27:43 +0000 Subject: [PATCH 161/169] Update indyscc24-bert.md --- docs/benchmarks/language/reproducibility/indyscc24-bert.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/benchmarks/language/reproducibility/indyscc24-bert.md b/docs/benchmarks/language/reproducibility/indyscc24-bert.md index 86f4f285d..463d1a299 100644 --- a/docs/benchmarks/language/reproducibility/indyscc24-bert.md +++ b/docs/benchmarks/language/reproducibility/indyscc24-bert.md @@ -37,7 +37,6 @@ All the needed files are automatically pushed to the GitHub repository if you ma ```bash cm run script --tags=generate,inference,submission \ --clean \ - --preprocess_submission=yes \ --run-checker \ --tar=yes \ --env.CM_TAR_OUTFILE=submission.tar.gz \ From 7094946e4c5e5d7e41f93c005202f311722303e8 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Thu, 7 Nov 2024 11:28:21 +0000 Subject: [PATCH 162/169] Update scc24.md --- docs/benchmarks/text_to_image/reproducibility/scc24.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/benchmarks/text_to_image/reproducibility/scc24.md b/docs/benchmarks/text_to_image/reproducibility/scc24.md index 6fe119b53..368f38dd2 100644 --- a/docs/benchmarks/text_to_image/reproducibility/scc24.md +++ b/docs/benchmarks/text_to_image/reproducibility/scc24.md @@ -62,7 +62,6 @@ or supporting multi-node execution) useful for the community and [MLCommons](htt ```bash cm run script --tags=generate,inference,submission \ --clean \ - --preprocess_submission=yes \ --run-checker \ --tar=yes \ --env.CM_TAR_OUTFILE=submission.tar.gz \ From cd1aa7824c8010324b9b63e00103af9ced101526 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sun, 10 Nov 2024 00:13:44 +0530 Subject: [PATCH 163/169] updated tip for reference implementation (#1912) --- main.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 6038131da..b68d2b0d8 100755 --- a/main.py +++ b/main.py @@ -237,12 +237,19 @@ def mlperf_inference_implementation_readme( extra_docker_input_string, ) + common_info = get_common_info( + spaces + 16, + implementation + ) + if ( execution_env == "Native" ): # Native implementation steps through virtual environment content += f"{cur_space3}####### Setup a virtual environment for Python\n" content += get_venv_command(spaces + 16) content += f"{cur_space3}####### Performance Estimation for Offline Scenario\n" + + content += common_info content += setup_run_cmd.replace( "--docker ", "") @@ -258,6 +265,9 @@ def mlperf_inference_implementation_readme( device, setup_tips, ) + + content += common_info + content += docker_info content += setup_run_cmd @@ -475,6 +485,23 @@ def get_venv_command(spaces): {pre_space}export CM_SCRIPT_EXTRA_CMD=\"--adr.python.name=mlperf\" {pre_space}```\n""" + # contains run command information which is common to both docker and native runs + def get_common_info(spaces, implementation): + info = "" + pre_space = "" + for i in range(1, spaces): + pre_space = pre_space + " " + pre_space += " " + # pre_space = " " + info += f"\n{pre_space}!!! tip\n\n" + info += f"{pre_space} - Batch size could be adjusted using `--batch_size=#`, where `#` is the desired batch size. This option works only if the implementation in use is supporting the given batch size.\n\n" + if implementation.lower() == "reference": + info += f"{pre_space} - Add `--adr.mlperf-implementation.tags=_branch.master,_repo.` if you are modifying the official MLPerf Inference implementation in a custom fork.\n\n" + info += f"{pre_space} - Add `--adr.inference-src.tags=_repo.` if you are modifying the model config accuracy script in the submission checker within a custom fork.\n\n" + info += f"{pre_space} - Add `--adr.inference-src.version=custom` if you are using the modified MLPerf Inference code or accuracy script on submission checker within a custom fork.\n\n" + + return info + def get_docker_info(spaces, model, implementation, device, setup_tips=True): info = "" @@ -489,7 +516,6 @@ def get_docker_info(spaces, model, implementation, if model == "sdxl": info += f"{pre_space} - `--env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes` option can be used to download the model on the host so that it can be reused across different container lanuches. \n\n" - info += f"{pre_space} - Batch size could be adjusted using `--batch_size=#`, where `#` is the desired batch size. This option works only if the implementation in use is supporting the given batch size.\n\n" if implementation.lower() == "nvidia": info += f"{pre_space} - Default batch size is assigned based on [GPU memory](https://github.com/mlcommons/cm4mlops/blob/dd0c35856969c68945524d5c80414c615f5fe42c/script/app-mlperf-inference-nvidia/_cm.yaml#L1129) or the [specified GPU](https://github.com/mlcommons/cm4mlops/blob/dd0c35856969c68945524d5c80414c615f5fe42c/script/app-mlperf-inference-nvidia/_cm.yaml#L1370). Please click more option for *docker launch* or *run command* to see how to specify the GPU name.\n\n" info += f"{pre_space} - When run with `--all_models=yes`, all the benchmark models of NVIDIA implementation can be executed within the same container.\n\n" @@ -501,6 +527,10 @@ def get_docker_info(spaces, model, implementation, info += f"\n{pre_space}!!! tip\n\n" info += f"{pre_space} - `--env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes` option can be used to download the model on the host so that it can be reused across different container lanuches. \n\n" + # return empty string if nothing is filled inside the tip + if info == f"\n{pre_space}!!! tip\n\n": + return "" + return info def get_readme_prefix(spaces, model, implementation, extra_variation_tags): From 69556399f99b41059f96fd90f50fa6171fb1d8b8 Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Sat, 9 Nov 2024 18:44:13 +0000 Subject: [PATCH 164/169] [Automated Commit] Format Codebase --- main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index b68d2b0d8..9966e6f6d 100755 --- a/main.py +++ b/main.py @@ -248,7 +248,7 @@ def mlperf_inference_implementation_readme( content += f"{cur_space3}####### Setup a virtual environment for Python\n" content += get_venv_command(spaces + 16) content += f"{cur_space3}####### Performance Estimation for Offline Scenario\n" - + content += common_info content += setup_run_cmd.replace( @@ -485,7 +485,8 @@ def get_venv_command(spaces): {pre_space}export CM_SCRIPT_EXTRA_CMD=\"--adr.python.name=mlperf\" {pre_space}```\n""" - # contains run command information which is common to both docker and native runs + # contains run command information which is common to both docker and + # native runs def get_common_info(spaces, implementation): info = "" pre_space = "" @@ -499,7 +500,7 @@ def get_common_info(spaces, implementation): info += f"{pre_space} - Add `--adr.mlperf-implementation.tags=_branch.master,_repo.` if you are modifying the official MLPerf Inference implementation in a custom fork.\n\n" info += f"{pre_space} - Add `--adr.inference-src.tags=_repo.` if you are modifying the model config accuracy script in the submission checker within a custom fork.\n\n" info += f"{pre_space} - Add `--adr.inference-src.version=custom` if you are using the modified MLPerf Inference code or accuracy script on submission checker within a custom fork.\n\n" - + return info def get_docker_info(spaces, model, implementation, From 387aa77245f89b638f8e179db7ea7813c9ea7def Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Sun, 10 Nov 2024 16:23:49 +0530 Subject: [PATCH 165/169] fix for run suffix (#1913) --- main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 9966e6f6d..1bbdc3dec 100755 --- a/main.py +++ b/main.py @@ -385,7 +385,8 @@ def mlperf_inference_implementation_readme( extra_input_string, ) content += run_cmd - content += run_suffix + + content += run_suffix readme_prefix = get_readme_prefix( spaces, model, implementation, extra_variation_tags From f46b6095b6e77e989d554daccae948617e3bef9d Mon Sep 17 00:00:00 2001 From: arjunsuresh Date: Sun, 10 Nov 2024 10:54:19 +0000 Subject: [PATCH 166/169] [Automated Commit] Format Codebase --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 1bbdc3dec..c8c64b8c3 100755 --- a/main.py +++ b/main.py @@ -385,7 +385,7 @@ def mlperf_inference_implementation_readme( extra_input_string, ) content += run_cmd - + content += run_suffix readme_prefix = get_readme_prefix( From 90b577f25c3e74074cde7bbe879b4b19b607c07b Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 14 Nov 2024 23:50:56 +0530 Subject: [PATCH 167/169] Updation for adding submission flow diagram --- docs/submission/index.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/submission/index.md b/docs/submission/index.md index adcd3df53..c99802420 100644 --- a/docs/submission/index.md +++ b/docs/submission/index.md @@ -2,7 +2,16 @@ hide: - toc --- -[![Streamline your MLPerf results using CM Framework](https://img.youtube.com/vi/eI1Hoecc3ho/0.jpg)](https://youtu.be/eI1Hoecc3ho) + +

    + Submission Generation Flow +

    + +

    Figure: MLPerf Inference Submission Generation Flow

    + + + +Click [here](https://youtu.be/eI1Hoecc3ho) to view the recording of the workshop: Streamlining your MLPerf Inference results using CM. === "CM based benchmark" If you have followed the `cm run` commands under the individual model pages in the [benchmarks](../index.md) directory, all the valid results will get aggregated to the `cm cache` folder. The following command could be used to browse the structure of inference results folder generated by CM. From bf19a97926c38b4b59859f9f26f906c7c3aa9e8a Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 14 Nov 2024 23:52:05 +0530 Subject: [PATCH 168/169] Added submission flow diagram --- docs/img/submission-flow.png | Bin 0 -> 10048 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/img/submission-flow.png diff --git a/docs/img/submission-flow.png b/docs/img/submission-flow.png new file mode 100644 index 0000000000000000000000000000000000000000..33a09a53530e112fd720b142f5033e06d38dad20 GIT binary patch literal 10048 zcmdU#Ra9Kfw(nOX3Bdv+xI+RV(6~14t^tBe2=3Cj1qlRq4Q|2R-QC?G1Zym~!zKIs z_P%G2v(J6F^ru~lbIP1*5!#F5i!N0~73*&^ zN|_;I!3!*y{VtCQ+$@QS5qe?VgQwTRTilF^ix%Cfw}b}skjQG(#X-51O+CWMmf_905W zdJ-wB{Pr3Uw&aVX)++C~$?UN7*SQ)VZo?iWq6u_wz&@}3G(*Wf`Sx>6fad~ZIYtswCkAa>G>& z;nwHgAvlm&&x{P>7Fcb~XY9d%%#5_D19F+K6*6aH#TkZaIu}~fGxxV6eLGr$dCW|h0+Xdor(Vm)Thm?dw0O6W=jIk9mx;Ru^`y3s z4GCin*!Xe}gs;C&H$d-iKss!Q-3NHU?`LKWQp;LCGzZf89y@w>gYSfv2Wx118&p}s zEfeWvYDXAYSy=_VQNrHGN*U6xb>-CtR@gk$v>1%d{jDpN zlAf42oL=W6#tp`ce3qHlep{<(wku7lnA*Rc@~GUXhBZRZZL==@FB|9@q;Kqco5kh{ zd-4u|bZ9n9ev9sIz*Gf1?dXL=xT;*{kJCy}dP(KhSQokDFgyJUE%Z`@h3Nz4$>@H# z&3QthBTTcY1czWsjnubH@M^a$QJ`tJ7GktMV1={~)i3R}1GSShro@uYn(Hh-&uhBv zUqvc>*Kk_}fAhxEK-Wv39xx%=l`BAOwA;z$A><8hp1$lH98O|0pV{X)$e3e@YiXvW zlz|;q8!Z5VoR>&!+8K4klt>*Gh`-IiZFvu-WO$qh#7LZoY1dg~sK1wW`M0ZNFtm`d zWi_9*CMy+_th_3?(4HPJk0wJ>_a5->v`IkdjN}y1IR-?(Z?W#Q%PZMhBj=7oW{w<7l+IV>C;|@FS ztL8=!Af(Zg%}Leo#)Xcjq-9D^5Fm5Fcs=rlAuafWB_+e5MNrf0jKk4wv6w^cgB>oa zX;pABuGWdmPQ=Hlp1DG?(V;uyewD3Yb5MD=uFQ{E%x6=N&VBsg*#VX6~ zN^?$fA)4QvS;k z{=JKbYkjf(nqZH`C1c*^IT0+ExrZKFvOj(s20If5RQYIHy5c8w@*`It)iZO;@PIL zH@AhK5r#$Yk482s=WI=W;l(Pv(D$`rGmUs9_W*DcYH}C91=?s;aO(5~KB9ild>=e0 zz8V-Tq=-a_(5I*fTOF3dmdcOe=C4efaKRLpKTMQVk$Nq&Cb=_z)tU=8M$|`hfrCJ< z31WS=_Tm>E7D9%HA_)fSm1mWsMLbsl3u~M;l8||N0cs6B8|dYDMxA@M&TBXym2)Qq z!L_Rk^cq8+`>Sd69A$J~z3jr)z^s!{jojaF&MaOpfXYBxS`Q!qpKQcbdkghBz}@zO z6MKcO$XP(>mK`i>Wl!T z0BtJ4c~nm1sYWL$datE*@!>De)a~ls4LM>>R9!6zW@;p$0tUP5o0~>E3R-+h49KQ1 zL`WIo3w4LtuWE|DNg7WH~GBm>3ix?X>^_w zMgMoNyJXFxh?y+)t?usC`$$t@%nzOB5^CjRP^%sPj)v=v!?OQdST^} zJ7>ei{%-;r9SO<9j&5e?2u5LOmqgrxDtL~$dBVN$ip@1%tOeq^Ejjc*Pa;$5-YK)! zcH^t5(N2#R4Q;0QV}l~f2^$4*1x`s zB`TgK*1SO|2-9(6Z?UkwzZk^S zld=@xvnB!bf^X8sI^PGo1K9B(0s7zV&o!h00WBhB%^Y4=z#k~E!Ypmoaf1->9h6Ek zr;zH6s*j*LqyBQ5hRurWKCGDkY>QZ>Xd76_BsGhWMi4-(}lA*$0!L$gbFCBNd=0KqjdQ8!?xM;BsgDPNd$nkFX;IMyOkoT$MBW{YYE3g zNqZ2*^Y27u`7x%4F(*3ts6R7nI>^li7R2Q_Zr9)`wdK22vRvJs?Qc6=cutfTiEEa4 z@OYkIlswQlu|+Rc1>TPkKgvQiu|2aPMrZ>N*1G<-ymnHT z&r;i3PRPB|QfRfn1Xiy+fvEKydxrhz7mX#cv&)20We5OdMQfjx<*dmU;-i#B`)JlQ zvT`~1EbWX7pR|-~C!@A-q_hzGRLk%n-nF%kl4ad=b>iGPl+bg*7n!}rF_<-4EAMkB zIMDw=dhG5{cY#hBaGlQq+*RkR9QH5b({yt4O+LCCwJ-i|L=%XvB`6F_R~}#hr!-rq zY|{ZZjj+3fkM-ZMU+nE7)Pa7U@rkqaqdl_=Y6hh@7@_?9ObfiKKHXGA0WMae3dX_Q zan3v@hVUY5PW;U>AF7x8FwIvy^VrgdHK)I|c~eW9-#Dz1086jZ{JQ)ma#6>4jM+Z@ zh`^GL(W*0Jf}>HE#FyqSCg7i5k)pGazYsZhj%f);=p}9N(sgLb1c> z{X#KGOVRi#V?iBhY*3t?aRgME$~09!u2!4m;cc?JYxmq}j%^k5@xBS}cW1nZl9V** z+C^#J`^us4;>66U;nh2vq-~(^Po^*wR!W-}p%x}N6SiHi2s4BoT8{~gk3i?E>uv|( zmNJGlJzOPlD8ZAu+v`Dx^j{u&gsOdQt=(jZa=CR?jpIjPvuv$UR`2q7$iii+{+fnm zt?AMtwx-4NrzhnB=^6HSciA!4{`$83*N->W0qpVWoEYi-&n!8-gmM<2<>%&g(jtOC z7db0>1U%cDrr&-(*IJWQZd8e905&TBp_o;udVj@dAQxP)$Z>7pi5CALO2?#S;6y(# zO}V8TXF51{@X(S<%)lgzu)k@TNZ|X^D+D9W&^XbEcrMD*h^I#U=s?HZcx(mNL@s~t$<*oJUPga56 z`K707gm1-)ReZ>hm|NSOQZ1Gpu}(%|s@M+D2hIEIGO{hc=|`!ko}H3YlS)A}C0Ug{ z9-1U4AMqDH>7K!AdvaRa<4+4zMTZiNe>(9sR@*LTcd!a{dr7=NSnGcEL7wwiBT~*q zdPcrfVi}1Ivd04(mZ5Iz7R^_FAG;b`F_N`_n-Uiv9{|B^IZybo5oq!yujus0e$;5s zFFz;nc&uU3Q-qKSWHdBOUW$u(k|pfYnM#^LF`sPZS6WYtq2?XOE`iQR>g(p4CrR3L z+<>_mxSB;Ncgo3-NNPmU<066hi2H&`2JyyOP4ok*#`iSmIa$$YW)T7qX5*dXQrs5b zZ)x;Ha%x1m1w({2H@~9nWfcrI!&Y5ap4qfqj!*kasc$RJ*V7ZaTcjDK7>~*`85Tdq zbTfI?@pxPFa!h!Mi=liTJmG1be0%ZsVyJiUBT=;cwY$!8^?L6r5b>xQ%Z6xR2TIAF zieIJwptT(tg^0E?LqqMZ(0V_nRNJy_=fuBn8DnlXORie*J~Mc(w1THcZL*FcnE{Wx zJp5iQlBp(-!eBkELfCl-%Ogf6F|rrWz@mMxXpB6TY34%M5h*+#@-R5Za1~Ym(1u4i z8%W$8-lOxCX%b38v*^a4RZ77VeSc$}pIcH(8i0+L)%WZ{=oJTR%8EHcBNF6rO zIRkWz0ByTsscp%z((cpIH~zdVKBJKE7&>>-z8@dz_s~%Bjtj&`ZzsKAZq7>Vasdr& zx7b`@zB98H(t6CpQW-=bxsTOA>w_1;H&Zx;7>q)mCS;V`0QiO-^x zUVQ6E>Q00I_?|}`*lnbLjOWb1u25LB-R9$X=0xAMPUS*OWl+%ivu&RTKw=xX75+Nu zoR*wK{eD2?b#_nip!QnwHyeg2l zo9XZnj#jpNU_<11?_W~ObZOsshwKETUoaVHP1&%m?B*J6FX=4pm*}us^+_JzL-9S) zCT_*37RO(pLt9rpP=S@Slf$##jrZfV-;-W*6eWji{f@!+5_`?4obUmTK6vu;5iL-niW%?g z=6SMB^$`$w$O;;DA|{p!(IE9i61a?QQWaMT_=?uZV4r}r|Lpx|(AAy&usWp}P=PQ_ zZ}|?sz!Gtc)KxOyFPqclojdmkk&?9%HGbS>DI}jI9bqg2yjQs;mxb~^E7`N7AvhdZ z6S7=Kq|eQ5N_;67<#kJZ3|GbPuaVE|e$s*fx%}-U(ZTJ34yxIUcc!09U-_Q&xNbi0 zdjsEQeGqCZdphjS$p8hmrdG;dEl^TD12DcL1INcEAXZK|K(G}aDwh#&#<9+?x5;dP zU{Q0e(QaN$>$KUuK)7~od>ty^WEJx_HL zJTb$vp5ZCkU>nM839l7`Rdy{ETxIw(p}cCS z)K_8DS6_C;v1znf)1dCXOp0IRZnlN`3H8=ZQACXvd8^vUB)F#<>B8R!#lJYE~ z0Ek?IV$u16ZeRfou?_+^p>abG6|0edM>qdOIDhr#J%--p4^cKWY8z&9y9)An_ypBR zauHSksTC!s3N1MLVG&gTnH&fR;>_PDdncg*L>v<4 z34A|7{@eUrKWe0Vx3?eyl&5+w)Kj*M>3LYr}WiczX1aR98S8N zGLev=c=TC!a8z87JYrP|Efj-+cv}tEbA$#Ym#un;-9So!T^T!5v(aD% zCX+=5>&Z9Ob!jF^>`RV2Q5#hw@(23We^aky7lPrT?qZ29U@@(ly0--#a_5A#M+8M%L)y4a~Y?Wf$Z6om8hHnGX6fSwDGY@L1w;Oa~p9R-xcX zcG`yCsTc^7@Fx?G!TlV6cH4RA)8|A$ian`J6Zye;lEi{EGO-Y#pXst5=a|A~c(%e% zG1yt>y8#y@@BiwXQKc@fiB5I+EVcPWTB8@t_K+JOk6>um7V_KaAN7120gixouFk>$ zSe?I_0;)&!^gB2?8C$QHS6SG^u2xJTQOVReHfxSjl`&<25{4pgn#ZEtcBW`zBh3p$ zMuhGpG94b4`xj8Qax&!aPn>Uge=Nv`12Uw?Bd&BizX<{e!)trabhN)WWKs ziuQ&Y#zB5_u?6Px-+I?!RhAE+Mh&!9c;ITtR(b*!Zp+c-a;hQf0E$p8-nB{!jmc{%COmF0YCA3W*RKe_x!3u-Iet+D2UK3atSMZ~h!Vo;w-{#!TI7_2Yf z4oEmvBl`aYfvNR6j`GL{bO@C z^Z@jdfq}!r;HgsjWq18!6cq))a5&V}D?IA_8x05^Zf>dw!sHmy=uD>I$SxBPBKhlN zDhHdtZicz8Uf$;uQ>`Zc+WheK{BH=UDFqU65Ewf1qGGkhTU?9gqu$uaIrKLEwQ|XH z*Rs4=k?kMbDIE0EI?j-7tfp|x1AQMIeUCi&yJj5J;u!apnW0~-5};RUEP_kUjQAzd zpp&;(E~4Rnv~T(C!L{U|d+V3$afRa#LCZr}=iwv`aZoddw!3%?Qe`h5%FY$P)?2~9 zQZ3lB1d;n=KIJ<}F^SdI>@dOibo*b`=!tY^tYmwdKHYYVC~ zJ;EnGdqu=};GaONM89{M~-QrykD zs;MsDb3+Y7HBijB5OU#%jp0qW2-{A`F8idt?aj~va*c7Pie77mY%UqG~Kz{o-e`|3Q_|& zdM^&88%zsS<_r7+MME`?UZ~ubn5j?*7HXh9q2{9k)(@m^NLCO>U#BUOYv~x&ww!=s zkY5T2ZV8ar`vaI^ua%}8;rBt%r#K&uUcT2EQ{8*ppuvsvw)melM1cYlQ>8K$RT%O! z$7Ke2R}9U^xEdB@w%weAlamvnr@dlu5Nl_~{9zk~`3!dtm0a7b(+d_Va5z-ORnW+O zuPJr-B{-5)L>n61KJMu?Nyisn7aDdCN>tu3bajB)mzBznfuR)nDHdoB1g!Gm2>005 zmz;4(^{t-1eevI@H@#<|tl(QM=ruH@lS=jI>`mhaca%2!AOTz%amt#)!@*N9 zunGvcTBMn-xjhKH8?&6!3L|EiaHPd8gi)6VxeN*DRhZ3A3C7aOKp^>!GOayJ3%2)OxK~5 zBP#LoKZpyzYpK1v;6^g%r*ku8oknpOMqu94Wzf<>a+TpkLUEOZW^UNOWY&`r$|e1z zxh4{-|J(_2`D1h>{7YStNpW)zNh z0ld>yPy3}pR)LcECrKn37?KRgp8*cp%qaIf8iFD1lZ^-jF1n)s@KA*@{zFm;tfQ>$ z2`UBXD@If)cUkg^`^MRlo8~KA`50;R_-DVxioXt?B3Hxv^USJBL~VSs|KGHZPuWn` ziXh|LQfXz8-uO?`c3(Y*SCDgV_vn5&W8+buAp-tnNInW&H!sIO{cuyQQSe4~%du+s z_K6k>i2W2%JA->n<8S8M+%?jM!Dg?}>6h)c@Rr=29N+D?O=!>LsUmp)GJ5x5ZZ~y= z4XmAI-VZ;Jm{CGIW$}#gQ9XQ_dQ?%o%*`@;ntqJ(SQv(s)6sQ9dgfH_`tS_)Y=W42 z=EAX6RBPO_ts1|5qT%Rzd{Bi8Tv_+~uSgO2H^ozI4E&XKx3iXc+?E7X5!dAQ3u9M@ z5tTBHm2FWUrAN;T%0Ui!*7F97X6iV# z!~HluV9cdbYhI&hC@gS=|0wyG%pt;cjuodh+~b20u5IAINRfAMh~e6a?RWhm4?F`d z&f$(IwaoA+8kUnFix$28F1{yJ6t;$HaJdv$_W4#bB=n_fqw+Ea$NI;(&?CIY0$3UB z@kiK$T4Z^a0`ns8{&IUrIWc=}&LR9{hmsh$iTyS{nbtZWWoo9X0z~nfd+oXN6atU( z^k2z&^DQA)X7{h>#h|q+C$hEI?$3~H;Car8N-y%AXXYu1fvA_nkOVEpfwcv$U<5%E z-5Ba8t%egvQEVj+%pTsRa1;dj0|)?fyPOKS3EiI7)r^5$D5V4+Vf3q&Y7zO@aCXAI zm+ZPI?!|}0%N)QI3YsNmd|7u}k73vohkkD^z5F8AKSyHkG6(qmu9e__BSE9Gg@2yg zz0O+tS~{rag(9G^qQ2ICoUTM{J;dLRcYAI7wV`AC(7+@4ZzTj0c|m8WZSHv4$g~o4 z{h+O2V5m0TwFrz{lDOJBc$F5q#<|c4EvJo!+Gsn14<9N3Cx9GmphOh?t62)#1Uu&d z5T56F(W!E-=f!w;OoihDceuiUigDnb89dpZUH>{%u=VSr>!g zCyw{h#BlnF6BIIV()Hw6RL}oI2LI0`<9{y`yB^^OoWhJ+=?Gb@CNk(QAW!pJz}$wB#V6D+Min;NFq004r#tFoGdrL&Q#g&_c_`5&k+S}k&7+!KHf ze9;su-fT0;5mmj z7T6oi9uUo0aAGaR`Q5m4>vhfRf1J3fLovI%aFoIEX8jdBZCI$ql`ycZAeW08w>(uD zvxmR_*0A30@T@@)_u4EQl}XDzl+s@YFC6KkLi5tS-h6zV#bTUW&Lv_Bk+JJ0Qq9cx zplNoQ&hOt`#KXe)-kfTTdpHCxX8c;r8Bjp6z&3E+puAs>5u1Tk@V zjuAMguMVl6wQK~5Wn0L;4$1Msv1I)0A}2K7!MJU)lFDvqx@Zm}T!ka45qdvf%8K<( z=r*;v=aGz6F2uMmlc2vWvFHnPJ7)4xzZ7@UccyYDXW|pusN7I@Q~tJOuU5xu*_KaN zqY1R7@f-TD>uWcz8Oc~Kk4M$V1$ku+zpFwTAk3dkJBHO`#YHqgjlM2Ih%WDJ-idDu zFt%Yu=8Sy^kWE7s#BrODn$$w8Gcx=6H(DHP;MxUiARA~S zcGn;a+1^1<{*~8d?>aSRgsu}l?u%gcu*sL^4r6j-t|P<-{&It>B7r%4{dHItA1AW2 zI%$0OK4HJ?|ANLfDg&mb!Eu;r@z!>M^js`-1}e_A2XTdsBMB3)n!8~o^os1L^hRci zhE9j+Buk8-Q(V)$v1T-cBFu!GJmx3h@5HcCXM6p^4o*TbBGM?oYi=dPTUmX^NZl_;^&vokEN1+;tLPA0!fyc+}Z_CSNGuu##`@ zskf5aSTMOe=M`;NB~9(Q>tk;rlV8MnQx!(LVu)ED%6&eH1hOCd=@C z-#&Z4Gt`khS)Xk_?-!b@%9?X0iyT&@glkzLfwBMSVFxh{A4eNNXzo&b!d_O}mWPdb1 zZLQUn|Gtsu9FjVlDS1S(bNGW{1wGSQ?sET!&&WpiBIanzXTfeYw}^vYv0DwQyraZa z8CgA7$_^TY`iCur(NXfX4WsSy`Fz=ad#jrXHF+aB_~eTxINxmV;&+?x6wGTZEWw+1 zAlbc+e}r?{aJElc$CEXic>bN2R57IXe7oRcmI;rIv~S8=M?u;q)ctvZc7nwL3h(Vx PIszm` Date: Mon, 18 Nov 2024 21:38:31 +0000 Subject: [PATCH 169/169] Update scc24.md --- docs/benchmarks/text_to_image/reproducibility/scc24.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/benchmarks/text_to_image/reproducibility/scc24.md b/docs/benchmarks/text_to_image/reproducibility/scc24.md index 368f38dd2..82a231b4d 100644 --- a/docs/benchmarks/text_to_image/reproducibility/scc24.md +++ b/docs/benchmarks/text_to_image/reproducibility/scc24.md @@ -59,6 +59,7 @@ or supporting multi-node execution) useful for the community and [MLCommons](htt ### Generate actual submission tree + ```bash cm run script --tags=generate,inference,submission \ --clean \