diff --git a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/README.md b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/README.md index f447aee6d..4a5d95d68 100644 --- a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/README.md +++ b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/README.md @@ -19,7 +19,7 @@ pip install -U cmind cm rm cache -f -cm pull repo mlcommons@mlperf-automations --checkout=a90475d2de72bf0622cebe8d5ca8eb8c9d872fbd +cm pull repo mlcommons@mlperf-automations --checkout=467517e4a572872046058e394a0d83512cfff38b cm run script \ --tags=app,mlperf,inference,generic,_nvidia,_bert-99,_tensorrt,_cuda,_valid,_r4.1-dev_default,_offline \ @@ -71,7 +71,7 @@ cm run script \ --env.CM_DOCKER_REUSE_EXISTING_CONTAINER=yes \ --env.CM_DOCKER_DETACHED_MODE=yes \ --env.CM_MLPERF_INFERENCE_RESULTS_DIR_=/home/arjun/gh_action_results/valid_results \ - --env.CM_DOCKER_CONTAINER_ID=c2a65ff58585 \ + --env.CM_DOCKER_CONTAINER_ID=0ea02743d854 \ --env.CM_MLPERF_LOADGEN_COMPLIANCE_TEST=TEST01 \ --add_deps_recursive.compiler.tags=gcc \ --add_deps_recursive.coco2014-original.tags=_full \ @@ -129,4 +129,4 @@ Model Precision: int8 `F1`: `90.15674`, Required accuracy for closed division `>= 89.96526` ### Performance Results -`Samples per second`: `8268.76` +`Samples per second`: `8277.86` diff --git a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy_console.out b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy_console.out index 16de83c02..4b354a0ea 100644 --- a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy_console.out +++ b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy_console.out @@ -1,7 +1,7 @@ -[2024-12-24 20:31:04,302 main.py:229 INFO] Detected system ID: KnownSystem.RTX4090x2 -[2024-12-24 20:31:04,852 generate_conf_files.py:107 INFO] Generated measurements/ entries for RTX4090x2_TRT/bert-99/Offline -[2024-12-24 20:31:04,853 __init__.py:46 INFO] Running command: ./build/bin/harness_bert --logfile_outdir="/cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy" --logfile_prefix="mlperf_log_" --performance_sample_count=10833 --test_mode="AccuracyOnly" --gpu_batch_size=256 --mlperf_conf_path="/home/cmuser/CM/repos/local/cache/af0a19318f524c01/inference/mlperf.conf" --tensor_path="build/preprocessed_data/squad_tokenized/input_ids.npy,build/preprocessed_data/squad_tokenized/segment_ids.npy,build/preprocessed_data/squad_tokenized/input_mask.npy" --use_graphs=false --user_conf_path="/home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/be76b677d3a64087a781cb7f69a2f3b4.conf" --gpu_inference_streams=2 --gpu_copy_streams=2 --gpu_engines="./build/engines/RTX4090x2/bert/Offline/bert-Offline-gpu-int8_S_384_B_256_P_2_vs.custom_k_99_MaxP.plan" --scenario Offline --model bert -[2024-12-24 20:31:04,853 __init__.py:53 INFO] Overriding Environment +[2024-12-27 20:25:43,536 main.py:229 INFO] Detected system ID: KnownSystem.RTX4090x2 +[2024-12-27 20:25:44,069 generate_conf_files.py:107 INFO] Generated measurements/ entries for RTX4090x2_TRT/bert-99/Offline +[2024-12-27 20:25:44,069 __init__.py:46 INFO] Running command: ./build/bin/harness_bert --logfile_outdir="/cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy" --logfile_prefix="mlperf_log_" --performance_sample_count=10833 --test_mode="AccuracyOnly" --gpu_batch_size=256 --mlperf_conf_path="/home/cmuser/CM/repos/local/cache/6c5d0d8c0f4f47c1/inference/mlperf.conf" --tensor_path="build/preprocessed_data/squad_tokenized/input_ids.npy,build/preprocessed_data/squad_tokenized/segment_ids.npy,build/preprocessed_data/squad_tokenized/input_mask.npy" --use_graphs=false --user_conf_path="/home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/3ef477688d004a39a48da5ba31ae9c98.conf" --gpu_inference_streams=2 --gpu_copy_streams=2 --gpu_engines="./build/engines/RTX4090x2/bert/Offline/bert-Offline-gpu-int8_S_384_B_256_P_2_vs.custom_k_99_MaxP.plan" --scenario Offline --model bert +[2024-12-27 20:25:44,069 __init__.py:53 INFO] Overriding Environment benchmark : Benchmark.BERT buffer_manager_thread_count : 0 coalesced_tensor : True @@ -11,8 +11,8 @@ gpu_copy_streams : 2 gpu_inference_streams : 2 input_dtype : int32 input_format : linear -log_dir : /home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/logs/2024.12.24-20.31.03 -mlperf_conf_path : /home/cmuser/CM/repos/local/cache/af0a19318f524c01/inference/mlperf.conf +log_dir : /home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/logs/2024.12.27-20.25.42 +mlperf_conf_path : /home/cmuser/CM/repos/local/cache/6c5d0d8c0f4f47c1/inference/mlperf.conf offline_expected_qps : 0.0 precision : int8 preprocessed_data_dir : /home/cmuser/CM/repos/local/cache/4db00c74da1e44c8/preprocessed_data @@ -21,7 +21,7 @@ system : SystemConfiguration(host_cpu_conf=CPUConfiguration(layout={CPU(name='In tensor_path : build/preprocessed_data/squad_tokenized/input_ids.npy,build/preprocessed_data/squad_tokenized/segment_ids.npy,build/preprocessed_data/squad_tokenized/input_mask.npy test_mode : AccuracyOnly use_graphs : False -user_conf_path : /home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/be76b677d3a64087a781cb7f69a2f3b4.conf +user_conf_path : /home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/3ef477688d004a39a48da5ba31ae9c98.conf system_id : RTX4090x2 config_name : RTX4090x2_bert_Offline workload_setting : WorkloadSetting(HarnessType.Custom, AccuracyTarget.k_99, PowerSetting.MaxP) @@ -34,64 +34,64 @@ skip_file_checks : True power_limit : None cpu_freq : None &&&& RUNNING BERT_HARNESS # ./build/bin/harness_bert -I1224 20:31:04.902976 20263 main_bert.cc:163] Found 2 GPUs -I1224 20:31:05.030550 20263 bert_server.cc:147] Engine Path: ./build/engines/RTX4090x2/bert/Offline/bert-Offline-gpu-int8_S_384_B_256_P_2_vs.custom_k_99_MaxP.plan +I1227 20:25:44.119817 20262 main_bert.cc:163] Found 2 GPUs +I1227 20:25:44.249424 20262 bert_server.cc:147] Engine Path: ./build/engines/RTX4090x2/bert/Offline/bert-Offline-gpu-int8_S_384_B_256_P_2_vs.custom_k_99_MaxP.plan [I] [TRT] Loaded engine size: 414 MiB [I] [TRT] Loaded engine size: 414 MiB [W] [TRT] Using an engine plan file across different models of devices is not recommended and is likely to affect performance or even cause errors. -[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +66, GPU +8, now: CPU 787, GPU 1225 (MiB) -[I] [TRT] [MemUsageChange] Init cuDNN: CPU +2, GPU +10, now: CPU 789, GPU 1235 (MiB) +[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +6, GPU +8, now: CPU 727, GPU 1225 (MiB) +[I] [TRT] [MemUsageChange] Init cuDNN: CPU +2, GPU +10, now: CPU 729, GPU 1235 (MiB) [I] [TRT] [MemUsageChange] TensorRT-managed allocation in engine deserialization: CPU +0, GPU +581, now: CPU 0, GPU 581 (MiB) [I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +6, GPU +8, now: CPU 737, GPU 969 (MiB) [I] [TRT] [MemUsageChange] Init cuDNN: CPU +1, GPU +10, now: CPU 738, GPU 979 (MiB) [I] [TRT] [MemUsageChange] TensorRT-managed allocation in engine deserialization: CPU +1, GPU +291, now: CPU 1, GPU 581 (MiB) -I1224 20:31:05.517578 20263 bert_server.cc:208] Engines Creation Completed -I1224 20:31:05.530956 20263 bert_core_vs.cc:385] Engine - Device Memory requirements: 704644608 -I1224 20:31:05.530967 20263 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 2 -I1224 20:31:05.530972 20263 bert_core_vs.cc:415] Engine - Profile 0 maxDims 98304 Bmax=256 Smax=384 +I1227 20:25:44.739281 20262 bert_server.cc:208] Engines Creation Completed +I1227 20:25:44.759653 20262 bert_core_vs.cc:385] Engine - Device Memory requirements: 704644608 +I1227 20:25:44.759660 20262 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 2 +I1227 20:25:44.759663 20262 bert_core_vs.cc:415] Engine - Profile 0 maxDims 98304 Bmax=256 Smax=384 [I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +8, now: CPU 324, GPU 1901 (MiB) [I] [TRT] [MemUsageChange] Init cuDNN: CPU +0, GPU +8, now: CPU 324, GPU 1909 (MiB) -I1224 20:31:05.595320 20263 bert_core_vs.cc:426] Setting Opt.Prof. to 0 +I1227 20:25:44.826346 20262 bert_core_vs.cc:426] Setting Opt.Prof. to 0 +I1227 20:25:44.826371 20262 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 256 [I] [TRT] [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +0, GPU +0, now: CPU 1, GPU 581 (MiB) -I1224 20:31:05.595347 20263 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 256 -I1224 20:31:05.596164 20263 bert_core_vs.cc:476] Setup complete -I1224 20:31:05.596310 20263 bert_core_vs.cc:385] Engine - Device Memory requirements: 704644608 -I1224 20:31:05.596315 20263 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 2 -I1224 20:31:05.596319 20263 bert_core_vs.cc:415] Engine - Profile 0 maxDims 98304 Bmax=256 Smax=384 +I1227 20:25:44.827178 20262 bert_core_vs.cc:476] Setup complete +I1227 20:25:44.827324 20262 bert_core_vs.cc:385] Engine - Device Memory requirements: 704644608 +I1227 20:25:44.827328 20262 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 2 +I1227 20:25:44.827332 20262 bert_core_vs.cc:415] Engine - Profile 0 maxDims 98304 Bmax=256 Smax=384 [I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +8, now: CPU 447, GPU 1645 (MiB) [I] [TRT] [MemUsageChange] Init cuDNN: CPU +0, GPU +8, now: CPU 447, GPU 1653 (MiB) -I1224 20:31:05.660576 20263 bert_core_vs.cc:426] Setting Opt.Prof. to 0 +I1227 20:25:44.893194 20262 bert_core_vs.cc:426] Setting Opt.Prof. to 0 +I1227 20:25:44.893208 20262 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 256 [I] [TRT] [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +0, GPU +0, now: CPU 1, GPU 581 (MiB) -I1224 20:31:05.660593 20263 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 256 -I1224 20:31:05.661386 20263 bert_core_vs.cc:476] Setup complete -I1224 20:31:05.661542 20263 bert_core_vs.cc:385] Engine - Device Memory requirements: 704644608 -I1224 20:31:05.661545 20263 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 2 -I1224 20:31:05.661548 20263 bert_core_vs.cc:415] Engine - Profile 1 maxDims 98304 Bmax=256 Smax=384 +I1227 20:25:44.894070 20262 bert_core_vs.cc:476] Setup complete +I1227 20:25:44.894234 20262 bert_core_vs.cc:385] Engine - Device Memory requirements: 704644608 +I1227 20:25:44.894239 20262 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 2 +I1227 20:25:44.894243 20262 bert_core_vs.cc:415] Engine - Profile 1 maxDims 98304 Bmax=256 Smax=384 [I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +8, now: CPU 570, GPU 2715 (MiB) -[I] [TRT] [MemUsageChange] Init cuDNN: CPU +1, GPU +10, now: CPU 571, GPU 2725 (MiB) -I1224 20:31:05.724439 20263 bert_core_vs.cc:426] Setting Opt.Prof. to 1 +[I] [TRT] [MemUsageChange] Init cuDNN: CPU +0, GPU +10, now: CPU 570, GPU 2725 (MiB) +I1227 20:25:44.957968 20262 bert_core_vs.cc:426] Setting Opt.Prof. to 1 [I] [TRT] Could not set default profile 0 for execution context. Profile index must be set explicitly. [I] [TRT] [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +1, GPU +0, now: CPU 2, GPU 581 (MiB) -I1224 20:31:05.724767 20263 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 256 -I1224 20:31:05.725576 20263 bert_core_vs.cc:476] Setup complete -I1224 20:31:05.725715 20263 bert_core_vs.cc:385] Engine - Device Memory requirements: 704644608 -I1224 20:31:05.725718 20263 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 2 -I1224 20:31:05.725721 20263 bert_core_vs.cc:415] Engine - Profile 1 maxDims 98304 Bmax=256 Smax=384 -[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +1, GPU +8, now: CPU 694, GPU 2459 (MiB) -[I] [TRT] [MemUsageChange] Init cuDNN: CPU +0, GPU +10, now: CPU 694, GPU 2469 (MiB) -I1224 20:31:05.788931 20263 bert_core_vs.cc:426] Setting Opt.Prof. to 1 +I1227 20:25:44.958278 20262 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 256 +I1227 20:25:44.959084 20262 bert_core_vs.cc:476] Setup complete +I1227 20:25:44.959231 20262 bert_core_vs.cc:385] Engine - Device Memory requirements: 704644608 +I1227 20:25:44.959236 20262 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 2 +I1227 20:25:44.959239 20262 bert_core_vs.cc:415] Engine - Profile 1 maxDims 98304 Bmax=256 Smax=384 +[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +8, now: CPU 693, GPU 2459 (MiB) +[I] [TRT] [MemUsageChange] Init cuDNN: CPU +0, GPU +10, now: CPU 693, GPU 2469 (MiB) +I1227 20:25:45.022871 20262 bert_core_vs.cc:426] Setting Opt.Prof. to 1 [I] [TRT] Could not set default profile 0 for execution context. Profile index must be set explicitly. [I] [TRT] [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +0, GPU +0, now: CPU 2, GPU 581 (MiB) -I1224 20:31:05.789244 20263 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 256 -I1224 20:31:05.790045 20263 bert_core_vs.cc:476] Setup complete -I1224 20:31:06.244817 20263 main_bert.cc:184] Starting running actual test. -I1224 20:31:07.623162 20263 main_bert.cc:190] Finished running actual test. +I1227 20:25:45.023195 20262 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 256 +I1227 20:25:45.024029 20262 bert_core_vs.cc:476] Setup complete +I1227 20:25:45.478945 20262 main_bert.cc:184] Starting running actual test. +I1227 20:25:46.866901 20262 main_bert.cc:190] Finished running actual test. No warnings encountered during test. No errors encountered during test. -[2024-12-24 20:31:07,851 run_harness.py:166 INFO] Result: Accuracy run detected. -[2024-12-24 20:31:07,851 __init__.py:46 INFO] Running command: PYTHONPATH=code/bert/tensorrt/helpers python3 /home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/language/bert/accuracy-squad.py --log_file /cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/mlperf_log_accuracy.json --vocab_file build/models/bert/vocab.txt --val_data /home/cmuser/CM/repos/local/cache/4db00c74da1e44c8/data/squad/dev-v1.1.json --out_file /cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/predictions.json --output_dtype float16 +[2024-12-27 20:25:47,087 run_harness.py:166 INFO] Result: Accuracy run detected. +[2024-12-27 20:25:47,088 __init__.py:46 INFO] Running command: PYTHONPATH=code/bert/tensorrt/helpers python3 /home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/language/bert/accuracy-squad.py --log_file /cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/mlperf_log_accuracy.json --vocab_file build/models/bert/vocab.txt --val_data /home/cmuser/CM/repos/local/cache/4db00c74da1e44c8/data/squad/dev-v1.1.json --out_file /cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/predictions.json --output_dtype float16 {"exact_match": 82.81929990539263, "f1": 90.15673510616978} Reading examples... Loading cached features from 'eval_features.pickle'... diff --git a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/cm-deps.mmd b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/cm-deps.mmd index fc0e19e7d..9a733ad1c 100644 --- a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/cm-deps.mmd +++ b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/cm-deps.mmd @@ -16,20 +16,20 @@ graph TD app-mlperf-inference,d775cac873ee4231_(_nvidia,_bert-99,_tensorrt,_cuda,_valid,_r4.1-dev_default,_offline_) --> get,cuda-devices,_with-pycuda app-mlperf-inference,d775cac873ee4231_(_nvidia,_bert-99,_tensorrt,_cuda,_valid,_r4.1-dev_default,_offline_) --> get,dataset,squad,language-processing app-mlperf-inference,d775cac873ee4231_(_nvidia,_bert-99,_tensorrt,_cuda,_valid,_r4.1-dev_default,_offline_) --> get,dataset-aux,squad-vocab - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> detect,os + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> detect,os detect-cpu,586c8a43320142f7 --> detect,os - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> detect,cpu - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> get,sys-utils-cm - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> get,mlperf,inference,nvidia,scratch,space + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> detect,cpu + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> get,sys-utils-cm + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> get,mlperf,inference,nvidia,scratch,space get-generic-python-lib,94b62a682bc44791_(_mlperf_logging_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> get,generic-python-lib,_mlperf_logging - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> get,ml-model,bert,_onnx,_fp32 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> get,ml-model,bert,_onnx,_int8 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> get,squad-vocab - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> get,mlcommons,inference,src,_deeplearningexamples - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> get,nvidia,mlperf,inference,common-code,_mlcommons + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> get,generic-python-lib,_mlperf_logging + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> get,ml-model,bert,_onnx,_fp32 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> get,ml-model,bert,_onnx,_int8 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> get,squad-vocab + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> get,mlcommons,inference,src,_deeplearningexamples + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> get,nvidia,mlperf,inference,common-code,_mlcommons pull-git-repo,c23132ed65c4421d --> detect,os - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> pull,git,repo + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> pull,git,repo generate-mlperf-inference-user-conf,3af4475745964b93 --> detect,os detect-cpu,586c8a43320142f7 --> detect,os generate-mlperf-inference-user-conf,3af4475745964b93 --> detect,cpu @@ -37,58 +37,58 @@ graph TD generate-mlperf-inference-user-conf,3af4475745964b93 --> get,mlcommons,inference,src,_deeplearningexamples get-mlperf-inference-sut-configs,c2fbf72009e2445b --> get,cache,dir,_name.mlperf-inference-sut-configs generate-mlperf-inference-user-conf,3af4475745964b93 --> get,sut,configs - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> generate,user-conf,mlperf,inference + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> generate,user-conf,mlperf,inference get-generic-python-lib,94b62a682bc44791_(_package.pycuda_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> get,generic-python-lib,_package.pycuda - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> get,nvidia,mitten - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> get,cuda,_cudnn - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> get,tensorrt - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> build,nvidia,inference,server,_mlcommons - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> detect,os + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> get,generic-python-lib,_package.pycuda + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> get,nvidia,mitten + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> get,cuda,_cudnn + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> get,tensorrt + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> build,nvidia,inference,server,_mlcommons + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev_) --> detect,os detect-cpu,586c8a43320142f7 --> detect,os - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> detect,cpu - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,sys-utils-cm - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,mlperf,inference,nvidia,scratch,space,_version.4_0 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev_) --> detect,cpu + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev_) --> get,sys-utils-cm + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev_) --> get,mlperf,inference,nvidia,scratch,space,_version.4_0 get-generic-python-lib,94b62a682bc44791_(_mlperf_logging_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_mlperf_logging - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,ml-model,bert,_onnx,_fp32 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,ml-model,bert,_onnx,_int8 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,squad-vocab - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,mlcommons,inference,src,_deeplearningexamples - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,nvidia,mlperf,inference,common-code,_mlcommons + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_mlperf_logging + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev_) --> get,ml-model,bert,_onnx,_fp32 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev_) --> get,ml-model,bert,_onnx,_int8 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev_) --> get,squad-vocab + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev_) --> get,mlcommons,inference,src,_deeplearningexamples + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev_) --> get,nvidia,mlperf,inference,common-code,_mlcommons pull-git-repo,c23132ed65c4421d --> detect,os - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> pull,git,repo + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev_) --> pull,git,repo get-generic-python-lib,94b62a682bc44791_(_package.pycuda_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_package.pycuda - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,nvidia,mitten - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,cuda,_cudnn - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,tensorrt - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> build,nvidia,inference,server,_mlcommons - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_package.pycuda + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev_) --> get,nvidia,mitten + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev_) --> get,cuda,_cudnn + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev_) --> get,tensorrt + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev_) --> build,nvidia,inference,server,_mlcommons + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev_) --> reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev get-generic-python-lib,94b62a682bc44791_(_transformers_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_transformers + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_transformers get-generic-python-lib,94b62a682bc44791_(_safetensors_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_safetensors + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_safetensors get-generic-python-lib,94b62a682bc44791_(_onnx_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_onnx + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_onnx get-generic-python-lib,94b62a682bc44791_(_package.sympy_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_package.sympy + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_package.sympy get-generic-python-lib,94b62a682bc44791_(_onnx-graphsurgeon_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_onnx-graphsurgeon - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> save,mlperf,inference,state - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> reproduce,mlperf,inference,nvidia,harness,_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_onnx-graphsurgeon + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev_) --> save,mlperf,inference,state + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> reproduce,mlperf,inference,nvidia,harness,_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev get-generic-python-lib,94b62a682bc44791_(_transformers_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> get,generic-python-lib,_transformers + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> get,generic-python-lib,_transformers get-generic-python-lib,94b62a682bc44791_(_safetensors_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> get,generic-python-lib,_safetensors + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> get,generic-python-lib,_safetensors get-generic-python-lib,94b62a682bc44791_(_onnx_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> get,generic-python-lib,_onnx + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> get,generic-python-lib,_onnx get-generic-python-lib,94b62a682bc44791_(_package.sympy_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> get,generic-python-lib,_package.sympy + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> get,generic-python-lib,_package.sympy get-generic-python-lib,94b62a682bc44791_(_onnx-graphsurgeon_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> get,generic-python-lib,_onnx-graphsurgeon + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> get,generic-python-lib,_onnx-graphsurgeon detect-cpu,586c8a43320142f7 --> detect,os benchmark-program,19f369ef47084895 --> detect,cpu benchmark-program-mlperf,cfff0132a8aa4018 --> benchmark-program,program - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090_) --> benchmark-mlperf \ No newline at end of file + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090_) --> benchmark-mlperf \ No newline at end of file diff --git a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/cm-deps.png b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/cm-deps.png index 2d92e9e07..03bcb1dd3 100644 Binary files a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/cm-deps.png and b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/cm-deps.png differ diff --git a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/cm-version-info.json b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/cm-version-info.json index 79beebfcc..795cc234f 100644 --- a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/cm-version-info.json +++ b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/cm-version-info.json @@ -177,7 +177,7 @@ "script_tags": "detect-os,detect,os,info", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } }, { @@ -197,7 +197,7 @@ "script_tags": "detect,cpu,detect-cpu,info", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } }, { @@ -207,7 +207,7 @@ "script_tags": "get,sys-utils-cm", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } }, { @@ -217,7 +217,7 @@ "script_tags": "get,mlperf,inference,nvidia,scratch,space", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } }, { @@ -237,7 +237,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "mlperf_logging", "version": "4.1.1", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } }, { @@ -247,7 +247,7 @@ "script_tags": "get,ml-model,raw,bert,bert-large,bert-squad,language,language-processing", "script_variations": "onnx,fp32", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } }, { @@ -257,7 +257,7 @@ "script_tags": "get,ml-model,raw,bert,bert-large,bert-squad,language,language-processing", "script_variations": "onnx,int8", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } }, { @@ -267,7 +267,7 @@ "script_tags": "get,aux,squad,dataset-aux,language-processing,squad-aux,vocab,squad-vocab", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } }, { @@ -277,7 +277,7 @@ "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", "script_variations": "deeplearningexamples", "version": "master-git-2fdb814e6a765d994872fb99ee38f8c43418d5b0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } }, { @@ -287,7 +287,7 @@ "script_tags": "get,nvidia,mlperf,inference,common-code", "script_variations": "mlcommons", "version": "r4.0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } }, { @@ -307,7 +307,7 @@ "script_tags": "pull,git,repo,repository", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } }, { @@ -387,7 +387,7 @@ "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } }, { @@ -407,7 +407,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "package.pycuda", "version": "2022.2.2", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } }, { @@ -417,7 +417,7 @@ "script_tags": "get,nvidia,mitten,nvidia-mitten", "script_variations": "", "version": "master", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } }, { @@ -427,7 +427,7 @@ "script_tags": "get,cuda,cuda-compiler,cuda-lib,toolkit,lib,nvcc,get-nvcc,get-cuda,46d133d9ef92422d", "script_variations": "cudnn", "version": "12.2", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } }, { @@ -437,7 +437,7 @@ "script_tags": "get,tensorrt,nvidia", "script_variations": "", "version": "vdetected", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } }, { @@ -447,7 +447,7 @@ "script_tags": "build,mlcommons,mlperf,inference,inference-server,server,nvidia-harness,nvidia", "script_variations": "mlcommons", "version": "r4.0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } }, { @@ -457,7 +457,7 @@ "script_tags": "detect-os,detect,os,info", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -477,7 +477,7 @@ "script_tags": "detect,cpu,detect-cpu,info", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -487,7 +487,7 @@ "script_tags": "get,sys-utils-cm", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -497,7 +497,7 @@ "script_tags": "get,mlperf,inference,nvidia,scratch,space", "script_variations": "version.4_0", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -517,7 +517,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "mlperf_logging", "version": "4.1.1", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -527,7 +527,7 @@ "script_tags": "get,ml-model,raw,bert,bert-large,bert-squad,language,language-processing", "script_variations": "onnx,fp32", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -537,7 +537,7 @@ "script_tags": "get,ml-model,raw,bert,bert-large,bert-squad,language,language-processing", "script_variations": "onnx,int8", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -547,7 +547,7 @@ "script_tags": "get,aux,squad,dataset-aux,language-processing,squad-aux,vocab,squad-vocab", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -557,7 +557,7 @@ "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", "script_variations": "deeplearningexamples", "version": "master-git-2fdb814e6a765d994872fb99ee38f8c43418d5b0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -567,7 +567,7 @@ "script_tags": "get,nvidia,mlperf,inference,common-code", "script_variations": "mlcommons", "version": "r4.0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -587,7 +587,7 @@ "script_tags": "pull,git,repo,repository", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -607,7 +607,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "package.pycuda", "version": "2022.2.2", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -617,7 +617,7 @@ "script_tags": "get,nvidia,mitten,nvidia-mitten", "script_variations": "", "version": "master", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -627,7 +627,7 @@ "script_tags": "get,cuda,cuda-compiler,cuda-lib,toolkit,lib,nvcc,get-nvcc,get-cuda,46d133d9ef92422d", "script_variations": "cudnn", "version": "12.2", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -637,7 +637,7 @@ "script_tags": "get,tensorrt,nvidia", "script_variations": "", "version": "vdetected", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -647,17 +647,17 @@ "script_tags": "build,mlcommons,mlperf,inference,inference-server,server,nvidia-harness,nvidia", "script_variations": "mlcommons", "version": "r4.0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev )" } }, { - "reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev": { + "reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev": { "script_uid": "bc3b17fb430f4732", "script_alias": "app-mlperf-inference-nvidia", "script_tags": "reproduce,mlcommons,mlperf,inference,harness,nvidia-harness,nvidia", - "script_variations": "preprocess_data,tensorrt,cuda,bert-99,bert_,v4.1-dev", + "script_variations": "preprocess_data,cuda,bert-99,tensorrt,bert_,v4.1-dev", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -677,7 +677,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "transformers", "version": "4.46.3", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -697,7 +697,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "safetensors", "version": "0.4.5", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -717,7 +717,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "onnx", "version": "1.13.1", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -737,7 +737,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "package.sympy", "version": "1.13.3", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -757,7 +757,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "onnx-graphsurgeon", "version": "0.3.27", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -767,27 +767,27 @@ "script_tags": "save,mlperf,inference,implementation,state", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev )" } }, { - "reproduce,mlperf,inference,nvidia,harness,_build_engine,_tensorrt,_cuda,_offline,_bert-99,_bert_,_batch_size.256,_v4.1-dev": { + "reproduce,mlperf,inference,nvidia,harness,_build_engine,_cuda,_bert-99,_tensorrt,_offline,_bert_,_batch_size.256,_v4.1-dev": { "script_uid": "bc3b17fb430f4732", "script_alias": "app-mlperf-inference-nvidia", "script_tags": "reproduce,mlcommons,mlperf,inference,harness,nvidia-harness,nvidia", - "script_variations": "build_engine,tensorrt,cuda,offline,bert-99,bert_,batch_size.256,v4.1-dev", + "script_variations": "build_engine,cuda,bert-99,tensorrt,offline,bert_,batch_size.256,v4.1-dev", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } }, { - "reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev": { + "reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev": { "script_uid": "bc3b17fb430f4732", "script_alias": "app-mlperf-inference-nvidia", "script_tags": "reproduce,mlcommons,mlperf,inference,harness,nvidia-harness,nvidia", - "script_variations": "preprocess_data,tensorrt,cuda,bert-99,bert_,v4.1-dev", + "script_variations": "preprocess_data,cuda,bert-99,tensorrt,bert_,v4.1-dev", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } }, { @@ -807,7 +807,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "transformers", "version": "4.46.3", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } }, { @@ -827,7 +827,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "safetensors", "version": "0.4.5", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } }, { @@ -847,7 +847,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "onnx", "version": "1.13.1", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } }, { @@ -867,7 +867,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "package.sympy", "version": "1.13.3", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } }, { @@ -887,7 +887,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "onnx-graphsurgeon", "version": "0.3.27", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } }, { @@ -927,7 +927,7 @@ "script_tags": "mlperf,benchmark-mlperf", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_cuda,_offline,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_tensorrt,_offline,_rtx_4090 )" } } ] diff --git a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/os_info.json b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/os_info.json index 7072460b5..2c7adbc09 100644 --- a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/os_info.json +++ b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/os_info.json @@ -26,5 +26,5 @@ ], "CM_HOST_PLATFORM_FLAVOR": "x86_64", "CM_HOST_PYTHON_BITS": "64", - "CM_HOST_SYSTEM_NAME": "c2a65ff58585" + "CM_HOST_SYSTEM_NAME": "0ea02743d854" } \ No newline at end of file diff --git a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance_console.out b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance_console.out index 26cf6e1a4..620d7b5d0 100644 --- a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance_console.out +++ b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance_console.out @@ -1,7 +1,7 @@ -[2024-12-24 20:19:58,461 main.py:229 INFO] Detected system ID: KnownSystem.RTX4090x2 -[2024-12-24 20:19:58,968 generate_conf_files.py:107 INFO] Generated measurements/ entries for RTX4090x2_TRT/bert-99/Offline -[2024-12-24 20:19:58,968 __init__.py:46 INFO] Running command: ./build/bin/harness_bert --logfile_outdir="/cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance/run_1" --logfile_prefix="mlperf_log_" --performance_sample_count=10833 --test_mode="PerformanceOnly" --gpu_batch_size=256 --mlperf_conf_path="/home/cmuser/CM/repos/local/cache/af0a19318f524c01/inference/mlperf.conf" --tensor_path="build/preprocessed_data/squad_tokenized/input_ids.npy,build/preprocessed_data/squad_tokenized/segment_ids.npy,build/preprocessed_data/squad_tokenized/input_mask.npy" --use_graphs=false --user_conf_path="/home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/1fb6735292a04f4e8245486c08572839.conf" --gpu_inference_streams=2 --gpu_copy_streams=2 --gpu_engines="./build/engines/RTX4090x2/bert/Offline/bert-Offline-gpu-int8_S_384_B_256_P_2_vs.custom_k_99_MaxP.plan" --scenario Offline --model bert -[2024-12-24 20:19:58,968 __init__.py:53 INFO] Overriding Environment +[2024-12-27 20:14:37,820 main.py:229 INFO] Detected system ID: KnownSystem.RTX4090x2 +[2024-12-27 20:14:38,336 generate_conf_files.py:107 INFO] Generated measurements/ entries for RTX4090x2_TRT/bert-99/Offline +[2024-12-27 20:14:38,336 __init__.py:46 INFO] Running command: ./build/bin/harness_bert --logfile_outdir="/cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance/run_1" --logfile_prefix="mlperf_log_" --performance_sample_count=10833 --test_mode="PerformanceOnly" --gpu_batch_size=256 --mlperf_conf_path="/home/cmuser/CM/repos/local/cache/6c5d0d8c0f4f47c1/inference/mlperf.conf" --tensor_path="build/preprocessed_data/squad_tokenized/input_ids.npy,build/preprocessed_data/squad_tokenized/segment_ids.npy,build/preprocessed_data/squad_tokenized/input_mask.npy" --use_graphs=false --user_conf_path="/home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/b6329a31809c422e8f3db6ab4757d418.conf" --gpu_inference_streams=2 --gpu_copy_streams=2 --gpu_engines="./build/engines/RTX4090x2/bert/Offline/bert-Offline-gpu-int8_S_384_B_256_P_2_vs.custom_k_99_MaxP.plan" --scenario Offline --model bert +[2024-12-27 20:14:38,336 __init__.py:53 INFO] Overriding Environment benchmark : Benchmark.BERT buffer_manager_thread_count : 0 coalesced_tensor : True @@ -11,8 +11,8 @@ gpu_copy_streams : 2 gpu_inference_streams : 2 input_dtype : int32 input_format : linear -log_dir : /home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/logs/2024.12.24-20.19.57 -mlperf_conf_path : /home/cmuser/CM/repos/local/cache/af0a19318f524c01/inference/mlperf.conf +log_dir : /home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/logs/2024.12.27-20.14.36 +mlperf_conf_path : /home/cmuser/CM/repos/local/cache/6c5d0d8c0f4f47c1/inference/mlperf.conf offline_expected_qps : 0.0 precision : int8 preprocessed_data_dir : /home/cmuser/CM/repos/local/cache/4db00c74da1e44c8/preprocessed_data @@ -21,7 +21,7 @@ system : SystemConfiguration(host_cpu_conf=CPUConfiguration(layout={CPU(name='In tensor_path : build/preprocessed_data/squad_tokenized/input_ids.npy,build/preprocessed_data/squad_tokenized/segment_ids.npy,build/preprocessed_data/squad_tokenized/input_mask.npy test_mode : PerformanceOnly use_graphs : False -user_conf_path : /home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/1fb6735292a04f4e8245486c08572839.conf +user_conf_path : /home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/b6329a31809c422e8f3db6ab4757d418.conf system_id : RTX4090x2 config_name : RTX4090x2_bert_Offline workload_setting : WorkloadSetting(HarnessType.Custom, AccuracyTarget.k_99, PowerSetting.MaxP) @@ -34,65 +34,65 @@ skip_file_checks : True power_limit : None cpu_freq : None &&&& RUNNING BERT_HARNESS # ./build/bin/harness_bert -I1224 20:19:59.012945 18583 main_bert.cc:163] Found 2 GPUs -I1224 20:19:59.129925 18583 bert_server.cc:147] Engine Path: ./build/engines/RTX4090x2/bert/Offline/bert-Offline-gpu-int8_S_384_B_256_P_2_vs.custom_k_99_MaxP.plan +I1227 20:14:38.381678 18582 main_bert.cc:163] Found 2 GPUs +I1227 20:14:38.499660 18582 bert_server.cc:147] Engine Path: ./build/engines/RTX4090x2/bert/Offline/bert-Offline-gpu-int8_S_384_B_256_P_2_vs.custom_k_99_MaxP.plan [I] [TRT] Loaded engine size: 414 MiB [I] [TRT] Loaded engine size: 414 MiB [W] [TRT] Using an engine plan file across different models of devices is not recommended and is likely to affect performance or even cause errors. -[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +7, GPU +8, now: CPU 728, GPU 1225 (MiB) -[I] [TRT] [MemUsageChange] Init cuDNN: CPU +61, GPU +10, now: CPU 789, GPU 1235 (MiB) +[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +66, GPU +8, now: CPU 787, GPU 1225 (MiB) +[I] [TRT] [MemUsageChange] Init cuDNN: CPU +2, GPU +10, now: CPU 789, GPU 1235 (MiB) [I] [TRT] [MemUsageChange] TensorRT-managed allocation in engine deserialization: CPU +0, GPU +581, now: CPU 0, GPU 581 (MiB) [I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +6, GPU +8, now: CPU 737, GPU 968 (MiB) [I] [TRT] [MemUsageChange] Init cuDNN: CPU +1, GPU +10, now: CPU 738, GPU 978 (MiB) [I] [TRT] [MemUsageChange] TensorRT-managed allocation in engine deserialization: CPU +1, GPU +291, now: CPU 1, GPU 581 (MiB) -I1224 20:19:59.619546 18583 bert_server.cc:208] Engines Creation Completed -I1224 20:19:59.635686 18583 bert_core_vs.cc:385] Engine - Device Memory requirements: 704644608 -I1224 20:19:59.635694 18583 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 2 -I1224 20:19:59.635699 18583 bert_core_vs.cc:415] Engine - Profile 0 maxDims 98304 Bmax=256 Smax=384 +I1227 20:14:38.983712 18582 bert_server.cc:208] Engines Creation Completed +I1227 20:14:39.003544 18582 bert_core_vs.cc:385] Engine - Device Memory requirements: 704644608 +I1227 20:14:39.003558 18582 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 2 +I1227 20:14:39.003564 18582 bert_core_vs.cc:415] Engine - Profile 0 maxDims 98304 Bmax=256 Smax=384 [I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +8, now: CPU 324, GPU 1901 (MiB) [I] [TRT] [MemUsageChange] Init cuDNN: CPU +0, GPU +8, now: CPU 324, GPU 1909 (MiB) -I1224 20:19:59.703217 18583 bert_core_vs.cc:426] Setting Opt.Prof. to 0 -I1224 20:19:59.703243 18583 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 256 +I1227 20:14:39.071316 18582 bert_core_vs.cc:426] Setting Opt.Prof. to 0 +I1227 20:14:39.071341 18582 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 256 [I] [TRT] [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +0, GPU +0, now: CPU 1, GPU 581 (MiB) -I1224 20:19:59.704025 18583 bert_core_vs.cc:476] Setup complete -I1224 20:19:59.704166 18583 bert_core_vs.cc:385] Engine - Device Memory requirements: 704644608 -I1224 20:19:59.704169 18583 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 2 -I1224 20:19:59.704172 18583 bert_core_vs.cc:415] Engine - Profile 0 maxDims 98304 Bmax=256 Smax=384 +I1227 20:14:39.072144 18582 bert_core_vs.cc:476] Setup complete +I1227 20:14:39.072288 18582 bert_core_vs.cc:385] Engine - Device Memory requirements: 704644608 +I1227 20:14:39.072292 18582 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 2 +I1227 20:14:39.072296 18582 bert_core_vs.cc:415] Engine - Profile 0 maxDims 98304 Bmax=256 Smax=384 [I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +8, now: CPU 447, GPU 1644 (MiB) [I] [TRT] [MemUsageChange] Init cuDNN: CPU +0, GPU +8, now: CPU 447, GPU 1652 (MiB) -I1224 20:19:59.767771 18583 bert_core_vs.cc:426] Setting Opt.Prof. to 0 -I1224 20:19:59.767786 18583 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 256 +I1227 20:14:39.137204 18582 bert_core_vs.cc:426] Setting Opt.Prof. to 0 +I1227 20:14:39.137216 18582 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 256 [I] [TRT] [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +0, GPU +0, now: CPU 1, GPU 581 (MiB) -I1224 20:19:59.768550 18583 bert_core_vs.cc:476] Setup complete -I1224 20:19:59.768707 18583 bert_core_vs.cc:385] Engine - Device Memory requirements: 704644608 -I1224 20:19:59.768710 18583 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 2 -I1224 20:19:59.768714 18583 bert_core_vs.cc:415] Engine - Profile 1 maxDims 98304 Bmax=256 Smax=384 +I1227 20:14:39.137998 18582 bert_core_vs.cc:476] Setup complete +I1227 20:14:39.138173 18582 bert_core_vs.cc:385] Engine - Device Memory requirements: 704644608 +I1227 20:14:39.138178 18582 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 2 +I1227 20:14:39.138182 18582 bert_core_vs.cc:415] Engine - Profile 1 maxDims 98304 Bmax=256 Smax=384 [I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +8, now: CPU 570, GPU 2715 (MiB) -[I] [TRT] [MemUsageChange] Init cuDNN: CPU +1, GPU +10, now: CPU 571, GPU 2725 (MiB) -I1224 20:19:59.834163 18583 bert_core_vs.cc:426] Setting Opt.Prof. to 1 +[I] [TRT] [MemUsageChange] Init cuDNN: CPU +0, GPU +10, now: CPU 570, GPU 2725 (MiB) +I1227 20:14:39.201488 18582 bert_core_vs.cc:426] Setting Opt.Prof. to 1 [I] [TRT] Could not set default profile 0 for execution context. Profile index must be set explicitly. [I] [TRT] [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +1, GPU +0, now: CPU 2, GPU 581 (MiB) -I1224 20:19:59.834482 18583 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 256 -I1224 20:19:59.835282 18583 bert_core_vs.cc:476] Setup complete -I1224 20:19:59.835424 18583 bert_core_vs.cc:385] Engine - Device Memory requirements: 704644608 -I1224 20:19:59.835427 18583 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 2 -I1224 20:19:59.835431 18583 bert_core_vs.cc:415] Engine - Profile 1 maxDims 98304 Bmax=256 Smax=384 -[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +1, GPU +8, now: CPU 694, GPU 2458 (MiB) -[I] [TRT] [MemUsageChange] Init cuDNN: CPU +0, GPU +10, now: CPU 694, GPU 2468 (MiB) -I1224 20:19:59.899272 18583 bert_core_vs.cc:426] Setting Opt.Prof. to 1 +I1227 20:14:39.201810 18582 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 256 +I1227 20:14:39.202615 18582 bert_core_vs.cc:476] Setup complete +I1227 20:14:39.202786 18582 bert_core_vs.cc:385] Engine - Device Memory requirements: 704644608 +I1227 20:14:39.202790 18582 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 2 +I1227 20:14:39.202795 18582 bert_core_vs.cc:415] Engine - Profile 1 maxDims 98304 Bmax=256 Smax=384 +[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +8, now: CPU 693, GPU 2458 (MiB) +[I] [TRT] [MemUsageChange] Init cuDNN: CPU +0, GPU +10, now: CPU 693, GPU 2468 (MiB) +I1227 20:14:39.266489 18582 bert_core_vs.cc:426] Setting Opt.Prof. to 1 [I] [TRT] Could not set default profile 0 for execution context. Profile index must be set explicitly. [I] [TRT] [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +0, GPU +0, now: CPU 2, GPU 581 (MiB) -I1224 20:19:59.899586 18583 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 256 -I1224 20:19:59.900385 18583 bert_core_vs.cc:476] Setup complete -I1224 20:20:00.352519 18583 main_bert.cc:184] Starting running actual test. -I1224 20:30:46.272568 18583 main_bert.cc:190] Finished running actual test. +I1227 20:14:39.266804 18582 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 256 +I1227 20:14:39.267608 18582 bert_core_vs.cc:476] Setup complete +I1227 20:14:39.719516 18582 main_bert.cc:184] Starting running actual test. +I1227 20:25:24.930007 18582 main_bert.cc:190] Finished running actual test. ================================================ MLPerf Results Summary ================================================ SUT name : BERT SERVER Scenario : Offline Mode : PerformanceOnly -Samples per second: 8268.76 +Samples per second: 8277.86 Result is : VALID Min duration satisfied : Yes Min queries satisfied : Yes @@ -101,15 +101,15 @@ Result is : VALID ================================================ Additional Stats ================================================ -Min latency (ns) : 1430163027 -Max latency (ns) : 644933169947 -Mean latency (ns) : 391145860524 -50.00 percentile latency (ns) : 416430237335 -90.00 percentile latency (ns) : 615068853269 -95.00 percentile latency (ns) : 632720953267 -97.00 percentile latency (ns) : 638542165122 -99.00 percentile latency (ns) : 643191943747 -99.90 percentile latency (ns) : 644798861050 +Min latency (ns) : 1468857306 +Max latency (ns) : 644224807583 +Mean latency (ns) : 390627257098 +50.00 percentile latency (ns) : 415939518318 +90.00 percentile latency (ns) : 614320528108 +95.00 percentile latency (ns) : 631979098623 +97.00 percentile latency (ns) : 637816603814 +99.00 percentile latency (ns) : 642475528110 +99.90 percentile latency (ns) : 644080347064 ================================================ Test Parameters Used @@ -137,7 +137,7 @@ performance_sample_count : 10833 No warnings encountered during test. No errors encountered during test. -[2024-12-24 20:30:46,496 run_harness.py:166 INFO] Result: result_samples_per_second: 8268.76, Result is VALID +[2024-12-27 20:25:25,189 run_harness.py:166 INFO] Result: result_samples_per_second: 8277.86, Result is VALID ======================== Result summaries: ======================== diff --git a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/system_info.txt b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/system_info.txt index a1635d0b3..585251579 100644 --- a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/system_info.txt +++ b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/system_info.txt @@ -2,10 +2,10 @@ Platform Details ------------------------------------------------------------ 1. uname -a -Linux c2a65ff58585 6.8.0-49-generic #49-Ubuntu SMP PREEMPT_DYNAMIC Mon Nov 4 02:06:24 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux +Linux 0ea02743d854 6.8.0-49-generic #49-Ubuntu SMP PREEMPT_DYNAMIC Mon Nov 4 02:06:24 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux ------------------------------------------------------------ 2. w - 20:30:48 up 14 days, 14:48, 0 users, load average: 3.86, 3.66, 2.53 + 20:25:26 up 17 days, 14:43, 0 users, load average: 4.00, 3.66, 2.56 USER TTY FROM LOGIN@ IDLE JCPU PCPU WHAT ------------------------------------------------------------ 3. Username @@ -40,7 +40,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 4020.833 +cpu MHz : 2100.000 cache size : 46080 KB physical id : 0 siblings : 48 @@ -68,7 +68,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 800.000 +cpu MHz : 1100.000 cache size : 46080 KB physical id : 0 siblings : 48 @@ -96,7 +96,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 800.000 +cpu MHz : 3500.000 cache size : 46080 KB physical id : 0 siblings : 48 @@ -124,7 +124,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 800.000 +cpu MHz : 3100.000 cache size : 46080 KB physical id : 0 siblings : 48 @@ -152,7 +152,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 800.000 +cpu MHz : 4276.756 cache size : 46080 KB physical id : 0 siblings : 48 @@ -180,7 +180,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 800.000 +cpu MHz : 799.991 cache size : 46080 KB physical id : 0 siblings : 48 @@ -404,7 +404,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 898.770 +cpu MHz : 1100.000 cache size : 46080 KB physical id : 0 siblings : 48 @@ -432,7 +432,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 898.339 +cpu MHz : 900.000 cache size : 46080 KB physical id : 0 siblings : 48 @@ -460,7 +460,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 801.431 +cpu MHz : 800.000 cache size : 46080 KB physical id : 0 siblings : 48 @@ -600,7 +600,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 3383.495 +cpu MHz : 800.000 cache size : 46080 KB physical id : 0 siblings : 48 @@ -684,7 +684,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 3300.000 +cpu MHz : 2165.753 cache size : 46080 KB physical id : 0 siblings : 48 @@ -824,7 +824,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 800.008 +cpu MHz : 800.000 cache size : 46080 KB physical id : 0 siblings : 48 @@ -1132,7 +1132,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 800.000 +cpu MHz : 861.538 cache size : 46080 KB physical id : 0 siblings : 48 @@ -1188,7 +1188,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 1389.324 +cpu MHz : 800.000 cache size : 46080 KB physical id : 0 siblings : 48 @@ -1216,7 +1216,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 2294.072 +cpu MHz : 800.000 cache size : 46080 KB physical id : 0 siblings : 48 @@ -1244,7 +1244,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 896.727 +cpu MHz : 800.000 cache size : 46080 KB physical id : 0 siblings : 48 @@ -1394,7 +1394,7 @@ CPU family: 6 Model: 143 Model name: Intel(R) Xeon(R) w7-2495X Stepping: 8 -CPU MHz: 3389.665 +CPU MHz: 1496.597 CPU max MHz: 4800.0000 CPU min MHz: 800.0000 BogoMIPS: 4992.00 @@ -1424,49 +1424,49 @@ Flags: fpu vme de pse tsc msr pae mce cx8 apic se available: 1 nodes (0) node 0 cpus: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 node 0 size: 192709 MB -node 0 free: 24183 MB +node 0 free: 23679 MB node distances: node 0 0: 10 ------------------------------------------------------------ 9. /proc/meminfo MemTotal: 197334532 kB -MemFree: 24763844 kB -MemAvailable: 193004968 kB -Buffers: 544844 kB -Cached: 163475360 kB -SwapCached: 37240 kB -Active: 4253440 kB -Inactive: 160224204 kB -Active(anon): 477828 kB -Inactive(anon): 18536 kB -Active(file): 3775612 kB -Inactive(file): 160205668 kB +MemFree: 24248236 kB +MemAvailable: 193175920 kB +Buffers: 265232 kB +Cached: 166285356 kB +SwapCached: 30032 kB +Active: 2133268 kB +Inactive: 164864504 kB +Active(anon): 344332 kB +Inactive(anon): 141396 kB +Active(file): 1788936 kB +Inactive(file): 164723108 kB Unevictable: 3088 kB Mlocked: 0 kB SwapTotal: 8388604 kB -SwapFree: 7977200 kB +SwapFree: 7960896 kB Zswap: 0 kB Zswapped: 0 kB -Dirty: 240 kB +Dirty: 108 kB Writeback: 0 kB -AnonPages: 454112 kB -Mapped: 332444 kB -Shmem: 38924 kB -KReclaimable: 6032140 kB -Slab: 7212944 kB -SReclaimable: 6032140 kB -SUnreclaim: 1180804 kB -KernelStack: 18336 kB -PageTables: 21404 kB +AnonPages: 438960 kB +Mapped: 325640 kB +Shmem: 38544 kB +KReclaimable: 4187936 kB +Slab: 5227464 kB +SReclaimable: 4187936 kB +SUnreclaim: 1039528 kB +KernelStack: 18400 kB +PageTables: 21832 kB SecPageTables: 0 kB NFS_Unstable: 0 kB Bounce: 0 kB WritebackTmp: 0 kB CommitLimit: 107055868 kB -Committed_AS: 5512188 kB +Committed_AS: 5504644 kB VmallocTotal: 13743895347199 kB -VmallocUsed: 156696 kB +VmallocUsed: 156576 kB VmallocChunk: 0 kB Percpu: 129600 kB HardwareCorrupted: 0 kB @@ -1482,8 +1482,8 @@ HugePages_Rsvd: 0 HugePages_Surp: 0 Hugepagesize: 2048 kB Hugetlb: 0 kB -DirectMap4k: 48871012 kB -DirectMap2M: 148789248 kB +DirectMap4k: 52756068 kB +DirectMap2M: 144904192 kB DirectMap1G: 5242880 kB ------------------------------------------------------------ 10. who -r @@ -1738,16 +1738,16 @@ dev.tty.ldisc_autoload = 1 dev.tty.legacy_tiocsti = 0 fs.aio-max-nr = 65536 fs.aio-nr = 0 -fs.dentry-state = 1645281 1524256 45 0 836765 0 +fs.dentry-state = 677469 599878 45 0 405274 0 fs.dir-notify-enable = 1 fs.epoll.max_user_watches = 43912503 fs.fanotify.max_queued_events = 16384 fs.fanotify.max_user_groups = 128 fs.fanotify.max_user_marks = 1048576 fs.file-max = 9223372036854775807 -fs.file-nr = 5792 0 9223372036854775807 -fs.inode-nr = 1234521 425807 -fs.inode-state = 1234521 425807 0 0 0 0 0 +fs.file-nr = 6848 0 9223372036854775807 +fs.inode-nr = 306124 61827 +fs.inode-state = 306124 61827 0 0 0 0 0 fs.inotify.max_queued_events = 16384 fs.inotify.max_user_instances = 128 fs.inotify.max_user_watches = 65536 @@ -1775,7 +1775,7 @@ fs.quota.drops = 0 fs.quota.free_dquots = 0 fs.quota.lookups = 0 fs.quota.reads = 0 -fs.quota.syncs = 1724 +fs.quota.syncs = 1906 fs.quota.writes = 0 fs.suid_dumpable = 2 fs.verity.require_signatures = 0 @@ -1802,7 +1802,7 @@ kernel.ftrace_dump_on_oops = 0 kernel.ftrace_enabled = 1 kernel.hardlockup_all_cpu_backtrace = 0 kernel.hardlockup_panic = 0 -kernel.hostname = c2a65ff58585 +kernel.hostname = 0ea02743d854 kernel.hotplug = kernel.hung_task_all_cpu_backtrace = 0 kernel.hung_task_check_count = 4194304 @@ -1834,7 +1834,7 @@ kernel.msgmnb = 16384 kernel.msgmni = 32000 kernel.ngroups_max = 65536 kernel.nmi_watchdog = 1 -kernel.ns_last_pid = 18969 +kernel.ns_last_pid = 18968 kernel.numa_balancing = 0 kernel.numa_balancing_promote_rate_limit_MBps = 65536 kernel.oops_all_cpu_backtrace = 0 @@ -1871,7 +1871,7 @@ kernel.random.boot_id = 29685405-01e1-41c8-8cf0-dcaddcc1862c kernel.random.entropy_avail = 256 kernel.random.poolsize = 256 kernel.random.urandom_min_reseed_secs = 60 -kernel.random.uuid = b46918a2-9a6c-4b03-a2ce-2210ffe40c5d +kernel.random.uuid = 992f989d-b236-401c-80e5-203bd0e6aac0 kernel.random.write_wakeup_threshold = 256 kernel.randomize_va_space = 2 kernel.real-root-dev = 0 diff --git a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/user.conf b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/user.conf index df137d76e..b6b330d32 100644 --- a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/user.conf +++ b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/user.conf @@ -1 +1 @@ -bert.Offline.target_qps = 8351.4476 +bert.Offline.target_qps = 8360.6386 diff --git a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/README.md b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/README.md index a80dcad7f..46b6618e3 100644 --- a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/README.md +++ b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/README.md @@ -19,7 +19,7 @@ pip install -U cmind cm rm cache -f -cm pull repo mlcommons@mlperf-automations --checkout=a90475d2de72bf0622cebe8d5ca8eb8c9d872fbd +cm pull repo mlcommons@mlperf-automations --checkout=467517e4a572872046058e394a0d83512cfff38b cm run script \ --tags=app,mlperf,inference,generic,_nvidia,_bert-99,_tensorrt,_cuda,_valid,_r4.1-dev_default,_singlestream \ @@ -126,4 +126,4 @@ Model Precision: int8 `F1`: `90.26682`, Required accuracy for closed division `>= 89.96526` ### Performance Results -`90th percentile latency (ns)`: `1029890.0` +`90th percentile latency (ns)`: `1032023.0` diff --git a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy_console.out b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy_console.out index 78ef85604..e930009d5 100644 --- a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy_console.out +++ b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy_console.out @@ -1,7 +1,7 @@ -[2024-12-24 20:04:34,721 main.py:229 INFO] Detected system ID: KnownSystem.RTX4090x2 -[2024-12-24 20:04:35,042 generate_conf_files.py:107 INFO] Generated measurements/ entries for RTX4090x2_TRT/bert-99/SingleStream -[2024-12-24 20:04:35,043 __init__.py:46 INFO] Running command: ./build/bin/harness_bert --logfile_outdir="/cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy" --logfile_prefix="mlperf_log_" --performance_sample_count=10833 --test_mode="AccuracyOnly" --gpu_batch_size=1 --mlperf_conf_path="/home/cmuser/CM/repos/local/cache/af0a19318f524c01/inference/mlperf.conf" --tensor_path="build/preprocessed_data/squad_tokenized/input_ids.npy,build/preprocessed_data/squad_tokenized/segment_ids.npy,build/preprocessed_data/squad_tokenized/input_mask.npy" --use_graphs=true --user_conf_path="/home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/f9892925a45948ed906f088639fb31b3.conf" --gpu_inference_streams=1 --gpu_copy_streams=1 --gpu_engines="./build/engines/RTX4090x2/bert/SingleStream/bert-SingleStream-gpu-int8_S_384_B_1_P_1_vs.custom_k_99_MaxP.plan" --scenario SingleStream --model bert -[2024-12-24 20:04:35,043 __init__.py:53 INFO] Overriding Environment +[2024-12-27 19:59:10,846 main.py:229 INFO] Detected system ID: KnownSystem.RTX4090x2 +[2024-12-27 19:59:11,171 generate_conf_files.py:107 INFO] Generated measurements/ entries for RTX4090x2_TRT/bert-99/SingleStream +[2024-12-27 19:59:11,171 __init__.py:46 INFO] Running command: ./build/bin/harness_bert --logfile_outdir="/cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy" --logfile_prefix="mlperf_log_" --performance_sample_count=10833 --test_mode="AccuracyOnly" --gpu_batch_size=1 --mlperf_conf_path="/home/cmuser/CM/repos/local/cache/6c5d0d8c0f4f47c1/inference/mlperf.conf" --tensor_path="build/preprocessed_data/squad_tokenized/input_ids.npy,build/preprocessed_data/squad_tokenized/segment_ids.npy,build/preprocessed_data/squad_tokenized/input_mask.npy" --use_graphs=true --user_conf_path="/home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/5eb3f0bdab6a4f31b7d7096279036de4.conf" --gpu_inference_streams=1 --gpu_copy_streams=1 --gpu_engines="./build/engines/RTX4090x2/bert/SingleStream/bert-SingleStream-gpu-int8_S_384_B_1_P_1_vs.custom_k_99_MaxP.plan" --scenario SingleStream --model bert +[2024-12-27 19:59:11,171 __init__.py:53 INFO] Overriding Environment benchmark : Benchmark.BERT buffer_manager_thread_count : 0 coalesced_tensor : True @@ -11,8 +11,8 @@ gpu_copy_streams : 1 gpu_inference_streams : 1 input_dtype : int32 input_format : linear -log_dir : /home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/logs/2024.12.24-20.04.33 -mlperf_conf_path : /home/cmuser/CM/repos/local/cache/af0a19318f524c01/inference/mlperf.conf +log_dir : /home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/logs/2024.12.27-19.59.09 +mlperf_conf_path : /home/cmuser/CM/repos/local/cache/6c5d0d8c0f4f47c1/inference/mlperf.conf precision : int8 preprocessed_data_dir : /home/cmuser/CM/repos/local/cache/4db00c74da1e44c8/preprocessed_data scenario : Scenario.SingleStream @@ -22,7 +22,7 @@ tensor_path : build/preprocessed_data/squad_tokenized/input_ids.npy,build/prepro test_mode : AccuracyOnly use_graphs : True use_small_tile_gemm_plugin : False -user_conf_path : /home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/f9892925a45948ed906f088639fb31b3.conf +user_conf_path : /home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/5eb3f0bdab6a4f31b7d7096279036de4.conf system_id : RTX4090x2 config_name : RTX4090x2_bert_SingleStream workload_setting : WorkloadSetting(HarnessType.Custom, AccuracyTarget.k_99, PowerSetting.MaxP) @@ -35,47 +35,47 @@ skip_file_checks : True power_limit : None cpu_freq : None &&&& RUNNING BERT_HARNESS # ./build/bin/harness_bert -I1224 20:04:35.088269 13811 main_bert.cc:163] Found 2 GPUs -I1224 20:04:35.205370 13811 bert_server.cc:147] Engine Path: ./build/engines/RTX4090x2/bert/SingleStream/bert-SingleStream-gpu-int8_S_384_B_1_P_1_vs.custom_k_99_MaxP.plan +I1227 19:59:11.219691 13810 main_bert.cc:163] Found 2 GPUs +I1227 19:59:11.336463 13810 bert_server.cc:147] Engine Path: ./build/engines/RTX4090x2/bert/SingleStream/bert-SingleStream-gpu-int8_S_384_B_1_P_1_vs.custom_k_99_MaxP.plan [I] [TRT] Loaded engine size: 352 MiB [I] [TRT] Loaded engine size: 352 MiB [W] [TRT] Using an engine plan file across different models of devices is not recommended and is likely to affect performance or even cause errors. [I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +66, GPU +8, now: CPU 600, GPU 1163 (MiB) -[I] [TRT] [MemUsageChange] Init cuDNN: CPU +2, GPU +10, now: CPU 602, GPU 1173 (MiB) +[I] [TRT] [MemUsageChange] Init cuDNN: CPU +1, GPU +10, now: CPU 601, GPU 1173 (MiB) [I] [TRT] [MemUsageChange] TensorRT-managed allocation in engine deserialization: CPU +0, GPU +581, now: CPU 0, GPU 581 (MiB) -[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +6, GPU +8, now: CPU 551, GPU 906 (MiB) -[I] [TRT] [MemUsageChange] Init cuDNN: CPU +2, GPU +10, now: CPU 553, GPU 916 (MiB) +[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +6, GPU +8, now: CPU 551, GPU 907 (MiB) +[I] [TRT] [MemUsageChange] Init cuDNN: CPU +1, GPU +10, now: CPU 552, GPU 917 (MiB) [I] [TRT] [MemUsageChange] TensorRT-managed allocation in engine deserialization: CPU +0, GPU +291, now: CPU 0, GPU 581 (MiB) -I1224 20:04:35.603152 13811 bert_server.cc:208] Engines Creation Completed -I1224 20:04:35.617327 13811 bert_server.cc:213] Use CUDA graphs -I1224 20:04:35.617455 13811 bert_core_vs.cc:385] Engine - Device Memory requirements: 2753024 -I1224 20:04:35.617457 13811 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 1 -I1224 20:04:35.617462 13811 bert_core_vs.cc:415] Engine - Profile 0 maxDims 384 Bmax=1 Smax=384 -[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +8, now: CPU 200, GPU 1169 (MiB) +I1227 19:59:11.723882 13810 bert_server.cc:208] Engines Creation Completed +I1227 19:59:11.734961 13810 bert_server.cc:213] Use CUDA graphs +I1227 19:59:11.735074 13810 bert_core_vs.cc:385] Engine - Device Memory requirements: 2753024 +I1227 19:59:11.735077 13810 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 1 +I1227 19:59:11.735081 13810 bert_core_vs.cc:415] Engine - Profile 0 maxDims 384 Bmax=1 Smax=384 +[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +1, GPU +8, now: CPU 200, GPU 1169 (MiB) [I] [TRT] [MemUsageChange] Init cuDNN: CPU +0, GPU +8, now: CPU 200, GPU 1177 (MiB) -I1224 20:04:35.650635 13811 bert_core_vs.cc:426] Setting Opt.Prof. to 0 -I1224 20:04:35.650657 13811 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 1 -I1224 20:04:35.650717 13811 bert_core_vs.cc:476] Setup complete +I1227 19:59:11.767561 13810 bert_core_vs.cc:426] Setting Opt.Prof. to 0 +I1227 19:59:11.767583 13810 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 1 +I1227 19:59:11.767644 13810 bert_core_vs.cc:476] Setup complete [I] [TRT] [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +1, GPU +0, now: CPU 1, GPU 581 (MiB) -I1224 20:04:35.650867 13811 bert_core_vs.cc:385] Engine - Device Memory requirements: 2753024 -I1224 20:04:35.650871 13811 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 1 -I1224 20:04:35.650873 13811 bert_core_vs.cc:415] Engine - Profile 0 maxDims 384 Bmax=1 Smax=384 -[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +8, now: CPU 262, GPU 912 (MiB) -[I] [TRT] [MemUsageChange] Init cuDNN: CPU +1, GPU +8, now: CPU 263, GPU 920 (MiB) -I1224 20:04:35.685195 13811 bert_core_vs.cc:426] Setting Opt.Prof. to 0 -I1224 20:04:35.685209 13811 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 1 -I1224 20:04:35.685272 13811 bert_core_vs.cc:476] Setup complete +I1227 19:59:11.767789 13810 bert_core_vs.cc:385] Engine - Device Memory requirements: 2753024 +I1227 19:59:11.767792 13810 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 1 +I1227 19:59:11.767794 13810 bert_core_vs.cc:415] Engine - Profile 0 maxDims 384 Bmax=1 Smax=384 +[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +8, now: CPU 262, GPU 913 (MiB) +[I] [TRT] [MemUsageChange] Init cuDNN: CPU +0, GPU +8, now: CPU 262, GPU 921 (MiB) +I1227 19:59:11.801242 13810 bert_core_vs.cc:426] Setting Opt.Prof. to 0 +I1227 19:59:11.801254 13810 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 1 [I] [TRT] [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +0, GPU +0, now: CPU 1, GPU 581 (MiB) -I1224 20:04:35.869999 13821 bert_core_vs.cc:352] Created 96 CUDA graphs -I1224 20:04:35.891862 13823 bert_core_vs.cc:352] Created 96 CUDA graphs -I1224 20:04:35.894897 13811 main_bert.cc:184] Starting running actual test. -I1224 20:04:46.288218 13811 main_bert.cc:190] Finished running actual test. +I1227 19:59:11.801311 13810 bert_core_vs.cc:476] Setup complete +I1227 19:59:11.983117 13820 bert_core_vs.cc:352] Created 96 CUDA graphs +I1227 19:59:12.005232 13822 bert_core_vs.cc:352] Created 96 CUDA graphs +I1227 19:59:12.008280 13810 main_bert.cc:184] Starting running actual test. +I1227 19:59:22.390895 13810 main_bert.cc:190] Finished running actual test. No warnings encountered during test. No errors encountered during test. -[2024-12-24 20:04:46,495 run_harness.py:166 INFO] Result: Accuracy run detected. -[2024-12-24 20:04:46,495 __init__.py:46 INFO] Running command: PYTHONPATH=code/bert/tensorrt/helpers python3 /home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/language/bert/accuracy-squad.py --log_file /cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/mlperf_log_accuracy.json --vocab_file build/models/bert/vocab.txt --val_data /home/cmuser/CM/repos/local/cache/4db00c74da1e44c8/data/squad/dev-v1.1.json --out_file /cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/predictions.json --output_dtype float16 +[2024-12-27 19:59:22,595 run_harness.py:166 INFO] Result: Accuracy run detected. +[2024-12-27 19:59:22,596 __init__.py:46 INFO] Running command: PYTHONPATH=code/bert/tensorrt/helpers python3 /home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/language/bert/accuracy-squad.py --log_file /cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/mlperf_log_accuracy.json --vocab_file build/models/bert/vocab.txt --val_data /home/cmuser/CM/repos/local/cache/4db00c74da1e44c8/data/squad/dev-v1.1.json --out_file /cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/predictions.json --output_dtype float16 {"exact_match": 82.81929990539263, "f1": 90.26682135974633} Reading examples... No cached features at 'eval_features.pickle'... converting from examples... diff --git a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/cm-deps.mmd b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/cm-deps.mmd index 6e38ec041..52362a091 100644 --- a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/cm-deps.mmd +++ b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/cm-deps.mmd @@ -33,28 +33,28 @@ graph TD download-and-extract,c67e81a4ce2649f5_(_wget,_url.https://zenodo.org/record/3733868/files/vocab.txt_) --> download,file,_wget,_url.https://zenodo.org/record/3733868/files/vocab.txt get-dataset-squad-vocab,e38874fff5094577 --> download-and-extract,_wget,_url.https://zenodo.org/record/3733868/files/vocab.txt app-mlperf-inference,d775cac873ee4231_(_nvidia,_bert-99,_tensorrt,_cuda,_valid,_r4.1-dev_default,_singlestream_) --> get,dataset-aux,squad-vocab - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> detect,os + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> detect,os detect-cpu,586c8a43320142f7 --> detect,os - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> detect,cpu - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> get,sys-utils-cm - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> get,mlperf,inference,nvidia,scratch,space + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> detect,cpu + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> get,sys-utils-cm + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> get,mlperf,inference,nvidia,scratch,space get-generic-python-lib,94b62a682bc44791_(_mlperf_logging_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> get,generic-python-lib,_mlperf_logging + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> get,generic-python-lib,_mlperf_logging download-file,9cdc8dc41aae437e_(_cmutil,_url.https://armi.in/files/model.onnx_) --> detect,os download-and-extract,c67e81a4ce2649f5_(_url.https://armi.in/files/model.onnx_) --> download,file,_cmutil,_url.https://armi.in/files/model.onnx get-ml-model-bert-large-squad,5e865dbdc65949d2_(_onnx,_fp32_) --> download-and-extract,_url.https://armi.in/files/model.onnx get-ml-model-bert-large-squad,5e865dbdc65949d2_(_onnx,_fp32_) --> get,dataset-aux,squad-vocab - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> get,ml-model,bert,_onnx,_fp32 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> get,ml-model,bert,_onnx,_fp32 download-file,9cdc8dc41aae437e_(_cmutil,_url.https://zenodo.org/record/3750364/files/bert_large_v1_1_fake_quant.onnx_) --> detect,os download-and-extract,c67e81a4ce2649f5_(_url.https://zenodo.org/record/3750364/files/bert_large_v1_1_fake_quant.onnx_) --> download,file,_cmutil,_url.https://zenodo.org/record/3750364/files/bert_large_v1_1_fake_quant.onnx get-ml-model-bert-large-squad,5e865dbdc65949d2_(_onnx,_int8_) --> download-and-extract,_url.https://zenodo.org/record/3750364/files/bert_large_v1_1_fake_quant.onnx get-ml-model-bert-large-squad,5e865dbdc65949d2_(_onnx,_int8_) --> get,dataset-aux,squad-vocab - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> get,ml-model,bert,_onnx,_int8 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> get,squad-vocab - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> get,mlcommons,inference,src,_deeplearningexamples - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> get,nvidia,mlperf,inference,common-code,_mlcommons + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> get,ml-model,bert,_onnx,_int8 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> get,squad-vocab + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> get,mlcommons,inference,src,_deeplearningexamples + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> get,nvidia,mlperf,inference,common-code,_mlcommons pull-git-repo,c23132ed65c4421d --> detect,os - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> pull,git,repo + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> pull,git,repo generate-mlperf-inference-user-conf,3af4475745964b93 --> detect,os detect-cpu,586c8a43320142f7 --> detect,os generate-mlperf-inference-user-conf,3af4475745964b93 --> detect,cpu @@ -62,9 +62,9 @@ graph TD generate-mlperf-inference-user-conf,3af4475745964b93 --> get,mlcommons,inference,src,_deeplearningexamples get-mlperf-inference-sut-configs,c2fbf72009e2445b --> get,cache,dir,_name.mlperf-inference-sut-configs generate-mlperf-inference-user-conf,3af4475745964b93 --> get,sut,configs - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> generate,user-conf,mlperf,inference + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> generate,user-conf,mlperf,inference get-generic-python-lib,94b62a682bc44791_(_package.pycuda_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> get,generic-python-lib,_package.pycuda + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> get,generic-python-lib,_package.pycuda get-nvidia-mitten,1c045f2902374de9 --> detect,os get-nvidia-mitten,1c045f2902374de9 --> get,python3 get-generic-python-lib,94b62a682bc44791_(_pycuda_) --> detect,os @@ -77,9 +77,9 @@ graph TD get-nvidia-mitten,1c045f2902374de9 --> get,generic-python-lib,_pycuda get-git-repo,ed603e7292974f10_(_repo.https://github.com/NVIDIA/mitten_) --> detect,os get-nvidia-mitten,1c045f2902374de9 --> get,git,_repo.https://github.com/NVIDIA/mitten - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> get,nvidia,mitten - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> get,cuda,_cudnn - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> get,tensorrt + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> get,nvidia,mitten + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> get,cuda,_cudnn + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> get,tensorrt build-mlperf-inference-server-nvidia,f37403af5e9f4541_(_mlcommons_) --> detect,os detect-cpu,586c8a43320142f7 --> detect,os build-mlperf-inference-server-nvidia,f37403af5e9f4541_(_mlcommons_) --> detect,cpu @@ -137,95 +137,95 @@ graph TD get-generic-python-lib,94b62a682bc44791_(_pycuda_) --> get,python3 add-custom-nvidia-system,b2e6c46c6e8745a3_(_mlcommons_) --> get,generic-python-lib,_pycuda build-mlperf-inference-server-nvidia,f37403af5e9f4541_(_mlcommons_) --> add,custom,system,nvidia,_mlcommons - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> build,nvidia,inference,server,_mlcommons - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev_) --> detect,os + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> build,nvidia,inference,server,_mlcommons + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> detect,os detect-cpu,586c8a43320142f7 --> detect,os - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev_) --> detect,cpu - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev_) --> get,sys-utils-cm - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev_) --> get,mlperf,inference,nvidia,scratch,space,_version.4_0 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> detect,cpu + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,sys-utils-cm + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,mlperf,inference,nvidia,scratch,space,_version.4_0 get-generic-python-lib,94b62a682bc44791_(_mlperf_logging_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_mlperf_logging - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev_) --> get,ml-model,bert,_onnx,_fp32 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev_) --> get,ml-model,bert,_onnx,_int8 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev_) --> get,squad-vocab - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev_) --> get,mlcommons,inference,src,_deeplearningexamples - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev_) --> get,nvidia,mlperf,inference,common-code,_mlcommons + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_mlperf_logging + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,ml-model,bert,_onnx,_fp32 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,ml-model,bert,_onnx,_int8 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,squad-vocab + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,mlcommons,inference,src,_deeplearningexamples + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,nvidia,mlperf,inference,common-code,_mlcommons pull-git-repo,c23132ed65c4421d --> detect,os - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev_) --> pull,git,repo + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> pull,git,repo get-generic-python-lib,94b62a682bc44791_(_package.pycuda_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_package.pycuda - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev_) --> get,nvidia,mitten - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev_) --> get,cuda,_cudnn - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev_) --> get,tensorrt - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev_) --> build,nvidia,inference,server,_mlcommons - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev_) --> detect,os + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_package.pycuda + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,nvidia,mitten + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,cuda,_cudnn + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,tensorrt + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> build,nvidia,inference,server,_mlcommons + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> detect,os detect-cpu,586c8a43320142f7 --> detect,os - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev_) --> detect,cpu - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev_) --> get,sys-utils-cm - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev_) --> get,mlperf,inference,nvidia,scratch,space,_version.4_0 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> detect,cpu + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,sys-utils-cm + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,mlperf,inference,nvidia,scratch,space,_version.4_0 get-generic-python-lib,94b62a682bc44791_(_mlperf_logging_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev_) --> get,generic-python-lib,_mlperf_logging - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev_) --> get,ml-model,bert,_onnx,_fp32 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev_) --> get,ml-model,bert,_onnx,_int8 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev_) --> get,squad-vocab - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev_) --> get,mlcommons,inference,src,_deeplearningexamples - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev_) --> get,nvidia,mlperf,inference,common-code,_mlcommons + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,generic-python-lib,_mlperf_logging + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,ml-model,bert,_onnx,_fp32 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,ml-model,bert,_onnx,_int8 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,squad-vocab + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,mlcommons,inference,src,_deeplearningexamples + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,nvidia,mlperf,inference,common-code,_mlcommons pull-git-repo,c23132ed65c4421d --> detect,os - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev_) --> pull,git,repo + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> pull,git,repo get-generic-python-lib,94b62a682bc44791_(_package.pycuda_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev_) --> get,generic-python-lib,_package.pycuda - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev_) --> get,nvidia,mitten + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,generic-python-lib,_package.pycuda + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,nvidia,mitten get-generic-python-lib,94b62a682bc44791_(_transformers_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev_) --> get,generic-python-lib,_transformers + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,generic-python-lib,_transformers get-generic-python-lib,94b62a682bc44791_(_safetensors_) --> detect,os detect-cpu,586c8a43320142f7 --> detect,os get-generic-python-lib,94b62a682bc44791_(_safetensors_) --> detect,cpu get-generic-python-lib,94b62a682bc44791_(_safetensors_) --> get,python3 get-generic-python-lib,94b62a682bc44791_(_pip_) --> get,python3 get-generic-python-lib,94b62a682bc44791_(_safetensors_) --> get,generic-python-lib,_pip - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev_) --> get,generic-python-lib,_safetensors + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,generic-python-lib,_safetensors get-generic-python-lib,94b62a682bc44791_(_onnx_) --> detect,os detect-cpu,586c8a43320142f7 --> detect,os get-generic-python-lib,94b62a682bc44791_(_onnx_) --> detect,cpu get-generic-python-lib,94b62a682bc44791_(_onnx_) --> get,python3 get-generic-python-lib,94b62a682bc44791_(_pip_) --> get,python3 get-generic-python-lib,94b62a682bc44791_(_onnx_) --> get,generic-python-lib,_pip - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev_) --> get,generic-python-lib,_onnx + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,generic-python-lib,_onnx get-generic-python-lib,94b62a682bc44791_(_package.sympy_) --> detect,os detect-cpu,586c8a43320142f7 --> detect,os get-generic-python-lib,94b62a682bc44791_(_package.sympy_) --> detect,cpu get-generic-python-lib,94b62a682bc44791_(_package.sympy_) --> get,python3 get-generic-python-lib,94b62a682bc44791_(_pip_) --> get,python3 get-generic-python-lib,94b62a682bc44791_(_package.sympy_) --> get,generic-python-lib,_pip - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev_) --> get,generic-python-lib,_package.sympy + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,generic-python-lib,_package.sympy get-generic-python-lib,94b62a682bc44791_(_onnx-graphsurgeon_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev_) --> get,generic-python-lib,_onnx-graphsurgeon - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev_) --> save,mlperf,inference,state - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev_) --> reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,generic-python-lib,_onnx-graphsurgeon + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> save,mlperf,inference,state + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev get-generic-python-lib,94b62a682bc44791_(_transformers_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_transformers + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_transformers get-generic-python-lib,94b62a682bc44791_(_safetensors_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_safetensors + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_safetensors get-generic-python-lib,94b62a682bc44791_(_onnx_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_onnx + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_onnx get-generic-python-lib,94b62a682bc44791_(_package.sympy_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_package.sympy + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_package.sympy get-generic-python-lib,94b62a682bc44791_(_onnx-graphsurgeon_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_onnx-graphsurgeon - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev_) --> save,mlperf,inference,state - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> reproduce,mlperf,inference,nvidia,harness,_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_onnx-graphsurgeon + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> save,mlperf,inference,state + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> reproduce,mlperf,inference,nvidia,harness,_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev get-generic-python-lib,94b62a682bc44791_(_transformers_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> get,generic-python-lib,_transformers + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> get,generic-python-lib,_transformers get-generic-python-lib,94b62a682bc44791_(_safetensors_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> get,generic-python-lib,_safetensors + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> get,generic-python-lib,_safetensors get-generic-python-lib,94b62a682bc44791_(_onnx_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> get,generic-python-lib,_onnx + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> get,generic-python-lib,_onnx get-generic-python-lib,94b62a682bc44791_(_package.sympy_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> get,generic-python-lib,_package.sympy + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> get,generic-python-lib,_package.sympy get-generic-python-lib,94b62a682bc44791_(_onnx-graphsurgeon_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> get,generic-python-lib,_onnx-graphsurgeon + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> get,generic-python-lib,_onnx-graphsurgeon detect-cpu,586c8a43320142f7 --> detect,os benchmark-program,19f369ef47084895 --> detect,cpu benchmark-program-mlperf,cfff0132a8aa4018 --> benchmark-program,program - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090_) --> benchmark-mlperf \ No newline at end of file + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090_) --> benchmark-mlperf \ No newline at end of file diff --git a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/cm-deps.png b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/cm-deps.png index 700e27270..c12912048 100644 Binary files a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/cm-deps.png and b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/cm-deps.png differ diff --git a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/cm-version-info.json b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/cm-version-info.json index b072125bd..d539269da 100644 --- a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/cm-version-info.json +++ b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/cm-version-info.json @@ -347,7 +347,7 @@ "script_tags": "detect-os,detect,os,info", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } }, { @@ -367,7 +367,7 @@ "script_tags": "detect,cpu,detect-cpu,info", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } }, { @@ -377,7 +377,7 @@ "script_tags": "get,sys-utils-cm", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } }, { @@ -387,7 +387,7 @@ "script_tags": "get,mlperf,inference,nvidia,scratch,space", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } }, { @@ -407,7 +407,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "mlperf_logging", "version": "4.1.1", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } }, { @@ -457,7 +457,7 @@ "script_tags": "get,ml-model,raw,bert,bert-large,bert-squad,language,language-processing", "script_variations": "onnx,fp32", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } }, { @@ -507,7 +507,7 @@ "script_tags": "get,ml-model,raw,bert,bert-large,bert-squad,language,language-processing", "script_variations": "onnx,int8", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } }, { @@ -517,7 +517,7 @@ "script_tags": "get,aux,squad,dataset-aux,language-processing,squad-aux,vocab,squad-vocab", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } }, { @@ -527,7 +527,7 @@ "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", "script_variations": "deeplearningexamples", "version": "master-git-2fdb814e6a765d994872fb99ee38f8c43418d5b0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } }, { @@ -537,7 +537,7 @@ "script_tags": "get,nvidia,mlperf,inference,common-code", "script_variations": "mlcommons", "version": "r4.0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } }, { @@ -557,7 +557,7 @@ "script_tags": "pull,git,repo,repository", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } }, { @@ -637,7 +637,7 @@ "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } }, { @@ -657,7 +657,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "package.pycuda", "version": "2022.2.2", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } }, { @@ -787,7 +787,7 @@ "script_tags": "get,nvidia,mitten,nvidia-mitten", "script_variations": "", "version": "master", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } }, { @@ -797,7 +797,7 @@ "script_tags": "get,cuda,cuda-compiler,cuda-lib,toolkit,lib,nvcc,get-nvcc,get-cuda,46d133d9ef92422d", "script_variations": "cudnn", "version": "12.2", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } }, { @@ -807,7 +807,7 @@ "script_tags": "get,tensorrt,nvidia", "script_variations": "", "version": "vdetected", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } }, { @@ -1387,7 +1387,7 @@ "script_tags": "build,mlcommons,mlperf,inference,inference-server,server,nvidia-harness,nvidia", "script_variations": "mlcommons", "version": "r4.0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } }, { @@ -1397,7 +1397,7 @@ "script_tags": "detect-os,detect,os,info", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1417,7 +1417,7 @@ "script_tags": "detect,cpu,detect-cpu,info", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1427,7 +1427,7 @@ "script_tags": "get,sys-utils-cm", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1437,7 +1437,7 @@ "script_tags": "get,mlperf,inference,nvidia,scratch,space", "script_variations": "version.4_0", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1457,7 +1457,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "mlperf_logging", "version": "4.1.1", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1467,7 +1467,7 @@ "script_tags": "get,ml-model,raw,bert,bert-large,bert-squad,language,language-processing", "script_variations": "onnx,fp32", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1477,7 +1477,7 @@ "script_tags": "get,ml-model,raw,bert,bert-large,bert-squad,language,language-processing", "script_variations": "onnx,int8", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1487,7 +1487,7 @@ "script_tags": "get,aux,squad,dataset-aux,language-processing,squad-aux,vocab,squad-vocab", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1497,7 +1497,7 @@ "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", "script_variations": "deeplearningexamples", "version": "master-git-2fdb814e6a765d994872fb99ee38f8c43418d5b0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1507,7 +1507,7 @@ "script_tags": "get,nvidia,mlperf,inference,common-code", "script_variations": "mlcommons", "version": "r4.0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1527,7 +1527,7 @@ "script_tags": "pull,git,repo,repository", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1547,7 +1547,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "package.pycuda", "version": "2022.2.2", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1557,7 +1557,7 @@ "script_tags": "get,nvidia,mitten,nvidia-mitten", "script_variations": "", "version": "master", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1567,7 +1567,7 @@ "script_tags": "get,cuda,cuda-compiler,cuda-lib,toolkit,lib,nvcc,get-nvcc,get-cuda,46d133d9ef92422d", "script_variations": "cudnn", "version": "12.2", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1577,7 +1577,7 @@ "script_tags": "get,tensorrt,nvidia", "script_variations": "", "version": "vdetected", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1587,7 +1587,7 @@ "script_tags": "build,mlcommons,mlperf,inference,inference-server,server,nvidia-harness,nvidia", "script_variations": "mlcommons", "version": "r4.0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1597,7 +1597,7 @@ "script_tags": "detect-os,detect,os,info", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" } }, { @@ -1617,7 +1617,7 @@ "script_tags": "detect,cpu,detect-cpu,info", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" } }, { @@ -1627,7 +1627,7 @@ "script_tags": "get,sys-utils-cm", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" } }, { @@ -1637,7 +1637,7 @@ "script_tags": "get,mlperf,inference,nvidia,scratch,space", "script_variations": "version.4_0", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" } }, { @@ -1657,7 +1657,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "mlperf_logging", "version": "4.1.1", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" } }, { @@ -1667,7 +1667,7 @@ "script_tags": "get,ml-model,raw,bert,bert-large,bert-squad,language,language-processing", "script_variations": "onnx,fp32", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" } }, { @@ -1677,7 +1677,7 @@ "script_tags": "get,ml-model,raw,bert,bert-large,bert-squad,language,language-processing", "script_variations": "onnx,int8", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" } }, { @@ -1687,7 +1687,7 @@ "script_tags": "get,aux,squad,dataset-aux,language-processing,squad-aux,vocab,squad-vocab", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" } }, { @@ -1697,7 +1697,7 @@ "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", "script_variations": "deeplearningexamples", "version": "master-git-2fdb814e6a765d994872fb99ee38f8c43418d5b0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" } }, { @@ -1707,7 +1707,7 @@ "script_tags": "get,nvidia,mlperf,inference,common-code", "script_variations": "mlcommons", "version": "r4.0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" } }, { @@ -1727,7 +1727,7 @@ "script_tags": "pull,git,repo,repository", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" } }, { @@ -1747,7 +1747,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "package.pycuda", "version": "2022.2.2", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" } }, { @@ -1757,7 +1757,7 @@ "script_tags": "get,nvidia,mitten,nvidia-mitten", "script_variations": "", "version": "master", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" } }, { @@ -1777,7 +1777,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "transformers", "version": "4.46.3", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" } }, { @@ -1847,7 +1847,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "safetensors", "version": "0.4.5", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" } }, { @@ -1917,7 +1917,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "onnx", "version": "1.13.1", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" } }, { @@ -1987,7 +1987,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "package.sympy", "version": "1.13.3", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" } }, { @@ -2007,7 +2007,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "onnx-graphsurgeon", "version": "0.3.27", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" } }, { @@ -2017,17 +2017,17 @@ "script_tags": "save,mlperf,inference,implementation,state", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" } }, { - "reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev": { + "reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev": { "script_uid": "bc3b17fb430f4732", "script_alias": "app-mlperf-inference-nvidia", "script_tags": "reproduce,mlcommons,mlperf,inference,harness,nvidia-harness,nvidia", - "script_variations": "preprocess_data,cuda,bert-99,tensorrt,bert_,v4.1-dev", + "script_variations": "preprocess_data,tensorrt,cuda,bert-99,bert_,v4.1-dev", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -2047,7 +2047,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "transformers", "version": "4.46.3", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -2067,7 +2067,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "safetensors", "version": "0.4.5", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -2087,7 +2087,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "onnx", "version": "1.13.1", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -2107,7 +2107,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "package.sympy", "version": "1.13.3", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -2127,7 +2127,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "onnx-graphsurgeon", "version": "0.3.27", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -2137,27 +2137,27 @@ "script_tags": "save,mlperf,inference,implementation,state", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" } }, { - "reproduce,mlperf,inference,nvidia,harness,_build_engine,_cuda,_bert-99,_singlestream,_tensorrt,_batch_size.1,_bert_,_v4.1-dev": { + "reproduce,mlperf,inference,nvidia,harness,_build_engine,_singlestream,_tensorrt,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev": { "script_uid": "bc3b17fb430f4732", "script_alias": "app-mlperf-inference-nvidia", "script_tags": "reproduce,mlcommons,mlperf,inference,harness,nvidia-harness,nvidia", - "script_variations": "build_engine,cuda,bert-99,singlestream,tensorrt,batch_size.1,bert_,v4.1-dev", + "script_variations": "build_engine,singlestream,tensorrt,cuda,bert-99,batch_size.1,bert_,v4.1-dev", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } }, { - "reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_cuda,_bert-99,_tensorrt,_bert_,_v4.1-dev": { + "reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev": { "script_uid": "bc3b17fb430f4732", "script_alias": "app-mlperf-inference-nvidia", "script_tags": "reproduce,mlcommons,mlperf,inference,harness,nvidia-harness,nvidia", - "script_variations": "preprocess_data,cuda,bert-99,tensorrt,bert_,v4.1-dev", + "script_variations": "preprocess_data,tensorrt,cuda,bert-99,bert_,v4.1-dev", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } }, { @@ -2177,7 +2177,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "transformers", "version": "4.46.3", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } }, { @@ -2197,7 +2197,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "safetensors", "version": "0.4.5", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } }, { @@ -2217,7 +2217,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "onnx", "version": "1.13.1", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } }, { @@ -2237,7 +2237,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "package.sympy", "version": "1.13.3", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } }, { @@ -2257,7 +2257,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "onnx-graphsurgeon", "version": "0.3.27", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } }, { @@ -2297,7 +2297,7 @@ "script_tags": "mlperf,benchmark-mlperf", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_cuda,_bert-99,_singlestream,_tensorrt,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_singlestream,_tensorrt,_cuda,_bert-99,_rtx_4090 )" } } ] diff --git a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/os_info.json b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/os_info.json index 7072460b5..2c7adbc09 100644 --- a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/os_info.json +++ b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/os_info.json @@ -26,5 +26,5 @@ ], "CM_HOST_PLATFORM_FLAVOR": "x86_64", "CM_HOST_PYTHON_BITS": "64", - "CM_HOST_SYSTEM_NAME": "c2a65ff58585" + "CM_HOST_SYSTEM_NAME": "0ea02743d854" } \ No newline at end of file diff --git a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance_console.out b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance_console.out index 69645b90d..dcd9bb435 100644 --- a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance_console.out +++ b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance_console.out @@ -1,7 +1,7 @@ -[2024-12-24 19:53:32,282 main.py:229 INFO] Detected system ID: KnownSystem.RTX4090x2 -[2024-12-24 19:53:32,612 generate_conf_files.py:107 INFO] Generated measurements/ entries for RTX4090x2_TRT/bert-99/SingleStream -[2024-12-24 19:53:32,612 __init__.py:46 INFO] Running command: ./build/bin/harness_bert --logfile_outdir="/cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance/run_1" --logfile_prefix="mlperf_log_" --performance_sample_count=10833 --test_mode="PerformanceOnly" --gpu_batch_size=1 --mlperf_conf_path="/home/cmuser/CM/repos/local/cache/af0a19318f524c01/inference/mlperf.conf" --tensor_path="build/preprocessed_data/squad_tokenized/input_ids.npy,build/preprocessed_data/squad_tokenized/segment_ids.npy,build/preprocessed_data/squad_tokenized/input_mask.npy" --use_graphs=true --user_conf_path="/home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/3b03f6bb152646c2baab6d43bbae4315.conf" --gpu_inference_streams=1 --gpu_copy_streams=1 --gpu_engines="./build/engines/RTX4090x2/bert/SingleStream/bert-SingleStream-gpu-int8_S_384_B_1_P_1_vs.custom_k_99_MaxP.plan" --scenario SingleStream --model bert -[2024-12-24 19:53:32,612 __init__.py:53 INFO] Overriding Environment +[2024-12-27 19:48:09,330 main.py:229 INFO] Detected system ID: KnownSystem.RTX4090x2 +[2024-12-27 19:48:09,676 generate_conf_files.py:107 INFO] Generated measurements/ entries for RTX4090x2_TRT/bert-99/SingleStream +[2024-12-27 19:48:09,676 __init__.py:46 INFO] Running command: ./build/bin/harness_bert --logfile_outdir="/cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance/run_1" --logfile_prefix="mlperf_log_" --performance_sample_count=10833 --test_mode="PerformanceOnly" --gpu_batch_size=1 --mlperf_conf_path="/home/cmuser/CM/repos/local/cache/6c5d0d8c0f4f47c1/inference/mlperf.conf" --tensor_path="build/preprocessed_data/squad_tokenized/input_ids.npy,build/preprocessed_data/squad_tokenized/segment_ids.npy,build/preprocessed_data/squad_tokenized/input_mask.npy" --use_graphs=true --user_conf_path="/home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/edbcf5a11f214c7f9ec0eeefcea40321.conf" --gpu_inference_streams=1 --gpu_copy_streams=1 --gpu_engines="./build/engines/RTX4090x2/bert/SingleStream/bert-SingleStream-gpu-int8_S_384_B_1_P_1_vs.custom_k_99_MaxP.plan" --scenario SingleStream --model bert +[2024-12-27 19:48:09,676 __init__.py:53 INFO] Overriding Environment benchmark : Benchmark.BERT buffer_manager_thread_count : 0 coalesced_tensor : True @@ -11,8 +11,8 @@ gpu_copy_streams : 1 gpu_inference_streams : 1 input_dtype : int32 input_format : linear -log_dir : /home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/logs/2024.12.24-19.53.31 -mlperf_conf_path : /home/cmuser/CM/repos/local/cache/af0a19318f524c01/inference/mlperf.conf +log_dir : /home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/logs/2024.12.27-19.48.08 +mlperf_conf_path : /home/cmuser/CM/repos/local/cache/6c5d0d8c0f4f47c1/inference/mlperf.conf precision : int8 preprocessed_data_dir : /home/cmuser/CM/repos/local/cache/4db00c74da1e44c8/preprocessed_data scenario : Scenario.SingleStream @@ -22,7 +22,7 @@ tensor_path : build/preprocessed_data/squad_tokenized/input_ids.npy,build/prepro test_mode : PerformanceOnly use_graphs : True use_small_tile_gemm_plugin : False -user_conf_path : /home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/3b03f6bb152646c2baab6d43bbae4315.conf +user_conf_path : /home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/edbcf5a11f214c7f9ec0eeefcea40321.conf system_id : RTX4090x2 config_name : RTX4090x2_bert_SingleStream workload_setting : WorkloadSetting(HarnessType.Custom, AccuracyTarget.k_99, PowerSetting.MaxP) @@ -35,73 +35,73 @@ skip_file_checks : True power_limit : None cpu_freq : None &&&& RUNNING BERT_HARNESS # ./build/bin/harness_bert -I1224 19:53:32.663496 10578 main_bert.cc:163] Found 2 GPUs -I1224 19:53:32.779435 10578 bert_server.cc:147] Engine Path: ./build/engines/RTX4090x2/bert/SingleStream/bert-SingleStream-gpu-int8_S_384_B_1_P_1_vs.custom_k_99_MaxP.plan +I1227 19:48:09.729212 10576 main_bert.cc:163] Found 2 GPUs +I1227 19:48:09.854475 10576 bert_server.cc:147] Engine Path: ./build/engines/RTX4090x2/bert/SingleStream/bert-SingleStream-gpu-int8_S_384_B_1_P_1_vs.custom_k_99_MaxP.plan [I] [TRT] Loaded engine size: 352 MiB [I] [TRT] Loaded engine size: 352 MiB [W] [TRT] Using an engine plan file across different models of devices is not recommended and is likely to affect performance or even cause errors. -[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +66, GPU +8, now: CPU 600, GPU 1163 (MiB) -[I] [TRT] [MemUsageChange] Init cuDNN: CPU +2, GPU +10, now: CPU 602, GPU 1173 (MiB) +[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +6, GPU +8, now: CPU 540, GPU 1163 (MiB) +[I] [TRT] [MemUsageChange] Init cuDNN: CPU +2, GPU +10, now: CPU 542, GPU 1173 (MiB) [I] [TRT] [MemUsageChange] TensorRT-managed allocation in engine deserialization: CPU +0, GPU +581, now: CPU 0, GPU 581 (MiB) -[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +6, GPU +8, now: CPU 551, GPU 907 (MiB) -[I] [TRT] [MemUsageChange] Init cuDNN: CPU +2, GPU +10, now: CPU 553, GPU 917 (MiB) +[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +6, GPU +8, now: CPU 551, GPU 906 (MiB) +[I] [TRT] [MemUsageChange] Init cuDNN: CPU +1, GPU +10, now: CPU 552, GPU 916 (MiB) [I] [TRT] [MemUsageChange] TensorRT-managed allocation in engine deserialization: CPU +0, GPU +291, now: CPU 0, GPU 581 (MiB) -I1224 19:53:33.181957 10578 bert_server.cc:208] Engines Creation Completed -I1224 19:53:33.193658 10578 bert_server.cc:213] Use CUDA graphs -I1224 19:53:33.193814 10578 bert_core_vs.cc:385] Engine - Device Memory requirements: 2753024 -I1224 19:53:33.193817 10578 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 1 -I1224 19:53:33.193822 10578 bert_core_vs.cc:415] Engine - Profile 0 maxDims 384 Bmax=1 Smax=384 -[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +8, now: CPU 200, GPU 1169 (MiB) +I1227 19:48:10.258617 10576 bert_server.cc:208] Engines Creation Completed +I1227 19:48:10.275656 10576 bert_server.cc:213] Use CUDA graphs +I1227 19:48:10.275812 10576 bert_core_vs.cc:385] Engine - Device Memory requirements: 2753024 +I1227 19:48:10.275815 10576 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 1 +I1227 19:48:10.275820 10576 bert_core_vs.cc:415] Engine - Profile 0 maxDims 384 Bmax=1 Smax=384 +[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +1, GPU +8, now: CPU 200, GPU 1169 (MiB) [I] [TRT] [MemUsageChange] Init cuDNN: CPU +0, GPU +8, now: CPU 200, GPU 1177 (MiB) -I1224 19:53:33.226703 10578 bert_core_vs.cc:426] Setting Opt.Prof. to 0 -I1224 19:53:33.226727 10578 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 1 -I1224 19:53:33.226791 10578 bert_core_vs.cc:476] Setup complete +I1227 19:48:10.311800 10576 bert_core_vs.cc:426] Setting Opt.Prof. to 0 +I1227 19:48:10.311825 10576 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 1 +I1227 19:48:10.311888 10576 bert_core_vs.cc:476] Setup complete [I] [TRT] [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +1, GPU +0, now: CPU 1, GPU 581 (MiB) -I1224 19:53:33.226934 10578 bert_core_vs.cc:385] Engine - Device Memory requirements: 2753024 -I1224 19:53:33.226938 10578 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 1 -I1224 19:53:33.226939 10578 bert_core_vs.cc:415] Engine - Profile 0 maxDims 384 Bmax=1 Smax=384 -[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +8, now: CPU 262, GPU 913 (MiB) -[I] [TRT] [MemUsageChange] Init cuDNN: CPU +0, GPU +8, now: CPU 262, GPU 921 (MiB) -I1224 19:53:33.260394 10578 bert_core_vs.cc:426] Setting Opt.Prof. to 0 -I1224 19:53:33.260406 10578 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 1 -I1224 19:53:33.260488 10578 bert_core_vs.cc:476] Setup complete +I1227 19:48:10.312022 10576 bert_core_vs.cc:385] Engine - Device Memory requirements: 2753024 +I1227 19:48:10.312026 10576 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 1 +I1227 19:48:10.312027 10576 bert_core_vs.cc:415] Engine - Profile 0 maxDims 384 Bmax=1 Smax=384 +[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +8, now: CPU 262, GPU 912 (MiB) +[I] [TRT] [MemUsageChange] Init cuDNN: CPU +0, GPU +8, now: CPU 262, GPU 920 (MiB) +I1227 19:48:10.347873 10576 bert_core_vs.cc:426] Setting Opt.Prof. to 0 +I1227 19:48:10.347885 10576 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 1 +I1227 19:48:10.347944 10576 bert_core_vs.cc:476] Setup complete [I] [TRT] [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +0, GPU +0, now: CPU 1, GPU 581 (MiB) -I1224 19:53:33.449940 10588 bert_core_vs.cc:352] Created 96 CUDA graphs -I1224 19:53:33.460364 10590 bert_core_vs.cc:352] Created 96 CUDA graphs -I1224 19:53:33.463522 10578 main_bert.cc:184] Starting running actual test. -I1224 20:03:35.459573 10578 main_bert.cc:190] Finished running actual test. +I1227 19:48:10.551548 10586 bert_core_vs.cc:352] Created 96 CUDA graphs +I1227 19:48:10.562552 10588 bert_core_vs.cc:352] Created 96 CUDA graphs +I1227 19:48:10.565043 10576 main_bert.cc:184] Starting running actual test. +I1227 19:58:12.244382 10576 main_bert.cc:190] Finished running actual test. ================================================ MLPerf Results Summary ================================================ SUT name : BERT SERVER Scenario : SingleStream Mode : PerformanceOnly -90th percentile latency (ns) : 1029406 +90th percentile latency (ns) : 1031143 Result is : VALID Min duration satisfied : Yes Min queries satisfied : Yes Early stopping satisfied: Yes Early Stopping Result: - * Processed at least 64 queries (634470). - * Would discard 62890 highest latency queries. - * Early stopping 90th percentile estimate: 1029890 - * Early stopping 99th percentile estimate: 1211301 + * Processed at least 64 queries (635928). + * Would discard 63035 highest latency queries. + * Early stopping 90th percentile estimate: 1032023 + * Early stopping 99th percentile estimate: 1208727 ================================================ Additional Stats ================================================ -QPS w/ loadgen overhead : 1057.45 -QPS w/o loadgen overhead : 1064.00 +QPS w/ loadgen overhead : 1059.88 +QPS w/o loadgen overhead : 1066.96 -Min latency (ns) : 860498 -Max latency (ns) : 1485171 -Mean latency (ns) : 939846 -50.00 percentile latency (ns) : 920898 -90.00 percentile latency (ns) : 1029406 -95.00 percentile latency (ns) : 1133784 -97.00 percentile latency (ns) : 1189113 -99.00 percentile latency (ns) : 1210866 -99.90 percentile latency (ns) : 1222247 +Min latency (ns) : 840130 +Max latency (ns) : 1457764 +Mean latency (ns) : 937243 +50.00 percentile latency (ns) : 918945 +90.00 percentile latency (ns) : 1031143 +95.00 percentile latency (ns) : 1127281 +97.00 percentile latency (ns) : 1184293 +99.00 percentile latency (ns) : 1208398 +99.90 percentile latency (ns) : 1222522 ================================================ Test Parameters Used @@ -129,7 +129,7 @@ performance_sample_count : 10833 No warnings encountered during test. No errors encountered during test. -[2024-12-24 20:03:35,687 run_harness.py:166 INFO] Result: result_90.00_percentile_latency_ns: 1029406, Result is VALID +[2024-12-27 19:58:12,466 run_harness.py:166 INFO] Result: result_90.00_percentile_latency_ns: 1031143, Result is VALID ======================== Result summaries: ======================== diff --git a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/system_info.txt b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/system_info.txt index 2708d9376..85584dcf2 100644 --- a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/system_info.txt +++ b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/system_info.txt @@ -2,10 +2,10 @@ Platform Details ------------------------------------------------------------ 1. uname -a -Linux c2a65ff58585 6.8.0-49-generic #49-Ubuntu SMP PREEMPT_DYNAMIC Mon Nov 4 02:06:24 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux +Linux 0ea02743d854 6.8.0-49-generic #49-Ubuntu SMP PREEMPT_DYNAMIC Mon Nov 4 02:06:24 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux ------------------------------------------------------------ 2. w - 20:04:01 up 14 days, 14:22, 0 users, load average: 0.91, 1.16, 1.39 + 19:58:35 up 17 days, 14:16, 0 users, load average: 0.89, 1.20, 1.52 USER TTY FROM LOGIN@ IDLE JCPU PCPU WHAT ------------------------------------------------------------ 3. Username @@ -40,7 +40,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 3200.000 +cpu MHz : 3416.915 cache size : 46080 KB physical id : 0 siblings : 48 @@ -68,7 +68,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 4000.000 +cpu MHz : 3483.333 cache size : 46080 KB physical id : 0 siblings : 48 @@ -96,7 +96,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 799.991 +cpu MHz : 3500.000 cache size : 46080 KB physical id : 0 siblings : 48 @@ -152,7 +152,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 800.000 +cpu MHz : 4121.026 cache size : 46080 KB physical id : 0 siblings : 48 @@ -180,7 +180,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 800.000 +cpu MHz : 799.991 cache size : 46080 KB physical id : 0 siblings : 48 @@ -404,7 +404,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 1100.000 +cpu MHz : 1310.747 cache size : 46080 KB physical id : 0 siblings : 48 @@ -432,7 +432,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 902.168 +cpu MHz : 900.268 cache size : 46080 KB physical id : 0 siblings : 48 @@ -600,7 +600,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 800.000 +cpu MHz : 900.000 cache size : 46080 KB physical id : 0 siblings : 48 @@ -628,7 +628,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 2800.000 +cpu MHz : 1202.678 cache size : 46080 KB physical id : 0 siblings : 48 @@ -656,7 +656,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 2900.000 +cpu MHz : 800.000 cache size : 46080 KB physical id : 0 siblings : 48 @@ -684,7 +684,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 800.000 +cpu MHz : 3911.337 cache size : 46080 KB physical id : 0 siblings : 48 @@ -1132,7 +1132,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 800.000 +cpu MHz : 797.066 cache size : 46080 KB physical id : 0 siblings : 48 @@ -1188,7 +1188,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 799.049 +cpu MHz : 800.000 cache size : 46080 KB physical id : 0 siblings : 48 @@ -1216,7 +1216,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 798.220 +cpu MHz : 798.748 cache size : 46080 KB physical id : 0 siblings : 48 @@ -1272,7 +1272,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 998.585 +cpu MHz : 800.000 cache size : 46080 KB physical id : 0 siblings : 48 @@ -1356,7 +1356,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 3400.000 +cpu MHz : 800.000 cache size : 46080 KB physical id : 0 siblings : 48 @@ -1394,7 +1394,7 @@ CPU family: 6 Model: 143 Model name: Intel(R) Xeon(R) w7-2495X Stepping: 8 -CPU MHz: 2950.185 +CPU MHz: 2500.000 CPU max MHz: 4800.0000 CPU min MHz: 800.0000 BogoMIPS: 4992.00 @@ -1424,49 +1424,49 @@ Flags: fpu vme de pse tsc msr pae mce cx8 apic se available: 1 nodes (0) node 0 cpus: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 node 0 size: 192709 MB -node 0 free: 24759 MB +node 0 free: 24207 MB node distances: node 0 0: 10 ------------------------------------------------------------ 9. /proc/meminfo MemTotal: 197334532 kB -MemFree: 25353604 kB -MemAvailable: 192868664 kB -Buffers: 541012 kB -Cached: 162784888 kB -SwapCached: 37236 kB -Active: 4228640 kB -Inactive: 159535420 kB -Active(anon): 457444 kB -Inactive(anon): 18536 kB -Active(file): 3771196 kB -Inactive(file): 159516884 kB +MemFree: 24788632 kB +MemAvailable: 192992516 kB +Buffers: 259032 kB +Cached: 165591368 kB +SwapCached: 30156 kB +Active: 2104720 kB +Inactive: 164172804 kB +Active(anon): 323124 kB +Inactive(anon): 141396 kB +Active(file): 1781596 kB +Inactive(file): 164031408 kB Unevictable: 3088 kB Mlocked: 0 kB SwapTotal: 8388604 kB -SwapFree: 7977200 kB +SwapFree: 7960128 kB Zswap: 0 kB Zswapped: 0 kB -Dirty: 21240 kB +Dirty: 21008 kB Writeback: 0 kB -AnonPages: 434860 kB -Mapped: 332596 kB -Shmem: 37816 kB -KReclaimable: 5999276 kB -Slab: 7179768 kB -SReclaimable: 5999276 kB -SUnreclaim: 1180492 kB +AnonPages: 418860 kB +Mapped: 323836 kB +Shmem: 37396 kB +KReclaimable: 4163176 kB +Slab: 5202288 kB +SReclaimable: 4163176 kB +SUnreclaim: 1039112 kB KernelStack: 18304 kB -PageTables: 21140 kB +PageTables: 21640 kB SecPageTables: 0 kB NFS_Unstable: 0 kB Bounce: 0 kB WritebackTmp: 0 kB CommitLimit: 107055868 kB -Committed_AS: 5472356 kB +Committed_AS: 5489140 kB VmallocTotal: 13743895347199 kB -VmallocUsed: 156580 kB +VmallocUsed: 156492 kB VmallocChunk: 0 kB Percpu: 129600 kB HardwareCorrupted: 0 kB @@ -1482,8 +1482,8 @@ HugePages_Rsvd: 0 HugePages_Surp: 0 Hugepagesize: 2048 kB Hugetlb: 0 kB -DirectMap4k: 48823908 kB -DirectMap2M: 148836352 kB +DirectMap4k: 52733540 kB +DirectMap2M: 144926720 kB DirectMap1G: 5242880 kB ------------------------------------------------------------ 10. who -r @@ -1738,16 +1738,16 @@ dev.tty.ldisc_autoload = 1 dev.tty.legacy_tiocsti = 0 fs.aio-max-nr = 65536 fs.aio-nr = 0 -fs.dentry-state = 1627383 1510419 45 0 829425 0 +fs.dentry-state = 656886 585010 45 0 396924 0 fs.dir-notify-enable = 1 fs.epoll.max_user_watches = 43912503 fs.fanotify.max_queued_events = 16384 fs.fanotify.max_user_groups = 128 fs.fanotify.max_user_marks = 1048576 fs.file-max = 9223372036854775807 -fs.file-nr = 5888 0 9223372036854775807 -fs.inode-nr = 1223971 425807 -fs.inode-state = 1223971 425807 0 0 0 0 0 +fs.file-nr = 6080 0 9223372036854775807 +fs.inode-nr = 295566 61827 +fs.inode-state = 295566 61827 0 0 0 0 0 fs.inotify.max_queued_events = 16384 fs.inotify.max_user_instances = 128 fs.inotify.max_user_watches = 65536 @@ -1775,7 +1775,7 @@ fs.quota.drops = 0 fs.quota.free_dquots = 0 fs.quota.lookups = 0 fs.quota.reads = 0 -fs.quota.syncs = 1724 +fs.quota.syncs = 1906 fs.quota.writes = 0 fs.suid_dumpable = 2 fs.verity.require_signatures = 0 @@ -1802,7 +1802,7 @@ kernel.ftrace_dump_on_oops = 0 kernel.ftrace_enabled = 1 kernel.hardlockup_all_cpu_backtrace = 0 kernel.hardlockup_panic = 0 -kernel.hostname = c2a65ff58585 +kernel.hostname = 0ea02743d854 kernel.hotplug = kernel.hung_task_all_cpu_backtrace = 0 kernel.hung_task_check_count = 4194304 @@ -1834,7 +1834,7 @@ kernel.msgmnb = 16384 kernel.msgmni = 32000 kernel.ngroups_max = 65536 kernel.nmi_watchdog = 1 -kernel.ns_last_pid = 12159 +kernel.ns_last_pid = 12157 kernel.numa_balancing = 0 kernel.numa_balancing_promote_rate_limit_MBps = 65536 kernel.oops_all_cpu_backtrace = 0 @@ -1871,7 +1871,7 @@ kernel.random.boot_id = 29685405-01e1-41c8-8cf0-dcaddcc1862c kernel.random.entropy_avail = 256 kernel.random.poolsize = 256 kernel.random.urandom_min_reseed_secs = 60 -kernel.random.uuid = a242e8bf-e59c-4be3-bd8b-2f23bad4435f +kernel.random.uuid = 6e15a61c-341d-4fb4-9a38-a4d1029062df kernel.random.write_wakeup_threshold = 256 kernel.randomize_va_space = 2 kernel.real-root-dev = 0 diff --git a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/user.conf b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/user.conf index 8cbdc716c..f149e644d 100644 --- a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/user.conf +++ b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/user.conf @@ -1,2 +1,2 @@ -bert.SingleStream.target_latency = 0.3759384 +bert.SingleStream.target_latency = 0.37489720000000004 bert.SingleStream.max_duration = 660000 diff --git a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/system_info.txt b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/system_info.txt index 2708d9376..85584dcf2 100644 --- a/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/system_info.txt +++ b/open/MLCommons/measurements/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/system_info.txt @@ -2,10 +2,10 @@ Platform Details ------------------------------------------------------------ 1. uname -a -Linux c2a65ff58585 6.8.0-49-generic #49-Ubuntu SMP PREEMPT_DYNAMIC Mon Nov 4 02:06:24 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux +Linux 0ea02743d854 6.8.0-49-generic #49-Ubuntu SMP PREEMPT_DYNAMIC Mon Nov 4 02:06:24 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux ------------------------------------------------------------ 2. w - 20:04:01 up 14 days, 14:22, 0 users, load average: 0.91, 1.16, 1.39 + 19:58:35 up 17 days, 14:16, 0 users, load average: 0.89, 1.20, 1.52 USER TTY FROM LOGIN@ IDLE JCPU PCPU WHAT ------------------------------------------------------------ 3. Username @@ -40,7 +40,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 3200.000 +cpu MHz : 3416.915 cache size : 46080 KB physical id : 0 siblings : 48 @@ -68,7 +68,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 4000.000 +cpu MHz : 3483.333 cache size : 46080 KB physical id : 0 siblings : 48 @@ -96,7 +96,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 799.991 +cpu MHz : 3500.000 cache size : 46080 KB physical id : 0 siblings : 48 @@ -152,7 +152,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 800.000 +cpu MHz : 4121.026 cache size : 46080 KB physical id : 0 siblings : 48 @@ -180,7 +180,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 800.000 +cpu MHz : 799.991 cache size : 46080 KB physical id : 0 siblings : 48 @@ -404,7 +404,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 1100.000 +cpu MHz : 1310.747 cache size : 46080 KB physical id : 0 siblings : 48 @@ -432,7 +432,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 902.168 +cpu MHz : 900.268 cache size : 46080 KB physical id : 0 siblings : 48 @@ -600,7 +600,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 800.000 +cpu MHz : 900.000 cache size : 46080 KB physical id : 0 siblings : 48 @@ -628,7 +628,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 2800.000 +cpu MHz : 1202.678 cache size : 46080 KB physical id : 0 siblings : 48 @@ -656,7 +656,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 2900.000 +cpu MHz : 800.000 cache size : 46080 KB physical id : 0 siblings : 48 @@ -684,7 +684,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 800.000 +cpu MHz : 3911.337 cache size : 46080 KB physical id : 0 siblings : 48 @@ -1132,7 +1132,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 800.000 +cpu MHz : 797.066 cache size : 46080 KB physical id : 0 siblings : 48 @@ -1188,7 +1188,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 799.049 +cpu MHz : 800.000 cache size : 46080 KB physical id : 0 siblings : 48 @@ -1216,7 +1216,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 798.220 +cpu MHz : 798.748 cache size : 46080 KB physical id : 0 siblings : 48 @@ -1272,7 +1272,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 998.585 +cpu MHz : 800.000 cache size : 46080 KB physical id : 0 siblings : 48 @@ -1356,7 +1356,7 @@ model : 143 model name : Intel(R) Xeon(R) w7-2495X stepping : 8 microcode : 0x2b0005c0 -cpu MHz : 3400.000 +cpu MHz : 800.000 cache size : 46080 KB physical id : 0 siblings : 48 @@ -1394,7 +1394,7 @@ CPU family: 6 Model: 143 Model name: Intel(R) Xeon(R) w7-2495X Stepping: 8 -CPU MHz: 2950.185 +CPU MHz: 2500.000 CPU max MHz: 4800.0000 CPU min MHz: 800.0000 BogoMIPS: 4992.00 @@ -1424,49 +1424,49 @@ Flags: fpu vme de pse tsc msr pae mce cx8 apic se available: 1 nodes (0) node 0 cpus: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 node 0 size: 192709 MB -node 0 free: 24759 MB +node 0 free: 24207 MB node distances: node 0 0: 10 ------------------------------------------------------------ 9. /proc/meminfo MemTotal: 197334532 kB -MemFree: 25353604 kB -MemAvailable: 192868664 kB -Buffers: 541012 kB -Cached: 162784888 kB -SwapCached: 37236 kB -Active: 4228640 kB -Inactive: 159535420 kB -Active(anon): 457444 kB -Inactive(anon): 18536 kB -Active(file): 3771196 kB -Inactive(file): 159516884 kB +MemFree: 24788632 kB +MemAvailable: 192992516 kB +Buffers: 259032 kB +Cached: 165591368 kB +SwapCached: 30156 kB +Active: 2104720 kB +Inactive: 164172804 kB +Active(anon): 323124 kB +Inactive(anon): 141396 kB +Active(file): 1781596 kB +Inactive(file): 164031408 kB Unevictable: 3088 kB Mlocked: 0 kB SwapTotal: 8388604 kB -SwapFree: 7977200 kB +SwapFree: 7960128 kB Zswap: 0 kB Zswapped: 0 kB -Dirty: 21240 kB +Dirty: 21008 kB Writeback: 0 kB -AnonPages: 434860 kB -Mapped: 332596 kB -Shmem: 37816 kB -KReclaimable: 5999276 kB -Slab: 7179768 kB -SReclaimable: 5999276 kB -SUnreclaim: 1180492 kB +AnonPages: 418860 kB +Mapped: 323836 kB +Shmem: 37396 kB +KReclaimable: 4163176 kB +Slab: 5202288 kB +SReclaimable: 4163176 kB +SUnreclaim: 1039112 kB KernelStack: 18304 kB -PageTables: 21140 kB +PageTables: 21640 kB SecPageTables: 0 kB NFS_Unstable: 0 kB Bounce: 0 kB WritebackTmp: 0 kB CommitLimit: 107055868 kB -Committed_AS: 5472356 kB +Committed_AS: 5489140 kB VmallocTotal: 13743895347199 kB -VmallocUsed: 156580 kB +VmallocUsed: 156492 kB VmallocChunk: 0 kB Percpu: 129600 kB HardwareCorrupted: 0 kB @@ -1482,8 +1482,8 @@ HugePages_Rsvd: 0 HugePages_Surp: 0 Hugepagesize: 2048 kB Hugetlb: 0 kB -DirectMap4k: 48823908 kB -DirectMap2M: 148836352 kB +DirectMap4k: 52733540 kB +DirectMap2M: 144926720 kB DirectMap1G: 5242880 kB ------------------------------------------------------------ 10. who -r @@ -1738,16 +1738,16 @@ dev.tty.ldisc_autoload = 1 dev.tty.legacy_tiocsti = 0 fs.aio-max-nr = 65536 fs.aio-nr = 0 -fs.dentry-state = 1627383 1510419 45 0 829425 0 +fs.dentry-state = 656886 585010 45 0 396924 0 fs.dir-notify-enable = 1 fs.epoll.max_user_watches = 43912503 fs.fanotify.max_queued_events = 16384 fs.fanotify.max_user_groups = 128 fs.fanotify.max_user_marks = 1048576 fs.file-max = 9223372036854775807 -fs.file-nr = 5888 0 9223372036854775807 -fs.inode-nr = 1223971 425807 -fs.inode-state = 1223971 425807 0 0 0 0 0 +fs.file-nr = 6080 0 9223372036854775807 +fs.inode-nr = 295566 61827 +fs.inode-state = 295566 61827 0 0 0 0 0 fs.inotify.max_queued_events = 16384 fs.inotify.max_user_instances = 128 fs.inotify.max_user_watches = 65536 @@ -1775,7 +1775,7 @@ fs.quota.drops = 0 fs.quota.free_dquots = 0 fs.quota.lookups = 0 fs.quota.reads = 0 -fs.quota.syncs = 1724 +fs.quota.syncs = 1906 fs.quota.writes = 0 fs.suid_dumpable = 2 fs.verity.require_signatures = 0 @@ -1802,7 +1802,7 @@ kernel.ftrace_dump_on_oops = 0 kernel.ftrace_enabled = 1 kernel.hardlockup_all_cpu_backtrace = 0 kernel.hardlockup_panic = 0 -kernel.hostname = c2a65ff58585 +kernel.hostname = 0ea02743d854 kernel.hotplug = kernel.hung_task_all_cpu_backtrace = 0 kernel.hung_task_check_count = 4194304 @@ -1834,7 +1834,7 @@ kernel.msgmnb = 16384 kernel.msgmni = 32000 kernel.ngroups_max = 65536 kernel.nmi_watchdog = 1 -kernel.ns_last_pid = 12159 +kernel.ns_last_pid = 12157 kernel.numa_balancing = 0 kernel.numa_balancing_promote_rate_limit_MBps = 65536 kernel.oops_all_cpu_backtrace = 0 @@ -1871,7 +1871,7 @@ kernel.random.boot_id = 29685405-01e1-41c8-8cf0-dcaddcc1862c kernel.random.entropy_avail = 256 kernel.random.poolsize = 256 kernel.random.urandom_min_reseed_secs = 60 -kernel.random.uuid = a242e8bf-e59c-4be3-bd8b-2f23bad4435f +kernel.random.uuid = 6e15a61c-341d-4fb4-9a38-a4d1029062df kernel.random.write_wakeup_threshold = 256 kernel.randomize_va_space = 2 kernel.real-root-dev = 0 diff --git a/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/accuracy.txt b/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/accuracy.txt index 80a25f4be..95ddc0be9 100644 --- a/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/accuracy.txt +++ b/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/accuracy.txt @@ -1,8 +1,8 @@ {"exact_match": 82.81929990539263, "f1": 90.15673510616978} Reading examples... -Loading cached features from '/home/cmuser/CM/repos/local/cache/af0a19318f524c01/inference/language/bert/eval_features.pickle'... +Loading cached features from '/home/cmuser/CM/repos/local/cache/6c5d0d8c0f4f47c1/inference/language/bert/eval_features.pickle'... Loading LoadGen logs... Post-processing predictions... Writing predictions to: /cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/predictions.json Evaluating predictions... -hash=48611ae77c64479b2880b4bcdc0a874fc3ad17af78c1c2f9750ac2adcecf5535 +hash=8b45d1f3a5e6a943610978c8992b162f9fae407d9f40ea02d884822036c9b5c7 diff --git a/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/mlperf_log_accuracy.json b/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/mlperf_log_accuracy.json index 4e1ec5728..9fa3931d0 100644 --- a/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/mlperf_log_accuracy.json +++ b/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/mlperf_log_accuracy.json @@ -1,6 +1,6 @@ [ -{ "seq_id" : 7928, "qsl_idx" : 10419, "data" : "A0C89EC732C866C875C910C991C85DC86DC86AC821C7DCC59AC9E4C819C9C4C713C97AC88EC886C792C939C9C1C848C84CC65FC64FC762C6EFC8F0C531C9B0C8C1C7F4C64EC9C8C8CAC81FC752C9BEC7AEC855C869C9E8C868C9D5C832C98BC735C9E8C758C913C9D7C5F6C45FC9B9C8FDC812C88BC77FC516C932C890C840C8BDC676C564C91AC9D0C889C733C9A2C877C8D6C72FC9CCC8ABC861C75AC90DC950C8DFC62FC973C85EC97FC869C875C63CC850C638C760C843C899C847C848C665C9F6C83AC905C984C8EBC888C8A8C834C900C9B7C878C8E2C81EC9CEC8AAC86DC93FC9EFC889C74AC9A1C896C890C805C9F4C8DBC8B4C80FC942C85CC92AC954C914C922C93EC841C9BBC8F1C84AC90EC915C92FC9E2C839C95DC902C92DC856C987C83DC93CC91CC933C98CC8A5C836C93FC951C9DEC84FC94CC920C921C9ABC896C73EC901C901C8C0C63EC957C985C81FC7E3C839C709C9E2C8C4C8FAC885C828C8C1C642C474C87CC736C862C8F0C753C664C9C9C8CBC80CC85AC905C87BC9EBC8A0C99BC873C88AC67BC9B7C856C9E3C836C995C8D0C8C2C60BC98AC706C9ABC802C945C9CEC89EC87AC939C973C933C88CC922C906C934C957C918C856C94FC950C9F3C8AEC88EC81AC982C72EC91FC866C86BC88BC6D8C538C992C8C0C87FC790C896C789C64BC741C5DEC5BDC499C221C912C803C936C8E1C88CC82DC970C87EC857C860C8E4C5E1C858C792C610C788C83CC8DCC874C615C907C972C870C878C764C85EC925C997C8B8C6E8C87FC7ABC883C8D9C8B7C8C0C8ACC8E0C6C5C697C7A8C505C999C8F5C8FDC8CEC8C3C826C932C792C915C925C905C913C958C912C9D2C884C92FC9D2C863C736C82CC749C536C761C920C9B3C838C85BC90AC9EEC7FFC84BC91AC9CEC8AAC77BC3BFC10CC861C783C818C8B1C89EC588C8E2C81CC851C6F4C8BCC863C7F9C55BC9CDC839C883C86BC8C1C704C99DC8CDC8B5C832C88AC82DC877C590C9F0C8ADC815C830C9F4C83EC91FC9FFC815C9E3C825C791C63FC7A9BD05C65BC57BC7A1C461C677C9D6C861C78FC5C3C492C79CC402C60AC90AC9BB427DC5C24767C00FC415C466C1B247E3C44A423DC966C738C795C8FAC5E6C724C99BC8E5C6FFC425C97FC846C961C888C804C819C840C331C867C776C88BC889C836C61AC95AC89CC8BDC844C8EBC89FC886C570C931C946C868C8A3C892C866C898C890C881C64CC98DC8DAC8C4C76AC735C3DBC630C04BC83DC956C337C398C896C875C736C413C966C961C83FC995C8B7C879C902C9A2C862C82AC722C5CAC8ACC887C839C9BAC867C91CC9C7C857C960C951C83BC9FFC862C91AC95EC867C946C953C93CC906C87AC486C90EC9C4C8E2C82CC970C86CC9E9C875C911C947C962C6AFC812C545C8FFC8BEC89CC86EC948C856C911C9EDC8A5C80EC9ECC862C909C92BC905C9D1C80BC9F3C8B7C72DC9F9C845C90FC9E0C87CC85EC9FCC804C9C0C877C84BC81BC991C773C939C920C988C86DC943C9ECC8A6C80CC9EBC87BC871C860C891C868C92EC9E8C85CC8E7C85DC862C935C9A3C856C630C992C822C9FAC882C83BC8E7C6F2C697C89AC74DC7E1C465C905C9C8C8B6C85FC93DC91AC99DC8A7C82FC73BC96DC80DC93EC919C9F7C8D2C8EFC843C945C9C2C8C2C8A1C800C753C95DC85FC944C9B5C870C813C9A6C8FAC8BBC831C82DC815C957C843C9EDC837C805C66BC917C9EBC8C9C8E8C7A0C7E5C8ADC777C9FCC8B4C856C85AC854C609C95CC894C8F0C863C914C92FC9F7C81FC99BC803C773C73BC80BC6F6C825C94DC9F8C858C903C952C8B4C74CC9F8C88CC7CAC5DDC83EC8C3C8E7C854C9E6C852C909C945C91BC934C9C1C809C77BC5D6C8D3C805C90BC9A5C8AFC8F5C8DCC83BC75AC78AC9DEC848C9C2C849C894C6E8C8A4C7A8C8B0C84DC902C9DCC8F3C852C8A4C841C86FC806C912C878C917C95FC8FDC793C8FAC667C977C80EC91BC967C9D5C86BC9FCC843C924C9A6C8CAC85DC9BCC854C917C932C9C9C85BC927C9D0C823C972C81BC82EC94BC84DC9E2C842C910C9D0C8D7C83BC8E1C651C999C8CDC8FEC849C983C869C9BCC870C8EEC813C9EDC873C93CC84CC9FAC858C915C93AC946C963C8F2C8BFC8D2C693C92CC96AC939C9" }, -{ "seq_id" : 2164, "qsl_idx" : 4386, "data" : "6DC113C1C8C859C99CC835C935C872C86DC63FC81EC9B5C80AC7B0C4D1C817C652C90AC938C955C766C858C957C878C95DC939C921C965C8E8C85AC942C943C9D5C843C9BCC8A0C889C81DC8A4C8E7C862C807C8E9C80FC91FC9A1C837C98BC8A2C9BBC8A6C83CC90AC929C899C92FC9ECC892C8AFC8FFC8A2C8C8C803C9D2C80EC919C98FC80EC9DAC869C8F7C8E6C84AC96EC836C9C3C8F2C814C9B7C83FC9DFC834C9E3C842C97BC844C9CBC808C9D5C838C81CC9F2C88DC82BC835C9C8C8BEC8F5C8D9C88BC71DC952C82FC909C90AC917C933C8F9C878C8D2C81CC9CAC871C80EC909C9B2C849C984C8DFC8D3C8F2C80EC97EC82FC920C828C9DAC84BC9F8C8EBC8F7C802C931C9E5C80EC983C841C995C845C995C825C9EFC8E6C83EC9D5C837C93CC938C904C9F8C81CC936C929C90CC9E8C84BC910C947C97DC998C85EC9E4C8FEC816C90AC922C9FCC830C941C846C97AC86FC7FDC86EC84CC939C972C71BC957C634C946C6D2C897C8CDC748C85FC9ABC6E9C805C720C971C8F6C823C8D6C76BC870C815C943C90CC951C936C6A5C8CCC8F3C715C9CAC8BFC6B0C812C7A1C8A9C847C722C9D1C866C723C96FC84FC99AC810C807C9E5C869C73BC983C8F6C7FDC8D5C8A5C7E5C807 +{ "seq_id" : 3150, "qsl_idx" : 6012, "data" : "A3C609C5D9C42DC5F9C8C8C892C734C899C886C816C9F9C8D0C86DC8F0C898C877C763C882C867C813C888C615C9AFC8E4C5ACC692C87CC814C887C81EC8B5C8E1B5FAC434C610C451C80BC700C992C891C5B7C4C0C742C623C95EC892C8D6C605C863C7D5C89DC825C9ABC8F2C871C83AC9A8C818C9B8C873C8FCC6F8C87EC7A3C6D8C6CCC7F8C615C8FAC619C8AAC607C832C6A5C8B5C799C5F3C546C832C7E7C7FCC5C6C721C6BBC859C8E7C873C843C9A8C80FC996C804C8CAC766C8BCC78AC896C73AC830C7AEC8B3C720C8C7C63FC866C8F7C8D0C80FC9BDC8E6C80CC8E8C856C861C6EEC64CC67BC65BC8D0C601C773C657C64AC44BC841C8CFC8A2C8B6C8C4C8F8C8AFC82FC9CFC841C913C971C891C852C9AFC816C908C945C979C803C884C600C56BC404C989C824C9B1C806C910C9D9C8F9C81AC9CCC82EC9F0C810C9D7C826C902C912C997C801C9D0C812C9DDC8EBC8CCC814C9F5C859C9E7C8C1C8BDC83EC9E6C8C7C8CFC860C9B2C807C9A5C8C4C8C0C810C9B1C829C9A9C808C988C811C9B7C895C85DC81FC9BDC819C9EAC825C9B0C82EC97BC82DC9D0C8CBC887C8E5C86DC831C950C8EEC859C8BDC8ADC8CDC839C81BC967C819C9A5C8E6C818C823C9AAC8D1C7F1C647C9CAC824C917C929C9E2C81FC973C816C9D8C8E0C8D8C83CC885C704C972C8C0C854C8FAC841C89FC833C896C8AAC830C8A1C7ECC838C821C9EEC825C9EDC83AC9FAC867C919C914C9FFC832C9D8C870C9C8C850C852C722C946C8FDC857C85FC973C88CC9A0C875C9D3C892C9C2C87BC9B0C874C99DC879C9B5C86AC917C94FC9E1C859C9B7C898C9BBC867C99FC845C9D4C80DC9C1C8F2C8EAC87EC9FDC8DEC8CCC8C3C851C810C935C8F9C887C88CC827C81EC955C882C8BBC692C818C883C81AC848C9B1C832C9C5C82EC983C86FC9CBC8FBC8ABC825C97BC815C937C8CFC8A0C73AC9D0C8C5C82DC8BDC615C732C874C6A0C840C75EC8ABC678C8ACC7B1C5F7C5F8C504C4E7C820C740C9C1C85DC9D8C82DC9DCC851C90EC927C9AFC809C9D7C881C9DCC803C9BAC83DC963C8BCC8A2C761C88DC8FBC8F3C89FC84DC805C9CFC884C896C87CC899C7F2C8ADC71EC744C61CC813C7F1C8DFC83AC74FC842C8EEC78BC896C755C709C78EC83CC6D5C889C83FC9C8C828C9C2C867C71FC849C9A2C835C9C9C8DDC838C934C91AC94EC914C992C813C9ABC8B7C81CC9CDC896C821C81DC972C878C86EC663C913C9D0C8FAC89BC867C9DDC8D1C80DC932C9BEC855C9B5C8B2C841C9D6C807C96DC853C99CC861C80EC970C8BEC826C695C8FBC7B8C729C8A0C88FC8B9C817C9C3C8CCC88BC80DC96BC821C9DEC8ADC80BC69FC8E4C89CC707C8C0C62CC821BF7AC26FC531C8C7C49DC773C63AC864C66CC890BD81C7EF45D83DB9C1CC44D2C571312FC9B9C8EBC8F1C805C9FAC8CFC88AC822C9DDC8CEC8BDC815C93AC82EC90AC8B7C86AC8F9C8BFC845C9CFC89CC8BEC88CC836C82BC9CEC8B8C748C882C8C4C717C979C8C8C8CCC70DC91EC814C9A8C827C9E7C816C9C4C807C9C3C8FDC8BEC8FEC88AC8FFC8AAC895C8AEC808C941C8D3C888C818C9D6C838C9FFC80FC9B4C855C9E2C81FC9BDC840C9A0C853C900C99BC8D3C8FCC863C8FCC856C87FC8E1C601C96BC7" }, +{ "seq_id" : 5827, "qsl_idx" : 10039, "data" : "7DC834C8DDC52FC649C96AC8BFC8E3C893C898C8E4C832C804C9B1C85BC9AAC84AC77DC8A0C8AEC8D0C778C8E5C858C7EBC882C5FCC877C836C90BC956C9A8C81FC9E0C85CC967C84DC9D2C84DC87CC80CC952C836C87BC884C831C823C99BC7CDC89BC8AFC881C8B8C861C70DC956C8C5C835C804C95FC8F0C84AC840C941C851C9C4C814C9B4C872C97CC807C81CC851C9E4C753C83AC8E9C8EAC7ACC8DAC7AAC80EC8DCC790C6B8C724C73EC8E0C703C89FC537C9A1C8F2C806C877C866C715C9A4C86EC625C5CDC8D5C79CC63BC8E6C653C81EC854C8C8C7BCC635C882C628C891C809C85CC8EFC79DC8D8C7CBC8CBC810C8D1C8E0C86CC8D5C88CC822C695C7E3C409C734C86FC48AC7BCBFDEC7E148AFBD9FC814C998C26AC6F8BEE64821C7CEC7CBC572C8A4C624BCF4C611BD79C7E0C599C806C879C8C7C8B5C8A5C885C8B6C835C8DEC544C8D1C403C82AC860C814C7D7C83EC89BC83CC8E0C822C8EFC818C81AC92BC77AC9CBC858C9DAC82DC9E0C8B4C869C826C9AEC819C6D5C545C8F1C577C86AC693C823C809C815C8ECC5DEC502C84CC681C8A7C6A4C843C8F6C712C821C9CEC8DFC7A6C723C815C6A3C889C6D4C872C80BC970C8EFC817C835C97FC811C9BAC866C83FC83FC9DFC81BC908C8BAC845C890C888C745C99CC81EC5A5C573C777C512C9A0C8E2C8B7C8C7C89FC8B8C8A1C8EBC893C88BC84BC703C9D5C7ADC84CC7EAC673C78AC8C0C7EAC875C8EAC827C8BFC811C74CC979C83DC97AC8F8C8CAC835C913C839C9B9C89EC70DC8A3C855C768C8E4C6E9C805C8D2C71AC8F9C8E0C788C8C5C630C99CC89CC876C77CC8F7C7A5C6F3C6B5C80FC88CC864C7D9C8EBC75BC8E5C7DAC872C836C869C66DC8AFC6F6C89FC8CCC87AC8C1C87BC8B9C863C83FC9ACC828C9B9C8CFC87BC81FC99AC818C94DC80FC9F8C729C9A9C7C8C897C74CC819C875C769C6BCC8C1C851C68EC652C8E0C6AFC845C8AEC823C832C9B3C8A3C899C8BDC8A3C86CC8B4C8AFC8CAC8CFC80DC807C9A5C739C96DC8D4C8D8C879C843C8CEC85CC86CC870C7B3C852C869C88EC7BDC8F5C782C5EDC43DC96BC81EC811C66BC8E8C517C9F0C7CEC84DC8D0C5E9C701C806C763C86CC816C875C855C8B8C88C ... diff --git a/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/mlperf_log_detail.txt b/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/mlperf_log_detail.txt index 8974bf5e6..dba845a62 100644 --- a/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/mlperf_log_detail.txt +++ b/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/mlperf_log_detail.txt @@ -1,68 +1,68 @@ -:::MLLOG {"key": "loadgen_version", "value": "4.0 @ 486a629ea4", "time_ms": 0.006050, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 53, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "loadgen_build_date_local", "value": "2024-12-24T19:50:51.839855", "time_ms": 0.006050, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 55, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "loadgen_build_date_utc", "value": "2024-12-25T03:50:51.839862", "time_ms": 0.006050, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 56, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "loadgen_git_commit_date", "value": "2024-02-15T15:46:50-05:00", "time_ms": 0.006050, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 57, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "loadgen_git_log_message", "value": "486a629ea4d5c5150f452d0b0a196bf71fd2021e Hotfix: DLRMv2 Audit Test01 fallback failure (#1626)\nf9a643c0a0e920588da1b51a1d822e1071a9dbec Add number of tokens to offline (#1623)\nf06b920eac2170df38b5502c7443f50c9afa6045 Bugfix: equal-issue mode on offline causing accuracy run to fail (3D-UNet) (#1624)\n0ed5190e82e1ca1cf95626e50e61cb6678c7f2ff Fix submission checker and TEST06 for Llama2 (#1616)\n817dd961221d3ca284b024368130bd5dc5e4b54a Turn equal issue mode off for TEST06 (#1615)\n840435a1abe8be823e0a5a18ad3173bc520aa24f Update README.md with CM commands to download stable-diffusion, gptj and dlrmv2 models (#1604)\n3190d09a148bc1d72c9d946e4cf3b54db0d0e60a Typo fix in README.md (#1588)\nb0777f0bd3b7e5cd6a51d670d8b4df656cf6fb29 Update README.md - remove unwanted lines in CM commands (#1601)\nd7dba0826ac96a0e8bcae525086961e47c409a38 Remove loadgen warnings (#1608)\nd45a66c33bb0b6548d6172b38645bf94ffe24b39 Add upper limit to tokens per sample (#1612)\n44285d9a44041c3d4f7df0acf1e72c375e9fcbd2 Set completed samples per second as llama metric (#1613)\n357ccefca4db55ff373e029d5d1e3e6246bde5d4 Enable equal issue mode for LLM benchmarks (#1610)\n104d85531af7ae2a07ded36dd6e8c7f2c7746d06 Add sample length check to test06 (#1603)\n473053fc9f1b8c5edf60e11ec74607126938fbc4 Fix loadgen token metrics latency constrains (#1596)\ncc3daae3c7f18d7eedc5ffbc271d324e5aca6233 #1598: fix token and sample logging for Llama2 when accuracy_log_sampling_target is enabled (#1599)\n3a902e598e5a17013c26fea975c0a60e6d7feae1 Add support to dump 10 compliance images during accuracy run for SDXL (#1591)", "time_ms": 0.006050, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 58, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "loadgen_git_status_message", "value": "", "time_ms": 0.006050, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 60, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"05d4a62d355a90363847358e58afeee4a54ad838","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/bindings/c_api.cc":"32181da9e161c285f8fe46ddaa49e6cba2f9f918","/bindings/c_api.h":"91f58bd79b83b278f3240174a9af747fc38aff74","/bindings/python_api.cc":"1dc0865130371c6a8ddb97cd8039244d74d6fe46","/demos/py_demo_multi_stream.py":"6ccf7dcbe08f8382ed2d3618370c2b21e205ca13","/demos/py_demo_offline.py":"6c3075538e0a3df5e93edee917f4537099759e11","/demos/py_demo_server.py":"40e37ea121ac08b21ed6af41aca8eb5eda641b89","/demos/py_demo_single_stream.py":"4ee5f048fb0dffc415342a2bc2049fb155bd2894","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"126e952d00f4ea9efd12405fb209aa3ed585e4b2","/issue_query_controller.h":"923d9d5cdf598e3ec33d7a1110a31f7e11527ec7","/loadgen.cc":"6650091ba7a918f343b06eb7a5aa540eae87275f","/loadgen.h":"e00fdc6dbc85a8c9a8485dbcbfe2944f81251c4e","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"197efc96d178e5d33a750d07fa7b2966417506ea","/logging.h":"ddb961df7bcc145bcd7cce8c21f7cf075350dcbe","/pyproject.toml":"974adb632af4b76a1c72b914cc2cf9b3cbc66aa3","/query_dispatch_library.h":"13ad6d842200cb161d6927eb74a3fafd79c46c75","/query_sample.h":"e9187c8612bbdc972305b789feb6e15c26e96cfe","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"a170f991de69f8a5e45616e871b39a30ffed5598","/results.h":"d82500c326c2de83db411f1146882aa4692b419c","/setup.py":"e3e707e1191478332c00f1b6ff806f0ef98e0f65","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"aaf3803e700f3431a5ada72559eef5cce996ea64","/test_settings_internal.cc":"42aa46b17ac3fb2748cd5cde60fa8742e916f3c8","/test_settings_internal.h":"73b11c02228f3afb4c924db7a86fd8a2b5b9e483","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"4c9d98360b507d5673049ed70154fcfb2934f8a6"}, "time_ms": 0.006050, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 67, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "test_datetime", "value": "2024-12-24T20:31:06Z", "time_ms": 0.011260, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1198, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "sut_name", "value": "BERT SERVER", "time_ms": 0.011260, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1199, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "get_sut_name_duration_ns", "value": 452, "time_ms": 0.011260, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1200, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "qsl_name", "value": "BERT QSL", "time_ms": 0.011260, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1201, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "qsl_reported_total_count", "value": 10833, "time_ms": 0.011260, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1202, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "qsl_reported_performance_count", "value": 10833, "time_ms": 0.011260, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1203, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "requested_scenario", "value": "Offline", "time_ms": 0.015883, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 268, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "requested_test_mode", "value": "AccuracyOnly", "time_ms": 0.015883, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 269, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "requested_offline_expected_qps", "value": 8351.45, "time_ms": 0.015883, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 306, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "requested_min_duration_ms", "value": 600000, "time_ms": 0.015883, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 312, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "requested_max_duration_ms", "value": 0, "time_ms": 0.015883, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 313, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "requested_min_query_count", "value": 10833, "time_ms": 0.015883, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 314, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "requested_max_query_count", "value": 0, "time_ms": 0.015883, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 315, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "requested_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.015883, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 316, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.015883, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 317, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "requested_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.015883, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 319, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.015883, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 320, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.015883, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 322, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.015883, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 324, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.015883, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 326, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.015883, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 327, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.015883, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 329, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.015883, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 331, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "requested_performance_sample_count_override", "value": 10833, "time_ms": 0.015883, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 333, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "effective_scenario", "value": "Offline", "time_ms": 0.015992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 409, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "effective_test_mode", "value": "AccuracyOnly", "time_ms": 0.015992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 410, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "effective_samples_per_query", "value": 5511955, "time_ms": 0.015992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 412, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "effective_target_qps", "value": 8351.45, "time_ms": 0.015992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 413, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.015992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 414, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.99, "time_ms": 0.015992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 415, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.015992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 417, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "effective_target_duration_ms", "value": 0, "time_ms": 0.015992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 418, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "effective_min_duration_ms", "value": 600000, "time_ms": 0.015992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 420, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "effective_max_duration_ms", "value": 0, "time_ms": 0.015992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 421, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "effective_min_query_count", "value": 1, "time_ms": 0.015992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 422, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "effective_max_query_count", "value": 0, "time_ms": 0.015992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 423, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "effective_min_sample_count", "value": 5511955, "time_ms": 0.015992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 424, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "effective_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.015992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 425, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.015992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 426, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "effective_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.015992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 428, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.015992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 429, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.015992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 431, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.015992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 433, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.015992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 435, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.015992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 436, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.015992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 438, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.015992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 440, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "effective_performance_sample_count", "value": 10833, "time_ms": 0.015992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 442, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "generic_message", "value": "Starting accuracy mode", "time_ms": 0.017035, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1090, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "loaded_qsl_set", "value": [4117,9486,2947,9270,8227,4487,9799,4629,8571,8839,7101,2191,3315,6375,207,10454,3122,9490,4083,6119,1890,7657,9347,344,201,4232,5741,5454,9533,7842,1596,5330,10615,8827,8094,6714,9169,977,6619,9924,4982,4894,6544,10118,783,766,2122,7163,396,4200,9864,4254,8440,1758,10722,2829,2788,9906,2719,5613,8726,7847,440,6103,8567,7497,4793,1184,8675,10587,8270,7074,7408,8740,9265,4855,1199,220,6556,6595,1226,6695,3066,2143,6264,9806,7122,5007,10075,2370,2185,7254,77,2806,6243,6687,1156,1802,7302,391,3348,8159,9708,251,3595,3947,2302,4690,122,3116,1300,3768,6388,2986,728,5127,3904,6807,6259,2218,7555,8821,3580,7804,1321,8460,5419,6542,7833,3365,2290,6346,7952,918,707,118,10469,9023,1304,5220,1891,6878,9524,6522,7867,4037,7698,8507,9634,2145,10057,6263,7486,7557,2401,6082,3434,5927,8708,10081,8114,1349,7742,2742,7157,5705,4638,1764,3651,920,4016,2744,5859,6740,5380,9649,2,5245,5230,6421,4071,8909,6319,288,9346,9671,9646,9360,4702,5044,9157,6296,8318,8971,8184,7702,10475,7318,4024,5530,3991,1171,1909,6230,8461,8099,10563,5398,10760,10693,3688,8170,4952,6827,5893,7767,2656,10032,9178,10480,7195,9359,10608,10261,8975,5432,363,3346,3049,3360,4432,9736,473,7587,9416,1021,6596,10352,2666,2325,973,2071,7368,4267,867,1040,5796,3491,8237,9597,3489,10568,3777,7899,856,4818,2858,4662,777,10776,10026,6065,6481,2590,6780,10553,8789,6032,930,1049,10178,6187,585,3359,5832,2014,480,5866,5017,5141,1525,8271,8132,1642,6270,2265,140,1174,4641,8017,8844,6569,5575,7081,3779,3845,467,9761,3314,8653,4250,7578,8907,664,217,8419,8757,683,4565,1638,5216,2004,6679,4485,7683,5180,7422,6070,6283,2909,6281,8933,10639,4340,3079,3513,10552,2291,7487,2680,8807,8370,6127,7434,1822,908,8101,451,10316,8282,10724,885,1834,1556,6121,1468,409,1908,10539,7238,2891,10312,9435,2178,10051,4522,9650,615,7193,7796,2008,6011,4347,435,6881,1396,844,971,4216,5001,7886,6875,1348,4300,9348,2709,10580,9993,6705,8784,10821,9232,7724,6581,9764,2913,10125,6914,7693,4568,10526,10231,8502,7825,1783,795,6828,8006,4595,3892,3075,1278,10050,8235,4679,9971,75,2164,1175,3684,9869,8414,9016,765,2721,5786,10703,1326,7463,237,836,3912,5482,7835,4374,7661,4070,7754,7180,9945,10415,3638,4208,939,1097,7628,5725,3723,4547,1994,5232,5279,4049,7309,5963,10478,9834,7758,9463,3666,4663,801,4030,3010,8236,9237,2470,3382,6336,7570,190,4552,4113,2229,9018,7038,5021,8701,8268,4490,1373,6092,10181,10445,3056,4333,10394,1640,2328,3349,10069,10683,6439,3191,2544,4760,1739,8900,3,258,5416,4671,2041,9375,2661,6376,2306,5897,6373,2125,1940,5109,4256,6203,6411,9311,8029,2200,6301,1734,6176,6443,3007,9583,1841,4627,2523,4306,2801,5208,8607,4961,1467,10700,4031,2181,9958,8194,10295,9043,5366,306,9056,162,3196,10759,4349,5401,7030,6184,9216,2069,6357,5640,3015,1752,406,10049,3983,791,5610,5944,304,6380,1937,7862,9366,9365,1607,9252,2698,10350,5221,4847,10707,3023,1615,7017,5990,4045,7231,1202,7182,5172,3176,9612,6693,1013,10334,1805,8730,8841,7372,10094,6010,3216,8177,7406,9472,9608,2533,7691,3959,1623,5144,1831,751,4912,2152,8144,241,4187,10001,1924,8402,7877,8928,7294,4915,3876,3242,8996,7153,1000,10658,3946,284,5088,959,10661,1018,8931,546,4262,6489,3786,5278,4308,8178,7638,5211,7142,5771,634,8273,263,2548,4674,9910,624,3290,5925,1524,2642,7198,10345,3539,6719,10735,2006,10233,3816,9578,8624,6769,9661,6468,8952,9702,8670,5175,3449,1773,6130,636,1622,1696,1414,6523,2898,10650,4420,1280,2538,7880,3743,1227,2757,6131,3371,7445,1680,2901,36,6868,10811,7140,1900,10398,3724,8765,8278,7379,6040,7914,1580,3535,8033,7536,6480,3861,2437,5082,2881,10596,4902,8162,9637,4886,411,7189,1789,6704,9176,10045,4107,6333,4583,3923,4843,9935,9955,9297,3222,8257,6365,3095,1995,7924,5428,8640,6628,4819,7152,5016,7932,3226,1267,1225,9025,7878,6321,2373,3801,1399,9063,6506,9436,5781,7500,423,3588,7110,4607,3476,2477,3635,2425,9394,10449,4054,8148,5510,2236,9543,10499,6153,780,8692,8105,10268,172,4128,841,8925,5453,8561,2135,2976,2845,2209,7418,10817,5620,7010,9960,474,6545,8223,9713,205,10548,905,2396,2914,5591,2887,3379,4865,3304,5849,1309,10171,608,7314,6228,6961,7068,6943,4613,1317,121,8611,5902,5827,8042,1139,3121,223,2984,2897,3144,5379,10590,6433,3871,5475,43,1026,1192,1599,5729,6194,5838,1254,2054,2706,7925,6060,2177,10647,7727,5681,10000,3549,10814,4433,2100,6869,6593,9404,5534,5801,10321,9145,8363,7451,9072,4600,5096,10372,3022,679,5631,2552,723,4668,9613,10153,7423,9114,2973,3071,1413,4956,2838,5084,6777,5196,539,5036,382,7480,9327,323,4766,2273,2684,8182,10134,9203,2667,9185,2658,9087,6286,4162,6620,10472,1095,6759,7216,8191,9738,9548,5029,10326,5478,4089,8710,5439,6126,6160,4828,1006,1677,3546,6930,425,2645,4573,2770,10248,358,4975,10603,8129,8614,10180,7902,4303,2447,3427,753,8920,9449,256,6859,5711,5246,623,370,10767,2563,8300,9591,9991,5126,9568,2647,484,8644,3654,9507,8632,10076,9813,10638,4326,3862,569,7020,4213,5628,9341,839,3341,8347,5163,5563,7339,6928,1197,9158,3099,6120,1483,9545,4608,4215,2761,1259,6802,4364,4001,1606,4222,4752,3057,10022,1108,10592,4398,10376,9888,10033,4273,1238,3428,6553,6145,5731,7273,604,2436,3170,8673,5901,10074,4513,8305,8697,7476,5015,2669,3100,6344,9338,10383,4972,8727,1363,3223,10495,897,1698,9553,5662,53,9328,3733,9932,7102,2212,7382,7159,4556,10748,9294,5895,2262,10512,5913,8314,10624,2530,7560,10464,2443,8787,3800,1510,2027,9795,9670,283,2351,4260,2577,1518,906,3377,4764,2113,10699,3948,8465,9264,9267,5600,2924,3783,9209,7966,6464,720,6973,155,6227,7994,2636,535,8846,2931,2046,4953,10058,8228,5200,2211,4234,2207,8121,2007,7268,3212,7772,6496,10201,2762,7534,5539,10555,10496,628,3653,702,8683,10325,8660,8411,9387,326,8842,837,9688,3855,7053,1826,7944,1053,3193,456,361,3206,10315,2059,6612,6173,4562,9003,8034,9162,377,7551,10769,5354,6299,5997,2080,6288,6068,653,9937,10825,8833,3247,3058,4923,9104,7983,3305,9322,8339,7711,131,7328,5212,8623,10761,9718,5402,948,10640,8659,6479,2768,4616,2223,10625,4252,9980,9807,5318,6188,159,10655,3565,1478,6860,3521,4646,9880,1608,10427,9804,2092,4897,10238,6020,7642,8091,7789,4058,6681,4173,7918,5523,2444,9898,7161,7736,4667,2603,9370,7403,10797,1857,3276,2276,3445,8197,991,1863,4266,2170,4331,3296,6658,1308,7542,8458,4533,10177,6873,4190,815,2726,5589,2846,269,2634,1560,7067,7554,3102,4908,10741,10086,5286,8734,3824,1744,4111,3264,7494,3825,8834,6139,8432,147,1589,1871,941,2911,6018,7906,2874,3713,9523,4023,7522,6116,6864,5698,5464,3924,4590,7345,7395,4790,10540,1020,5309,5950,2956,4983,6163,9194,3848,3612,10525,9069,5253,7232,6876,5003,5067,2350,10375,10523,850,6399,10002,9689,7255,1016,3707,3384,2137,4242,4077,5170,8388,2387,10577,2482,3735,4313,8862,2029,8452,10528,6058,6871,5233,1195,3301,8738,3194,2638,2488,5009,6981,7935,10684,10631,1043,1962,3541,4973,9581,4285,9118,9770,8279,1316,6284,1161,877,7894,3981,2423,6872,5033,1798,6181,1255,7517,4067,3319,7457,5164,3977,1719,4436,9317,4765,4137,6515,10025,4628,3745,7739,9445,7016,1980,4481,8580,175,10704,10653,245,8163,3834,7006,9135,1950,4401,2289,1412,7325,4821,6917,1361,7569,6782,2474,8075,5757,1034,3353,8570,2678,4922,4356,6831,3401,10393,10508,5161,2635,1472,3331,3048,9526,5383,5670,7508,5113,10148,2912,8547,1183,5496,5139,1153,6897,7997,6625,9819,5766,7460,10124,6637,327,3466,1848,2455,5939,2282,6371,4812,6721,5314,4780,3112,5154,250,7103,7158,463,6920,8368,6024,6877,342,6799,437,2876,9690,4889,5129,2631,3366,8380,1528,10135,584,6292,9180,5863,10031,2432,980,7211,3117,6909,1248,3761,3601,4976,9420,10602,178,10755,1150,2009,6653,7378,4422,8445,1978,2390,5417,4353,6452,2002,10595,1306,2817,8735,8424,8976,4396,4919,9488,534,4478,8211,8264,385,8288,3063,9825,6996,9726,7239,668,2896,6212,5761,6271,618,8216,6838,195,216,465,8870,6774,8556,3392,786,2791,1096,10565,9399,7603,1723,4133,6106,5491,10443,335,8387,5006,8131,966,7321,4664,5361,8778,8085,10370,512,4408,9227,4676,927,5805,10457,5444,5333,2951,6804,7955,888,3706,8788,2895,1902,9874,7221,5222,10628,6300,3522,65,10112,7284,6826,9762,10226,8642,8030,609,3812,9914,8276,5946,4130,767,4876,3831,9441,5515,7928,4838,4550,78,6682,63,600,3798,8226,3577,8618,8919,4716,10385,5560,7695,9677,9470,6351,10585,6031,2941,2010,7623,1735,6574,1513,2774,1490,2303,5429,141,1915,7737,9086,2205,32,7165,4564,3715,8209,8559,4979,4985,7143,1656,6789,8025,7208,6845,503,3687,7028,5471,9975,5664,3404,9068,5248,2433,322,10390,3840,8058,1297,3551,3819,3342,7265,3266,8914,4394,3583,9786,9198,5609,1444,8947,3060,10252,5594,1291,7319,9559,3166,4669,9592,661,6602,4789,9415,2424,4265,7098,1876,7535,1203,10053,9809,3104,1941,2705,6635,3148,3598,1502,7383,9073,10425,6590,5162,3529,7306,10800,6393,8751,387,10172,2692,5719,2741,7855,4845,7039,2459,6901,6624,3554,4237,3867,10566,2116,531,5943,6690,792,8441,6260,682,3995,6154,4165,9066,2717,1842,5340,8261,8539,1074,4259,701,9500,8648,513,3417,2155,9041,116,4010,1218,7514,3738,10176,8592,4659,8491,291,10417,8338,7244,862,5005,4735,1362,964,9714,10691,9829,10779,351,4930,16,3561,5388,5985,5239,6821,4320,10682,3154,4367,8961,50,8733,4288,9893,824,2504,404,10529,6560,1489,8125,48,76,909,2237,2691,2416,1987,7699,4599,5585,10503,7807,3639,3994,5708,7135,4625,4516,5540,2916,5528,5975,7413,8915,1347,10065,706,5304,3436,6607,8262,5645,3625,1376,4476,2934,1592,1935,3958,5821,2456,3362,5367,4151,981,9573,10610,9207,6190,5275,4631,1708,1921,9287,699,1447,1636,696,3740,4597,3373,9444,1221,3780,8124,5381,7648,3536,10571,654,9064,9044,3503,9854,9626,3378,10675,10629,6080,4445,9605,2646,6982,5074,3618,52,6561,6185,3026,5968,9742,989,5867,9190,1178,2689,2001,6149,8334,2049,3180,5759,111,4691,2650,8777,3149,804,1868,5328,840,8687,1382,57,2419,808,10355,5588,5215,1403,5701,9,7853,8520,8669,8019,4006,9097,226,2827,1135,1901,979,819,2561,7049,504,4391,4400,6739,1402,5264,2230,10666,2566,7430,8459,10720,8794,10310,10174,443,6349,7310,533,7609,6200,4108,613,7558,7709,1553,5076,10645,300,2842,6261,2324,588,2193,2945,7510,7396,3094,1889,10781,3791,554,9284,9148,3519,9995,3209,5810,8322,9519,2560,3073,947,2781,7004,5548,7116,10152,10121,552,1887,2952,8739,9225,5819,3712,6550,5446,574,3090,8327,428,1266,8433,8383,4645,8754,6898,2652,6822,1455,2176,1519,8566,10303,6049,10068,4977,3894,7561,7488,5536,7224,7793,1989,3985,8695,3424,7035,7612,9938,6775,39,1850,7803,9174,3031,6441,2974,1603,7941,7223,7923,2831,2977,3080,2075,9236,312,8565,4634,8814,1558,10484,1272,7389,8872,10418,4755,7869,2610,10633,5046,6960,4193,7852,1754,2878,1936,3784,10314,4025,4805,4997,10007,9863,7155,10101,1342,9395,10291,9385,10218,799,1123,5843,2562,4905,1377,9226,5937,1986,7591,9962,1189,8902,619,9067,3458,7586,7210,328,8506,3000,1004,9212,4591,8020,8887,4012,592,101,3415,2527,72,7452,9400,7809,1561,7032,4840,7008,1056,8,5752,9588,3101,3106,3609,4503,8312,6536,3446,8745,1437,7547,10287,4816,1795,8004,5837,1042,9329,9008,9433,2047,1087,3421,4324,10588,6699,10747,7765,9719,10438,887,9062,6820,2946,4741,527,3043,1648,1133,2918,882,684,4939,4386,8641,8154,1323,6436,7873,10752,3686,10680,807,9886,549,3230,6147,294,7120,354,8171,8493,2663,6863,7489,6037,410,9913,124,6312,541,3159,10304,2117,735,9882,199,4315,9250,4055,3739,3826,9558,10782,3699,542,6645,7593,5123,6413,7532,8805,1517,8514,3396,9239,8883,6748,6002,5319,8690,6308,1691,4655,589,10278,5680,9489,6806,10465,4251,10142,538,5740,194,9167,9974,2107,1480,8439,5722,5674,5712,7528,3385,1906,10567,9057,340,5707,9308,6207,3078,2818,7518,6794,9555,9983,1676,1678,6079,2038,1670,2608,2335,10021,10243,7257,9405,8165,2242,1247,8620,7479,1305,8818,8790,7865,4998,5653,2097,261,5270,2140,2105,789,6984,7501,8748,9538,8167,7566,3307,8768,5195,3932,1130,3930,6535,3665,6818,870,6657,7938,8656,2975,8108,4377,1332,1792,10027,3395,5078,10380,724,6034,3423,4788,4511,9362,2462,7760,7023,2497,9802,5504,6817,4263,3361,8164,8418,2478,2287,3457,1264,9796,7218,6591,6401,1683,3534,7474,5945,9722,7233,3719,7072,9915,42,6386,1481,3426,8808,4075,9665,132,3679,1418,6327,4854,5667,1228,5621,10063,9685,9641,3792,8509,10725,1726,3510,6733,5311,6890,1127,1372,4557,9076,4493,1319,4639,6503,5373,319,6252,5205,2568,3262,5611,3692,8016,2058,10708,1990,10246,5149,6730,4745,9617,4496,8093,3764,4587,9046,8240,5654,7484,9304,9356,9122,9012,2409,2879,1728,8443,6110,2374,2695,6156,6123,7266,9525,1770,9102,10660,6745,5128,5806,10298,1081,5852,10365,5171,3316,1949,2796,5938,2888,505,9040,10137,2063,8296,703,7982,5184,5822,6713,8356,1212,10214,1643,1344,1740,6170,2765,8513,7491,313,7468,5830,5188,7420,8246,7931,5592,2300,5803,1333,1904,10524,2557,1320,9801,3887,3372,4648,8123,10207,10570,2782,9208,3493,8993,4688,9635,2367,890,4834,4750,2186,4059,9823,383,5746,6601,6995,3696,10626,6202,4034,2588,1415,3128,5342,9471,8634,6712,7283,5689,7596,5728,9160,6465,1823,8542,2381,129,3703,5297,5845,10090,9892,8109,10676,1417,6171,9518,2483,5993,6066,10657,4131,7317,4665,10546,7047,6337,6183,3537,253,7720,5316,4382,2720,8286,7478,3902,7237,3088,5066,2131,2830,10694,3644,1407,5650,5733,7970,8970,2283,8951,9288,6643,10349,4772,6849,3295,4178,1354,8522,10736,1720,4463,10409,7172,6220,1998,6367,7464,1973,6738,4238,9065,9537,2528,6911,4247,8425,183,7919,4791,5494,3274,6583,4995,7128,869,5889,7655,5199,1555,6754,10636,3874,5094,545,3805,1611,9596,2168,6048,7912,1253,3675,10190,6051,6887,5414,2919,2397,8982,2189,5410,2589,6458,7272,8280,6078,2349,9268,9229,8372,9571,2129,9070,7371,5048,8593,4925,5490,1503,6584,4713,6442,3119,687,3013,10757,8401,6102,6842,6324,7114,10127,9879,9783,4963,1626,2785,3640,9163,4835,1390,5087,8073,10203,3068,5747,1958,49,4916,620,1548,3018,9306,184,10272,3878,3053,2239,10187,1477,2902,9456,7260,5051,7639,5386,4477,80,1151,7277,5579,5072,5596,2204,10199,8267,2624,5110,6272,1330,10059,757,9947,1357,7745,6159,9580,6926,1486,10062,10488,8712,6582,7733,8905,3935,8265,7334,5495,9170,1657,4880,1542,9546,5577,3169,2880,8287,2763,1465,1041,10545,7951,2814,10729,1234,7881,6985,6670,5055,4329,9749,4744,7621,9946,665,475,6655,6976,1327,9703,33,9513,9857,3248,2158,7640,3830,6910,6599,10785,3908,9303,6403,858,2074,3941,4068,1488,4929,9954,4872,10514,7838,4677,1877,7014,8113,6883,1511,7174,381,1523,10229,9627,6966,9767,4283,1167,4168,6977,3543,6953,2737,4698,7908,8759,2429,7706,9642,7199,2064,7087,557,8591,4609,3962,2795,8283,6449,9092,9964,4970,4529,3616,6475,7523,56,2238,6431,8256,640,3416,688,10471,8348,5541,2371,994,5570,82,6166,2259,7589,10479,5713,449,2308,130,6882,6298,6517,9870,2353,4316,5452,2792,7556,3803,822,9084,798,9477,9082,657,495,8781,10619,4328,2710,460,10133,3234,2555,6069,5255,348,5511,6934,9594,5120,6729,3956,6290,830,9948,5911,1731,6291,4993,1533,9758,5693,8152,4988,514,10270,6000,9682,6946,8344,8051,5320,8247,9020,3751,7864,10163,540,5933,3650,6107,2840,9372,3701,8955,4269,8511,1024,2165,4739,5474,10239,2466,4157,6533,10780,9408,3312,2549,7113,4480,7741,3467,5710,10381,2392,4166,7485,9902,10116,1089,5281,2157,9182,412,6056,2281,697,4383,6531,7473,9231,7411,7070,1493,3570,4122,3461,4849,5119,3174,3139,8068,7162,3184,3691,2316,18,650,2382,8612,9692,6483,2442,7718,488,6642,8681,5271,9004,3942,7986,5406,6744,9014,6975,8403,7449,8721,3240,8297,752,4471,2833,10812,202,7701,10064,9164,1565,2172,3613,6589,2843,1052,8359,1932,2389,127,5508,4782,1048,2729,6216,5692,5219,4730,8987,940,7810,3907,4399,1220,9901,1549,1242,9815,673,7459,3322,8550,6865,5229,8557,2257,898,2893,8674,2446,7645,6528,2809,4711,2106,6267,1110,6415,7998,2094,9509,6269,4475,3096,1567,7066,2464,8516,1843,9885,8995,2345,4940,1077,903,6886,347,8766,3343,8534,5871,5310,5738,10217,7539,2823,9867,8238,7093,10667,2332,5535,1419,3203,7540,2417,1619,10744,4592,8817,9219,2565,4276,9380,6329,10426,8853,8106,6765,6932,3822,5864,825,8244,9566,8333,5954,5351,2978,3376,6275,10290,1604,1067,7901,3143,7979,3630,5252,5642,6098,3077,1464,325,10481,5853,10012,4658,8315,10029,2812,4123,4967,6097,4038,1755,2873,2905,8098,6941,3681,1070,8555,4875,10340,9944,9694,2180,7441,2904,8054,2531,9330,6750,6945,10067,6012,5092,10772,73,6213,1504,6425,8835,2520,1401,10768,3368,4375,4941,685,9479,453,1268,2245,1694,4325,7705,892,10775,7450,2679,842,3996,6241,1117,7713,10343,2124,5857,5617,6594,7305,8770,5928,2197,9166,5956,9299,10095,6524,10531,4335,6622,10823,802,3120,2073,3864,3490,6957,9845,3972,6935,5543,9774,154,5250,10342,8137,8040,4224,10500,7327,5779,2755,5085,8243,4632,1952,8047,7056,6304,3241,6219,5353,10320,4559,6552,4748,9475,203,4551,8552,7187,3748,4271,3232,5903,9402,1566,6381,1569,6192,5338,2965,8023,2360,10386,6383,923,10618,9172,421,3412,7582,7685,7332,4138,2820,876,1356,1325,1012,3814,9584,3799,5041,9273,5764,9757,1655,6991,8588,9423,8001,3853,6191,8755,576,6629,4115,8878,4253,3245,1885,2505,5426,2621,260,8848,3873,2060,8352,4842,7686,5359,3600,6732,857,5014,9386,9173,8150,3065,7127,10754,2987,9907,7347,1051,6114,1609,221,6666,7851,5093,2471,2465,398,2077,6459,5799,8462,7943,10731,2227,9619,3813,9031,4753,7797,6347,3836,1806,9134,6150,5643,497,3868,6915,5071,2769,3717,8575,4426,5358,1587,7752,4942,8857,742,8464,3442,3050,9660,1165,3309,5293,3171,5099,1154,1827,7728,3140,5995,7719,601,9119,2053,10056,10487,4949,5307,7866,7076,8052,4372,6045,9586,7799,4800,4575,5458,316,5425,6361,4965,1187,10816,1907,7808,10551,4531,146,3884,8978,1271,3568,10668,179,6571,163,408,1190,10175,10716,3837,9340,303,6792,5936,5979,6330,8574,6129,5555,2385,10594,4980,10728,5842,4321,10444,427,5890,1776,4598,5235,7900,602,4763,6907,6379,4804,1960,924,6133,5932,8041,98,3622,476,2517,8161,1374,10787,556,493,5804,9109,8786,1872,3138,1337,2822,6289,7562,3005,1392,6866,2518,1140,5258,133,10331,8306,5313,9575,10717,3156,2103,8495,4799,4672,838,2249,4686,2233,4281,1475,9108,206,9453,5941,8379,5103,2868,9988,9732,7417,7071,1231,2515,9371,4498,3081,9352,6529,2313,1529,4229,771,1134,3690,9005,1128,9503,9776,2571,8369,10271,4387,3408,5533,8666,9035,8390,712,9255,6634,8299,3221,7005,6850,2854,3921,4548,3501,1593,1050,1692,3685,4879,4277,2805,8468,2867,46,1258,2272,1118,816,6796,7965,2036,1722,9448,9851,6940,6862,10538,2547,6437,5343,6727,1359,6444,6573,852,7574,10336,61,7085,6247,5134,106,9192,2369,7786,9009,670,8311,638,5288,8446,5463,296,7337,5018,3103,10423,9970,10008,944,3205,5435,5332,7921,871,967,4777,271,7214,4652,6630,8199,6526,1695,5140,6215,9277,5174,8504,5181,8193,2863,126,2377,691,9662,6495,233,565,6466,1771,5348,9411,5026,5537,4801,656,1384,8396,10541,2326,6958,675,402,10098,4203,6835,6258,6969,7950,5704,4047,10642,5038,3340,1820,6525,7354,8752,5131,3937,4369,3486,586,5305,4971,9773,5948,7618,9361,6790,8743,4309,3722,7721,3944,8836,4154,700,2935,8003,7351,5521,9464,4718,4888,5124,7841,3135,5697,4530,7276,1395,9615,4825,6224,198,7805,633,8442,1840,1261,1170,879,9725,3626,3870,5050,3901,3954,9332,2906,10073,7674,722,6843,4363,9289,5077,6598,3059,3928,1114,10034,6377,1380,4378,4418,10505,4869,7246,1292,9942,7092,9139,861,7416,2747,1245,6423,9095,9653,3130,8948,6737,2404,4026,726,6101,7726,4540,10308,4243,478,2980,3227,8985,6913,2675,4504,3753,4931,2434,6753,1045,7477,834,4181,7134,10507,9514,1747,714,7790,6755,1971,8301,6009,3584,8877,4361,2927,9027,599,8160,9059,4675,8090,10091,8192,5283,719,3145,8671,8569,6824,8639,9253,5815,10361,6988,2510,2585,8904,7405,2398,1768,7989,8696,9899,9805,2439,9412,2727,7927,6257,8746,3982,5679,2013,4579,4141,7634,7553,346,603,1522,8764,7001,3603,7775,9715,7678,8482,8630,10530,7905,9211,5308,6760,5266,7308,7377,3957,7058,6340,3602,3218,4945,239,8867,9918,8130,8000,4950,8913,275,8097,1029,4630,10774,3173,2808,3754,1591,9115,3593,755,3318,6902,5762,2104,8893,3749,10401,8946,2616,8510,8937,7975,1078,6335,6238,10733,2044,7885,4233,7426,3606,5280,9756,810,9844,9695,8053,4704,3852,1484,1031,8399,5467,7672,7590,8215,5263,8028,2657,8117,6142,6273,5561,9748,672,7229,3098,84,10405,4272,2800,6889,8792,6691,4204,5646,5345,5615,4278,5315,4486,1543,9467,5355,2797,5400,1953,8917,7806,7073,3807,7312,1406,5716,7086,1738,6847,9781,10212,9631,10442,7548,10115,4678,4097,5030,2247,3002,10293,492,10429,8944,5106,4009,10269,5760,859,1201,10771,2026,6508,2957,1780,317,6445,8139,1810,1520,8983,2354,490,7375,9866,6035,5393,4751,9810,7710,5155,6576,7367,4368,332,5870,7762,7292,10020,6076,8355,1216,2119,2362,6470,6180,500,9775,298,4319,4935,7585,9563,4182,5326,1970,9266,845,6756,2256,8541,8972,7094,2748,7353,5185,10664,10743,9565,6942,7743,9912,5135,1452,1618,7592,3844,7037,2048,946,7036,5780,3146,5973,5505,8810,2892,3682,9607,931,6852,8823,10136,10440,2615,10764,9421,7909,8964,8449,3160,2153,181,10421,3357,5638,9478,9572,5797,2020,5368,10014,2415,9659,1772,9831,7340,10777,2584,1967,7190,5789,9905,3909,5499,5879,9787,2921,2683,462,2535,10730,7492,6718,10492,7124,1148,1079,3986,6891,5696,10414,5855,5737,4214,9389,3253,1273,3869,2641,8329,7323,4120,7723,4798,1387,5285,6983,7132,8731,5299,3663,4249,3438,365,5100,8684,8801,1993,4506,1671,4566,5122,1686,3195,7916,3880,1101,9672,8036,4397,2701,1786,8120,10726,3429,5114,4588,5217,1057,2735,10794,1331,8815,2261,4703,1667,9574,10803,10054,2435,8067,4393,6362,2937,985,8750,5374,770,2463,1749,5922,1063,10335,4743,2305,10396,7546,257,3297,8553,1775,10197,7279,2690,496,3454,8616,10612,7857,7126,3107,2298,4072,8906,151,1581,8061,439,6856,7903,2142,4526,5244,9029,4714,8335,881,10363,7503,6647,935,5133,7649,10561,6108,933,2551,6870,4673,2826,10129,5906,626,7717,1082,3575,1172,9417,3456,5568,5914,5856,1851,2953,4881,5851,6239,2050,4371,6477,7366,4452,9849,2246,7930,988,1951,10789,5878,9853,2323,2850,1942,1136,7421,10230,5130,6313,6929,4846,10267,7890,4964,6829,4357,8582,649,7241,6716,8258,1176,9263,5390,6994,5921,3192,1069,10796,5908,6486,6138,8562,8578,6925,1408,7917,8045,10286,6671,2352,1312,10309,8210,800,6050,2686,6214,8456,10302,8858,3443,418,8005,1445,7115,1824,5690,2569,2917,5197,6749,8643,1537,9977,5648,3997,2600,10395,8360,2861,1014,9790,9865,4095,6124,1666,9189,5160,8186,5137,10828,6025,1847,4932,1249,1299,9530,7282,6990,9223,3720,3818,7987,3137,7670,286,4731,7654,805,9668,10018,230,2467,8638,10084,8422,3481,9811,3285,4877,9281,10307,10801,102,530,1819,4918,6546,928,2930,6678,10253,10151,4094,8430,5056,7400,1313,10461,6039,7992,6664,6502,10644,6424,3383,4302,7186,8342,3559,310,10379,9446,2277,1116,5778,6538,1963,3185,1487,3294,10362,1207,6921,8181,5236,10353,5709,7359,10011,4147,910,2476,5337,9213,9916,2696,2391,7286,8797,1796,2411,2570,7990,2889,10723,10150,171,472,6246,7280,10100,8298,4517,2775,7831,10582,2365,3183,3670,6432,9074,7993,4852,646,3645,4052,5323,5556,10807,4014,5349,7109,1296,7125,8147,10043,10085,10709,6906,2251,1058,3842,1132,10328,6005,7725,1853,1862,4528,10824,2837,9129,2214,4209,3557,8758,2605,6667,8761,7620,2156,8505,4322,5269,9818,4447,3186,4709,3920,10087,568,7985,8911,2331,6494,10654,7631,2875,5363,6151,6896,5481,9705,3286,7753,5449,6013,6874,5153,1779,1494,4722,6728,4726,8008,6316,430,4403,6359,8762,6936,8704,8677,1111,8153,8680,7700,2622,582,1462,1621,4373,5430,6218,1825,10038,5597,5578,6064,7907,7892,1469,9038,6763,7197,10543,9461,3881,1536,8992,7465,235,4240,7502,1541,1616,7207,6608,4245,3470,2961,5423,9460,3993,1979,3680,8998,6844,6370,8959,3488,3217,2604,3891,3369,6467,6320,4458,9248,1793,8676,7581,10502,730,566,3694,4699,4416,2284,8110,1933,9333,10388,9873,7454,9254,8881,775,3497,1712,7090,5158,10122,7483,10818,9973,9704,1969,4606,2682,3354,8869,1338,10215,4379,9793,694,3725,7365,4431,8302,2752,3255,5392,5468,68,10751,4871,390,457,4705,3181,4081,1491,9246,5982,2524,3721,2372,727,2163,4734,6689,10005,5237,5583,280,5207,3034,4022,5934,6326,2334,7209,2702,4643,7498,10119,5825,8962,8185,5440,2169,1803,1742,7316,5183,1177,2936,5020,6959,5063,698,8234,9307,3086,8831,919,2234,3787,5714,8263,8991,6104,8554,3350,5187,4092,8071,1898,1394,4992,5371,9200,486,7787,8939,6047,2190,4210,3142,4261,1583,1121,1927,270,4948,6204,10255,6811,315,8896,3477,3895,9606,8851,4623,4244,893,7009,10344,5404,2539,3448,8682,6105,8965,9706,2862,5375,3273,242,9218,3585,7027,1112,8232,7751,8332,9015,6507,8544,4428,884,3857,1086,8115,949,4160,4268,6867,9803,8337,6903,10165,5744,3284,5854,1785,2319,3089,8448,389,2066,7236,3351,1466,123,1956,10341,8785,1741,2310,1544,10547,526,4421,8916,7915,161,2972,5424,1301,222,4661,6438,2496,3109,8488,9354,743,9345,3615,10721,9622,51,8829,9000,4890,10182,3760,8921,1867,5754,3265,2399,6089,7313,6339,9285,212,3308,1886,10441,2341,2626,1284,8138,10424,10204,3572,3966,7296,7303,6846,1875,2758,9663,7173,2959,7261,6295,8303,9088,7669,2112,1328,809,4456,3132,2346,1241,958,339,7663,7531,2519,7509,6949,8340,9959,7063,3055,4636,4650,7680,7203,6030,2264,8979,8819,2500,7559,5606,9434,3256,10689,2321,3324,523,8078,992,2736,7513,6385,7052,5970,5483,278,6350,1730,9367,729,157,7960,4446,2110,10575,3544,10795,6554,4384,6279,6659,2623,2082,145,6372,3582,4127,1878,10826,4129,1017,1837,9788,3281,1663,1929,1597,7393,10473,5167,9599,1482,5394,4906,952,2098,3465,9240,5524,3459,1724,9454,6189,5210,8609,3447,252,20,8014,560,7346,8756,9939,6463,3810,8537,7240,6703,3435,2252,8404,4582,4011,285,3744,6435,8127,6676,3492,9245,7358,4079,3555,3329,9868,10154,1435,5189,8220,1275,3926,7007,4464,8647,10250,7756,10798,10221,8891,4723,10161,6297,9083,3765,95,3530,10346,9909,1532,3700,2541,9989,6816,10311,9640,6547,6384,2081,7112,5666,3974,10374,617,5691,6310,1702,2487,2285,4644,3336,3756,1224,4343,987,369,1093,5223,1610,9981,7227,4114,915,1186,3856,583,10701,6518,7343,860,956,3165,4762,2529,10281,1429,3413,4135,483,1449,739,4878,3697,769,5841,4621,999,10557,9149,6955,1997,1073,4172,7653,5518,1485,3287,5466,9746,7146,8953,8830,7521,8289,2198,5300,4404,1659,336,5723,2857,1499,8658,5105,6122,9601,7954,7573,5243,762,3418,6234,3161,2681,4441,4649,8345,3804,1818,3177,833,9727,10225,965,7635,3006,2772,1142,3208,1098,150,3943,4761,3460,148,4784,7496,7263,612,3524,984,1983,5502,1120,7200,4104,28,9675,9544,7936,8763,9603,4304,6014,8498,10131,1367,2580,5959,2016,1205,1351,3288,4457,10673,8116,4406,1206,7046,9177,5683,7707,3547,2045,4376,1516,2865,4622,3207,3659,9175,5121,2777,2653,8524,1157,218,6418,3321,10102,2380,7980,4617,4220,8859,2550,9473,3495,5408,10677,343,7398,380,2894,5325,10235,1329,2869,7202,10437,6613,3509,2703,6314,7226,5218,10184,10627,3526,7179,6801,6853,8560,962,10511,7729,4125,4134,4392,2766,2492,5971,6144,2358,2583,1122,9632,2992,747,5145,7409,8392,543,3133,7946,10450,754,4981,5441,2618,6462,1229,8587,6498,9512,954,5848,5688,4759,5331,356,8088,7814,4350,7795,8176,3631,2773,4999,8259,7381,744,6609,2564,5940,1594,3755,6476,6540,5057,10616,9221,621,597,7887,2606,81,3980,2220,6640,518,2659,4538,9926,1733,7948,8018,3004,7363,2021,6992,2342,8483,9373,4527,4419,10224,5816,10623,2693,2160,2963,6773,74,4684,104,4898,4809,1674,7499,10696,4695,3978,2699,5214,7315,829,10410,1145,803,5951,301,3763,4196,7746,8328,3070,7407,6055,938,10522,7977,9896,2243,7033,9814,4494,9676,6036,6100,4775,12,2195,10196,1103,5862,3333,3471,6764,10222,4174,8568,8249,8362,5492,4053,5630,7000,1966,3422,1711,7682,295,865,7361,9652,5790,5626,3821,8409,9859,6363,8860,5989,4087,6395,9848,8308,4341,7520,7616,2671,6836,4860,3014,9789,9205,1918,5053,3335,9542,320,8884,5204,1442,5558,8250,9872,5952,1424,3709,8195,6140,4057,6633,659,2383,5834,1508,4246,10010,7287,8076,4554,3599,10685,1431,4050,7264,8535,5166,8086,8133,6541,4201,3545,9923,5647,4035,7740,10183,4169,1689,2664,5770,1882,1295,6978,4921,1812,5412,5387,8668,8408,4781,3579,732,3899,9510,3082,1914,7242,1557,1912,3961,3250,1039,9611,6683,9820,7647,10220,5720,10173,10330,4909,2337,2654,7133,1896,2883,2448,10727,2793,578,3199,8572,2743,10223,10662,9378,10004,4008,9331,1675,718,4682,4274,6762,1240,3261,4046,1934,79,1988,9624,5601,6621,3153,3882,10144,7429,6430,8986,5860,3225,5873,3960,9397,7687,5080,10674,10778,6193,5531,5793,88,7583,4601,2248,5748,3393,6702,6857,6950,8635,2375,1141,4236,4145,2733,5629,401,8954,5777,6783,3337,1943,5918,7821,255,10692,1434,9058,9629,3168,360,1690,1230,9458,951,9271,6469,5967,3061,7579,1003,3108,7681,8486,6265,4409,6109,8412,5618,6487,5562,6276,5907,7575,5529,10601,1725,9096,6905,7801,8610,5277,2627,5035,9569,3157,9957,7322,1600,6157,796,4460,6627,5685,10805,394,3669,7330,2426,7744,10520,8645,8932,5209,5820,8391,4769,6132,8613,1836,3841,3041,1085,521,5186,9462,7136,6429,9951,5409,1269,8128,7516,9188,8354,10167,10216,9497,9972,7399,8910,6456,2810,10339,10819,1646,9616,2174,4175,6091,748,3661,2366,6654,587,3627,10097,8367,1109,1844,8652,1364,8151,5060,2480,7817,4284,1076,5257,4861,7355,5872,9709,1298,8145,8519,806,2427,4524,3660,6293,4444,1660,3989,9374,7192,5369,8292,3914,5608,210,1002,7154,1302,2855,2412,10211,8799,4612,9744,267,5877,7137,8426,2828,4192,2378,6236,4924,9921,2440,6378,3556,9511,1897,357,8104,2400,10609,2970,3499,1830,8321,4543,7794,2971,254,6223,27,9480,7003,8856,6954,7704,318,2407,1182,4624,644,9496,10318,7961,9257,5695,6231,8894,4186,7976,812,7331,4217,7106,8092,1126,4102,8980,10591,3734,1213,2670,9679,3172,10459,8626,6492,3323,6646,9539,355,9337,8716,3808,6165,10562,5658,818,9707,8958,2134,4866,10317,7818,274,5457,3388,4282,4330,6482,109,561,4955,4874,117,8038,8866,826,3938,7069,9357,4806,4903,8741,2688,6343,2402,9895,9561,4100,10491,2597,3533,1849,3190,8737,8967,9007,4468,1068,10277,2258,4360,59,107,1036,5769,8437,9843,9700,9127,3525,1,7297,1809,5972,2484,9840,9609,431,5800,671,3009,10159,481,5025,6254,9602,2025,1001,8679,7472,8929,3164,10786,5165,7177,9987,733,4959,10276,6199,8022,9001,10576,9036,8938,10189,9238,297,3231,9080,6512,6509,1568,997,3773,9643,1030,22,2192,9407,7798,2813,10039,2088,5173,7170,6652,8536,10436,3325,10338,9437,1977,1919,9808,2430,10240,1107,6451,2784,1817,8037,9633,143,7117,6964,721,4194,4954,9730,8203,9491,1807,5659,9116,9816,6808,1160,1358,4106,1166,7206,371,3636,7656,2148,1938,7394,3634,8204,4770,4938,3896,4577,8548,7530,6016,7461,324,10646,7307,7338,8436,4381,6615,6087,5668,942,9953,1899,5546,9406,15,9684,3552,7335,7392,450,4470,6201,4040,10641,8713,953,5675,4947,8715,3092,5422,6023,5783,3950,1423,677,4710,878,7130,10822,5512,7204,9929,7326,2019,7774,3197,10810,7732,8471,5052,10750,4482,5156,334,8526,6222,9214,6614,7258,6488,6680,8320,5259,3757,9549,9196,5715,3581,5413,8031,5224,6497,7651,1100,10110,4797,7839,3091,10123,10358,8188,2208,5447,4542,373,434,3293,4904,214,8494,5450,1721,9828,9961,8747,4365,110,4910,704,9155,3504,7235,7734,873,10448,7667,8824,3796,7971,4605,10003,917,4066,705,779,1612,8861,5260,7295,3027,8487,3823,9826,4459,8941,6715,2085,3517,4808,7075,8272,4500,9656,6927,8405,1386,4737,8475,8365,1813,10088,4996,6029,6993,3364,4491,4885,4287,3667,10170,9325,2314,1129,3911,7830,5924,2676,4934,9403,7940,1982,5054,3028,9693,8633,454,1944,8599,9833,8595,41,1400,9291,25,3594,2697,8945,8840,7290,8963,2043,8431,6618,7978,4635,1669,7879,4019,3811,8168,1791,7099,6588,9120,8107,2633,7467,2490,3905,4697,3652,6742,1718,3987,6206,8467,3716,8875,9504,7822,1874,2899,2537,1515,7730,9667,9388,8007,5294,7185,9778,9451,446,1512,4152,9894,6948,6416,8189,10042,7024,232,648,1425,9531,10391,10715,6568,6353,1441,4423,470,7962,5376,2725,4610,9079,678,24,794,963,5479,2339,5023,1627,9260,19,811,1410,2506,2086,4218,7015,9600,2022,9466,7222,7410,438,2732,4654,7252,8826,1766,3483,9081,3998,2315,2767,5503,9724,4060,4507,4857,5360,8310,10600,3211,6448,7386,3790,3872,1343,5636,2280,2384,4015,5068,6328,7697,7538,1588,2760,4701,5706,2338,3925,9966,2674,5599,1985,3879,6636,10366,9877,3042,544,2724,429,1957,8924,1634,10106,6242,9852,1884,1799,3610,299,7696,1144,5923,5194,4467,5813,1925,5177,5554,9889,886,6244,8968,9883,1910,1661,4438,7766,2194,5792,7610,8874,9729,7166,5700,8477,2091,4076,4962,9797,10400,2179,6539,1243,3452,6322,7091,10164,6564,8879,1637,2222,8434,6026,9485,4235,10564,6164,10599,186,7953,1223,6484,630,2034,6225,5991,5787,2344,7376,6979,4462,204,10621,6747,13,302,7329,4339,5983,6338,3113,3397,2213,5772,6603,4140,8357,8048,9752,3875,4177,1106,4179,2789,2995,3774,625,797,4188,333,4093,7495,6043,8057,5433,9839,1038,9750,1763,8427,10470,8601,1471,7956,8118,6848,6237,3794,5899,7245,1913,8242,6565,5329,442,2644,986,2983,5684,7060,368,89,10710,8027,2481,10632,9061,3313,10698,5357,6054,2221,4615,1578,9878,2509,4810,8126,848,1303,2489,2441,764,8255,7065,1527,553,2628,6282,4518,6331,8096,3017,2039,5350,3657,6795,2357,6577,4296,510,5627,338,393,10407,6951,9090,2798,4589,8606,6434,868,4778,631,2964,231,4594,6665,3945,5427,1293,8655,8499,388,3633,1393,2877,8319,6980,736,622,8813,3797,5883,3433,10160,2786,9567,9022,5739,7920,8966,1083,4771,3532,10697,4826,137,2182,522,8984,9982,10264,4017,1974,5652,1981,2910,7029,2990,8724,3505,4651,7819,4345,5497,1928,7782,8949,433,2403,10288,6751,6342,4712,7684,9683,1035,2844,2317,10157,3858,4370,3883,1554,7061,8373,8015,5686,8795,6278,3683,8686,5493,5168,4032,1748,10790,1458,1432,7436,8032,7625,8651,10815,5998,10506,5202,8688,3678,9343,6510,1629,3182,1353,7419,3785,2472,8393,7761,1355,6332,9681,9931,364,10678,854,9098,9691,10792,5289,7764,4230,5069,7055,3955,3576,4567,10194,6414,6766,8956,3269,2997,3888,144,1777,5477,7013,5192,3642,2311,1336,849,1601,4968,7482,8135,6404,3124,8622,2949,5403,9269,1256,10521,9091,5486,6970,2815,4311,1893,7183,85,2292,9856,7984,855,1750,4279,1713,7002,7626,1705,2090,105,4687,9153,2294,10411,7939,6707,1672,10284,8545,2609,10714,8589,1757,7469,7217,2263,5027,1745,853,4429,6758,9651,7144,9286,7676,8527,8155,420,4443,1845,8082,6611,0,2794,9547,639,1222,6115,1572,9903,9495,5545,3243,9621,5182,247,5581,3289,3729,776,5095,2379,243,8744,2295,3732,4725,10544,2745,1246,10024,3345,2076,1155,627,6939,2454,2839,10259,641,5988,1808,7401,9723,6734,1294,10155,5818,1260,7904,10671,7750,1061,8224,4013,7194,9030,9482,1505,8012,10185,2293,8551,520,4831,10451,5002,1628,244,23,5459,3152,3566,2923,4389,2508,5984,9363,9138,4756,9871,3200,1729,8922,3064,5090,4626,491,1968,7768,4039,7882,6317,4822,10244,6947,6559,5,188,4424,6963,1879,10447,8269,9576,3865,2632,4003,9368,3484,8089,2790,5472,1922,6723,5553,6057,2734,4989,4706,397,8384,8002,6233,3469,1576,8470,9846,2836,9521,1829,7442,8771,7731,2751,813,9051,5637,863,329,8056,5574,9779,10558,9234,8923,366,10036,7320,8997,9522,5301,3292,3327,1113,5242,5602,5876,3817,6972,1054,7870,4310,211,1290,4005,6888,8732,7384,9564,6837,6044,6205,5655,10017,4824,2938,7281,10462,5091,6918,5488,8481,7828,5498,7613,3405,3648,6007,5634,10209,10109,10516,10466,10688,6501,2023,400,10517,10399,2297,5079,4148,6248,3482,5231,441,7759,9244,1064,8564,8973,711,10711,5525,3420,2526,4757,3759,8142,5750,1787,6232,10806,3730,4483,4255,2811,2707,8295,9699,10739,9911,7622,1784,6592,9590,9556,1687,7643,10265,1620,1119,8451,4198,10519,10607,6446,4227,4943,6810,5061,9364,4570,2738,2640,10126,8694,9666,5043,1883,8233,904,6081,4532,3553,7533,3507,7636,6788,9128,995,4867,9560,5190,1115,2771,731,9777,1653,6919,6776,10262,9881,1196,3386,6390,3574,8693,913,2714,2713,8780,674,3478,550,686,4786,8990,5885,139,10416,1684,1811,5147,5614,5178,2662,4080,3141,6885,4211,307,1232,7565,2704,7598,9383,5930,7348,9999,8415,5980,6965,3236,6685,8251,2065,1573,4893,10245,2051,1239,4581,3425,2594,2740,3394,2607,384,3646,3567,5732,10820,7380,9440,10827,7490,1194,9984,4435,5587,901,5828,3008,7045,8749,6974,1547,8225,2202,6968,1602,9124,8466,509,10066,934,5022,2542,10193,1852,5339,8180,6639,3263,4159,5632,1590,8059,3952,7225,158,3494,2723,6447,5625,10732,2864,6698,3906,7974,4002,10044,2673,1008,266,9293,1999,7714,7968,4112,7779,9610,2266,7515,7274,8558,2139,8725,6067,4642,8930,374,8485,4717,173,2575,3832,34,2449,1346,7854,7675,8378,1743,7178,7022,2495,5542,1668,40,8230,3849,3363,10006,90,1143,8636,3591,2779,2994,9997,8281,10483,3463,4899,7722,10830,8492,4136,10368,7437,5013,2783,2592,9110,10227,6638,6650,7827,5455,10583,4338,7387,4158,1318,6757,4126,5434,1931,7856,9721,8597,1209,737,4474,2718,93,4337,5102,3317,10432,83,1339,10534,572,3788,3298,8604,4176,8889,7836,9505,8400,3246,9552,3390,7599,9630,7850,6075,6854,6077,2687,866,6668,9206,3781,3587,717,2312,6600,6407,6341,264,6952,1579,2558,2532,6562,5389,3970,7034,293,7777,9506,4873,7247,9140,10493,464,4225,5639,3134,9195,2386,5603,2578,8533,9782,436,6893,5590,3596,2083,9908,5891,5064,9589,8760,1693,5571,8837,168,6382,8512,4334,3711,10474,9655,1704,7057,5724,5673,9735,5522,7289,6944,2787,1709,6198,1131,4555,2999,555,7781,8812,632,5743,6513,3414,2336,1682,8988,8087,2343,4412,9424,99,444,5874,10460,9017,6094,7614,2147,5125,10117,7840,4637,5104,1285,8602,1091,10113,9887,7576,10605,3258,4018,276,9324,7641,9276,10477,3054,9256,9450,7105,3953,5909,8717,6504,2715,2154,9197,8828,5929,3621,3016,7042,419,3847,9320,1334,3782,9312,8699,3228,7715,7370,9313,8346,4425,3047,8838,9049,6352,1340,4492,3356,7095,663,4660,4119,10146,3129,7369,2040,6674,5527,9259,2322,64,7858,7019,8417,1746,7285,3332,7129,6814,4796,8274,3083,2824,3001,8621,3746,5344,1575,10357,4758,5436,4896,1060,8371,426,5031,7991,5916,1335,1345,8444,2162,10791,4539,9850,7051,2270,961,94,4578,6521,9290,6724,3809,6306,1644,447,7084,9623,7811,8895,2410,2493,4839,6003,7889,8892,3674,1283,790,2118,1473,5070,9296,469,9447,2128,3252,3793,10408,466,9013,2776,10452,6135,9187,6662,8183,1605,3695,8077,9430,5955,2376,7511,1894,7304,8703,6063,6672,5462,4405,8742,10120,5593,2475,208,2431,2672,1288,2582,5519,2586,7843,353,8981,896,10275,647,10456,3689,3802,4317,6663,6118,4185,4708,395,4829,3538,9278,471,5010,8531,6345,3586,6688,120,847,8804,2494,6987,1125,7031,6879,2730,6717,1033,6709,1562,2274,9686,7735,5942,4863,10572,10404,71,2968,2067,5717,92,10412,3311,6833,5509,5794,4785,5042,3656,8366,3020,10494,2405,5758,10536,9541,1920,6696,6287,2756,31,10430,349,3571,7708,3037,5256,5321,10554,7529,9733,10179,5227,3702,9785,680,10140,4195,3381,9638,7385,3643,9165,10604,3355,8044,4603,580,7455,9274,6389,6169,8254,1630,4696,7633,10327,2173,8039,9220,10247,7041,6706,1492,2217,9186,9024,2127,7947,3550,7563,4143,570,5633,1816,4510,5549,10139,4149,4450,9112,4078,5910,10132,4414,3806,9204,10713,9384,7250,3444,2330,10071,4990,7959,5900,8010,695,2418,9349,2079,793,6022,8438,3569,305,8863,8888,1397,596,6355,5604,3795,4291,2731,5203,2908,7568,3939,8175,4569,1762,2017,3766,3623,8062,138,54,5884,2240,6735,4455,5961,7527,6660,9060,8855,5785,6392,7688,551,4768,10367,2546,432,6143,5703,7278,547,4440,4614,2031,760,9107,164,921,1262,1858,6701,4692,9105,4354,768,2359,2012,6566,7820,759,2144,10251,9156,6096,3370,2226,708,1514,5835,8811,10060,4184,9763,9838,2553,6095,2739,7630,4521,10515,8615,5831,2939,7650,6491,8999,311,8060,8381,9298,7748,4314,10356,6823,788,3150,5169,2525,7913,8479,2035,9235,5396,3833,6781,5501,2126,8389,8729,5767,8543,8899,149,3676,5028,9026,8523,3229,1778,2279,3562,6931,3968,2388,1586,2084,7646,331,2309,9358,9410,6419,8661,7981,2872,5966,3249,1438,1179,5514,9442,1421,7544,10463,10092,6813,7402,10433,2096,4264,3254,2188,7571,4693,9111,2183,3233,1270,9930,7201,6177,9326,6128,9272,10283,1420,482,8918,5904,1717,10166,4410,7447,10504,1500,7996,3302,62,3277,9151,7220,9469,3629,3965,3506,508,8207,7776,8825,6989,831,3439,8361,1092,1570,7605,3019,7205,3437,6454,3328,5547,6280,10300,5808,9936,4473,7111,8532,1368,10527,3775,10439,8290,3605,10169,2393,4774,6741,7802,7435,1375,4064,9210,1010,1274,662,6711,8865,3214,4099,10299,7104,571,3178,8753,2625,6461,2033,1379,5641,2095,5965,7249,6797,1947,9280,2643,8791,4864,9821,749,10294,5385,4132,8897,5726,3564,3558,6499,2032,2834,10634,10389,4163,8820,3038,6585,7673,5247,7352,9830,4171,7151,7824,2244,4848,10371,1211,8876,9133,689,7397,4352,1769,10832,6356,8081,2637,4926,7169,3409,8450,3934,1080,10061,693,3030,4740,5784,2907,5576,7288,10670,9125,8903,5886,6548,907,9950,268,4553,7424,750,5947,5377,5663,2235,3769,7615,4239,2501,6572,1559,745,1495,10273,10681,8489,4395,10550,4749,7145,2825,8650,10485,8134,10023,8463,3291,6412,9078,9224,9431,4944,3105,1422,10455,5507,4180,9075,9123,9438,6986,3671,8386,7507,7716,9927,5284,3589,1448,8503,7832,1584,8714,362,5802,5564,4604,10718,5437,4307,7506,8890,4913,4146,7823,7181,4280,7362,5384,2890,3886,9117,9728,10687,182,4183,10770,3859,10228,676,5251,690,1443,4927,2540,5520,9837,598,2513,8190,637,823,5487,4275,4862,9484,6360,8063,3762,1474,5032,6158,10419,3431,7602,9279,2841,1635,9928,8617,1959,5836,3440,4197,6061,2932,3375,7550,9791,5273,7964,10249,2579,452,2271,10518,6111,4096,1647,1753,4212,6798,5378,1700,7876,8594,4033,1869,5024,2620,9011,8095,6478,5881,9319,8205,5978,6779,8200,4074,8850,2269,4073,9222,3487,4519,2885,7160,6172,936,2458,7834,9468,7763,3430,1707,187,6770,7770,1055,352,7783,10737,9401,7167,1991,5240,1180,9121,1861,8395,2268,8957,5774,996,8605,3772,8940,4836,891,321,7988,6534,5671,9144,9301,9754,8849,2554,1546,7425,4746,10070,10742,2061,1782,6038,6250,591,1257,1391,1954,8206,6307,1903,9941,3093,4348,7666,2996,6904,9100,5694,10143,10663,1426,607,8201,10035,7107,1371,7747,2042,10206,5327,1992,8420,5391,6752,8538,9396,3479,7849,6514,6892,477,8969,1585,192,5669,516,1237,1833,2979,4488,9753,4362,176,1916,7215,5365,3213,1563,7973,5557,8685,2619,1200,3502,10104,4851,7267,8100,6858,3062,4062,9876,2821,645,9604,4742,7118,507,5987,2668,579,9459,6125,10813,10046,10202,70,227,9483,10606,10332,3198,3044,10672,774,2601,1251,5953,559,4109,6563,8803,4832,1094,4911,4514,4549,5225,9428,3931,9381,10578,8473,10305,1506,10,5012,2587,10195,10434,1162,3347,2556,976,5152,414,5445,7588,4680,1210,5566,4027,1651,10013,180,9734,4150,2216,5584,9168,6426,10533,403,6391,8317,7471,8773,6772,1072,2835,5935,4951,8382,2799,9835,8079,629,2596,8525,7123,9678,113,3992,10149,8364,761,3111,519,8672,1521,7119,1011,2057,3940,7139,1236,1450,1930,1307,1191,1147,17,8074,8428,1509,9136,7293,4515,975,1388,4537,5672,5513,2611,4156,1071,10105,4453,2998,9717,875,8084,3750,10030,9142,9784,7311,6895,2514,494,415,6325,8698,6626,2460,9933,108,282,10589,6610,4323,1427,4830,6348,26,9527,2832,2015,1383,7344,1961,1088,2149,3496,8083,4294,4105,5850,1265,4969,8187,1137,8324,1625,2413,193,5964,9355,8629,4366,7933,7456,1821,4327,9053,7859,6520,103,2420,4882,3084,4653,562,9979,1381,10099,2925,8667,10048,10482,6210,10637,5861,5958,7601,1765,5176,6366,7512,10788,4048,10513,8935,10186,1460,8398,6532,6028,3029,10258,7891,3903,4402,537,2062,1281,4813,740,7324,3040,4299,7668,3275,9071,8275,10532,9628,4580,7969,6649,5485,6042,4596,4602,125,8722,6015,758,5287,3237,3391,9664,4563,2981,7427,3951,4618,4312,2803,5268,9261,5992,8960,8376,375,1688,610,9393,2210,8111,912,4994,1214,4355,4733,3278,1168,5352,7026,5812,7390,3929,3718,9800,9249,6656,4098,7543,5415,5334,5034,2521,5228,1828,273,8886,3604,9425,7156,4574,9965,10712,2886,7298,10613,9052,4820,778,135,2132,10831,1761,6851,5039,3179,9452,2929,2982,9037,4683,9106,5150,7958,4585,5298,2395,10130,4221,4738,287,4891,2203,4572,4508,1697,5651,9191,1832,4295,10556,8454,1378,9741,4110,4301,3155,9897,713,5136,4681,9141,5138,6088,5157,10773,2485,9499,5644,1664,536,7580,8598,4088,6422,10257,6400,5551,1440,1534,4084,4633,3051,3949,3767,1946,2780,4946,2940,899,2278,10289,4063,2660,9418,9045,2250,9494,309,8245,8691,6616,642,2639,2985,1459,3235,7388,455,6579,1315,10260,9618,6197,6460,498,5290,5322,337,3025,6245,7440,3922,7141,10354,4576,4933,10292,9582,9429,2102,7462,5919,5996,4978,2333,8043,8149,2296,10028,9657,3012,6768,7692,1550,9824,3528,8871,5347,9179,10579,10297,5159,1654,7082,4167,6812,8540,11,4207,345,3210,2576,5075,10114,605,2340,7664,6736,4747,3473,3747,9183,9171,3850,5276,8994,9310,7149,7253,4103,3573,1185,1633,773,8528,246,3890,10347,9305,5312,9520,9751,8898,9154,10337,4966,9391,3889,1790,3035,2516,9318,8323,7868,1023,10077,1727,3032,10329,2922,4571,3464,9759,2545,1277,6285,4957,8397,616,7861,3542,9680,4689,7088,8521,1895,3741,8457,2856,5112,10635,7624,7665,10808,2450,4297,2950,6168,6967,4584,6179,10420,3187,943,7443,1215,2962,5896,5484,8689,4469,7147,7945,3607,2111,4479,3136,5047,4051,8050,8336,10015,2394,2573,4305,9963,4509,6916,3462,5254,4318,265,58,422,8927,279,5718,8069,6527,10620,4795,9766,4535,3189,7712,6998,10263,7096,6186,960,8809,5324,4206,4858,5143,4427,9101,4118,3352,2848,4776,10649,3175,9536,10208,2591,2944,7230,8912,7660,1614,228,2115,1385,7043,4437,8662,8349,914,10319,2130,8649,5249,8294,9152,1461,10162,1169,5198,9535,7018,399,7658,9755,4451,10107,3866,506,1526,3283,6274,7184,7926,2870,611,2884,1865,1984,10758,4449,7617,1706,2728,9443,7374,651,3592,8476,9028,9516,7453,3752,8774,6731,7011,6912,9047,10763,8654,3789,6093,2543,1009,3647,2101,3518,8563,1662,6686,2716,10072,4495,8603,10387,5306,9696,8508,8775,998,3658,21,9772,7972,10274,9585,2572,4289,564,8579,970,6209,6358,1235,1250,894,9579,5949,7248,1856,3268,3202,525,3406,6090,872,3916,9262,5442,55,7089,8429,215,4461,5749,9351,5559,8783,3624,2598,7611,8141,3432,2807,5399,1645,7458,4004,9925,7703,6684,8374,5238,9540,5565,6226,8421,5782,9113,6632,5582,1428,5438,1350,2694,3338,9419,1501,7360,1800,10573,6001,3238,9847,7769,7848,9593,7594,10254,5191,7481,7963,4161,9551,2355,6651,7846,9455,3480,6086,376,6604,9858,2286,4041,5267,6315,5875,6999,2967,9344,4336,8490,1314,3827,3736,8628,6305,152,5586,4720,8351,7064,7373,4091,3303,4390,4727,3815,900,3115,142,8009,359,590,341,10413,1075,5869,9595,2711,5920,9769,4541,4986,3219,6761,7431,1470,573,5976,5840,593,9598,10829,3984,9532,972,9550,9126,8584,2866,2593,6841,1923,7999,1463,2602,1905,4707,7600,200,10237,2819,5960,6033,10489,9243,1059,9768,9033,7097,3127,5065,2215,10745,6364,1545,10598,7391,7412,9577,4228,4783,9342,874,6,4732,725,6323,8143,8478,563,1737,7679,4868,7541,3339,10428,8035,8307,3828,5292,2491,6861,3474,9376,4984,8926,5745,9562,594,4802,1598,3637,8779,4358,5108,6053,8709,6580,7910,10266,8253,1736,9390,4560,489,5768,2368,6743,262,10490,3851,2991,1027,2993,581,3300,6809,3358,8496,3655,7176,8501,2559,236,1577,10019,1233,1926,9426,9050,4332,10236,9089,9992,2133,9230,2617,1685,2422,2900,4736,5111,8720,7062,1289,4021,3123,7415,9150,8806,2136,9501,1873,2225,5894,7800,9377,7845,9006,3649,8266,5101,5469,6318,4219,5098,5847,4170,2093,2536,6398,6137,5912,1015,8718,9812,7629,1341,292,8172,136,8723,511,6933,4351,3976,1641,5420,189,9427,827,5370,9382,9529,4914,3563,7957,6710,7138,756,6017,1047,4293,6606,3988,7432,9822,197,8112,1405,2121,4199,4715,10476,4870,5735,6099,1370,5826,2507,7893,6648,10108,8484,1360,6661,6402,7414,7349,7433,9251,38,4499,4619,3399,6136,5986,1976,5833,2301,1948,167,4844,2327,2438,7444,3919,6396,5456,8590,1446,9861,4611,5058,10406,5656,6771,3516,902,8252,8212,851,2759,9021,6530,7577,1972,9201,9295,1219,5407,6493,2629,69,828,6146,9740,4086,3705,281,3975,5756,1244,5418,3704,8989,86,2926,1859,8158,3617,8800,4685,6262,1679,9940,937,5550,8102,1276,7350,1571,9969,4958,4434,8343,5607,4620,166,1456,567,6311,10542,8218,6767,9392,7219,7671,5179,2052,8174,8196,67,781,9413,5362,6840,643,4987,2612,6334,4817,5097,9712,955,2933,9275,8013,9334,10397,10535,37,1613,4085,2753,10784,6405,6631,2175,5059,1539,191,10242,6235,7228,9103,3131,6708,8213,10648,6410,5213,3334,4472,6277,2802,7632,4657,4442,2056,4153,2267,3632,1451,1476,3641,8313,2099,6923,8627,1022,6041,6387,9817,1756,379,8140,4887,6268,1917,9978,6062,10498,4029,1975,9922,10569,2665,2254,5206,3326,9282,3374,7677,8413,1855,2299,2408,7897,3778,6217,7077,2816,2473,5573,6249,9184,10079,7537,7300,3419,8664,289,7896,2468,249,4841,3267,7504,7545,4286,5552,128,6726,8577,2406,314,5839,3893,9215,5132,7875,2651,10559,8024,1497,9697,2241,8407,1964,499,6587,932,7341,2089,10213,1286,229,1433,479,7755,2581,6570,4124,969,7251,501,8260,2028,4534,6417,97,10145,8847,8816,5296,4292,1552,9161,3846,10765,1311,10296,8146,6473,7175,3693,330,4928,6052,9130,1208,3918,6785,4164,5763,4497,6397,3917,1198,8173,4090,3498,517,2969,10168,8156,1104,5622,10497,3515,8222,6551,10128,3380,3979,8882,10467,7829,3410,883,10431,9639,5791,9720,8700,4792,9247,5727,3771,5858,8546,468,548,3500,8646,6453,746,10446,459,9674,6253,2120,8472,1044,1650,3860,7778,9998,5829,10804,2498,9094,8331,10089,4561,2108,6815,667,4892,3389,6221,5341,2955,6174,10630,2451,6956,7773,10232,6549,248,528,6597,1411,60,6669,2114,8455,4920,6027,3726,6516,8474,9508,2363,9716,308,3039,4895,96,5476,10205,916,6195,5262,7131,3511,5516,1430,9217,4191,7301,4448,5598,9093,6161,7025,9710,4767,8055,5773,3843,4960,8705,7826,9457,7524,4359,4346,1102,4466,3310,9339,5282,10191,5460,7694,4042,7844,4502,9644,10333,3838,669,7428,515,7564,8854,6567,6725,177,635,367,6803,8358,10734,4586,10313,1124,9949,2453,7949,7050,5678,577,4640,44,5107,5661,4439,6409,8136,5356,1025,7595,8736,10384,7525,4728,8608,4290,2712,5538,1217,7872,5702,5635,1479,8072,6834,2928,9143,3770,9283,10200,3468,7333,6922,8663,5660,2595,9920,6428,3036,7012,5957,9315,3118,5788,10378,10659,595,6255,4779,3224,10510,4991,6175,4936,386,8901,6586,4837,4694,1324,8350,9647,3829,763,784,5687,4380,5969,10198,1732,8326,7191,1788,3282,880,2502,2461,2018,5962,5272,3776,7171,2003,1911,8239,2915,1794,5811,6839,3163,7475,2860,9085,889,1774,2958,5517,5532,10285,5775,9919,5335,7888,10738,8497,3758,5004,1498,9309,91,3677,9557,1617,9314,7597,9792,5037,8793,7644,7967,1365,5994,7079,3162,710,5926,2196,10651,5008,2852,5981,7911,9780,4811,3863,4833,4155,4298,4028,6294,5193,558,2574,7150,8423,2954,821,9669,9181,9241,4593,8702,3220,4505,10210,10705,846,1804,10746,9747,6938,2055,3611,1716,7874,6617,9745,5081,10009,1945,1152,8377,1658,929,5765,7567,4724,6374,7,10138,1007,5086,8179,2255,9614,6784,2853,2512,9010,2920,7871,6825,6697,10373,3485,8277,7100,3344,2037,782,7048,8166,9967,3614,4043,4385,8769,3204,9353,8285,10679,10219,10586,1066,6420,6537,6511,7608,2871,6819,8934,6083,1710,114,10793,4859,3472,6354,8229,9302,8873,9515,213,6472,3069,7188,7021,6505,5882,5336,8157,3067,1158,1496,8596,3936,7812,30,4850,10234,6368,2522,153,8049,7619,5241,6557,3087,10706,2166,6722,3835,2943,47,6519,9917,1866,5795,5448,6046,1439,3708,2882,3215,7291,4729,6786,2648,3608,3933,1703,7121,8625,5303,10597,814,100,3820,372,119,7357,8070,2649,10617,4856,3024,9534,7791,1436,8375,10695,5721,1105,6266,9054,10593,3512,7883,2072,8500,8309,5612,925,7466,10192,10040,5915,7078,3628,4342,1767,10377,3877,9587,461,9985,5500,8868,10111,2000,8248,2206,9409,655,8586,9321,8341,8832,1781,2078,6787,35,6558,9146,10082,2452,5000,5699,6485,3151,3011,5526,8469,6555,9335,3279,10665,6256,185,8316,277,7637,5619,4223,234,4388,8198,10435,5905,6021,10753,2847,3698,2253,1352,5624,832,6471,3662,134,1540,5657,4901,4205,715,4241,1715,7196,9765,4512,5736,10537,8573,7234,5730,10549,9968,926,9986,4413,5880,10809,5544,5892,3085,1880,10306,3259,3973,3367,7780,1839,982,9099,8581,8600,772,5302,2219,7863,3913,6085,5572,10369,4142,6134,2764,5291,9476,1551,6303,10702,4007,5317,2989,6937,1854,2201,6182,3074,4344,8782,6641,3999,5461,2150,8908,2677,5364,1965,1531,115,7549,10581,6019,8943,2749,2161,7470,7275,2151,5470,2851,1814,2445,10360,3520,5814,10055,1019,7607,3620,502,4883,10574,1389,3260,7108,2171,5411,1681,1409,8719,5755,10749,2655,9300,2511,10037,5011,9528,3508,4794,1084,3548,3125,2030,10643,6004,2534,1457,10256,8066,2232,4069,445,10766,5265,8169,9654,6113,2942,7995,6605,3403,8576,8291,9193,4036,2599,945,3244,4656,787,9645,2746,7749,10324,7299,6793,5089,8885,10348,7792,8529,3527,1888,9002,4666,7269,10622,7815,5846,3578,6406,692,3045,6369,8776,3072,6071,6746,7526,6900,5062,8011,9658,392,5817,5623,10453,8214,2722,4411,5443,1181,9498,1028,2138,66,5073,5605,3540,6440,7271,170,1188,1163,7934,1751,2224,6073,4,3052,1065,1369,1801,2499,7895,5346,10802,350,2988,2849,3619,4937,6455,10156,10041,7690,8103,9369,8619,4257,1282,5676,9827,1138,8631,5201,1099,9032,3668,9673,2146,5226,6251,895,9841,2005,6152,3927,7243,7448,4544,4231,7083,3731,3272,8293,5116,9350,8711,9137,2167,4407,4065,10047,7584,5931,8064,259,10301,9884,6692,6500,5844,1835,8021,1881,1673,6677,240,785,5465,4258,957,3742,4189,7040,1892,5083,606,9132,5506,2320,10501,3915,8530,9147,666,2187,225,10096,6074,4974,734,8852,8515,6694,7059,4056,2024,5798,10686,7659,3402,9990,9934,7364,3257,7438,165,8637,6240,4484,3126,3714,405,196,3514,3900,2804,8284,5045,1564,2754,10158,974,1173,2859,9233,10103,3839,3885,7606,9202,4044,2364,9798,7922,2318,3737,7164,8480,6543,9952,7860,1149,6778,4773,9474,6924,3407,983,9379,8728,7336,3076,8772,3110,8065,7080,10509,5148,7256,9648,9701,9832,8517,817,4202,3455,8453,681,458,8549,9836,3897,2356,10783,8974,9048,417,6302,8208,4721,5451,9620,1996,4139,7816,5421,8796,8119,3147,9465,6450,2068,6112,3167,10486,4807,532,5753,5824,9034,2304,6309,485,272,8802,416,5807,5742,8325,9739,9890,9625,7342,8585,3967,7757,843,4900,575,4814,9771,4523,6800,8942,5117,8707,10279,4430,6997,9199,1582,9292,7404,9976,3531,10093,3523,219,6155,6006,1632,9042,10422,10083,3673,9698,4546,7662,6059,1699,8880,8435,9994,922,2486,911,3114,660,29,1005,4520,1815,8202,10382,5234,8950,2503,8330,9019,5295,1860,1366,8221,614,7884,1955,174,4536,5974,2708,2307,7738,9242,8706,9039,7446,2159,10756,112,3003,9493,6167,9862,2903,9159,7519,7439,2614,6084,6072,8822,1838,3239,6675,9422,3964,3033,6899,4754,3963,4853,3710,5040,8080,4061,3597,2109,2123,1595,6211,5146,6791,4827,9996,4144,5569,7942,1159,9943,7572,1639,3672,10359,10690,4525,6196,9481,738,716,4489,2685,10147,4248,8583,6457,3387,10614,10364,993,1037,1404,10080,9258,5580,10403,3854,4116,5118,10669,9891,9737,5665,2469,10241,5489,6971,8936,6880,5977,5649,9228,3158,7784,5115,1574,9316,1797,8385,835,169,2288,8864,4121,6575,4101,407,2421,3898,5395,5616,3453,1454,5431,8122,156,1864,87,6008,7652,6427,7356,2260,1090,4823,8447,2457,10280,1631,1538,950,8406,1279,2966,7552,2347,7259,6394,10560,5868,3560,5823,3451,3046,5019,2414,7785,7929,8217,3320,4226,864,6148,6673,6490,3400,1310,9492,2348,4415,968,10323,3990,5776,9900,9336,1046,7148,8798,5151,2960,6178,6623,1204,4454,1760,7168,6117,1146,10719,10458,1649,5397,10282,10584,9794,3475,10402,10392,10762,3270,413,1759,9711,3664,3306,5274,8678,652,8665,8416,6408,8046,1624,5865,9687,8657,5888,1062,2184,1164,3280,9055,3097,5372,424,1287,5887,4647,7493,6962,10740,5382,6805,4082,8845,3590,7270,2228,2630,3021,9731,524,4700,378,7771,10656,3398,6644,990,1322,10016,6855,5261,10799,9323,6720,6162,1252,7898,487,4787,9439,5898,7813,3728,7689,8353,45,8231,2275,1665,709,2700,7044,6229,9875,3411,1846,5567,9855,6894,658,448,8410,1416,978,9570,5480,2329,290,4719,6830,7788,5473,8767,6832,2750,4270,2778,4501,1530,4417,7262,9398,3441,8026,2141,6700,4803,4545,3201,8518,9432,5734,3969,9904,8241,9760,10652,4558,5677,1535,3251,3971,10052,2199,5751,9956,7627,3299,10078,6578,4000,2567,1193,1453,2231,1398,1714,1507,2613,8977,1701,4465,5809,820,5049,9517,5595,5917,529,4815,224,3910,7054,3727,1939,7937,6884,9636,8394,5682,5405,9487,4907,9554,9743,10322,6208,9414,8843,7213,7505,10188,7604,7837,9860,10351,2087,14,7212,5999,6474,209,9842,10468,3271,741,10611,5142,4020,1870,10141,2070,238,4670,3450,3188,2479,8304,1263,9502,3330,2361,9077,160,4884,2948,2011,1032,1652,9131,2428,4917,6908,8219,6141], "time_ms": 0.139556, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 613, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "generated_query_count", "value": 1, "time_ms": 44.074907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 428, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "generated_samples_per_query", "value": 5511955, "time_ms": 44.074907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 429, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "generated_query_duration", "value": 119739, "time_ms": 44.074907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 430, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 1357.863124, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 898, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 1357.863124, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 900, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 1357.863124, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 902, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 1357.863124, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 904, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 1357.863124, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 906, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 1357.863124, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 908, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "power_begin", "value": "12-25-2024 04:31:06.289", "time_ms": 1357.864196, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 564, "pid": 20263, "tid": 20263}} -:::MLLOG {"key": "power_end", "value": "12-25-2024 04:31:07.593", "time_ms": 1357.864196, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 566, "pid": 20263, "tid": 20263}} +:::MLLOG {"key": "loadgen_version", "value": "4.0 @ 486a629ea4", "time_ms": 0.007014, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 53, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "loadgen_build_date_local", "value": "2024-12-27T19:45:31.956819", "time_ms": 0.007014, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 55, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "loadgen_build_date_utc", "value": "2024-12-28T03:45:31.956827", "time_ms": 0.007014, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 56, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "loadgen_git_commit_date", "value": "2024-02-15T15:46:50-05:00", "time_ms": 0.007014, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 57, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "loadgen_git_log_message", "value": "486a629ea4d5c5150f452d0b0a196bf71fd2021e Hotfix: DLRMv2 Audit Test01 fallback failure (#1626)\nf9a643c0a0e920588da1b51a1d822e1071a9dbec Add number of tokens to offline (#1623)\nf06b920eac2170df38b5502c7443f50c9afa6045 Bugfix: equal-issue mode on offline causing accuracy run to fail (3D-UNet) (#1624)\n0ed5190e82e1ca1cf95626e50e61cb6678c7f2ff Fix submission checker and TEST06 for Llama2 (#1616)\n817dd961221d3ca284b024368130bd5dc5e4b54a Turn equal issue mode off for TEST06 (#1615)\n840435a1abe8be823e0a5a18ad3173bc520aa24f Update README.md with CM commands to download stable-diffusion, gptj and dlrmv2 models (#1604)\n3190d09a148bc1d72c9d946e4cf3b54db0d0e60a Typo fix in README.md (#1588)\nb0777f0bd3b7e5cd6a51d670d8b4df656cf6fb29 Update README.md - remove unwanted lines in CM commands (#1601)\nd7dba0826ac96a0e8bcae525086961e47c409a38 Remove loadgen warnings (#1608)\nd45a66c33bb0b6548d6172b38645bf94ffe24b39 Add upper limit to tokens per sample (#1612)\n44285d9a44041c3d4f7df0acf1e72c375e9fcbd2 Set completed samples per second as llama metric (#1613)\n357ccefca4db55ff373e029d5d1e3e6246bde5d4 Enable equal issue mode for LLM benchmarks (#1610)\n104d85531af7ae2a07ded36dd6e8c7f2c7746d06 Add sample length check to test06 (#1603)\n473053fc9f1b8c5edf60e11ec74607126938fbc4 Fix loadgen token metrics latency constrains (#1596)\ncc3daae3c7f18d7eedc5ffbc271d324e5aca6233 #1598: fix token and sample logging for Llama2 when accuracy_log_sampling_target is enabled (#1599)\n3a902e598e5a17013c26fea975c0a60e6d7feae1 Add support to dump 10 compliance images during accuracy run for SDXL (#1591)", "time_ms": 0.007014, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 58, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "loadgen_git_status_message", "value": "", "time_ms": 0.007014, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 60, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"05d4a62d355a90363847358e58afeee4a54ad838","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/bindings/c_api.cc":"32181da9e161c285f8fe46ddaa49e6cba2f9f918","/bindings/c_api.h":"91f58bd79b83b278f3240174a9af747fc38aff74","/bindings/python_api.cc":"1dc0865130371c6a8ddb97cd8039244d74d6fe46","/demos/py_demo_multi_stream.py":"6ccf7dcbe08f8382ed2d3618370c2b21e205ca13","/demos/py_demo_offline.py":"6c3075538e0a3df5e93edee917f4537099759e11","/demos/py_demo_server.py":"40e37ea121ac08b21ed6af41aca8eb5eda641b89","/demos/py_demo_single_stream.py":"4ee5f048fb0dffc415342a2bc2049fb155bd2894","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"126e952d00f4ea9efd12405fb209aa3ed585e4b2","/issue_query_controller.h":"923d9d5cdf598e3ec33d7a1110a31f7e11527ec7","/loadgen.cc":"6650091ba7a918f343b06eb7a5aa540eae87275f","/loadgen.h":"e00fdc6dbc85a8c9a8485dbcbfe2944f81251c4e","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"197efc96d178e5d33a750d07fa7b2966417506ea","/logging.h":"ddb961df7bcc145bcd7cce8c21f7cf075350dcbe","/pyproject.toml":"974adb632af4b76a1c72b914cc2cf9b3cbc66aa3","/query_dispatch_library.h":"13ad6d842200cb161d6927eb74a3fafd79c46c75","/query_sample.h":"e9187c8612bbdc972305b789feb6e15c26e96cfe","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"a170f991de69f8a5e45616e871b39a30ffed5598","/results.h":"d82500c326c2de83db411f1146882aa4692b419c","/setup.py":"e3e707e1191478332c00f1b6ff806f0ef98e0f65","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"aaf3803e700f3431a5ada72559eef5cce996ea64","/test_settings_internal.cc":"42aa46b17ac3fb2748cd5cde60fa8742e916f3c8","/test_settings_internal.h":"73b11c02228f3afb4c924db7a86fd8a2b5b9e483","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"4c9d98360b507d5673049ed70154fcfb2934f8a6"}, "time_ms": 0.007014, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 67, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "test_datetime", "value": "2024-12-27T20:25:45Z", "time_ms": 0.011314, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1198, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "sut_name", "value": "BERT SERVER", "time_ms": 0.011314, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1199, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "get_sut_name_duration_ns", "value": 359, "time_ms": 0.011314, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1200, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "qsl_name", "value": "BERT QSL", "time_ms": 0.011314, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1201, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "qsl_reported_total_count", "value": 10833, "time_ms": 0.011314, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1202, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "qsl_reported_performance_count", "value": 10833, "time_ms": 0.011314, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1203, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "requested_scenario", "value": "Offline", "time_ms": 0.015878, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 268, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "requested_test_mode", "value": "AccuracyOnly", "time_ms": 0.015878, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 269, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "requested_offline_expected_qps", "value": 8360.64, "time_ms": 0.015878, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 306, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "requested_min_duration_ms", "value": 600000, "time_ms": 0.015878, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 312, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "requested_max_duration_ms", "value": 0, "time_ms": 0.015878, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 313, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "requested_min_query_count", "value": 10833, "time_ms": 0.015878, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 314, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "requested_max_query_count", "value": 0, "time_ms": 0.015878, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 315, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "requested_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.015878, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 316, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.015878, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 317, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "requested_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.015878, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 319, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.015878, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 320, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.015878, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 322, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.015878, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 324, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.015878, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 326, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.015878, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 327, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.015878, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 329, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.015878, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 331, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "requested_performance_sample_count_override", "value": 10833, "time_ms": 0.015878, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 333, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "effective_scenario", "value": "Offline", "time_ms": 0.015977, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 409, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "effective_test_mode", "value": "AccuracyOnly", "time_ms": 0.015977, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 410, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "effective_samples_per_query", "value": 5518021, "time_ms": 0.015977, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 412, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "effective_target_qps", "value": 8360.64, "time_ms": 0.015977, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 413, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.015977, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 414, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.99, "time_ms": 0.015977, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 415, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.015977, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 417, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "effective_target_duration_ms", "value": 0, "time_ms": 0.015977, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 418, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "effective_min_duration_ms", "value": 600000, "time_ms": 0.015977, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 420, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "effective_max_duration_ms", "value": 0, "time_ms": 0.015977, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 421, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "effective_min_query_count", "value": 1, "time_ms": 0.015977, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 422, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "effective_max_query_count", "value": 0, "time_ms": 0.015977, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 423, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "effective_min_sample_count", "value": 5518021, "time_ms": 0.015977, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 424, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "effective_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.015977, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 425, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.015977, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 426, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "effective_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.015977, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 428, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.015977, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 429, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.015977, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 431, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.015977, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 433, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.015977, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 435, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.015977, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 436, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.015977, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 438, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.015977, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 440, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "effective_performance_sample_count", "value": 10833, "time_ms": 0.015977, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 442, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "generic_message", "value": "Starting accuracy mode", "time_ms": 0.016975, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1090, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "loaded_qsl_set", "value": [4117,9486,2947,9270,8227,4487,9799,4629,8571,8839,7101,2191,3315,6375,207,10454,3122,9490,4083,6119,1890,7657,9347,344,201,4232,5741,5454,9533,7842,1596,5330,10615,8827,8094,6714,9169,977,6619,9924,4982,4894,6544,10118,783,766,2122,7163,396,4200,9864,4254,8440,1758,10722,2829,2788,9906,2719,5613,8726,7847,440,6103,8567,7497,4793,1184,8675,10587,8270,7074,7408,8740,9265,4855,1199,220,6556,6595,1226,6695,3066,2143,6264,9806,7122,5007,10075,2370,2185,7254,77,2806,6243,6687,1156,1802,7302,391,3348,8159,9708,251,3595,3947,2302,4690,122,3116,1300,3768,6388,2986,728,5127,3904,6807,6259,2218,7555,8821,3580,7804,1321,8460,5419,6542,7833,3365,2290,6346,7952,918,707,118,10469,9023,1304,5220,1891,6878,9524,6522,7867,4037,7698,8507,9634,2145,10057,6263,7486,7557,2401,6082,3434,5927,8708,10081,8114,1349,7742,2742,7157,5705,4638,1764,3651,920,4016,2744,5859,6740,5380,9649,2,5245,5230,6421,4071,8909,6319,288,9346,9671,9646,9360,4702,5044,9157,6296,8318,8971,8184,7702,10475,7318,4024,5530,3991,1171,1909,6230,8461,8099,10563,5398,10760,10693,3688,8170,4952,6827,5893,7767,2656,10032,9178,10480,7195,9359,10608,10261,8975,5432,363,3346,3049,3360,4432,9736,473,7587,9416,1021,6596,10352,2666,2325,973,2071,7368,4267,867,1040,5796,3491,8237,9597,3489,10568,3777,7899,856,4818,2858,4662,777,10776,10026,6065,6481,2590,6780,10553,8789,6032,930,1049,10178,6187,585,3359,5832,2014,480,5866,5017,5141,1525,8271,8132,1642,6270,2265,140,1174,4641,8017,8844,6569,5575,7081,3779,3845,467,9761,3314,8653,4250,7578,8907,664,217,8419,8757,683,4565,1638,5216,2004,6679,4485,7683,5180,7422,6070,6283,2909,6281,8933,10639,4340,3079,3513,10552,2291,7487,2680,8807,8370,6127,7434,1822,908,8101,451,10316,8282,10724,885,1834,1556,6121,1468,409,1908,10539,7238,2891,10312,9435,2178,10051,4522,9650,615,7193,7796,2008,6011,4347,435,6881,1396,844,971,4216,5001,7886,6875,1348,4300,9348,2709,10580,9993,6705,8784,10821,9232,7724,6581,9764,2913,10125,6914,7693,4568,10526,10231,8502,7825,1783,795,6828,8006,4595,3892,3075,1278,10050,8235,4679,9971,75,2164,1175,3684,9869,8414,9016,765,2721,5786,10703,1326,7463,237,836,3912,5482,7835,4374,7661,4070,7754,7180,9945,10415,3638,4208,939,1097,7628,5725,3723,4547,1994,5232,5279,4049,7309,5963,10478,9834,7758,9463,3666,4663,801,4030,3010,8236,9237,2470,3382,6336,7570,190,4552,4113,2229,9018,7038,5021,8701,8268,4490,1373,6092,10181,10445,3056,4333,10394,1640,2328,3349,10069,10683,6439,3191,2544,4760,1739,8900,3,258,5416,4671,2041,9375,2661,6376,2306,5897,6373,2125,1940,5109,4256,6203,6411,9311,8029,2200,6301,1734,6176,6443,3007,9583,1841,4627,2523,4306,2801,5208,8607,4961,1467,10700,4031,2181,9958,8194,10295,9043,5366,306,9056,162,3196,10759,4349,5401,7030,6184,9216,2069,6357,5640,3015,1752,406,10049,3983,791,5610,5944,304,6380,1937,7862,9366,9365,1607,9252,2698,10350,5221,4847,10707,3023,1615,7017,5990,4045,7231,1202,7182,5172,3176,9612,6693,1013,10334,1805,8730,8841,7372,10094,6010,3216,8177,7406,9472,9608,2533,7691,3959,1623,5144,1831,751,4912,2152,8144,241,4187,10001,1924,8402,7877,8928,7294,4915,3876,3242,8996,7153,1000,10658,3946,284,5088,959,10661,1018,8931,546,4262,6489,3786,5278,4308,8178,7638,5211,7142,5771,634,8273,263,2548,4674,9910,624,3290,5925,1524,2642,7198,10345,3539,6719,10735,2006,10233,3816,9578,8624,6769,9661,6468,8952,9702,8670,5175,3449,1773,6130,636,1622,1696,1414,6523,2898,10650,4420,1280,2538,7880,3743,1227,2757,6131,3371,7445,1680,2901,36,6868,10811,7140,1900,10398,3724,8765,8278,7379,6040,7914,1580,3535,8033,7536,6480,3861,2437,5082,2881,10596,4902,8162,9637,4886,411,7189,1789,6704,9176,10045,4107,6333,4583,3923,4843,9935,9955,9297,3222,8257,6365,3095,1995,7924,5428,8640,6628,4819,7152,5016,7932,3226,1267,1225,9025,7878,6321,2373,3801,1399,9063,6506,9436,5781,7500,423,3588,7110,4607,3476,2477,3635,2425,9394,10449,4054,8148,5510,2236,9543,10499,6153,780,8692,8105,10268,172,4128,841,8925,5453,8561,2135,2976,2845,2209,7418,10817,5620,7010,9960,474,6545,8223,9713,205,10548,905,2396,2914,5591,2887,3379,4865,3304,5849,1309,10171,608,7314,6228,6961,7068,6943,4613,1317,121,8611,5902,5827,8042,1139,3121,223,2984,2897,3144,5379,10590,6433,3871,5475,43,1026,1192,1599,5729,6194,5838,1254,2054,2706,7925,6060,2177,10647,7727,5681,10000,3549,10814,4433,2100,6869,6593,9404,5534,5801,10321,9145,8363,7451,9072,4600,5096,10372,3022,679,5631,2552,723,4668,9613,10153,7423,9114,2973,3071,1413,4956,2838,5084,6777,5196,539,5036,382,7480,9327,323,4766,2273,2684,8182,10134,9203,2667,9185,2658,9087,6286,4162,6620,10472,1095,6759,7216,8191,9738,9548,5029,10326,5478,4089,8710,5439,6126,6160,4828,1006,1677,3546,6930,425,2645,4573,2770,10248,358,4975,10603,8129,8614,10180,7902,4303,2447,3427,753,8920,9449,256,6859,5711,5246,623,370,10767,2563,8300,9591,9991,5126,9568,2647,484,8644,3654,9507,8632,10076,9813,10638,4326,3862,569,7020,4213,5628,9341,839,3341,8347,5163,5563,7339,6928,1197,9158,3099,6120,1483,9545,4608,4215,2761,1259,6802,4364,4001,1606,4222,4752,3057,10022,1108,10592,4398,10376,9888,10033,4273,1238,3428,6553,6145,5731,7273,604,2436,3170,8673,5901,10074,4513,8305,8697,7476,5015,2669,3100,6344,9338,10383,4972,8727,1363,3223,10495,897,1698,9553,5662,53,9328,3733,9932,7102,2212,7382,7159,4556,10748,9294,5895,2262,10512,5913,8314,10624,2530,7560,10464,2443,8787,3800,1510,2027,9795,9670,283,2351,4260,2577,1518,906,3377,4764,2113,10699,3948,8465,9264,9267,5600,2924,3783,9209,7966,6464,720,6973,155,6227,7994,2636,535,8846,2931,2046,4953,10058,8228,5200,2211,4234,2207,8121,2007,7268,3212,7772,6496,10201,2762,7534,5539,10555,10496,628,3653,702,8683,10325,8660,8411,9387,326,8842,837,9688,3855,7053,1826,7944,1053,3193,456,361,3206,10315,2059,6612,6173,4562,9003,8034,9162,377,7551,10769,5354,6299,5997,2080,6288,6068,653,9937,10825,8833,3247,3058,4923,9104,7983,3305,9322,8339,7711,131,7328,5212,8623,10761,9718,5402,948,10640,8659,6479,2768,4616,2223,10625,4252,9980,9807,5318,6188,159,10655,3565,1478,6860,3521,4646,9880,1608,10427,9804,2092,4897,10238,6020,7642,8091,7789,4058,6681,4173,7918,5523,2444,9898,7161,7736,4667,2603,9370,7403,10797,1857,3276,2276,3445,8197,991,1863,4266,2170,4331,3296,6658,1308,7542,8458,4533,10177,6873,4190,815,2726,5589,2846,269,2634,1560,7067,7554,3102,4908,10741,10086,5286,8734,3824,1744,4111,3264,7494,3825,8834,6139,8432,147,1589,1871,941,2911,6018,7906,2874,3713,9523,4023,7522,6116,6864,5698,5464,3924,4590,7345,7395,4790,10540,1020,5309,5950,2956,4983,6163,9194,3848,3612,10525,9069,5253,7232,6876,5003,5067,2350,10375,10523,850,6399,10002,9689,7255,1016,3707,3384,2137,4242,4077,5170,8388,2387,10577,2482,3735,4313,8862,2029,8452,10528,6058,6871,5233,1195,3301,8738,3194,2638,2488,5009,6981,7935,10684,10631,1043,1962,3541,4973,9581,4285,9118,9770,8279,1316,6284,1161,877,7894,3981,2423,6872,5033,1798,6181,1255,7517,4067,3319,7457,5164,3977,1719,4436,9317,4765,4137,6515,10025,4628,3745,7739,9445,7016,1980,4481,8580,175,10704,10653,245,8163,3834,7006,9135,1950,4401,2289,1412,7325,4821,6917,1361,7569,6782,2474,8075,5757,1034,3353,8570,2678,4922,4356,6831,3401,10393,10508,5161,2635,1472,3331,3048,9526,5383,5670,7508,5113,10148,2912,8547,1183,5496,5139,1153,6897,7997,6625,9819,5766,7460,10124,6637,327,3466,1848,2455,5939,2282,6371,4812,6721,5314,4780,3112,5154,250,7103,7158,463,6920,8368,6024,6877,342,6799,437,2876,9690,4889,5129,2631,3366,8380,1528,10135,584,6292,9180,5863,10031,2432,980,7211,3117,6909,1248,3761,3601,4976,9420,10602,178,10755,1150,2009,6653,7378,4422,8445,1978,2390,5417,4353,6452,2002,10595,1306,2817,8735,8424,8976,4396,4919,9488,534,4478,8211,8264,385,8288,3063,9825,6996,9726,7239,668,2896,6212,5761,6271,618,8216,6838,195,216,465,8870,6774,8556,3392,786,2791,1096,10565,9399,7603,1723,4133,6106,5491,10443,335,8387,5006,8131,966,7321,4664,5361,8778,8085,10370,512,4408,9227,4676,927,5805,10457,5444,5333,2951,6804,7955,888,3706,8788,2895,1902,9874,7221,5222,10628,6300,3522,65,10112,7284,6826,9762,10226,8642,8030,609,3812,9914,8276,5946,4130,767,4876,3831,9441,5515,7928,4838,4550,78,6682,63,600,3798,8226,3577,8618,8919,4716,10385,5560,7695,9677,9470,6351,10585,6031,2941,2010,7623,1735,6574,1513,2774,1490,2303,5429,141,1915,7737,9086,2205,32,7165,4564,3715,8209,8559,4979,4985,7143,1656,6789,8025,7208,6845,503,3687,7028,5471,9975,5664,3404,9068,5248,2433,322,10390,3840,8058,1297,3551,3819,3342,7265,3266,8914,4394,3583,9786,9198,5609,1444,8947,3060,10252,5594,1291,7319,9559,3166,4669,9592,661,6602,4789,9415,2424,4265,7098,1876,7535,1203,10053,9809,3104,1941,2705,6635,3148,3598,1502,7383,9073,10425,6590,5162,3529,7306,10800,6393,8751,387,10172,2692,5719,2741,7855,4845,7039,2459,6901,6624,3554,4237,3867,10566,2116,531,5943,6690,792,8441,6260,682,3995,6154,4165,9066,2717,1842,5340,8261,8539,1074,4259,701,9500,8648,513,3417,2155,9041,116,4010,1218,7514,3738,10176,8592,4659,8491,291,10417,8338,7244,862,5005,4735,1362,964,9714,10691,9829,10779,351,4930,16,3561,5388,5985,5239,6821,4320,10682,3154,4367,8961,50,8733,4288,9893,824,2504,404,10529,6560,1489,8125,48,76,909,2237,2691,2416,1987,7699,4599,5585,10503,7807,3639,3994,5708,7135,4625,4516,5540,2916,5528,5975,7413,8915,1347,10065,706,5304,3436,6607,8262,5645,3625,1376,4476,2934,1592,1935,3958,5821,2456,3362,5367,4151,981,9573,10610,9207,6190,5275,4631,1708,1921,9287,699,1447,1636,696,3740,4597,3373,9444,1221,3780,8124,5381,7648,3536,10571,654,9064,9044,3503,9854,9626,3378,10675,10629,6080,4445,9605,2646,6982,5074,3618,52,6561,6185,3026,5968,9742,989,5867,9190,1178,2689,2001,6149,8334,2049,3180,5759,111,4691,2650,8777,3149,804,1868,5328,840,8687,1382,57,2419,808,10355,5588,5215,1403,5701,9,7853,8520,8669,8019,4006,9097,226,2827,1135,1901,979,819,2561,7049,504,4391,4400,6739,1402,5264,2230,10666,2566,7430,8459,10720,8794,10310,10174,443,6349,7310,533,7609,6200,4108,613,7558,7709,1553,5076,10645,300,2842,6261,2324,588,2193,2945,7510,7396,3094,1889,10781,3791,554,9284,9148,3519,9995,3209,5810,8322,9519,2560,3073,947,2781,7004,5548,7116,10152,10121,552,1887,2952,8739,9225,5819,3712,6550,5446,574,3090,8327,428,1266,8433,8383,4645,8754,6898,2652,6822,1455,2176,1519,8566,10303,6049,10068,4977,3894,7561,7488,5536,7224,7793,1989,3985,8695,3424,7035,7612,9938,6775,39,1850,7803,9174,3031,6441,2974,1603,7941,7223,7923,2831,2977,3080,2075,9236,312,8565,4634,8814,1558,10484,1272,7389,8872,10418,4755,7869,2610,10633,5046,6960,4193,7852,1754,2878,1936,3784,10314,4025,4805,4997,10007,9863,7155,10101,1342,9395,10291,9385,10218,799,1123,5843,2562,4905,1377,9226,5937,1986,7591,9962,1189,8902,619,9067,3458,7586,7210,328,8506,3000,1004,9212,4591,8020,8887,4012,592,101,3415,2527,72,7452,9400,7809,1561,7032,4840,7008,1056,8,5752,9588,3101,3106,3609,4503,8312,6536,3446,8745,1437,7547,10287,4816,1795,8004,5837,1042,9329,9008,9433,2047,1087,3421,4324,10588,6699,10747,7765,9719,10438,887,9062,6820,2946,4741,527,3043,1648,1133,2918,882,684,4939,4386,8641,8154,1323,6436,7873,10752,3686,10680,807,9886,549,3230,6147,294,7120,354,8171,8493,2663,6863,7489,6037,410,9913,124,6312,541,3159,10304,2117,735,9882,199,4315,9250,4055,3739,3826,9558,10782,3699,542,6645,7593,5123,6413,7532,8805,1517,8514,3396,9239,8883,6748,6002,5319,8690,6308,1691,4655,589,10278,5680,9489,6806,10465,4251,10142,538,5740,194,9167,9974,2107,1480,8439,5722,5674,5712,7528,3385,1906,10567,9057,340,5707,9308,6207,3078,2818,7518,6794,9555,9983,1676,1678,6079,2038,1670,2608,2335,10021,10243,7257,9405,8165,2242,1247,8620,7479,1305,8818,8790,7865,4998,5653,2097,261,5270,2140,2105,789,6984,7501,8748,9538,8167,7566,3307,8768,5195,3932,1130,3930,6535,3665,6818,870,6657,7938,8656,2975,8108,4377,1332,1792,10027,3395,5078,10380,724,6034,3423,4788,4511,9362,2462,7760,7023,2497,9802,5504,6817,4263,3361,8164,8418,2478,2287,3457,1264,9796,7218,6591,6401,1683,3534,7474,5945,9722,7233,3719,7072,9915,42,6386,1481,3426,8808,4075,9665,132,3679,1418,6327,4854,5667,1228,5621,10063,9685,9641,3792,8509,10725,1726,3510,6733,5311,6890,1127,1372,4557,9076,4493,1319,4639,6503,5373,319,6252,5205,2568,3262,5611,3692,8016,2058,10708,1990,10246,5149,6730,4745,9617,4496,8093,3764,4587,9046,8240,5654,7484,9304,9356,9122,9012,2409,2879,1728,8443,6110,2374,2695,6156,6123,7266,9525,1770,9102,10660,6745,5128,5806,10298,1081,5852,10365,5171,3316,1949,2796,5938,2888,505,9040,10137,2063,8296,703,7982,5184,5822,6713,8356,1212,10214,1643,1344,1740,6170,2765,8513,7491,313,7468,5830,5188,7420,8246,7931,5592,2300,5803,1333,1904,10524,2557,1320,9801,3887,3372,4648,8123,10207,10570,2782,9208,3493,8993,4688,9635,2367,890,4834,4750,2186,4059,9823,383,5746,6601,6995,3696,10626,6202,4034,2588,1415,3128,5342,9471,8634,6712,7283,5689,7596,5728,9160,6465,1823,8542,2381,129,3703,5297,5845,10090,9892,8109,10676,1417,6171,9518,2483,5993,6066,10657,4131,7317,4665,10546,7047,6337,6183,3537,253,7720,5316,4382,2720,8286,7478,3902,7237,3088,5066,2131,2830,10694,3644,1407,5650,5733,7970,8970,2283,8951,9288,6643,10349,4772,6849,3295,4178,1354,8522,10736,1720,4463,10409,7172,6220,1998,6367,7464,1973,6738,4238,9065,9537,2528,6911,4247,8425,183,7919,4791,5494,3274,6583,4995,7128,869,5889,7655,5199,1555,6754,10636,3874,5094,545,3805,1611,9596,2168,6048,7912,1253,3675,10190,6051,6887,5414,2919,2397,8982,2189,5410,2589,6458,7272,8280,6078,2349,9268,9229,8372,9571,2129,9070,7371,5048,8593,4925,5490,1503,6584,4713,6442,3119,687,3013,10757,8401,6102,6842,6324,7114,10127,9879,9783,4963,1626,2785,3640,9163,4835,1390,5087,8073,10203,3068,5747,1958,49,4916,620,1548,3018,9306,184,10272,3878,3053,2239,10187,1477,2902,9456,7260,5051,7639,5386,4477,80,1151,7277,5579,5072,5596,2204,10199,8267,2624,5110,6272,1330,10059,757,9947,1357,7745,6159,9580,6926,1486,10062,10488,8712,6582,7733,8905,3935,8265,7334,5495,9170,1657,4880,1542,9546,5577,3169,2880,8287,2763,1465,1041,10545,7951,2814,10729,1234,7881,6985,6670,5055,4329,9749,4744,7621,9946,665,475,6655,6976,1327,9703,33,9513,9857,3248,2158,7640,3830,6910,6599,10785,3908,9303,6403,858,2074,3941,4068,1488,4929,9954,4872,10514,7838,4677,1877,7014,8113,6883,1511,7174,381,1523,10229,9627,6966,9767,4283,1167,4168,6977,3543,6953,2737,4698,7908,8759,2429,7706,9642,7199,2064,7087,557,8591,4609,3962,2795,8283,6449,9092,9964,4970,4529,3616,6475,7523,56,2238,6431,8256,640,3416,688,10471,8348,5541,2371,994,5570,82,6166,2259,7589,10479,5713,449,2308,130,6882,6298,6517,9870,2353,4316,5452,2792,7556,3803,822,9084,798,9477,9082,657,495,8781,10619,4328,2710,460,10133,3234,2555,6069,5255,348,5511,6934,9594,5120,6729,3956,6290,830,9948,5911,1731,6291,4993,1533,9758,5693,8152,4988,514,10270,6000,9682,6946,8344,8051,5320,8247,9020,3751,7864,10163,540,5933,3650,6107,2840,9372,3701,8955,4269,8511,1024,2165,4739,5474,10239,2466,4157,6533,10780,9408,3312,2549,7113,4480,7741,3467,5710,10381,2392,4166,7485,9902,10116,1089,5281,2157,9182,412,6056,2281,697,4383,6531,7473,9231,7411,7070,1493,3570,4122,3461,4849,5119,3174,3139,8068,7162,3184,3691,2316,18,650,2382,8612,9692,6483,2442,7718,488,6642,8681,5271,9004,3942,7986,5406,6744,9014,6975,8403,7449,8721,3240,8297,752,4471,2833,10812,202,7701,10064,9164,1565,2172,3613,6589,2843,1052,8359,1932,2389,127,5508,4782,1048,2729,6216,5692,5219,4730,8987,940,7810,3907,4399,1220,9901,1549,1242,9815,673,7459,3322,8550,6865,5229,8557,2257,898,2893,8674,2446,7645,6528,2809,4711,2106,6267,1110,6415,7998,2094,9509,6269,4475,3096,1567,7066,2464,8516,1843,9885,8995,2345,4940,1077,903,6886,347,8766,3343,8534,5871,5310,5738,10217,7539,2823,9867,8238,7093,10667,2332,5535,1419,3203,7540,2417,1619,10744,4592,8817,9219,2565,4276,9380,6329,10426,8853,8106,6765,6932,3822,5864,825,8244,9566,8333,5954,5351,2978,3376,6275,10290,1604,1067,7901,3143,7979,3630,5252,5642,6098,3077,1464,325,10481,5853,10012,4658,8315,10029,2812,4123,4967,6097,4038,1755,2873,2905,8098,6941,3681,1070,8555,4875,10340,9944,9694,2180,7441,2904,8054,2531,9330,6750,6945,10067,6012,5092,10772,73,6213,1504,6425,8835,2520,1401,10768,3368,4375,4941,685,9479,453,1268,2245,1694,4325,7705,892,10775,7450,2679,842,3996,6241,1117,7713,10343,2124,5857,5617,6594,7305,8770,5928,2197,9166,5956,9299,10095,6524,10531,4335,6622,10823,802,3120,2073,3864,3490,6957,9845,3972,6935,5543,9774,154,5250,10342,8137,8040,4224,10500,7327,5779,2755,5085,8243,4632,1952,8047,7056,6304,3241,6219,5353,10320,4559,6552,4748,9475,203,4551,8552,7187,3748,4271,3232,5903,9402,1566,6381,1569,6192,5338,2965,8023,2360,10386,6383,923,10618,9172,421,3412,7582,7685,7332,4138,2820,876,1356,1325,1012,3814,9584,3799,5041,9273,5764,9757,1655,6991,8588,9423,8001,3853,6191,8755,576,6629,4115,8878,4253,3245,1885,2505,5426,2621,260,8848,3873,2060,8352,4842,7686,5359,3600,6732,857,5014,9386,9173,8150,3065,7127,10754,2987,9907,7347,1051,6114,1609,221,6666,7851,5093,2471,2465,398,2077,6459,5799,8462,7943,10731,2227,9619,3813,9031,4753,7797,6347,3836,1806,9134,6150,5643,497,3868,6915,5071,2769,3717,8575,4426,5358,1587,7752,4942,8857,742,8464,3442,3050,9660,1165,3309,5293,3171,5099,1154,1827,7728,3140,5995,7719,601,9119,2053,10056,10487,4949,5307,7866,7076,8052,4372,6045,9586,7799,4800,4575,5458,316,5425,6361,4965,1187,10816,1907,7808,10551,4531,146,3884,8978,1271,3568,10668,179,6571,163,408,1190,10175,10716,3837,9340,303,6792,5936,5979,6330,8574,6129,5555,2385,10594,4980,10728,5842,4321,10444,427,5890,1776,4598,5235,7900,602,4763,6907,6379,4804,1960,924,6133,5932,8041,98,3622,476,2517,8161,1374,10787,556,493,5804,9109,8786,1872,3138,1337,2822,6289,7562,3005,1392,6866,2518,1140,5258,133,10331,8306,5313,9575,10717,3156,2103,8495,4799,4672,838,2249,4686,2233,4281,1475,9108,206,9453,5941,8379,5103,2868,9988,9732,7417,7071,1231,2515,9371,4498,3081,9352,6529,2313,1529,4229,771,1134,3690,9005,1128,9503,9776,2571,8369,10271,4387,3408,5533,8666,9035,8390,712,9255,6634,8299,3221,7005,6850,2854,3921,4548,3501,1593,1050,1692,3685,4879,4277,2805,8468,2867,46,1258,2272,1118,816,6796,7965,2036,1722,9448,9851,6940,6862,10538,2547,6437,5343,6727,1359,6444,6573,852,7574,10336,61,7085,6247,5134,106,9192,2369,7786,9009,670,8311,638,5288,8446,5463,296,7337,5018,3103,10423,9970,10008,944,3205,5435,5332,7921,871,967,4777,271,7214,4652,6630,8199,6526,1695,5140,6215,9277,5174,8504,5181,8193,2863,126,2377,691,9662,6495,233,565,6466,1771,5348,9411,5026,5537,4801,656,1384,8396,10541,2326,6958,675,402,10098,4203,6835,6258,6969,7950,5704,4047,10642,5038,3340,1820,6525,7354,8752,5131,3937,4369,3486,586,5305,4971,9773,5948,7618,9361,6790,8743,4309,3722,7721,3944,8836,4154,700,2935,8003,7351,5521,9464,4718,4888,5124,7841,3135,5697,4530,7276,1395,9615,4825,6224,198,7805,633,8442,1840,1261,1170,879,9725,3626,3870,5050,3901,3954,9332,2906,10073,7674,722,6843,4363,9289,5077,6598,3059,3928,1114,10034,6377,1380,4378,4418,10505,4869,7246,1292,9942,7092,9139,861,7416,2747,1245,6423,9095,9653,3130,8948,6737,2404,4026,726,6101,7726,4540,10308,4243,478,2980,3227,8985,6913,2675,4504,3753,4931,2434,6753,1045,7477,834,4181,7134,10507,9514,1747,714,7790,6755,1971,8301,6009,3584,8877,4361,2927,9027,599,8160,9059,4675,8090,10091,8192,5283,719,3145,8671,8569,6824,8639,9253,5815,10361,6988,2510,2585,8904,7405,2398,1768,7989,8696,9899,9805,2439,9412,2727,7927,6257,8746,3982,5679,2013,4579,4141,7634,7553,346,603,1522,8764,7001,3603,7775,9715,7678,8482,8630,10530,7905,9211,5308,6760,5266,7308,7377,3957,7058,6340,3602,3218,4945,239,8867,9918,8130,8000,4950,8913,275,8097,1029,4630,10774,3173,2808,3754,1591,9115,3593,755,3318,6902,5762,2104,8893,3749,10401,8946,2616,8510,8937,7975,1078,6335,6238,10733,2044,7885,4233,7426,3606,5280,9756,810,9844,9695,8053,4704,3852,1484,1031,8399,5467,7672,7590,8215,5263,8028,2657,8117,6142,6273,5561,9748,672,7229,3098,84,10405,4272,2800,6889,8792,6691,4204,5646,5345,5615,4278,5315,4486,1543,9467,5355,2797,5400,1953,8917,7806,7073,3807,7312,1406,5716,7086,1738,6847,9781,10212,9631,10442,7548,10115,4678,4097,5030,2247,3002,10293,492,10429,8944,5106,4009,10269,5760,859,1201,10771,2026,6508,2957,1780,317,6445,8139,1810,1520,8983,2354,490,7375,9866,6035,5393,4751,9810,7710,5155,6576,7367,4368,332,5870,7762,7292,10020,6076,8355,1216,2119,2362,6470,6180,500,9775,298,4319,4935,7585,9563,4182,5326,1970,9266,845,6756,2256,8541,8972,7094,2748,7353,5185,10664,10743,9565,6942,7743,9912,5135,1452,1618,7592,3844,7037,2048,946,7036,5780,3146,5973,5505,8810,2892,3682,9607,931,6852,8823,10136,10440,2615,10764,9421,7909,8964,8449,3160,2153,181,10421,3357,5638,9478,9572,5797,2020,5368,10014,2415,9659,1772,9831,7340,10777,2584,1967,7190,5789,9905,3909,5499,5879,9787,2921,2683,462,2535,10730,7492,6718,10492,7124,1148,1079,3986,6891,5696,10414,5855,5737,4214,9389,3253,1273,3869,2641,8329,7323,4120,7723,4798,1387,5285,6983,7132,8731,5299,3663,4249,3438,365,5100,8684,8801,1993,4506,1671,4566,5122,1686,3195,7916,3880,1101,9672,8036,4397,2701,1786,8120,10726,3429,5114,4588,5217,1057,2735,10794,1331,8815,2261,4703,1667,9574,10803,10054,2435,8067,4393,6362,2937,985,8750,5374,770,2463,1749,5922,1063,10335,4743,2305,10396,7546,257,3297,8553,1775,10197,7279,2690,496,3454,8616,10612,7857,7126,3107,2298,4072,8906,151,1581,8061,439,6856,7903,2142,4526,5244,9029,4714,8335,881,10363,7503,6647,935,5133,7649,10561,6108,933,2551,6870,4673,2826,10129,5906,626,7717,1082,3575,1172,9417,3456,5568,5914,5856,1851,2953,4881,5851,6239,2050,4371,6477,7366,4452,9849,2246,7930,988,1951,10789,5878,9853,2323,2850,1942,1136,7421,10230,5130,6313,6929,4846,10267,7890,4964,6829,4357,8582,649,7241,6716,8258,1176,9263,5390,6994,5921,3192,1069,10796,5908,6486,6138,8562,8578,6925,1408,7917,8045,10286,6671,2352,1312,10309,8210,800,6050,2686,6214,8456,10302,8858,3443,418,8005,1445,7115,1824,5690,2569,2917,5197,6749,8643,1537,9977,5648,3997,2600,10395,8360,2861,1014,9790,9865,4095,6124,1666,9189,5160,8186,5137,10828,6025,1847,4932,1249,1299,9530,7282,6990,9223,3720,3818,7987,3137,7670,286,4731,7654,805,9668,10018,230,2467,8638,10084,8422,3481,9811,3285,4877,9281,10307,10801,102,530,1819,4918,6546,928,2930,6678,10253,10151,4094,8430,5056,7400,1313,10461,6039,7992,6664,6502,10644,6424,3383,4302,7186,8342,3559,310,10379,9446,2277,1116,5778,6538,1963,3185,1487,3294,10362,1207,6921,8181,5236,10353,5709,7359,10011,4147,910,2476,5337,9213,9916,2696,2391,7286,8797,1796,2411,2570,7990,2889,10723,10150,171,472,6246,7280,10100,8298,4517,2775,7831,10582,2365,3183,3670,6432,9074,7993,4852,646,3645,4052,5323,5556,10807,4014,5349,7109,1296,7125,8147,10043,10085,10709,6906,2251,1058,3842,1132,10328,6005,7725,1853,1862,4528,10824,2837,9129,2214,4209,3557,8758,2605,6667,8761,7620,2156,8505,4322,5269,9818,4447,3186,4709,3920,10087,568,7985,8911,2331,6494,10654,7631,2875,5363,6151,6896,5481,9705,3286,7753,5449,6013,6874,5153,1779,1494,4722,6728,4726,8008,6316,430,4403,6359,8762,6936,8704,8677,1111,8153,8680,7700,2622,582,1462,1621,4373,5430,6218,1825,10038,5597,5578,6064,7907,7892,1469,9038,6763,7197,10543,9461,3881,1536,8992,7465,235,4240,7502,1541,1616,7207,6608,4245,3470,2961,5423,9460,3993,1979,3680,8998,6844,6370,8959,3488,3217,2604,3891,3369,6467,6320,4458,9248,1793,8676,7581,10502,730,566,3694,4699,4416,2284,8110,1933,9333,10388,9873,7454,9254,8881,775,3497,1712,7090,5158,10122,7483,10818,9973,9704,1969,4606,2682,3354,8869,1338,10215,4379,9793,694,3725,7365,4431,8302,2752,3255,5392,5468,68,10751,4871,390,457,4705,3181,4081,1491,9246,5982,2524,3721,2372,727,2163,4734,6689,10005,5237,5583,280,5207,3034,4022,5934,6326,2334,7209,2702,4643,7498,10119,5825,8962,8185,5440,2169,1803,1742,7316,5183,1177,2936,5020,6959,5063,698,8234,9307,3086,8831,919,2234,3787,5714,8263,8991,6104,8554,3350,5187,4092,8071,1898,1394,4992,5371,9200,486,7787,8939,6047,2190,4210,3142,4261,1583,1121,1927,270,4948,6204,10255,6811,315,8896,3477,3895,9606,8851,4623,4244,893,7009,10344,5404,2539,3448,8682,6105,8965,9706,2862,5375,3273,242,9218,3585,7027,1112,8232,7751,8332,9015,6507,8544,4428,884,3857,1086,8115,949,4160,4268,6867,9803,8337,6903,10165,5744,3284,5854,1785,2319,3089,8448,389,2066,7236,3351,1466,123,1956,10341,8785,1741,2310,1544,10547,526,4421,8916,7915,161,2972,5424,1301,222,4661,6438,2496,3109,8488,9354,743,9345,3615,10721,9622,51,8829,9000,4890,10182,3760,8921,1867,5754,3265,2399,6089,7313,6339,9285,212,3308,1886,10441,2341,2626,1284,8138,10424,10204,3572,3966,7296,7303,6846,1875,2758,9663,7173,2959,7261,6295,8303,9088,7669,2112,1328,809,4456,3132,2346,1241,958,339,7663,7531,2519,7509,6949,8340,9959,7063,3055,4636,4650,7680,7203,6030,2264,8979,8819,2500,7559,5606,9434,3256,10689,2321,3324,523,8078,992,2736,7513,6385,7052,5970,5483,278,6350,1730,9367,729,157,7960,4446,2110,10575,3544,10795,6554,4384,6279,6659,2623,2082,145,6372,3582,4127,1878,10826,4129,1017,1837,9788,3281,1663,1929,1597,7393,10473,5167,9599,1482,5394,4906,952,2098,3465,9240,5524,3459,1724,9454,6189,5210,8609,3447,252,20,8014,560,7346,8756,9939,6463,3810,8537,7240,6703,3435,2252,8404,4582,4011,285,3744,6435,8127,6676,3492,9245,7358,4079,3555,3329,9868,10154,1435,5189,8220,1275,3926,7007,4464,8647,10250,7756,10798,10221,8891,4723,10161,6297,9083,3765,95,3530,10346,9909,1532,3700,2541,9989,6816,10311,9640,6547,6384,2081,7112,5666,3974,10374,617,5691,6310,1702,2487,2285,4644,3336,3756,1224,4343,987,369,1093,5223,1610,9981,7227,4114,915,1186,3856,583,10701,6518,7343,860,956,3165,4762,2529,10281,1429,3413,4135,483,1449,739,4878,3697,769,5841,4621,999,10557,9149,6955,1997,1073,4172,7653,5518,1485,3287,5466,9746,7146,8953,8830,7521,8289,2198,5300,4404,1659,336,5723,2857,1499,8658,5105,6122,9601,7954,7573,5243,762,3418,6234,3161,2681,4441,4649,8345,3804,1818,3177,833,9727,10225,965,7635,3006,2772,1142,3208,1098,150,3943,4761,3460,148,4784,7496,7263,612,3524,984,1983,5502,1120,7200,4104,28,9675,9544,7936,8763,9603,4304,6014,8498,10131,1367,2580,5959,2016,1205,1351,3288,4457,10673,8116,4406,1206,7046,9177,5683,7707,3547,2045,4376,1516,2865,4622,3207,3659,9175,5121,2777,2653,8524,1157,218,6418,3321,10102,2380,7980,4617,4220,8859,2550,9473,3495,5408,10677,343,7398,380,2894,5325,10235,1329,2869,7202,10437,6613,3509,2703,6314,7226,5218,10184,10627,3526,7179,6801,6853,8560,962,10511,7729,4125,4134,4392,2766,2492,5971,6144,2358,2583,1122,9632,2992,747,5145,7409,8392,543,3133,7946,10450,754,4981,5441,2618,6462,1229,8587,6498,9512,954,5848,5688,4759,5331,356,8088,7814,4350,7795,8176,3631,2773,4999,8259,7381,744,6609,2564,5940,1594,3755,6476,6540,5057,10616,9221,621,597,7887,2606,81,3980,2220,6640,518,2659,4538,9926,1733,7948,8018,3004,7363,2021,6992,2342,8483,9373,4527,4419,10224,5816,10623,2693,2160,2963,6773,74,4684,104,4898,4809,1674,7499,10696,4695,3978,2699,5214,7315,829,10410,1145,803,5951,301,3763,4196,7746,8328,3070,7407,6055,938,10522,7977,9896,2243,7033,9814,4494,9676,6036,6100,4775,12,2195,10196,1103,5862,3333,3471,6764,10222,4174,8568,8249,8362,5492,4053,5630,7000,1966,3422,1711,7682,295,865,7361,9652,5790,5626,3821,8409,9859,6363,8860,5989,4087,6395,9848,8308,4341,7520,7616,2671,6836,4860,3014,9789,9205,1918,5053,3335,9542,320,8884,5204,1442,5558,8250,9872,5952,1424,3709,8195,6140,4057,6633,659,2383,5834,1508,4246,10010,7287,8076,4554,3599,10685,1431,4050,7264,8535,5166,8086,8133,6541,4201,3545,9923,5647,4035,7740,10183,4169,1689,2664,5770,1882,1295,6978,4921,1812,5412,5387,8668,8408,4781,3579,732,3899,9510,3082,1914,7242,1557,1912,3961,3250,1039,9611,6683,9820,7647,10220,5720,10173,10330,4909,2337,2654,7133,1896,2883,2448,10727,2793,578,3199,8572,2743,10223,10662,9378,10004,4008,9331,1675,718,4682,4274,6762,1240,3261,4046,1934,79,1988,9624,5601,6621,3153,3882,10144,7429,6430,8986,5860,3225,5873,3960,9397,7687,5080,10674,10778,6193,5531,5793,88,7583,4601,2248,5748,3393,6702,6857,6950,8635,2375,1141,4236,4145,2733,5629,401,8954,5777,6783,3337,1943,5918,7821,255,10692,1434,9058,9629,3168,360,1690,1230,9458,951,9271,6469,5967,3061,7579,1003,3108,7681,8486,6265,4409,6109,8412,5618,6487,5562,6276,5907,7575,5529,10601,1725,9096,6905,7801,8610,5277,2627,5035,9569,3157,9957,7322,1600,6157,796,4460,6627,5685,10805,394,3669,7330,2426,7744,10520,8645,8932,5209,5820,8391,4769,6132,8613,1836,3841,3041,1085,521,5186,9462,7136,6429,9951,5409,1269,8128,7516,9188,8354,10167,10216,9497,9972,7399,8910,6456,2810,10339,10819,1646,9616,2174,4175,6091,748,3661,2366,6654,587,3627,10097,8367,1109,1844,8652,1364,8151,5060,2480,7817,4284,1076,5257,4861,7355,5872,9709,1298,8145,8519,806,2427,4524,3660,6293,4444,1660,3989,9374,7192,5369,8292,3914,5608,210,1002,7154,1302,2855,2412,10211,8799,4612,9744,267,5877,7137,8426,2828,4192,2378,6236,4924,9921,2440,6378,3556,9511,1897,357,8104,2400,10609,2970,3499,1830,8321,4543,7794,2971,254,6223,27,9480,7003,8856,6954,7704,318,2407,1182,4624,644,9496,10318,7961,9257,5695,6231,8894,4186,7976,812,7331,4217,7106,8092,1126,4102,8980,10591,3734,1213,2670,9679,3172,10459,8626,6492,3323,6646,9539,355,9337,8716,3808,6165,10562,5658,818,9707,8958,2134,4866,10317,7818,274,5457,3388,4282,4330,6482,109,561,4955,4874,117,8038,8866,826,3938,7069,9357,4806,4903,8741,2688,6343,2402,9895,9561,4100,10491,2597,3533,1849,3190,8737,8967,9007,4468,1068,10277,2258,4360,59,107,1036,5769,8437,9843,9700,9127,3525,1,7297,1809,5972,2484,9840,9609,431,5800,671,3009,10159,481,5025,6254,9602,2025,1001,8679,7472,8929,3164,10786,5165,7177,9987,733,4959,10276,6199,8022,9001,10576,9036,8938,10189,9238,297,3231,9080,6512,6509,1568,997,3773,9643,1030,22,2192,9407,7798,2813,10039,2088,5173,7170,6652,8536,10436,3325,10338,9437,1977,1919,9808,2430,10240,1107,6451,2784,1817,8037,9633,143,7117,6964,721,4194,4954,9730,8203,9491,1807,5659,9116,9816,6808,1160,1358,4106,1166,7206,371,3636,7656,2148,1938,7394,3634,8204,4770,4938,3896,4577,8548,7530,6016,7461,324,10646,7307,7338,8436,4381,6615,6087,5668,942,9953,1899,5546,9406,15,9684,3552,7335,7392,450,4470,6201,4040,10641,8713,953,5675,4947,8715,3092,5422,6023,5783,3950,1423,677,4710,878,7130,10822,5512,7204,9929,7326,2019,7774,3197,10810,7732,8471,5052,10750,4482,5156,334,8526,6222,9214,6614,7258,6488,6680,8320,5259,3757,9549,9196,5715,3581,5413,8031,5224,6497,7651,1100,10110,4797,7839,3091,10123,10358,8188,2208,5447,4542,373,434,3293,4904,214,8494,5450,1721,9828,9961,8747,4365,110,4910,704,9155,3504,7235,7734,873,10448,7667,8824,3796,7971,4605,10003,917,4066,705,779,1612,8861,5260,7295,3027,8487,3823,9826,4459,8941,6715,2085,3517,4808,7075,8272,4500,9656,6927,8405,1386,4737,8475,8365,1813,10088,4996,6029,6993,3364,4491,4885,4287,3667,10170,9325,2314,1129,3911,7830,5924,2676,4934,9403,7940,1982,5054,3028,9693,8633,454,1944,8599,9833,8595,41,1400,9291,25,3594,2697,8945,8840,7290,8963,2043,8431,6618,7978,4635,1669,7879,4019,3811,8168,1791,7099,6588,9120,8107,2633,7467,2490,3905,4697,3652,6742,1718,3987,6206,8467,3716,8875,9504,7822,1874,2899,2537,1515,7730,9667,9388,8007,5294,7185,9778,9451,446,1512,4152,9894,6948,6416,8189,10042,7024,232,648,1425,9531,10391,10715,6568,6353,1441,4423,470,7962,5376,2725,4610,9079,678,24,794,963,5479,2339,5023,1627,9260,19,811,1410,2506,2086,4218,7015,9600,2022,9466,7222,7410,438,2732,4654,7252,8826,1766,3483,9081,3998,2315,2767,5503,9724,4060,4507,4857,5360,8310,10600,3211,6448,7386,3790,3872,1343,5636,2280,2384,4015,5068,6328,7697,7538,1588,2760,4701,5706,2338,3925,9966,2674,5599,1985,3879,6636,10366,9877,3042,544,2724,429,1957,8924,1634,10106,6242,9852,1884,1799,3610,299,7696,1144,5923,5194,4467,5813,1925,5177,5554,9889,886,6244,8968,9883,1910,1661,4438,7766,2194,5792,7610,8874,9729,7166,5700,8477,2091,4076,4962,9797,10400,2179,6539,1243,3452,6322,7091,10164,6564,8879,1637,2222,8434,6026,9485,4235,10564,6164,10599,186,7953,1223,6484,630,2034,6225,5991,5787,2344,7376,6979,4462,204,10621,6747,13,302,7329,4339,5983,6338,3113,3397,2213,5772,6603,4140,8357,8048,9752,3875,4177,1106,4179,2789,2995,3774,625,797,4188,333,4093,7495,6043,8057,5433,9839,1038,9750,1763,8427,10470,8601,1471,7956,8118,6848,6237,3794,5899,7245,1913,8242,6565,5329,442,2644,986,2983,5684,7060,368,89,10710,8027,2481,10632,9061,3313,10698,5357,6054,2221,4615,1578,9878,2509,4810,8126,848,1303,2489,2441,764,8255,7065,1527,553,2628,6282,4518,6331,8096,3017,2039,5350,3657,6795,2357,6577,4296,510,5627,338,393,10407,6951,9090,2798,4589,8606,6434,868,4778,631,2964,231,4594,6665,3945,5427,1293,8655,8499,388,3633,1393,2877,8319,6980,736,622,8813,3797,5883,3433,10160,2786,9567,9022,5739,7920,8966,1083,4771,3532,10697,4826,137,2182,522,8984,9982,10264,4017,1974,5652,1981,2910,7029,2990,8724,3505,4651,7819,4345,5497,1928,7782,8949,433,2403,10288,6751,6342,4712,7684,9683,1035,2844,2317,10157,3858,4370,3883,1554,7061,8373,8015,5686,8795,6278,3683,8686,5493,5168,4032,1748,10790,1458,1432,7436,8032,7625,8651,10815,5998,10506,5202,8688,3678,9343,6510,1629,3182,1353,7419,3785,2472,8393,7761,1355,6332,9681,9931,364,10678,854,9098,9691,10792,5289,7764,4230,5069,7055,3955,3576,4567,10194,6414,6766,8956,3269,2997,3888,144,1777,5477,7013,5192,3642,2311,1336,849,1601,4968,7482,8135,6404,3124,8622,2949,5403,9269,1256,10521,9091,5486,6970,2815,4311,1893,7183,85,2292,9856,7984,855,1750,4279,1713,7002,7626,1705,2090,105,4687,9153,2294,10411,7939,6707,1672,10284,8545,2609,10714,8589,1757,7469,7217,2263,5027,1745,853,4429,6758,9651,7144,9286,7676,8527,8155,420,4443,1845,8082,6611,0,2794,9547,639,1222,6115,1572,9903,9495,5545,3243,9621,5182,247,5581,3289,3729,776,5095,2379,243,8744,2295,3732,4725,10544,2745,1246,10024,3345,2076,1155,627,6939,2454,2839,10259,641,5988,1808,7401,9723,6734,1294,10155,5818,1260,7904,10671,7750,1061,8224,4013,7194,9030,9482,1505,8012,10185,2293,8551,520,4831,10451,5002,1628,244,23,5459,3152,3566,2923,4389,2508,5984,9363,9138,4756,9871,3200,1729,8922,3064,5090,4626,491,1968,7768,4039,7882,6317,4822,10244,6947,6559,5,188,4424,6963,1879,10447,8269,9576,3865,2632,4003,9368,3484,8089,2790,5472,1922,6723,5553,6057,2734,4989,4706,397,8384,8002,6233,3469,1576,8470,9846,2836,9521,1829,7442,8771,7731,2751,813,9051,5637,863,329,8056,5574,9779,10558,9234,8923,366,10036,7320,8997,9522,5301,3292,3327,1113,5242,5602,5876,3817,6972,1054,7870,4310,211,1290,4005,6888,8732,7384,9564,6837,6044,6205,5655,10017,4824,2938,7281,10462,5091,6918,5488,8481,7828,5498,7613,3405,3648,6007,5634,10209,10109,10516,10466,10688,6501,2023,400,10517,10399,2297,5079,4148,6248,3482,5231,441,7759,9244,1064,8564,8973,711,10711,5525,3420,2526,4757,3759,8142,5750,1787,6232,10806,3730,4483,4255,2811,2707,8295,9699,10739,9911,7622,1784,6592,9590,9556,1687,7643,10265,1620,1119,8451,4198,10519,10607,6446,4227,4943,6810,5061,9364,4570,2738,2640,10126,8694,9666,5043,1883,8233,904,6081,4532,3553,7533,3507,7636,6788,9128,995,4867,9560,5190,1115,2771,731,9777,1653,6919,6776,10262,9881,1196,3386,6390,3574,8693,913,2714,2713,8780,674,3478,550,686,4786,8990,5885,139,10416,1684,1811,5147,5614,5178,2662,4080,3141,6885,4211,307,1232,7565,2704,7598,9383,5930,7348,9999,8415,5980,6965,3236,6685,8251,2065,1573,4893,10245,2051,1239,4581,3425,2594,2740,3394,2607,384,3646,3567,5732,10820,7380,9440,10827,7490,1194,9984,4435,5587,901,5828,3008,7045,8749,6974,1547,8225,2202,6968,1602,9124,8466,509,10066,934,5022,2542,10193,1852,5339,8180,6639,3263,4159,5632,1590,8059,3952,7225,158,3494,2723,6447,5625,10732,2864,6698,3906,7974,4002,10044,2673,1008,266,9293,1999,7714,7968,4112,7779,9610,2266,7515,7274,8558,2139,8725,6067,4642,8930,374,8485,4717,173,2575,3832,34,2449,1346,7854,7675,8378,1743,7178,7022,2495,5542,1668,40,8230,3849,3363,10006,90,1143,8636,3591,2779,2994,9997,8281,10483,3463,4899,7722,10830,8492,4136,10368,7437,5013,2783,2592,9110,10227,6638,6650,7827,5455,10583,4338,7387,4158,1318,6757,4126,5434,1931,7856,9721,8597,1209,737,4474,2718,93,4337,5102,3317,10432,83,1339,10534,572,3788,3298,8604,4176,8889,7836,9505,8400,3246,9552,3390,7599,9630,7850,6075,6854,6077,2687,866,6668,9206,3781,3587,717,2312,6600,6407,6341,264,6952,1579,2558,2532,6562,5389,3970,7034,293,7777,9506,4873,7247,9140,10493,464,4225,5639,3134,9195,2386,5603,2578,8533,9782,436,6893,5590,3596,2083,9908,5891,5064,9589,8760,1693,5571,8837,168,6382,8512,4334,3711,10474,9655,1704,7057,5724,5673,9735,5522,7289,6944,2787,1709,6198,1131,4555,2999,555,7781,8812,632,5743,6513,3414,2336,1682,8988,8087,2343,4412,9424,99,444,5874,10460,9017,6094,7614,2147,5125,10117,7840,4637,5104,1285,8602,1091,10113,9887,7576,10605,3258,4018,276,9324,7641,9276,10477,3054,9256,9450,7105,3953,5909,8717,6504,2715,2154,9197,8828,5929,3621,3016,7042,419,3847,9320,1334,3782,9312,8699,3228,7715,7370,9313,8346,4425,3047,8838,9049,6352,1340,4492,3356,7095,663,4660,4119,10146,3129,7369,2040,6674,5527,9259,2322,64,7858,7019,8417,1746,7285,3332,7129,6814,4796,8274,3083,2824,3001,8621,3746,5344,1575,10357,4758,5436,4896,1060,8371,426,5031,7991,5916,1335,1345,8444,2162,10791,4539,9850,7051,2270,961,94,4578,6521,9290,6724,3809,6306,1644,447,7084,9623,7811,8895,2410,2493,4839,6003,7889,8892,3674,1283,790,2118,1473,5070,9296,469,9447,2128,3252,3793,10408,466,9013,2776,10452,6135,9187,6662,8183,1605,3695,8077,9430,5955,2376,7511,1894,7304,8703,6063,6672,5462,4405,8742,10120,5593,2475,208,2431,2672,1288,2582,5519,2586,7843,353,8981,896,10275,647,10456,3689,3802,4317,6663,6118,4185,4708,395,4829,3538,9278,471,5010,8531,6345,3586,6688,120,847,8804,2494,6987,1125,7031,6879,2730,6717,1033,6709,1562,2274,9686,7735,5942,4863,10572,10404,71,2968,2067,5717,92,10412,3311,6833,5509,5794,4785,5042,3656,8366,3020,10494,2405,5758,10536,9541,1920,6696,6287,2756,31,10430,349,3571,7708,3037,5256,5321,10554,7529,9733,10179,5227,3702,9785,680,10140,4195,3381,9638,7385,3643,9165,10604,3355,8044,4603,580,7455,9274,6389,6169,8254,1630,4696,7633,10327,2173,8039,9220,10247,7041,6706,1492,2217,9186,9024,2127,7947,3550,7563,4143,570,5633,1816,4510,5549,10139,4149,4450,9112,4078,5910,10132,4414,3806,9204,10713,9384,7250,3444,2330,10071,4990,7959,5900,8010,695,2418,9349,2079,793,6022,8438,3569,305,8863,8888,1397,596,6355,5604,3795,4291,2731,5203,2908,7568,3939,8175,4569,1762,2017,3766,3623,8062,138,54,5884,2240,6735,4455,5961,7527,6660,9060,8855,5785,6392,7688,551,4768,10367,2546,432,6143,5703,7278,547,4440,4614,2031,760,9107,164,921,1262,1858,6701,4692,9105,4354,768,2359,2012,6566,7820,759,2144,10251,9156,6096,3370,2226,708,1514,5835,8811,10060,4184,9763,9838,2553,6095,2739,7630,4521,10515,8615,5831,2939,7650,6491,8999,311,8060,8381,9298,7748,4314,10356,6823,788,3150,5169,2525,7913,8479,2035,9235,5396,3833,6781,5501,2126,8389,8729,5767,8543,8899,149,3676,5028,9026,8523,3229,1778,2279,3562,6931,3968,2388,1586,2084,7646,331,2309,9358,9410,6419,8661,7981,2872,5966,3249,1438,1179,5514,9442,1421,7544,10463,10092,6813,7402,10433,2096,4264,3254,2188,7571,4693,9111,2183,3233,1270,9930,7201,6177,9326,6128,9272,10283,1420,482,8918,5904,1717,10166,4410,7447,10504,1500,7996,3302,62,3277,9151,7220,9469,3629,3965,3506,508,8207,7776,8825,6989,831,3439,8361,1092,1570,7605,3019,7205,3437,6454,3328,5547,6280,10300,5808,9936,4473,7111,8532,1368,10527,3775,10439,8290,3605,10169,2393,4774,6741,7802,7435,1375,4064,9210,1010,1274,662,6711,8865,3214,4099,10299,7104,571,3178,8753,2625,6461,2033,1379,5641,2095,5965,7249,6797,1947,9280,2643,8791,4864,9821,749,10294,5385,4132,8897,5726,3564,3558,6499,2032,2834,10634,10389,4163,8820,3038,6585,7673,5247,7352,9830,4171,7151,7824,2244,4848,10371,1211,8876,9133,689,7397,4352,1769,10832,6356,8081,2637,4926,7169,3409,8450,3934,1080,10061,693,3030,4740,5784,2907,5576,7288,10670,9125,8903,5886,6548,907,9950,268,4553,7424,750,5947,5377,5663,2235,3769,7615,4239,2501,6572,1559,745,1495,10273,10681,8489,4395,10550,4749,7145,2825,8650,10485,8134,10023,8463,3291,6412,9078,9224,9431,4944,3105,1422,10455,5507,4180,9075,9123,9438,6986,3671,8386,7507,7716,9927,5284,3589,1448,8503,7832,1584,8714,362,5802,5564,4604,10718,5437,4307,7506,8890,4913,4146,7823,7181,4280,7362,5384,2890,3886,9117,9728,10687,182,4183,10770,3859,10228,676,5251,690,1443,4927,2540,5520,9837,598,2513,8190,637,823,5487,4275,4862,9484,6360,8063,3762,1474,5032,6158,10419,3431,7602,9279,2841,1635,9928,8617,1959,5836,3440,4197,6061,2932,3375,7550,9791,5273,7964,10249,2579,452,2271,10518,6111,4096,1647,1753,4212,6798,5378,1700,7876,8594,4033,1869,5024,2620,9011,8095,6478,5881,9319,8205,5978,6779,8200,4074,8850,2269,4073,9222,3487,4519,2885,7160,6172,936,2458,7834,9468,7763,3430,1707,187,6770,7770,1055,352,7783,10737,9401,7167,1991,5240,1180,9121,1861,8395,2268,8957,5774,996,8605,3772,8940,4836,891,321,7988,6534,5671,9144,9301,9754,8849,2554,1546,7425,4746,10070,10742,2061,1782,6038,6250,591,1257,1391,1954,8206,6307,1903,9941,3093,4348,7666,2996,6904,9100,5694,10143,10663,1426,607,8201,10035,7107,1371,7747,2042,10206,5327,1992,8420,5391,6752,8538,9396,3479,7849,6514,6892,477,8969,1585,192,5669,516,1237,1833,2979,4488,9753,4362,176,1916,7215,5365,3213,1563,7973,5557,8685,2619,1200,3502,10104,4851,7267,8100,6858,3062,4062,9876,2821,645,9604,4742,7118,507,5987,2668,579,9459,6125,10813,10046,10202,70,227,9483,10606,10332,3198,3044,10672,774,2601,1251,5953,559,4109,6563,8803,4832,1094,4911,4514,4549,5225,9428,3931,9381,10578,8473,10305,1506,10,5012,2587,10195,10434,1162,3347,2556,976,5152,414,5445,7588,4680,1210,5566,4027,1651,10013,180,9734,4150,2216,5584,9168,6426,10533,403,6391,8317,7471,8773,6772,1072,2835,5935,4951,8382,2799,9835,8079,629,2596,8525,7123,9678,113,3992,10149,8364,761,3111,519,8672,1521,7119,1011,2057,3940,7139,1236,1450,1930,1307,1191,1147,17,8074,8428,1509,9136,7293,4515,975,1388,4537,5672,5513,2611,4156,1071,10105,4453,2998,9717,875,8084,3750,10030,9142,9784,7311,6895,2514,494,415,6325,8698,6626,2460,9933,108,282,10589,6610,4323,1427,4830,6348,26,9527,2832,2015,1383,7344,1961,1088,2149,3496,8083,4294,4105,5850,1265,4969,8187,1137,8324,1625,2413,193,5964,9355,8629,4366,7933,7456,1821,4327,9053,7859,6520,103,2420,4882,3084,4653,562,9979,1381,10099,2925,8667,10048,10482,6210,10637,5861,5958,7601,1765,5176,6366,7512,10788,4048,10513,8935,10186,1460,8398,6532,6028,3029,10258,7891,3903,4402,537,2062,1281,4813,740,7324,3040,4299,7668,3275,9071,8275,10532,9628,4580,7969,6649,5485,6042,4596,4602,125,8722,6015,758,5287,3237,3391,9664,4563,2981,7427,3951,4618,4312,2803,5268,9261,5992,8960,8376,375,1688,610,9393,2210,8111,912,4994,1214,4355,4733,3278,1168,5352,7026,5812,7390,3929,3718,9800,9249,6656,4098,7543,5415,5334,5034,2521,5228,1828,273,8886,3604,9425,7156,4574,9965,10712,2886,7298,10613,9052,4820,778,135,2132,10831,1761,6851,5039,3179,9452,2929,2982,9037,4683,9106,5150,7958,4585,5298,2395,10130,4221,4738,287,4891,2203,4572,4508,1697,5651,9191,1832,4295,10556,8454,1378,9741,4110,4301,3155,9897,713,5136,4681,9141,5138,6088,5157,10773,2485,9499,5644,1664,536,7580,8598,4088,6422,10257,6400,5551,1440,1534,4084,4633,3051,3949,3767,1946,2780,4946,2940,899,2278,10289,4063,2660,9418,9045,2250,9494,309,8245,8691,6616,642,2639,2985,1459,3235,7388,455,6579,1315,10260,9618,6197,6460,498,5290,5322,337,3025,6245,7440,3922,7141,10354,4576,4933,10292,9582,9429,2102,7462,5919,5996,4978,2333,8043,8149,2296,10028,9657,3012,6768,7692,1550,9824,3528,8871,5347,9179,10579,10297,5159,1654,7082,4167,6812,8540,11,4207,345,3210,2576,5075,10114,605,2340,7664,6736,4747,3473,3747,9183,9171,3850,5276,8994,9310,7149,7253,4103,3573,1185,1633,773,8528,246,3890,10347,9305,5312,9520,9751,8898,9154,10337,4966,9391,3889,1790,3035,2516,9318,8323,7868,1023,10077,1727,3032,10329,2922,4571,3464,9759,2545,1277,6285,4957,8397,616,7861,3542,9680,4689,7088,8521,1895,3741,8457,2856,5112,10635,7624,7665,10808,2450,4297,2950,6168,6967,4584,6179,10420,3187,943,7443,1215,2962,5896,5484,8689,4469,7147,7945,3607,2111,4479,3136,5047,4051,8050,8336,10015,2394,2573,4305,9963,4509,6916,3462,5254,4318,265,58,422,8927,279,5718,8069,6527,10620,4795,9766,4535,3189,7712,6998,10263,7096,6186,960,8809,5324,4206,4858,5143,4427,9101,4118,3352,2848,4776,10649,3175,9536,10208,2591,2944,7230,8912,7660,1614,228,2115,1385,7043,4437,8662,8349,914,10319,2130,8649,5249,8294,9152,1461,10162,1169,5198,9535,7018,399,7658,9755,4451,10107,3866,506,1526,3283,6274,7184,7926,2870,611,2884,1865,1984,10758,4449,7617,1706,2728,9443,7374,651,3592,8476,9028,9516,7453,3752,8774,6731,7011,6912,9047,10763,8654,3789,6093,2543,1009,3647,2101,3518,8563,1662,6686,2716,10072,4495,8603,10387,5306,9696,8508,8775,998,3658,21,9772,7972,10274,9585,2572,4289,564,8579,970,6209,6358,1235,1250,894,9579,5949,7248,1856,3268,3202,525,3406,6090,872,3916,9262,5442,55,7089,8429,215,4461,5749,9351,5559,8783,3624,2598,7611,8141,3432,2807,5399,1645,7458,4004,9925,7703,6684,8374,5238,9540,5565,6226,8421,5782,9113,6632,5582,1428,5438,1350,2694,3338,9419,1501,7360,1800,10573,6001,3238,9847,7769,7848,9593,7594,10254,5191,7481,7963,4161,9551,2355,6651,7846,9455,3480,6086,376,6604,9858,2286,4041,5267,6315,5875,6999,2967,9344,4336,8490,1314,3827,3736,8628,6305,152,5586,4720,8351,7064,7373,4091,3303,4390,4727,3815,900,3115,142,8009,359,590,341,10413,1075,5869,9595,2711,5920,9769,4541,4986,3219,6761,7431,1470,573,5976,5840,593,9598,10829,3984,9532,972,9550,9126,8584,2866,2593,6841,1923,7999,1463,2602,1905,4707,7600,200,10237,2819,5960,6033,10489,9243,1059,9768,9033,7097,3127,5065,2215,10745,6364,1545,10598,7391,7412,9577,4228,4783,9342,874,6,4732,725,6323,8143,8478,563,1737,7679,4868,7541,3339,10428,8035,8307,3828,5292,2491,6861,3474,9376,4984,8926,5745,9562,594,4802,1598,3637,8779,4358,5108,6053,8709,6580,7910,10266,8253,1736,9390,4560,489,5768,2368,6743,262,10490,3851,2991,1027,2993,581,3300,6809,3358,8496,3655,7176,8501,2559,236,1577,10019,1233,1926,9426,9050,4332,10236,9089,9992,2133,9230,2617,1685,2422,2900,4736,5111,8720,7062,1289,4021,3123,7415,9150,8806,2136,9501,1873,2225,5894,7800,9377,7845,9006,3649,8266,5101,5469,6318,4219,5098,5847,4170,2093,2536,6398,6137,5912,1015,8718,9812,7629,1341,292,8172,136,8723,511,6933,4351,3976,1641,5420,189,9427,827,5370,9382,9529,4914,3563,7957,6710,7138,756,6017,1047,4293,6606,3988,7432,9822,197,8112,1405,2121,4199,4715,10476,4870,5735,6099,1370,5826,2507,7893,6648,10108,8484,1360,6661,6402,7414,7349,7433,9251,38,4499,4619,3399,6136,5986,1976,5833,2301,1948,167,4844,2327,2438,7444,3919,6396,5456,8590,1446,9861,4611,5058,10406,5656,6771,3516,902,8252,8212,851,2759,9021,6530,7577,1972,9201,9295,1219,5407,6493,2629,69,828,6146,9740,4086,3705,281,3975,5756,1244,5418,3704,8989,86,2926,1859,8158,3617,8800,4685,6262,1679,9940,937,5550,8102,1276,7350,1571,9969,4958,4434,8343,5607,4620,166,1456,567,6311,10542,8218,6767,9392,7219,7671,5179,2052,8174,8196,67,781,9413,5362,6840,643,4987,2612,6334,4817,5097,9712,955,2933,9275,8013,9334,10397,10535,37,1613,4085,2753,10784,6405,6631,2175,5059,1539,191,10242,6235,7228,9103,3131,6708,8213,10648,6410,5213,3334,4472,6277,2802,7632,4657,4442,2056,4153,2267,3632,1451,1476,3641,8313,2099,6923,8627,1022,6041,6387,9817,1756,379,8140,4887,6268,1917,9978,6062,10498,4029,1975,9922,10569,2665,2254,5206,3326,9282,3374,7677,8413,1855,2299,2408,7897,3778,6217,7077,2816,2473,5573,6249,9184,10079,7537,7300,3419,8664,289,7896,2468,249,4841,3267,7504,7545,4286,5552,128,6726,8577,2406,314,5839,3893,9215,5132,7875,2651,10559,8024,1497,9697,2241,8407,1964,499,6587,932,7341,2089,10213,1286,229,1433,479,7755,2581,6570,4124,969,7251,501,8260,2028,4534,6417,97,10145,8847,8816,5296,4292,1552,9161,3846,10765,1311,10296,8146,6473,7175,3693,330,4928,6052,9130,1208,3918,6785,4164,5763,4497,6397,3917,1198,8173,4090,3498,517,2969,10168,8156,1104,5622,10497,3515,8222,6551,10128,3380,3979,8882,10467,7829,3410,883,10431,9639,5791,9720,8700,4792,9247,5727,3771,5858,8546,468,548,3500,8646,6453,746,10446,459,9674,6253,2120,8472,1044,1650,3860,7778,9998,5829,10804,2498,9094,8331,10089,4561,2108,6815,667,4892,3389,6221,5341,2955,6174,10630,2451,6956,7773,10232,6549,248,528,6597,1411,60,6669,2114,8455,4920,6027,3726,6516,8474,9508,2363,9716,308,3039,4895,96,5476,10205,916,6195,5262,7131,3511,5516,1430,9217,4191,7301,4448,5598,9093,6161,7025,9710,4767,8055,5773,3843,4960,8705,7826,9457,7524,4359,4346,1102,4466,3310,9339,5282,10191,5460,7694,4042,7844,4502,9644,10333,3838,669,7428,515,7564,8854,6567,6725,177,635,367,6803,8358,10734,4586,10313,1124,9949,2453,7949,7050,5678,577,4640,44,5107,5661,4439,6409,8136,5356,1025,7595,8736,10384,7525,4728,8608,4290,2712,5538,1217,7872,5702,5635,1479,8072,6834,2928,9143,3770,9283,10200,3468,7333,6922,8663,5660,2595,9920,6428,3036,7012,5957,9315,3118,5788,10378,10659,595,6255,4779,3224,10510,4991,6175,4936,386,8901,6586,4837,4694,1324,8350,9647,3829,763,784,5687,4380,5969,10198,1732,8326,7191,1788,3282,880,2502,2461,2018,5962,5272,3776,7171,2003,1911,8239,2915,1794,5811,6839,3163,7475,2860,9085,889,1774,2958,5517,5532,10285,5775,9919,5335,7888,10738,8497,3758,5004,1498,9309,91,3677,9557,1617,9314,7597,9792,5037,8793,7644,7967,1365,5994,7079,3162,710,5926,2196,10651,5008,2852,5981,7911,9780,4811,3863,4833,4155,4298,4028,6294,5193,558,2574,7150,8423,2954,821,9669,9181,9241,4593,8702,3220,4505,10210,10705,846,1804,10746,9747,6938,2055,3611,1716,7874,6617,9745,5081,10009,1945,1152,8377,1658,929,5765,7567,4724,6374,7,10138,1007,5086,8179,2255,9614,6784,2853,2512,9010,2920,7871,6825,6697,10373,3485,8277,7100,3344,2037,782,7048,8166,9967,3614,4043,4385,8769,3204,9353,8285,10679,10219,10586,1066,6420,6537,6511,7608,2871,6819,8934,6083,1710,114,10793,4859,3472,6354,8229,9302,8873,9515,213,6472,3069,7188,7021,6505,5882,5336,8157,3067,1158,1496,8596,3936,7812,30,4850,10234,6368,2522,153,8049,7619,5241,6557,3087,10706,2166,6722,3835,2943,47,6519,9917,1866,5795,5448,6046,1439,3708,2882,3215,7291,4729,6786,2648,3608,3933,1703,7121,8625,5303,10597,814,100,3820,372,119,7357,8070,2649,10617,4856,3024,9534,7791,1436,8375,10695,5721,1105,6266,9054,10593,3512,7883,2072,8500,8309,5612,925,7466,10192,10040,5915,7078,3628,4342,1767,10377,3877,9587,461,9985,5500,8868,10111,2000,8248,2206,9409,655,8586,9321,8341,8832,1781,2078,6787,35,6558,9146,10082,2452,5000,5699,6485,3151,3011,5526,8469,6555,9335,3279,10665,6256,185,8316,277,7637,5619,4223,234,4388,8198,10435,5905,6021,10753,2847,3698,2253,1352,5624,832,6471,3662,134,1540,5657,4901,4205,715,4241,1715,7196,9765,4512,5736,10537,8573,7234,5730,10549,9968,926,9986,4413,5880,10809,5544,5892,3085,1880,10306,3259,3973,3367,7780,1839,982,9099,8581,8600,772,5302,2219,7863,3913,6085,5572,10369,4142,6134,2764,5291,9476,1551,6303,10702,4007,5317,2989,6937,1854,2201,6182,3074,4344,8782,6641,3999,5461,2150,8908,2677,5364,1965,1531,115,7549,10581,6019,8943,2749,2161,7470,7275,2151,5470,2851,1814,2445,10360,3520,5814,10055,1019,7607,3620,502,4883,10574,1389,3260,7108,2171,5411,1681,1409,8719,5755,10749,2655,9300,2511,10037,5011,9528,3508,4794,1084,3548,3125,2030,10643,6004,2534,1457,10256,8066,2232,4069,445,10766,5265,8169,9654,6113,2942,7995,6605,3403,8576,8291,9193,4036,2599,945,3244,4656,787,9645,2746,7749,10324,7299,6793,5089,8885,10348,7792,8529,3527,1888,9002,4666,7269,10622,7815,5846,3578,6406,692,3045,6369,8776,3072,6071,6746,7526,6900,5062,8011,9658,392,5817,5623,10453,8214,2722,4411,5443,1181,9498,1028,2138,66,5073,5605,3540,6440,7271,170,1188,1163,7934,1751,2224,6073,4,3052,1065,1369,1801,2499,7895,5346,10802,350,2988,2849,3619,4937,6455,10156,10041,7690,8103,9369,8619,4257,1282,5676,9827,1138,8631,5201,1099,9032,3668,9673,2146,5226,6251,895,9841,2005,6152,3927,7243,7448,4544,4231,7083,3731,3272,8293,5116,9350,8711,9137,2167,4407,4065,10047,7584,5931,8064,259,10301,9884,6692,6500,5844,1835,8021,1881,1673,6677,240,785,5465,4258,957,3742,4189,7040,1892,5083,606,9132,5506,2320,10501,3915,8530,9147,666,2187,225,10096,6074,4974,734,8852,8515,6694,7059,4056,2024,5798,10686,7659,3402,9990,9934,7364,3257,7438,165,8637,6240,4484,3126,3714,405,196,3514,3900,2804,8284,5045,1564,2754,10158,974,1173,2859,9233,10103,3839,3885,7606,9202,4044,2364,9798,7922,2318,3737,7164,8480,6543,9952,7860,1149,6778,4773,9474,6924,3407,983,9379,8728,7336,3076,8772,3110,8065,7080,10509,5148,7256,9648,9701,9832,8517,817,4202,3455,8453,681,458,8549,9836,3897,2356,10783,8974,9048,417,6302,8208,4721,5451,9620,1996,4139,7816,5421,8796,8119,3147,9465,6450,2068,6112,3167,10486,4807,532,5753,5824,9034,2304,6309,485,272,8802,416,5807,5742,8325,9739,9890,9625,7342,8585,3967,7757,843,4900,575,4814,9771,4523,6800,8942,5117,8707,10279,4430,6997,9199,1582,9292,7404,9976,3531,10093,3523,219,6155,6006,1632,9042,10422,10083,3673,9698,4546,7662,6059,1699,8880,8435,9994,922,2486,911,3114,660,29,1005,4520,1815,8202,10382,5234,8950,2503,8330,9019,5295,1860,1366,8221,614,7884,1955,174,4536,5974,2708,2307,7738,9242,8706,9039,7446,2159,10756,112,3003,9493,6167,9862,2903,9159,7519,7439,2614,6084,6072,8822,1838,3239,6675,9422,3964,3033,6899,4754,3963,4853,3710,5040,8080,4061,3597,2109,2123,1595,6211,5146,6791,4827,9996,4144,5569,7942,1159,9943,7572,1639,3672,10359,10690,4525,6196,9481,738,716,4489,2685,10147,4248,8583,6457,3387,10614,10364,993,1037,1404,10080,9258,5580,10403,3854,4116,5118,10669,9891,9737,5665,2469,10241,5489,6971,8936,6880,5977,5649,9228,3158,7784,5115,1574,9316,1797,8385,835,169,2288,8864,4121,6575,4101,407,2421,3898,5395,5616,3453,1454,5431,8122,156,1864,87,6008,7652,6427,7356,2260,1090,4823,8447,2457,10280,1631,1538,950,8406,1279,2966,7552,2347,7259,6394,10560,5868,3560,5823,3451,3046,5019,2414,7785,7929,8217,3320,4226,864,6148,6673,6490,3400,1310,9492,2348,4415,968,10323,3990,5776,9900,9336,1046,7148,8798,5151,2960,6178,6623,1204,4454,1760,7168,6117,1146,10719,10458,1649,5397,10282,10584,9794,3475,10402,10392,10762,3270,413,1759,9711,3664,3306,5274,8678,652,8665,8416,6408,8046,1624,5865,9687,8657,5888,1062,2184,1164,3280,9055,3097,5372,424,1287,5887,4647,7493,6962,10740,5382,6805,4082,8845,3590,7270,2228,2630,3021,9731,524,4700,378,7771,10656,3398,6644,990,1322,10016,6855,5261,10799,9323,6720,6162,1252,7898,487,4787,9439,5898,7813,3728,7689,8353,45,8231,2275,1665,709,2700,7044,6229,9875,3411,1846,5567,9855,6894,658,448,8410,1416,978,9570,5480,2329,290,4719,6830,7788,5473,8767,6832,2750,4270,2778,4501,1530,4417,7262,9398,3441,8026,2141,6700,4803,4545,3201,8518,9432,5734,3969,9904,8241,9760,10652,4558,5677,1535,3251,3971,10052,2199,5751,9956,7627,3299,10078,6578,4000,2567,1193,1453,2231,1398,1714,1507,2613,8977,1701,4465,5809,820,5049,9517,5595,5917,529,4815,224,3910,7054,3727,1939,7937,6884,9636,8394,5682,5405,9487,4907,9554,9743,10322,6208,9414,8843,7213,7505,10188,7604,7837,9860,10351,2087,14,7212,5999,6474,209,9842,10468,3271,741,10611,5142,4020,1870,10141,2070,238,4670,3450,3188,2479,8304,1263,9502,3330,2361,9077,160,4884,2948,2011,1032,1652,9131,2428,4917,6908,8219,6141], "time_ms": 0.132813, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 613, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "generated_query_count", "value": 1, "time_ms": 43.819412, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 428, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "generated_samples_per_query", "value": 5518021, "time_ms": 43.819412, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 429, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "generated_query_duration", "value": 119608, "time_ms": 43.819412, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 430, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 1367.170391, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 898, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 1367.170391, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 900, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 1367.170391, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 902, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 1367.170391, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 904, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 1367.170391, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 906, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 1367.170391, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 908, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "power_begin", "value": "12-28-2024 04:25:45.522", "time_ms": 1367.171915, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 564, "pid": 20262, "tid": 20262}} +:::MLLOG {"key": "power_end", "value": "12-28-2024 04:25:46.837", "time_ms": 1367.171915, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 566, "pid": 20262, "tid": 20262}} diff --git a/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance/run_1/mlperf_log_detail.txt b/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance/run_1/mlperf_log_detail.txt index f2966f19f..a7de80d40 100644 --- a/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance/run_1/mlperf_log_detail.txt +++ b/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance/run_1/mlperf_log_detail.txt @@ -1,85 +1,85 @@ -:::MLLOG {"key": "loadgen_version", "value": "4.0 @ 486a629ea4", "time_ms": 0.004506, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 53, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "loadgen_build_date_local", "value": "2024-12-24T19:50:51.839855", "time_ms": 0.004506, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 55, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "loadgen_build_date_utc", "value": "2024-12-25T03:50:51.839862", "time_ms": 0.004506, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 56, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "loadgen_git_commit_date", "value": "2024-02-15T15:46:50-05:00", "time_ms": 0.004506, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 57, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "loadgen_git_log_message", "value": "486a629ea4d5c5150f452d0b0a196bf71fd2021e Hotfix: DLRMv2 Audit Test01 fallback failure (#1626)\nf9a643c0a0e920588da1b51a1d822e1071a9dbec Add number of tokens to offline (#1623)\nf06b920eac2170df38b5502c7443f50c9afa6045 Bugfix: equal-issue mode on offline causing accuracy run to fail (3D-UNet) (#1624)\n0ed5190e82e1ca1cf95626e50e61cb6678c7f2ff Fix submission checker and TEST06 for Llama2 (#1616)\n817dd961221d3ca284b024368130bd5dc5e4b54a Turn equal issue mode off for TEST06 (#1615)\n840435a1abe8be823e0a5a18ad3173bc520aa24f Update README.md with CM commands to download stable-diffusion, gptj and dlrmv2 models (#1604)\n3190d09a148bc1d72c9d946e4cf3b54db0d0e60a Typo fix in README.md (#1588)\nb0777f0bd3b7e5cd6a51d670d8b4df656cf6fb29 Update README.md - remove unwanted lines in CM commands (#1601)\nd7dba0826ac96a0e8bcae525086961e47c409a38 Remove loadgen warnings (#1608)\nd45a66c33bb0b6548d6172b38645bf94ffe24b39 Add upper limit to tokens per sample (#1612)\n44285d9a44041c3d4f7df0acf1e72c375e9fcbd2 Set completed samples per second as llama metric (#1613)\n357ccefca4db55ff373e029d5d1e3e6246bde5d4 Enable equal issue mode for LLM benchmarks (#1610)\n104d85531af7ae2a07ded36dd6e8c7f2c7746d06 Add sample length check to test06 (#1603)\n473053fc9f1b8c5edf60e11ec74607126938fbc4 Fix loadgen token metrics latency constrains (#1596)\ncc3daae3c7f18d7eedc5ffbc271d324e5aca6233 #1598: fix token and sample logging for Llama2 when accuracy_log_sampling_target is enabled (#1599)\n3a902e598e5a17013c26fea975c0a60e6d7feae1 Add support to dump 10 compliance images during accuracy run for SDXL (#1591)", "time_ms": 0.004506, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 58, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "loadgen_git_status_message", "value": "", "time_ms": 0.004506, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 60, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"05d4a62d355a90363847358e58afeee4a54ad838","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/bindings/c_api.cc":"32181da9e161c285f8fe46ddaa49e6cba2f9f918","/bindings/c_api.h":"91f58bd79b83b278f3240174a9af747fc38aff74","/bindings/python_api.cc":"1dc0865130371c6a8ddb97cd8039244d74d6fe46","/demos/py_demo_multi_stream.py":"6ccf7dcbe08f8382ed2d3618370c2b21e205ca13","/demos/py_demo_offline.py":"6c3075538e0a3df5e93edee917f4537099759e11","/demos/py_demo_server.py":"40e37ea121ac08b21ed6af41aca8eb5eda641b89","/demos/py_demo_single_stream.py":"4ee5f048fb0dffc415342a2bc2049fb155bd2894","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"126e952d00f4ea9efd12405fb209aa3ed585e4b2","/issue_query_controller.h":"923d9d5cdf598e3ec33d7a1110a31f7e11527ec7","/loadgen.cc":"6650091ba7a918f343b06eb7a5aa540eae87275f","/loadgen.h":"e00fdc6dbc85a8c9a8485dbcbfe2944f81251c4e","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"197efc96d178e5d33a750d07fa7b2966417506ea","/logging.h":"ddb961df7bcc145bcd7cce8c21f7cf075350dcbe","/pyproject.toml":"974adb632af4b76a1c72b914cc2cf9b3cbc66aa3","/query_dispatch_library.h":"13ad6d842200cb161d6927eb74a3fafd79c46c75","/query_sample.h":"e9187c8612bbdc972305b789feb6e15c26e96cfe","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"a170f991de69f8a5e45616e871b39a30ffed5598","/results.h":"d82500c326c2de83db411f1146882aa4692b419c","/setup.py":"e3e707e1191478332c00f1b6ff806f0ef98e0f65","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"aaf3803e700f3431a5ada72559eef5cce996ea64","/test_settings_internal.cc":"42aa46b17ac3fb2748cd5cde60fa8742e916f3c8","/test_settings_internal.h":"73b11c02228f3afb4c924db7a86fd8a2b5b9e483","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"4c9d98360b507d5673049ed70154fcfb2934f8a6"}, "time_ms": 0.004506, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 67, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "test_datetime", "value": "2024-12-24T20:20:00Z", "time_ms": 0.012956, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1198, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "sut_name", "value": "BERT SERVER", "time_ms": 0.012956, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1199, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "get_sut_name_duration_ns", "value": 335, "time_ms": 0.012956, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1200, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "qsl_name", "value": "BERT QSL", "time_ms": 0.012956, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1201, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "qsl_reported_total_count", "value": 10833, "time_ms": 0.012956, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1202, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "qsl_reported_performance_count", "value": 10833, "time_ms": 0.012956, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1203, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "requested_scenario", "value": "Offline", "time_ms": 0.018012, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 268, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "requested_test_mode", "value": "PerformanceOnly", "time_ms": 0.018012, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 269, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "requested_offline_expected_qps", "value": 8080, "time_ms": 0.018012, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 306, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "requested_min_duration_ms", "value": 600000, "time_ms": 0.018012, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 312, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "requested_max_duration_ms", "value": 0, "time_ms": 0.018012, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 313, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "requested_min_query_count", "value": 10833, "time_ms": 0.018012, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 314, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "requested_max_query_count", "value": 0, "time_ms": 0.018012, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 315, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "requested_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.018012, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 316, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.018012, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 317, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "requested_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.018012, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 319, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.018012, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 320, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.018012, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 322, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.018012, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 324, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.018012, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 326, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.018012, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 327, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.018012, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 329, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.018012, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 331, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "requested_performance_sample_count_override", "value": 10833, "time_ms": 0.018012, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 333, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "effective_scenario", "value": "Offline", "time_ms": 0.018271, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 409, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "effective_test_mode", "value": "PerformanceOnly", "time_ms": 0.018271, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 410, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "effective_samples_per_query", "value": 5332800, "time_ms": 0.018271, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 412, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "effective_target_qps", "value": 8080, "time_ms": 0.018271, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 413, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.018271, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 414, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.99, "time_ms": 0.018271, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 415, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.018271, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 417, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "effective_target_duration_ms", "value": 0, "time_ms": 0.018271, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 418, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "effective_min_duration_ms", "value": 600000, "time_ms": 0.018271, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 420, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "effective_max_duration_ms", "value": 0, "time_ms": 0.018271, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 421, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "effective_min_query_count", "value": 1, "time_ms": 0.018271, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 422, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "effective_max_query_count", "value": 0, "time_ms": 0.018271, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 423, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "effective_min_sample_count", "value": 5332800, "time_ms": 0.018271, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 424, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "effective_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.018271, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 425, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.018271, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 426, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "effective_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.018271, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 428, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.018271, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 429, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.018271, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 431, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.018271, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 433, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.018271, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 435, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.018271, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 436, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.018271, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 438, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.018271, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 440, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "effective_performance_sample_count", "value": 10833, "time_ms": 0.018271, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 442, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "generic_message", "value": "Starting performance mode", "time_ms": 0.019390, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 841, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "loaded_qsl_set", "value": [4117,9486,2947,9270,8227,4487,9799,4629,8571,8839,7101,2191,3315,6375,207,10454,3122,9490,4083,6119,1890,7657,9347,344,201,4232,5741,5454,9533,7842,1596,5330,10615,8827,8094,6714,9169,977,6619,9924,4982,4894,6544,10118,783,766,2122,7163,396,4200,9864,4254,8440,1758,10722,2829,2788,9906,2719,5613,8726,7847,440,6103,8567,7497,4793,1184,8675,10587,8270,7074,7408,8740,9265,4855,1199,220,6556,6595,1226,6695,3066,2143,6264,9806,7122,5007,10075,2370,2185,7254,77,2806,6243,6687,1156,1802,7302,391,3348,8159,9708,251,3595,3947,2302,4690,122,3116,1300,3768,6388,2986,728,5127,3904,6807,6259,2218,7555,8821,3580,7804,1321,8460,5419,6542,7833,3365,2290,6346,7952,918,707,118,10469,9023,1304,5220,1891,6878,9524,6522,7867,4037,7698,8507,9634,2145,10057,6263,7486,7557,2401,6082,3434,5927,8708,10081,8114,1349,7742,2742,7157,5705,4638,1764,3651,920,4016,2744,5859,6740,5380,9649,2,5245,5230,6421,4071,8909,6319,288,9346,9671,9646,9360,4702,5044,9157,6296,8318,8971,8184,7702,10475,7318,4024,5530,3991,1171,1909,6230,8461,8099,10563,5398,10760,10693,3688,8170,4952,6827,5893,7767,2656,10032,9178,10480,7195,9359,10608,10261,8975,5432,363,3346,3049,3360,4432,9736,473,7587,9416,1021,6596,10352,2666,2325,973,2071,7368,4267,867,1040,5796,3491,8237,9597,3489,10568,3777,7899,856,4818,2858,4662,777,10776,10026,6065,6481,2590,6780,10553,8789,6032,930,1049,10178,6187,585,3359,5832,2014,480,5866,5017,5141,1525,8271,8132,1642,6270,2265,140,1174,4641,8017,8844,6569,5575,7081,3779,3845,467,9761,3314,8653,4250,7578,8907,664,217,8419,8757,683,4565,1638,5216,2004,6679,4485,7683,5180,7422,6070,6283,2909,6281,8933,10639,4340,3079,3513,10552,2291,7487,2680,8807,8370,6127,7434,1822,908,8101,451,10316,8282,10724,885,1834,1556,6121,1468,409,1908,10539,7238,2891,10312,9435,2178,10051,4522,9650,615,7193,7796,2008,6011,4347,435,6881,1396,844,971,4216,5001,7886,6875,1348,4300,9348,2709,10580,9993,6705,8784,10821,9232,7724,6581,9764,2913,10125,6914,7693,4568,10526,10231,8502,7825,1783,795,6828,8006,4595,3892,3075,1278,10050,8235,4679,9971,75,2164,1175,3684,9869,8414,9016,765,2721,5786,10703,1326,7463,237,836,3912,5482,7835,4374,7661,4070,7754,7180,9945,10415,3638,4208,939,1097,7628,5725,3723,4547,1994,5232,5279,4049,7309,5963,10478,9834,7758,9463,3666,4663,801,4030,3010,8236,9237,2470,3382,6336,7570,190,4552,4113,2229,9018,7038,5021,8701,8268,4490,1373,6092,10181,10445,3056,4333,10394,1640,2328,3349,10069,10683,6439,3191,2544,4760,1739,8900,3,258,5416,4671,2041,9375,2661,6376,2306,5897,6373,2125,1940,5109,4256,6203,6411,9311,8029,2200,6301,1734,6176,6443,3007,9583,1841,4627,2523,4306,2801,5208,8607,4961,1467,10700,4031,2181,9958,8194,10295,9043,5366,306,9056,162,3196,10759,4349,5401,7030,6184,9216,2069,6357,5640,3015,1752,406,10049,3983,791,5610,5944,304,6380,1937,7862,9366,9365,1607,9252,2698,10350,5221,4847,10707,3023,1615,7017,5990,4045,7231,1202,7182,5172,3176,9612,6693,1013,10334,1805,8730,8841,7372,10094,6010,3216,8177,7406,9472,9608,2533,7691,3959,1623,5144,1831,751,4912,2152,8144,241,4187,10001,1924,8402,7877,8928,7294,4915,3876,3242,8996,7153,1000,10658,3946,284,5088,959,10661,1018,8931,546,4262,6489,3786,5278,4308,8178,7638,5211,7142,5771,634,8273,263,2548,4674,9910,624,3290,5925,1524,2642,7198,10345,3539,6719,10735,2006,10233,3816,9578,8624,6769,9661,6468,8952,9702,8670,5175,3449,1773,6130,636,1622,1696,1414,6523,2898,10650,4420,1280,2538,7880,3743,1227,2757,6131,3371,7445,1680,2901,36,6868,10811,7140,1900,10398,3724,8765,8278,7379,6040,7914,1580,3535,8033,7536,6480,3861,2437,5082,2881,10596,4902,8162,9637,4886,411,7189,1789,6704,9176,10045,4107,6333,4583,3923,4843,9935,9955,9297,3222,8257,6365,3095,1995,7924,5428,8640,6628,4819,7152,5016,7932,3226,1267,1225,9025,7878,6321,2373,3801,1399,9063,6506,9436,5781,7500,423,3588,7110,4607,3476,2477,3635,2425,9394,10449,4054,8148,5510,2236,9543,10499,6153,780,8692,8105,10268,172,4128,841,8925,5453,8561,2135,2976,2845,2209,7418,10817,5620,7010,9960,474,6545,8223,9713,205,10548,905,2396,2914,5591,2887,3379,4865,3304,5849,1309,10171,608,7314,6228,6961,7068,6943,4613,1317,121,8611,5902,5827,8042,1139,3121,223,2984,2897,3144,5379,10590,6433,3871,5475,43,1026,1192,1599,5729,6194,5838,1254,2054,2706,7925,6060,2177,10647,7727,5681,10000,3549,10814,4433,2100,6869,6593,9404,5534,5801,10321,9145,8363,7451,9072,4600,5096,10372,3022,679,5631,2552,723,4668,9613,10153,7423,9114,2973,3071,1413,4956,2838,5084,6777,5196,539,5036,382,7480,9327,323,4766,2273,2684,8182,10134,9203,2667,9185,2658,9087,6286,4162,6620,10472,1095,6759,7216,8191,9738,9548,5029,10326,5478,4089,8710,5439,6126,6160,4828,1006,1677,3546,6930,425,2645,4573,2770,10248,358,4975,10603,8129,8614,10180,7902,4303,2447,3427,753,8920,9449,256,6859,5711,5246,623,370,10767,2563,8300,9591,9991,5126,9568,2647,484,8644,3654,9507,8632,10076,9813,10638,4326,3862,569,7020,4213,5628,9341,839,3341,8347,5163,5563,7339,6928,1197,9158,3099,6120,1483,9545,4608,4215,2761,1259,6802,4364,4001,1606,4222,4752,3057,10022,1108,10592,4398,10376,9888,10033,4273,1238,3428,6553,6145,5731,7273,604,2436,3170,8673,5901,10074,4513,8305,8697,7476,5015,2669,3100,6344,9338,10383,4972,8727,1363,3223,10495,897,1698,9553,5662,53,9328,3733,9932,7102,2212,7382,7159,4556,10748,9294,5895,2262,10512,5913,8314,10624,2530,7560,10464,2443,8787,3800,1510,2027,9795,9670,283,2351,4260,2577,1518,906,3377,4764,2113,10699,3948,8465,9264,9267,5600,2924,3783,9209,7966,6464,720,6973,155,6227,7994,2636,535,8846,2931,2046,4953,10058,8228,5200,2211,4234,2207,8121,2007,7268,3212,7772,6496,10201,2762,7534,5539,10555,10496,628,3653,702,8683,10325,8660,8411,9387,326,8842,837,9688,3855,7053,1826,7944,1053,3193,456,361,3206,10315,2059,6612,6173,4562,9003,8034,9162,377,7551,10769,5354,6299,5997,2080,6288,6068,653,9937,10825,8833,3247,3058,4923,9104,7983,3305,9322,8339,7711,131,7328,5212,8623,10761,9718,5402,948,10640,8659,6479,2768,4616,2223,10625,4252,9980,9807,5318,6188,159,10655,3565,1478,6860,3521,4646,9880,1608,10427,9804,2092,4897,10238,6020,7642,8091,7789,4058,6681,4173,7918,5523,2444,9898,7161,7736,4667,2603,9370,7403,10797,1857,3276,2276,3445,8197,991,1863,4266,2170,4331,3296,6658,1308,7542,8458,4533,10177,6873,4190,815,2726,5589,2846,269,2634,1560,7067,7554,3102,4908,10741,10086,5286,8734,3824,1744,4111,3264,7494,3825,8834,6139,8432,147,1589,1871,941,2911,6018,7906,2874,3713,9523,4023,7522,6116,6864,5698,5464,3924,4590,7345,7395,4790,10540,1020,5309,5950,2956,4983,6163,9194,3848,3612,10525,9069,5253,7232,6876,5003,5067,2350,10375,10523,850,6399,10002,9689,7255,1016,3707,3384,2137,4242,4077,5170,8388,2387,10577,2482,3735,4313,8862,2029,8452,10528,6058,6871,5233,1195,3301,8738,3194,2638,2488,5009,6981,7935,10684,10631,1043,1962,3541,4973,9581,4285,9118,9770,8279,1316,6284,1161,877,7894,3981,2423,6872,5033,1798,6181,1255,7517,4067,3319,7457,5164,3977,1719,4436,9317,4765,4137,6515,10025,4628,3745,7739,9445,7016,1980,4481,8580,175,10704,10653,245,8163,3834,7006,9135,1950,4401,2289,1412,7325,4821,6917,1361,7569,6782,2474,8075,5757,1034,3353,8570,2678,4922,4356,6831,3401,10393,10508,5161,2635,1472,3331,3048,9526,5383,5670,7508,5113,10148,2912,8547,1183,5496,5139,1153,6897,7997,6625,9819,5766,7460,10124,6637,327,3466,1848,2455,5939,2282,6371,4812,6721,5314,4780,3112,5154,250,7103,7158,463,6920,8368,6024,6877,342,6799,437,2876,9690,4889,5129,2631,3366,8380,1528,10135,584,6292,9180,5863,10031,2432,980,7211,3117,6909,1248,3761,3601,4976,9420,10602,178,10755,1150,2009,6653,7378,4422,8445,1978,2390,5417,4353,6452,2002,10595,1306,2817,8735,8424,8976,4396,4919,9488,534,4478,8211,8264,385,8288,3063,9825,6996,9726,7239,668,2896,6212,5761,6271,618,8216,6838,195,216,465,8870,6774,8556,3392,786,2791,1096,10565,9399,7603,1723,4133,6106,5491,10443,335,8387,5006,8131,966,7321,4664,5361,8778,8085,10370,512,4408,9227,4676,927,5805,10457,5444,5333,2951,6804,7955,888,3706,8788,2895,1902,9874,7221,5222,10628,6300,3522,65,10112,7284,6826,9762,10226,8642,8030,609,3812,9914,8276,5946,4130,767,4876,3831,9441,5515,7928,4838,4550,78,6682,63,600,3798,8226,3577,8618,8919,4716,10385,5560,7695,9677,9470,6351,10585,6031,2941,2010,7623,1735,6574,1513,2774,1490,2303,5429,141,1915,7737,9086,2205,32,7165,4564,3715,8209,8559,4979,4985,7143,1656,6789,8025,7208,6845,503,3687,7028,5471,9975,5664,3404,9068,5248,2433,322,10390,3840,8058,1297,3551,3819,3342,7265,3266,8914,4394,3583,9786,9198,5609,1444,8947,3060,10252,5594,1291,7319,9559,3166,4669,9592,661,6602,4789,9415,2424,4265,7098,1876,7535,1203,10053,9809,3104,1941,2705,6635,3148,3598,1502,7383,9073,10425,6590,5162,3529,7306,10800,6393,8751,387,10172,2692,5719,2741,7855,4845,7039,2459,6901,6624,3554,4237,3867,10566,2116,531,5943,6690,792,8441,6260,682,3995,6154,4165,9066,2717,1842,5340,8261,8539,1074,4259,701,9500,8648,513,3417,2155,9041,116,4010,1218,7514,3738,10176,8592,4659,8491,291,10417,8338,7244,862,5005,4735,1362,964,9714,10691,9829,10779,351,4930,16,3561,5388,5985,5239,6821,4320,10682,3154,4367,8961,50,8733,4288,9893,824,2504,404,10529,6560,1489,8125,48,76,909,2237,2691,2416,1987,7699,4599,5585,10503,7807,3639,3994,5708,7135,4625,4516,5540,2916,5528,5975,7413,8915,1347,10065,706,5304,3436,6607,8262,5645,3625,1376,4476,2934,1592,1935,3958,5821,2456,3362,5367,4151,981,9573,10610,9207,6190,5275,4631,1708,1921,9287,699,1447,1636,696,3740,4597,3373,9444,1221,3780,8124,5381,7648,3536,10571,654,9064,9044,3503,9854,9626,3378,10675,10629,6080,4445,9605,2646,6982,5074,3618,52,6561,6185,3026,5968,9742,989,5867,9190,1178,2689,2001,6149,8334,2049,3180,5759,111,4691,2650,8777,3149,804,1868,5328,840,8687,1382,57,2419,808,10355,5588,5215,1403,5701,9,7853,8520,8669,8019,4006,9097,226,2827,1135,1901,979,819,2561,7049,504,4391,4400,6739,1402,5264,2230,10666,2566,7430,8459,10720,8794,10310,10174,443,6349,7310,533,7609,6200,4108,613,7558,7709,1553,5076,10645,300,2842,6261,2324,588,2193,2945,7510,7396,3094,1889,10781,3791,554,9284,9148,3519,9995,3209,5810,8322,9519,2560,3073,947,2781,7004,5548,7116,10152,10121,552,1887,2952,8739,9225,5819,3712,6550,5446,574,3090,8327,428,1266,8433,8383,4645,8754,6898,2652,6822,1455,2176,1519,8566,10303,6049,10068,4977,3894,7561,7488,5536,7224,7793,1989,3985,8695,3424,7035,7612,9938,6775,39,1850,7803,9174,3031,6441,2974,1603,7941,7223,7923,2831,2977,3080,2075,9236,312,8565,4634,8814,1558,10484,1272,7389,8872,10418,4755,7869,2610,10633,5046,6960,4193,7852,1754,2878,1936,3784,10314,4025,4805,4997,10007,9863,7155,10101,1342,9395,10291,9385,10218,799,1123,5843,2562,4905,1377,9226,5937,1986,7591,9962,1189,8902,619,9067,3458,7586,7210,328,8506,3000,1004,9212,4591,8020,8887,4012,592,101,3415,2527,72,7452,9400,7809,1561,7032,4840,7008,1056,8,5752,9588,3101,3106,3609,4503,8312,6536,3446,8745,1437,7547,10287,4816,1795,8004,5837,1042,9329,9008,9433,2047,1087,3421,4324,10588,6699,10747,7765,9719,10438,887,9062,6820,2946,4741,527,3043,1648,1133,2918,882,684,4939,4386,8641,8154,1323,6436,7873,10752,3686,10680,807,9886,549,3230,6147,294,7120,354,8171,8493,2663,6863,7489,6037,410,9913,124,6312,541,3159,10304,2117,735,9882,199,4315,9250,4055,3739,3826,9558,10782,3699,542,6645,7593,5123,6413,7532,8805,1517,8514,3396,9239,8883,6748,6002,5319,8690,6308,1691,4655,589,10278,5680,9489,6806,10465,4251,10142,538,5740,194,9167,9974,2107,1480,8439,5722,5674,5712,7528,3385,1906,10567,9057,340,5707,9308,6207,3078,2818,7518,6794,9555,9983,1676,1678,6079,2038,1670,2608,2335,10021,10243,7257,9405,8165,2242,1247,8620,7479,1305,8818,8790,7865,4998,5653,2097,261,5270,2140,2105,789,6984,7501,8748,9538,8167,7566,3307,8768,5195,3932,1130,3930,6535,3665,6818,870,6657,7938,8656,2975,8108,4377,1332,1792,10027,3395,5078,10380,724,6034,3423,4788,4511,9362,2462,7760,7023,2497,9802,5504,6817,4263,3361,8164,8418,2478,2287,3457,1264,9796,7218,6591,6401,1683,3534,7474,5945,9722,7233,3719,7072,9915,42,6386,1481,3426,8808,4075,9665,132,3679,1418,6327,4854,5667,1228,5621,10063,9685,9641,3792,8509,10725,1726,3510,6733,5311,6890,1127,1372,4557,9076,4493,1319,4639,6503,5373,319,6252,5205,2568,3262,5611,3692,8016,2058,10708,1990,10246,5149,6730,4745,9617,4496,8093,3764,4587,9046,8240,5654,7484,9304,9356,9122,9012,2409,2879,1728,8443,6110,2374,2695,6156,6123,7266,9525,1770,9102,10660,6745,5128,5806,10298,1081,5852,10365,5171,3316,1949,2796,5938,2888,505,9040,10137,2063,8296,703,7982,5184,5822,6713,8356,1212,10214,1643,1344,1740,6170,2765,8513,7491,313,7468,5830,5188,7420,8246,7931,5592,2300,5803,1333,1904,10524,2557,1320,9801,3887,3372,4648,8123,10207,10570,2782,9208,3493,8993,4688,9635,2367,890,4834,4750,2186,4059,9823,383,5746,6601,6995,3696,10626,6202,4034,2588,1415,3128,5342,9471,8634,6712,7283,5689,7596,5728,9160,6465,1823,8542,2381,129,3703,5297,5845,10090,9892,8109,10676,1417,6171,9518,2483,5993,6066,10657,4131,7317,4665,10546,7047,6337,6183,3537,253,7720,5316,4382,2720,8286,7478,3902,7237,3088,5066,2131,2830,10694,3644,1407,5650,5733,7970,8970,2283,8951,9288,6643,10349,4772,6849,3295,4178,1354,8522,10736,1720,4463,10409,7172,6220,1998,6367,7464,1973,6738,4238,9065,9537,2528,6911,4247,8425,183,7919,4791,5494,3274,6583,4995,7128,869,5889,7655,5199,1555,6754,10636,3874,5094,545,3805,1611,9596,2168,6048,7912,1253,3675,10190,6051,6887,5414,2919,2397,8982,2189,5410,2589,6458,7272,8280,6078,2349,9268,9229,8372,9571,2129,9070,7371,5048,8593,4925,5490,1503,6584,4713,6442,3119,687,3013,10757,8401,6102,6842,6324,7114,10127,9879,9783,4963,1626,2785,3640,9163,4835,1390,5087,8073,10203,3068,5747,1958,49,4916,620,1548,3018,9306,184,10272,3878,3053,2239,10187,1477,2902,9456,7260,5051,7639,5386,4477,80,1151,7277,5579,5072,5596,2204,10199,8267,2624,5110,6272,1330,10059,757,9947,1357,7745,6159,9580,6926,1486,10062,10488,8712,6582,7733,8905,3935,8265,7334,5495,9170,1657,4880,1542,9546,5577,3169,2880,8287,2763,1465,1041,10545,7951,2814,10729,1234,7881,6985,6670,5055,4329,9749,4744,7621,9946,665,475,6655,6976,1327,9703,33,9513,9857,3248,2158,7640,3830,6910,6599,10785,3908,9303,6403,858,2074,3941,4068,1488,4929,9954,4872,10514,7838,4677,1877,7014,8113,6883,1511,7174,381,1523,10229,9627,6966,9767,4283,1167,4168,6977,3543,6953,2737,4698,7908,8759,2429,7706,9642,7199,2064,7087,557,8591,4609,3962,2795,8283,6449,9092,9964,4970,4529,3616,6475,7523,56,2238,6431,8256,640,3416,688,10471,8348,5541,2371,994,5570,82,6166,2259,7589,10479,5713,449,2308,130,6882,6298,6517,9870,2353,4316,5452,2792,7556,3803,822,9084,798,9477,9082,657,495,8781,10619,4328,2710,460,10133,3234,2555,6069,5255,348,5511,6934,9594,5120,6729,3956,6290,830,9948,5911,1731,6291,4993,1533,9758,5693,8152,4988,514,10270,6000,9682,6946,8344,8051,5320,8247,9020,3751,7864,10163,540,5933,3650,6107,2840,9372,3701,8955,4269,8511,1024,2165,4739,5474,10239,2466,4157,6533,10780,9408,3312,2549,7113,4480,7741,3467,5710,10381,2392,4166,7485,9902,10116,1089,5281,2157,9182,412,6056,2281,697,4383,6531,7473,9231,7411,7070,1493,3570,4122,3461,4849,5119,3174,3139,8068,7162,3184,3691,2316,18,650,2382,8612,9692,6483,2442,7718,488,6642,8681,5271,9004,3942,7986,5406,6744,9014,6975,8403,7449,8721,3240,8297,752,4471,2833,10812,202,7701,10064,9164,1565,2172,3613,6589,2843,1052,8359,1932,2389,127,5508,4782,1048,2729,6216,5692,5219,4730,8987,940,7810,3907,4399,1220,9901,1549,1242,9815,673,7459,3322,8550,6865,5229,8557,2257,898,2893,8674,2446,7645,6528,2809,4711,2106,6267,1110,6415,7998,2094,9509,6269,4475,3096,1567,7066,2464,8516,1843,9885,8995,2345,4940,1077,903,6886,347,8766,3343,8534,5871,5310,5738,10217,7539,2823,9867,8238,7093,10667,2332,5535,1419,3203,7540,2417,1619,10744,4592,8817,9219,2565,4276,9380,6329,10426,8853,8106,6765,6932,3822,5864,825,8244,9566,8333,5954,5351,2978,3376,6275,10290,1604,1067,7901,3143,7979,3630,5252,5642,6098,3077,1464,325,10481,5853,10012,4658,8315,10029,2812,4123,4967,6097,4038,1755,2873,2905,8098,6941,3681,1070,8555,4875,10340,9944,9694,2180,7441,2904,8054,2531,9330,6750,6945,10067,6012,5092,10772,73,6213,1504,6425,8835,2520,1401,10768,3368,4375,4941,685,9479,453,1268,2245,1694,4325,7705,892,10775,7450,2679,842,3996,6241,1117,7713,10343,2124,5857,5617,6594,7305,8770,5928,2197,9166,5956,9299,10095,6524,10531,4335,6622,10823,802,3120,2073,3864,3490,6957,9845,3972,6935,5543,9774,154,5250,10342,8137,8040,4224,10500,7327,5779,2755,5085,8243,4632,1952,8047,7056,6304,3241,6219,5353,10320,4559,6552,4748,9475,203,4551,8552,7187,3748,4271,3232,5903,9402,1566,6381,1569,6192,5338,2965,8023,2360,10386,6383,923,10618,9172,421,3412,7582,7685,7332,4138,2820,876,1356,1325,1012,3814,9584,3799,5041,9273,5764,9757,1655,6991,8588,9423,8001,3853,6191,8755,576,6629,4115,8878,4253,3245,1885,2505,5426,2621,260,8848,3873,2060,8352,4842,7686,5359,3600,6732,857,5014,9386,9173,8150,3065,7127,10754,2987,9907,7347,1051,6114,1609,221,6666,7851,5093,2471,2465,398,2077,6459,5799,8462,7943,10731,2227,9619,3813,9031,4753,7797,6347,3836,1806,9134,6150,5643,497,3868,6915,5071,2769,3717,8575,4426,5358,1587,7752,4942,8857,742,8464,3442,3050,9660,1165,3309,5293,3171,5099,1154,1827,7728,3140,5995,7719,601,9119,2053,10056,10487,4949,5307,7866,7076,8052,4372,6045,9586,7799,4800,4575,5458,316,5425,6361,4965,1187,10816,1907,7808,10551,4531,146,3884,8978,1271,3568,10668,179,6571,163,408,1190,10175,10716,3837,9340,303,6792,5936,5979,6330,8574,6129,5555,2385,10594,4980,10728,5842,4321,10444,427,5890,1776,4598,5235,7900,602,4763,6907,6379,4804,1960,924,6133,5932,8041,98,3622,476,2517,8161,1374,10787,556,493,5804,9109,8786,1872,3138,1337,2822,6289,7562,3005,1392,6866,2518,1140,5258,133,10331,8306,5313,9575,10717,3156,2103,8495,4799,4672,838,2249,4686,2233,4281,1475,9108,206,9453,5941,8379,5103,2868,9988,9732,7417,7071,1231,2515,9371,4498,3081,9352,6529,2313,1529,4229,771,1134,3690,9005,1128,9503,9776,2571,8369,10271,4387,3408,5533,8666,9035,8390,712,9255,6634,8299,3221,7005,6850,2854,3921,4548,3501,1593,1050,1692,3685,4879,4277,2805,8468,2867,46,1258,2272,1118,816,6796,7965,2036,1722,9448,9851,6940,6862,10538,2547,6437,5343,6727,1359,6444,6573,852,7574,10336,61,7085,6247,5134,106,9192,2369,7786,9009,670,8311,638,5288,8446,5463,296,7337,5018,3103,10423,9970,10008,944,3205,5435,5332,7921,871,967,4777,271,7214,4652,6630,8199,6526,1695,5140,6215,9277,5174,8504,5181,8193,2863,126,2377,691,9662,6495,233,565,6466,1771,5348,9411,5026,5537,4801,656,1384,8396,10541,2326,6958,675,402,10098,4203,6835,6258,6969,7950,5704,4047,10642,5038,3340,1820,6525,7354,8752,5131,3937,4369,3486,586,5305,4971,9773,5948,7618,9361,6790,8743,4309,3722,7721,3944,8836,4154,700,2935,8003,7351,5521,9464,4718,4888,5124,7841,3135,5697,4530,7276,1395,9615,4825,6224,198,7805,633,8442,1840,1261,1170,879,9725,3626,3870,5050,3901,3954,9332,2906,10073,7674,722,6843,4363,9289,5077,6598,3059,3928,1114,10034,6377,1380,4378,4418,10505,4869,7246,1292,9942,7092,9139,861,7416,2747,1245,6423,9095,9653,3130,8948,6737,2404,4026,726,6101,7726,4540,10308,4243,478,2980,3227,8985,6913,2675,4504,3753,4931,2434,6753,1045,7477,834,4181,7134,10507,9514,1747,714,7790,6755,1971,8301,6009,3584,8877,4361,2927,9027,599,8160,9059,4675,8090,10091,8192,5283,719,3145,8671,8569,6824,8639,9253,5815,10361,6988,2510,2585,8904,7405,2398,1768,7989,8696,9899,9805,2439,9412,2727,7927,6257,8746,3982,5679,2013,4579,4141,7634,7553,346,603,1522,8764,7001,3603,7775,9715,7678,8482,8630,10530,7905,9211,5308,6760,5266,7308,7377,3957,7058,6340,3602,3218,4945,239,8867,9918,8130,8000,4950,8913,275,8097,1029,4630,10774,3173,2808,3754,1591,9115,3593,755,3318,6902,5762,2104,8893,3749,10401,8946,2616,8510,8937,7975,1078,6335,6238,10733,2044,7885,4233,7426,3606,5280,9756,810,9844,9695,8053,4704,3852,1484,1031,8399,5467,7672,7590,8215,5263,8028,2657,8117,6142,6273,5561,9748,672,7229,3098,84,10405,4272,2800,6889,8792,6691,4204,5646,5345,5615,4278,5315,4486,1543,9467,5355,2797,5400,1953,8917,7806,7073,3807,7312,1406,5716,7086,1738,6847,9781,10212,9631,10442,7548,10115,4678,4097,5030,2247,3002,10293,492,10429,8944,5106,4009,10269,5760,859,1201,10771,2026,6508,2957,1780,317,6445,8139,1810,1520,8983,2354,490,7375,9866,6035,5393,4751,9810,7710,5155,6576,7367,4368,332,5870,7762,7292,10020,6076,8355,1216,2119,2362,6470,6180,500,9775,298,4319,4935,7585,9563,4182,5326,1970,9266,845,6756,2256,8541,8972,7094,2748,7353,5185,10664,10743,9565,6942,7743,9912,5135,1452,1618,7592,3844,7037,2048,946,7036,5780,3146,5973,5505,8810,2892,3682,9607,931,6852,8823,10136,10440,2615,10764,9421,7909,8964,8449,3160,2153,181,10421,3357,5638,9478,9572,5797,2020,5368,10014,2415,9659,1772,9831,7340,10777,2584,1967,7190,5789,9905,3909,5499,5879,9787,2921,2683,462,2535,10730,7492,6718,10492,7124,1148,1079,3986,6891,5696,10414,5855,5737,4214,9389,3253,1273,3869,2641,8329,7323,4120,7723,4798,1387,5285,6983,7132,8731,5299,3663,4249,3438,365,5100,8684,8801,1993,4506,1671,4566,5122,1686,3195,7916,3880,1101,9672,8036,4397,2701,1786,8120,10726,3429,5114,4588,5217,1057,2735,10794,1331,8815,2261,4703,1667,9574,10803,10054,2435,8067,4393,6362,2937,985,8750,5374,770,2463,1749,5922,1063,10335,4743,2305,10396,7546,257,3297,8553,1775,10197,7279,2690,496,3454,8616,10612,7857,7126,3107,2298,4072,8906,151,1581,8061,439,6856,7903,2142,4526,5244,9029,4714,8335,881,10363,7503,6647,935,5133,7649,10561,6108,933,2551,6870,4673,2826,10129,5906,626,7717,1082,3575,1172,9417,3456,5568,5914,5856,1851,2953,4881,5851,6239,2050,4371,6477,7366,4452,9849,2246,7930,988,1951,10789,5878,9853,2323,2850,1942,1136,7421,10230,5130,6313,6929,4846,10267,7890,4964,6829,4357,8582,649,7241,6716,8258,1176,9263,5390,6994,5921,3192,1069,10796,5908,6486,6138,8562,8578,6925,1408,7917,8045,10286,6671,2352,1312,10309,8210,800,6050,2686,6214,8456,10302,8858,3443,418,8005,1445,7115,1824,5690,2569,2917,5197,6749,8643,1537,9977,5648,3997,2600,10395,8360,2861,1014,9790,9865,4095,6124,1666,9189,5160,8186,5137,10828,6025,1847,4932,1249,1299,9530,7282,6990,9223,3720,3818,7987,3137,7670,286,4731,7654,805,9668,10018,230,2467,8638,10084,8422,3481,9811,3285,4877,9281,10307,10801,102,530,1819,4918,6546,928,2930,6678,10253,10151,4094,8430,5056,7400,1313,10461,6039,7992,6664,6502,10644,6424,3383,4302,7186,8342,3559,310,10379,9446,2277,1116,5778,6538,1963,3185,1487,3294,10362,1207,6921,8181,5236,10353,5709,7359,10011,4147,910,2476,5337,9213,9916,2696,2391,7286,8797,1796,2411,2570,7990,2889,10723,10150,171,472,6246,7280,10100,8298,4517,2775,7831,10582,2365,3183,3670,6432,9074,7993,4852,646,3645,4052,5323,5556,10807,4014,5349,7109,1296,7125,8147,10043,10085,10709,6906,2251,1058,3842,1132,10328,6005,7725,1853,1862,4528,10824,2837,9129,2214,4209,3557,8758,2605,6667,8761,7620,2156,8505,4322,5269,9818,4447,3186,4709,3920,10087,568,7985,8911,2331,6494,10654,7631,2875,5363,6151,6896,5481,9705,3286,7753,5449,6013,6874,5153,1779,1494,4722,6728,4726,8008,6316,430,4403,6359,8762,6936,8704,8677,1111,8153,8680,7700,2622,582,1462,1621,4373,5430,6218,1825,10038,5597,5578,6064,7907,7892,1469,9038,6763,7197,10543,9461,3881,1536,8992,7465,235,4240,7502,1541,1616,7207,6608,4245,3470,2961,5423,9460,3993,1979,3680,8998,6844,6370,8959,3488,3217,2604,3891,3369,6467,6320,4458,9248,1793,8676,7581,10502,730,566,3694,4699,4416,2284,8110,1933,9333,10388,9873,7454,9254,8881,775,3497,1712,7090,5158,10122,7483,10818,9973,9704,1969,4606,2682,3354,8869,1338,10215,4379,9793,694,3725,7365,4431,8302,2752,3255,5392,5468,68,10751,4871,390,457,4705,3181,4081,1491,9246,5982,2524,3721,2372,727,2163,4734,6689,10005,5237,5583,280,5207,3034,4022,5934,6326,2334,7209,2702,4643,7498,10119,5825,8962,8185,5440,2169,1803,1742,7316,5183,1177,2936,5020,6959,5063,698,8234,9307,3086,8831,919,2234,3787,5714,8263,8991,6104,8554,3350,5187,4092,8071,1898,1394,4992,5371,9200,486,7787,8939,6047,2190,4210,3142,4261,1583,1121,1927,270,4948,6204,10255,6811,315,8896,3477,3895,9606,8851,4623,4244,893,7009,10344,5404,2539,3448,8682,6105,8965,9706,2862,5375,3273,242,9218,3585,7027,1112,8232,7751,8332,9015,6507,8544,4428,884,3857,1086,8115,949,4160,4268,6867,9803,8337,6903,10165,5744,3284,5854,1785,2319,3089,8448,389,2066,7236,3351,1466,123,1956,10341,8785,1741,2310,1544,10547,526,4421,8916,7915,161,2972,5424,1301,222,4661,6438,2496,3109,8488,9354,743,9345,3615,10721,9622,51,8829,9000,4890,10182,3760,8921,1867,5754,3265,2399,6089,7313,6339,9285,212,3308,1886,10441,2341,2626,1284,8138,10424,10204,3572,3966,7296,7303,6846,1875,2758,9663,7173,2959,7261,6295,8303,9088,7669,2112,1328,809,4456,3132,2346,1241,958,339,7663,7531,2519,7509,6949,8340,9959,7063,3055,4636,4650,7680,7203,6030,2264,8979,8819,2500,7559,5606,9434,3256,10689,2321,3324,523,8078,992,2736,7513,6385,7052,5970,5483,278,6350,1730,9367,729,157,7960,4446,2110,10575,3544,10795,6554,4384,6279,6659,2623,2082,145,6372,3582,4127,1878,10826,4129,1017,1837,9788,3281,1663,1929,1597,7393,10473,5167,9599,1482,5394,4906,952,2098,3465,9240,5524,3459,1724,9454,6189,5210,8609,3447,252,20,8014,560,7346,8756,9939,6463,3810,8537,7240,6703,3435,2252,8404,4582,4011,285,3744,6435,8127,6676,3492,9245,7358,4079,3555,3329,9868,10154,1435,5189,8220,1275,3926,7007,4464,8647,10250,7756,10798,10221,8891,4723,10161,6297,9083,3765,95,3530,10346,9909,1532,3700,2541,9989,6816,10311,9640,6547,6384,2081,7112,5666,3974,10374,617,5691,6310,1702,2487,2285,4644,3336,3756,1224,4343,987,369,1093,5223,1610,9981,7227,4114,915,1186,3856,583,10701,6518,7343,860,956,3165,4762,2529,10281,1429,3413,4135,483,1449,739,4878,3697,769,5841,4621,999,10557,9149,6955,1997,1073,4172,7653,5518,1485,3287,5466,9746,7146,8953,8830,7521,8289,2198,5300,4404,1659,336,5723,2857,1499,8658,5105,6122,9601,7954,7573,5243,762,3418,6234,3161,2681,4441,4649,8345,3804,1818,3177,833,9727,10225,965,7635,3006,2772,1142,3208,1098,150,3943,4761,3460,148,4784,7496,7263,612,3524,984,1983,5502,1120,7200,4104,28,9675,9544,7936,8763,9603,4304,6014,8498,10131,1367,2580,5959,2016,1205,1351,3288,4457,10673,8116,4406,1206,7046,9177,5683,7707,3547,2045,4376,1516,2865,4622,3207,3659,9175,5121,2777,2653,8524,1157,218,6418,3321,10102,2380,7980,4617,4220,8859,2550,9473,3495,5408,10677,343,7398,380,2894,5325,10235,1329,2869,7202,10437,6613,3509,2703,6314,7226,5218,10184,10627,3526,7179,6801,6853,8560,962,10511,7729,4125,4134,4392,2766,2492,5971,6144,2358,2583,1122,9632,2992,747,5145,7409,8392,543,3133,7946,10450,754,4981,5441,2618,6462,1229,8587,6498,9512,954,5848,5688,4759,5331,356,8088,7814,4350,7795,8176,3631,2773,4999,8259,7381,744,6609,2564,5940,1594,3755,6476,6540,5057,10616,9221,621,597,7887,2606,81,3980,2220,6640,518,2659,4538,9926,1733,7948,8018,3004,7363,2021,6992,2342,8483,9373,4527,4419,10224,5816,10623,2693,2160,2963,6773,74,4684,104,4898,4809,1674,7499,10696,4695,3978,2699,5214,7315,829,10410,1145,803,5951,301,3763,4196,7746,8328,3070,7407,6055,938,10522,7977,9896,2243,7033,9814,4494,9676,6036,6100,4775,12,2195,10196,1103,5862,3333,3471,6764,10222,4174,8568,8249,8362,5492,4053,5630,7000,1966,3422,1711,7682,295,865,7361,9652,5790,5626,3821,8409,9859,6363,8860,5989,4087,6395,9848,8308,4341,7520,7616,2671,6836,4860,3014,9789,9205,1918,5053,3335,9542,320,8884,5204,1442,5558,8250,9872,5952,1424,3709,8195,6140,4057,6633,659,2383,5834,1508,4246,10010,7287,8076,4554,3599,10685,1431,4050,7264,8535,5166,8086,8133,6541,4201,3545,9923,5647,4035,7740,10183,4169,1689,2664,5770,1882,1295,6978,4921,1812,5412,5387,8668,8408,4781,3579,732,3899,9510,3082,1914,7242,1557,1912,3961,3250,1039,9611,6683,9820,7647,10220,5720,10173,10330,4909,2337,2654,7133,1896,2883,2448,10727,2793,578,3199,8572,2743,10223,10662,9378,10004,4008,9331,1675,718,4682,4274,6762,1240,3261,4046,1934,79,1988,9624,5601,6621,3153,3882,10144,7429,6430,8986,5860,3225,5873,3960,9397,7687,5080,10674,10778,6193,5531,5793,88,7583,4601,2248,5748,3393,6702,6857,6950,8635,2375,1141,4236,4145,2733,5629,401,8954,5777,6783,3337,1943,5918,7821,255,10692,1434,9058,9629,3168,360,1690,1230,9458,951,9271,6469,5967,3061,7579,1003,3108,7681,8486,6265,4409,6109,8412,5618,6487,5562,6276,5907,7575,5529,10601,1725,9096,6905,7801,8610,5277,2627,5035,9569,3157,9957,7322,1600,6157,796,4460,6627,5685,10805,394,3669,7330,2426,7744,10520,8645,8932,5209,5820,8391,4769,6132,8613,1836,3841,3041,1085,521,5186,9462,7136,6429,9951,5409,1269,8128,7516,9188,8354,10167,10216,9497,9972,7399,8910,6456,2810,10339,10819,1646,9616,2174,4175,6091,748,3661,2366,6654,587,3627,10097,8367,1109,1844,8652,1364,8151,5060,2480,7817,4284,1076,5257,4861,7355,5872,9709,1298,8145,8519,806,2427,4524,3660,6293,4444,1660,3989,9374,7192,5369,8292,3914,5608,210,1002,7154,1302,2855,2412,10211,8799,4612,9744,267,5877,7137,8426,2828,4192,2378,6236,4924,9921,2440,6378,3556,9511,1897,357,8104,2400,10609,2970,3499,1830,8321,4543,7794,2971,254,6223,27,9480,7003,8856,6954,7704,318,2407,1182,4624,644,9496,10318,7961,9257,5695,6231,8894,4186,7976,812,7331,4217,7106,8092,1126,4102,8980,10591,3734,1213,2670,9679,3172,10459,8626,6492,3323,6646,9539,355,9337,8716,3808,6165,10562,5658,818,9707,8958,2134,4866,10317,7818,274,5457,3388,4282,4330,6482,109,561,4955,4874,117,8038,8866,826,3938,7069,9357,4806,4903,8741,2688,6343,2402,9895,9561,4100,10491,2597,3533,1849,3190,8737,8967,9007,4468,1068,10277,2258,4360,59,107,1036,5769,8437,9843,9700,9127,3525,1,7297,1809,5972,2484,9840,9609,431,5800,671,3009,10159,481,5025,6254,9602,2025,1001,8679,7472,8929,3164,10786,5165,7177,9987,733,4959,10276,6199,8022,9001,10576,9036,8938,10189,9238,297,3231,9080,6512,6509,1568,997,3773,9643,1030,22,2192,9407,7798,2813,10039,2088,5173,7170,6652,8536,10436,3325,10338,9437,1977,1919,9808,2430,10240,1107,6451,2784,1817,8037,9633,143,7117,6964,721,4194,4954,9730,8203,9491,1807,5659,9116,9816,6808,1160,1358,4106,1166,7206,371,3636,7656,2148,1938,7394,3634,8204,4770,4938,3896,4577,8548,7530,6016,7461,324,10646,7307,7338,8436,4381,6615,6087,5668,942,9953,1899,5546,9406,15,9684,3552,7335,7392,450,4470,6201,4040,10641,8713,953,5675,4947,8715,3092,5422,6023,5783,3950,1423,677,4710,878,7130,10822,5512,7204,9929,7326,2019,7774,3197,10810,7732,8471,5052,10750,4482,5156,334,8526,6222,9214,6614,7258,6488,6680,8320,5259,3757,9549,9196,5715,3581,5413,8031,5224,6497,7651,1100,10110,4797,7839,3091,10123,10358,8188,2208,5447,4542,373,434,3293,4904,214,8494,5450,1721,9828,9961,8747,4365,110,4910,704,9155,3504,7235,7734,873,10448,7667,8824,3796,7971,4605,10003,917,4066,705,779,1612,8861,5260,7295,3027,8487,3823,9826,4459,8941,6715,2085,3517,4808,7075,8272,4500,9656,6927,8405,1386,4737,8475,8365,1813,10088,4996,6029,6993,3364,4491,4885,4287,3667,10170,9325,2314,1129,3911,7830,5924,2676,4934,9403,7940,1982,5054,3028,9693,8633,454,1944,8599,9833,8595,41,1400,9291,25,3594,2697,8945,8840,7290,8963,2043,8431,6618,7978,4635,1669,7879,4019,3811,8168,1791,7099,6588,9120,8107,2633,7467,2490,3905,4697,3652,6742,1718,3987,6206,8467,3716,8875,9504,7822,1874,2899,2537,1515,7730,9667,9388,8007,5294,7185,9778,9451,446,1512,4152,9894,6948,6416,8189,10042,7024,232,648,1425,9531,10391,10715,6568,6353,1441,4423,470,7962,5376,2725,4610,9079,678,24,794,963,5479,2339,5023,1627,9260,19,811,1410,2506,2086,4218,7015,9600,2022,9466,7222,7410,438,2732,4654,7252,8826,1766,3483,9081,3998,2315,2767,5503,9724,4060,4507,4857,5360,8310,10600,3211,6448,7386,3790,3872,1343,5636,2280,2384,4015,5068,6328,7697,7538,1588,2760,4701,5706,2338,3925,9966,2674,5599,1985,3879,6636,10366,9877,3042,544,2724,429,1957,8924,1634,10106,6242,9852,1884,1799,3610,299,7696,1144,5923,5194,4467,5813,1925,5177,5554,9889,886,6244,8968,9883,1910,1661,4438,7766,2194,5792,7610,8874,9729,7166,5700,8477,2091,4076,4962,9797,10400,2179,6539,1243,3452,6322,7091,10164,6564,8879,1637,2222,8434,6026,9485,4235,10564,6164,10599,186,7953,1223,6484,630,2034,6225,5991,5787,2344,7376,6979,4462,204,10621,6747,13,302,7329,4339,5983,6338,3113,3397,2213,5772,6603,4140,8357,8048,9752,3875,4177,1106,4179,2789,2995,3774,625,797,4188,333,4093,7495,6043,8057,5433,9839,1038,9750,1763,8427,10470,8601,1471,7956,8118,6848,6237,3794,5899,7245,1913,8242,6565,5329,442,2644,986,2983,5684,7060,368,89,10710,8027,2481,10632,9061,3313,10698,5357,6054,2221,4615,1578,9878,2509,4810,8126,848,1303,2489,2441,764,8255,7065,1527,553,2628,6282,4518,6331,8096,3017,2039,5350,3657,6795,2357,6577,4296,510,5627,338,393,10407,6951,9090,2798,4589,8606,6434,868,4778,631,2964,231,4594,6665,3945,5427,1293,8655,8499,388,3633,1393,2877,8319,6980,736,622,8813,3797,5883,3433,10160,2786,9567,9022,5739,7920,8966,1083,4771,3532,10697,4826,137,2182,522,8984,9982,10264,4017,1974,5652,1981,2910,7029,2990,8724,3505,4651,7819,4345,5497,1928,7782,8949,433,2403,10288,6751,6342,4712,7684,9683,1035,2844,2317,10157,3858,4370,3883,1554,7061,8373,8015,5686,8795,6278,3683,8686,5493,5168,4032,1748,10790,1458,1432,7436,8032,7625,8651,10815,5998,10506,5202,8688,3678,9343,6510,1629,3182,1353,7419,3785,2472,8393,7761,1355,6332,9681,9931,364,10678,854,9098,9691,10792,5289,7764,4230,5069,7055,3955,3576,4567,10194,6414,6766,8956,3269,2997,3888,144,1777,5477,7013,5192,3642,2311,1336,849,1601,4968,7482,8135,6404,3124,8622,2949,5403,9269,1256,10521,9091,5486,6970,2815,4311,1893,7183,85,2292,9856,7984,855,1750,4279,1713,7002,7626,1705,2090,105,4687,9153,2294,10411,7939,6707,1672,10284,8545,2609,10714,8589,1757,7469,7217,2263,5027,1745,853,4429,6758,9651,7144,9286,7676,8527,8155,420,4443,1845,8082,6611,0,2794,9547,639,1222,6115,1572,9903,9495,5545,3243,9621,5182,247,5581,3289,3729,776,5095,2379,243,8744,2295,3732,4725,10544,2745,1246,10024,3345,2076,1155,627,6939,2454,2839,10259,641,5988,1808,7401,9723,6734,1294,10155,5818,1260,7904,10671,7750,1061,8224,4013,7194,9030,9482,1505,8012,10185,2293,8551,520,4831,10451,5002,1628,244,23,5459,3152,3566,2923,4389,2508,5984,9363,9138,4756,9871,3200,1729,8922,3064,5090,4626,491,1968,7768,4039,7882,6317,4822,10244,6947,6559,5,188,4424,6963,1879,10447,8269,9576,3865,2632,4003,9368,3484,8089,2790,5472,1922,6723,5553,6057,2734,4989,4706,397,8384,8002,6233,3469,1576,8470,9846,2836,9521,1829,7442,8771,7731,2751,813,9051,5637,863,329,8056,5574,9779,10558,9234,8923,366,10036,7320,8997,9522,5301,3292,3327,1113,5242,5602,5876,3817,6972,1054,7870,4310,211,1290,4005,6888,8732,7384,9564,6837,6044,6205,5655,10017,4824,2938,7281,10462,5091,6918,5488,8481,7828,5498,7613,3405,3648,6007,5634,10209,10109,10516,10466,10688,6501,2023,400,10517,10399,2297,5079,4148,6248,3482,5231,441,7759,9244,1064,8564,8973,711,10711,5525,3420,2526,4757,3759,8142,5750,1787,6232,10806,3730,4483,4255,2811,2707,8295,9699,10739,9911,7622,1784,6592,9590,9556,1687,7643,10265,1620,1119,8451,4198,10519,10607,6446,4227,4943,6810,5061,9364,4570,2738,2640,10126,8694,9666,5043,1883,8233,904,6081,4532,3553,7533,3507,7636,6788,9128,995,4867,9560,5190,1115,2771,731,9777,1653,6919,6776,10262,9881,1196,3386,6390,3574,8693,913,2714,2713,8780,674,3478,550,686,4786,8990,5885,139,10416,1684,1811,5147,5614,5178,2662,4080,3141,6885,4211,307,1232,7565,2704,7598,9383,5930,7348,9999,8415,5980,6965,3236,6685,8251,2065,1573,4893,10245,2051,1239,4581,3425,2594,2740,3394,2607,384,3646,3567,5732,10820,7380,9440,10827,7490,1194,9984,4435,5587,901,5828,3008,7045,8749,6974,1547,8225,2202,6968,1602,9124,8466,509,10066,934,5022,2542,10193,1852,5339,8180,6639,3263,4159,5632,1590,8059,3952,7225,158,3494,2723,6447,5625,10732,2864,6698,3906,7974,4002,10044,2673,1008,266,9293,1999,7714,7968,4112,7779,9610,2266,7515,7274,8558,2139,8725,6067,4642,8930,374,8485,4717,173,2575,3832,34,2449,1346,7854,7675,8378,1743,7178,7022,2495,5542,1668,40,8230,3849,3363,10006,90,1143,8636,3591,2779,2994,9997,8281,10483,3463,4899,7722,10830,8492,4136,10368,7437,5013,2783,2592,9110,10227,6638,6650,7827,5455,10583,4338,7387,4158,1318,6757,4126,5434,1931,7856,9721,8597,1209,737,4474,2718,93,4337,5102,3317,10432,83,1339,10534,572,3788,3298,8604,4176,8889,7836,9505,8400,3246,9552,3390,7599,9630,7850,6075,6854,6077,2687,866,6668,9206,3781,3587,717,2312,6600,6407,6341,264,6952,1579,2558,2532,6562,5389,3970,7034,293,7777,9506,4873,7247,9140,10493,464,4225,5639,3134,9195,2386,5603,2578,8533,9782,436,6893,5590,3596,2083,9908,5891,5064,9589,8760,1693,5571,8837,168,6382,8512,4334,3711,10474,9655,1704,7057,5724,5673,9735,5522,7289,6944,2787,1709,6198,1131,4555,2999,555,7781,8812,632,5743,6513,3414,2336,1682,8988,8087,2343,4412,9424,99,444,5874,10460,9017,6094,7614,2147,5125,10117,7840,4637,5104,1285,8602,1091,10113,9887,7576,10605,3258,4018,276,9324,7641,9276,10477,3054,9256,9450,7105,3953,5909,8717,6504,2715,2154,9197,8828,5929,3621,3016,7042,419,3847,9320,1334,3782,9312,8699,3228,7715,7370,9313,8346,4425,3047,8838,9049,6352,1340,4492,3356,7095,663,4660,4119,10146,3129,7369,2040,6674,5527,9259,2322,64,7858,7019,8417,1746,7285,3332,7129,6814,4796,8274,3083,2824,3001,8621,3746,5344,1575,10357,4758,5436,4896,1060,8371,426,5031,7991,5916,1335,1345,8444,2162,10791,4539,9850,7051,2270,961,94,4578,6521,9290,6724,3809,6306,1644,447,7084,9623,7811,8895,2410,2493,4839,6003,7889,8892,3674,1283,790,2118,1473,5070,9296,469,9447,2128,3252,3793,10408,466,9013,2776,10452,6135,9187,6662,8183,1605,3695,8077,9430,5955,2376,7511,1894,7304,8703,6063,6672,5462,4405,8742,10120,5593,2475,208,2431,2672,1288,2582,5519,2586,7843,353,8981,896,10275,647,10456,3689,3802,4317,6663,6118,4185,4708,395,4829,3538,9278,471,5010,8531,6345,3586,6688,120,847,8804,2494,6987,1125,7031,6879,2730,6717,1033,6709,1562,2274,9686,7735,5942,4863,10572,10404,71,2968,2067,5717,92,10412,3311,6833,5509,5794,4785,5042,3656,8366,3020,10494,2405,5758,10536,9541,1920,6696,6287,2756,31,10430,349,3571,7708,3037,5256,5321,10554,7529,9733,10179,5227,3702,9785,680,10140,4195,3381,9638,7385,3643,9165,10604,3355,8044,4603,580,7455,9274,6389,6169,8254,1630,4696,7633,10327,2173,8039,9220,10247,7041,6706,1492,2217,9186,9024,2127,7947,3550,7563,4143,570,5633,1816,4510,5549,10139,4149,4450,9112,4078,5910,10132,4414,3806,9204,10713,9384,7250,3444,2330,10071,4990,7959,5900,8010,695,2418,9349,2079,793,6022,8438,3569,305,8863,8888,1397,596,6355,5604,3795,4291,2731,5203,2908,7568,3939,8175,4569,1762,2017,3766,3623,8062,138,54,5884,2240,6735,4455,5961,7527,6660,9060,8855,5785,6392,7688,551,4768,10367,2546,432,6143,5703,7278,547,4440,4614,2031,760,9107,164,921,1262,1858,6701,4692,9105,4354,768,2359,2012,6566,7820,759,2144,10251,9156,6096,3370,2226,708,1514,5835,8811,10060,4184,9763,9838,2553,6095,2739,7630,4521,10515,8615,5831,2939,7650,6491,8999,311,8060,8381,9298,7748,4314,10356,6823,788,3150,5169,2525,7913,8479,2035,9235,5396,3833,6781,5501,2126,8389,8729,5767,8543,8899,149,3676,5028,9026,8523,3229,1778,2279,3562,6931,3968,2388,1586,2084,7646,331,2309,9358,9410,6419,8661,7981,2872,5966,3249,1438,1179,5514,9442,1421,7544,10463,10092,6813,7402,10433,2096,4264,3254,2188,7571,4693,9111,2183,3233,1270,9930,7201,6177,9326,6128,9272,10283,1420,482,8918,5904,1717,10166,4410,7447,10504,1500,7996,3302,62,3277,9151,7220,9469,3629,3965,3506,508,8207,7776,8825,6989,831,3439,8361,1092,1570,7605,3019,7205,3437,6454,3328,5547,6280,10300,5808,9936,4473,7111,8532,1368,10527,3775,10439,8290,3605,10169,2393,4774,6741,7802,7435,1375,4064,9210,1010,1274,662,6711,8865,3214,4099,10299,7104,571,3178,8753,2625,6461,2033,1379,5641,2095,5965,7249,6797,1947,9280,2643,8791,4864,9821,749,10294,5385,4132,8897,5726,3564,3558,6499,2032,2834,10634,10389,4163,8820,3038,6585,7673,5247,7352,9830,4171,7151,7824,2244,4848,10371,1211,8876,9133,689,7397,4352,1769,10832,6356,8081,2637,4926,7169,3409,8450,3934,1080,10061,693,3030,4740,5784,2907,5576,7288,10670,9125,8903,5886,6548,907,9950,268,4553,7424,750,5947,5377,5663,2235,3769,7615,4239,2501,6572,1559,745,1495,10273,10681,8489,4395,10550,4749,7145,2825,8650,10485,8134,10023,8463,3291,6412,9078,9224,9431,4944,3105,1422,10455,5507,4180,9075,9123,9438,6986,3671,8386,7507,7716,9927,5284,3589,1448,8503,7832,1584,8714,362,5802,5564,4604,10718,5437,4307,7506,8890,4913,4146,7823,7181,4280,7362,5384,2890,3886,9117,9728,10687,182,4183,10770,3859,10228,676,5251,690,1443,4927,2540,5520,9837,598,2513,8190,637,823,5487,4275,4862,9484,6360,8063,3762,1474,5032,6158,10419,3431,7602,9279,2841,1635,9928,8617,1959,5836,3440,4197,6061,2932,3375,7550,9791,5273,7964,10249,2579,452,2271,10518,6111,4096,1647,1753,4212,6798,5378,1700,7876,8594,4033,1869,5024,2620,9011,8095,6478,5881,9319,8205,5978,6779,8200,4074,8850,2269,4073,9222,3487,4519,2885,7160,6172,936,2458,7834,9468,7763,3430,1707,187,6770,7770,1055,352,7783,10737,9401,7167,1991,5240,1180,9121,1861,8395,2268,8957,5774,996,8605,3772,8940,4836,891,321,7988,6534,5671,9144,9301,9754,8849,2554,1546,7425,4746,10070,10742,2061,1782,6038,6250,591,1257,1391,1954,8206,6307,1903,9941,3093,4348,7666,2996,6904,9100,5694,10143,10663,1426,607,8201,10035,7107,1371,7747,2042,10206,5327,1992,8420,5391,6752,8538,9396,3479,7849,6514,6892,477,8969,1585,192,5669,516,1237,1833,2979,4488,9753,4362,176,1916,7215,5365,3213,1563,7973,5557,8685,2619,1200,3502,10104,4851,7267,8100,6858,3062,4062,9876,2821,645,9604,4742,7118,507,5987,2668,579,9459,6125,10813,10046,10202,70,227,9483,10606,10332,3198,3044,10672,774,2601,1251,5953,559,4109,6563,8803,4832,1094,4911,4514,4549,5225,9428,3931,9381,10578,8473,10305,1506,10,5012,2587,10195,10434,1162,3347,2556,976,5152,414,5445,7588,4680,1210,5566,4027,1651,10013,180,9734,4150,2216,5584,9168,6426,10533,403,6391,8317,7471,8773,6772,1072,2835,5935,4951,8382,2799,9835,8079,629,2596,8525,7123,9678,113,3992,10149,8364,761,3111,519,8672,1521,7119,1011,2057,3940,7139,1236,1450,1930,1307,1191,1147,17,8074,8428,1509,9136,7293,4515,975,1388,4537,5672,5513,2611,4156,1071,10105,4453,2998,9717,875,8084,3750,10030,9142,9784,7311,6895,2514,494,415,6325,8698,6626,2460,9933,108,282,10589,6610,4323,1427,4830,6348,26,9527,2832,2015,1383,7344,1961,1088,2149,3496,8083,4294,4105,5850,1265,4969,8187,1137,8324,1625,2413,193,5964,9355,8629,4366,7933,7456,1821,4327,9053,7859,6520,103,2420,4882,3084,4653,562,9979,1381,10099,2925,8667,10048,10482,6210,10637,5861,5958,7601,1765,5176,6366,7512,10788,4048,10513,8935,10186,1460,8398,6532,6028,3029,10258,7891,3903,4402,537,2062,1281,4813,740,7324,3040,4299,7668,3275,9071,8275,10532,9628,4580,7969,6649,5485,6042,4596,4602,125,8722,6015,758,5287,3237,3391,9664,4563,2981,7427,3951,4618,4312,2803,5268,9261,5992,8960,8376,375,1688,610,9393,2210,8111,912,4994,1214,4355,4733,3278,1168,5352,7026,5812,7390,3929,3718,9800,9249,6656,4098,7543,5415,5334,5034,2521,5228,1828,273,8886,3604,9425,7156,4574,9965,10712,2886,7298,10613,9052,4820,778,135,2132,10831,1761,6851,5039,3179,9452,2929,2982,9037,4683,9106,5150,7958,4585,5298,2395,10130,4221,4738,287,4891,2203,4572,4508,1697,5651,9191,1832,4295,10556,8454,1378,9741,4110,4301,3155,9897,713,5136,4681,9141,5138,6088,5157,10773,2485,9499,5644,1664,536,7580,8598,4088,6422,10257,6400,5551,1440,1534,4084,4633,3051,3949,3767,1946,2780,4946,2940,899,2278,10289,4063,2660,9418,9045,2250,9494,309,8245,8691,6616,642,2639,2985,1459,3235,7388,455,6579,1315,10260,9618,6197,6460,498,5290,5322,337,3025,6245,7440,3922,7141,10354,4576,4933,10292,9582,9429,2102,7462,5919,5996,4978,2333,8043,8149,2296,10028,9657,3012,6768,7692,1550,9824,3528,8871,5347,9179,10579,10297,5159,1654,7082,4167,6812,8540,11,4207,345,3210,2576,5075,10114,605,2340,7664,6736,4747,3473,3747,9183,9171,3850,5276,8994,9310,7149,7253,4103,3573,1185,1633,773,8528,246,3890,10347,9305,5312,9520,9751,8898,9154,10337,4966,9391,3889,1790,3035,2516,9318,8323,7868,1023,10077,1727,3032,10329,2922,4571,3464,9759,2545,1277,6285,4957,8397,616,7861,3542,9680,4689,7088,8521,1895,3741,8457,2856,5112,10635,7624,7665,10808,2450,4297,2950,6168,6967,4584,6179,10420,3187,943,7443,1215,2962,5896,5484,8689,4469,7147,7945,3607,2111,4479,3136,5047,4051,8050,8336,10015,2394,2573,4305,9963,4509,6916,3462,5254,4318,265,58,422,8927,279,5718,8069,6527,10620,4795,9766,4535,3189,7712,6998,10263,7096,6186,960,8809,5324,4206,4858,5143,4427,9101,4118,3352,2848,4776,10649,3175,9536,10208,2591,2944,7230,8912,7660,1614,228,2115,1385,7043,4437,8662,8349,914,10319,2130,8649,5249,8294,9152,1461,10162,1169,5198,9535,7018,399,7658,9755,4451,10107,3866,506,1526,3283,6274,7184,7926,2870,611,2884,1865,1984,10758,4449,7617,1706,2728,9443,7374,651,3592,8476,9028,9516,7453,3752,8774,6731,7011,6912,9047,10763,8654,3789,6093,2543,1009,3647,2101,3518,8563,1662,6686,2716,10072,4495,8603,10387,5306,9696,8508,8775,998,3658,21,9772,7972,10274,9585,2572,4289,564,8579,970,6209,6358,1235,1250,894,9579,5949,7248,1856,3268,3202,525,3406,6090,872,3916,9262,5442,55,7089,8429,215,4461,5749,9351,5559,8783,3624,2598,7611,8141,3432,2807,5399,1645,7458,4004,9925,7703,6684,8374,5238,9540,5565,6226,8421,5782,9113,6632,5582,1428,5438,1350,2694,3338,9419,1501,7360,1800,10573,6001,3238,9847,7769,7848,9593,7594,10254,5191,7481,7963,4161,9551,2355,6651,7846,9455,3480,6086,376,6604,9858,2286,4041,5267,6315,5875,6999,2967,9344,4336,8490,1314,3827,3736,8628,6305,152,5586,4720,8351,7064,7373,4091,3303,4390,4727,3815,900,3115,142,8009,359,590,341,10413,1075,5869,9595,2711,5920,9769,4541,4986,3219,6761,7431,1470,573,5976,5840,593,9598,10829,3984,9532,972,9550,9126,8584,2866,2593,6841,1923,7999,1463,2602,1905,4707,7600,200,10237,2819,5960,6033,10489,9243,1059,9768,9033,7097,3127,5065,2215,10745,6364,1545,10598,7391,7412,9577,4228,4783,9342,874,6,4732,725,6323,8143,8478,563,1737,7679,4868,7541,3339,10428,8035,8307,3828,5292,2491,6861,3474,9376,4984,8926,5745,9562,594,4802,1598,3637,8779,4358,5108,6053,8709,6580,7910,10266,8253,1736,9390,4560,489,5768,2368,6743,262,10490,3851,2991,1027,2993,581,3300,6809,3358,8496,3655,7176,8501,2559,236,1577,10019,1233,1926,9426,9050,4332,10236,9089,9992,2133,9230,2617,1685,2422,2900,4736,5111,8720,7062,1289,4021,3123,7415,9150,8806,2136,9501,1873,2225,5894,7800,9377,7845,9006,3649,8266,5101,5469,6318,4219,5098,5847,4170,2093,2536,6398,6137,5912,1015,8718,9812,7629,1341,292,8172,136,8723,511,6933,4351,3976,1641,5420,189,9427,827,5370,9382,9529,4914,3563,7957,6710,7138,756,6017,1047,4293,6606,3988,7432,9822,197,8112,1405,2121,4199,4715,10476,4870,5735,6099,1370,5826,2507,7893,6648,10108,8484,1360,6661,6402,7414,7349,7433,9251,38,4499,4619,3399,6136,5986,1976,5833,2301,1948,167,4844,2327,2438,7444,3919,6396,5456,8590,1446,9861,4611,5058,10406,5656,6771,3516,902,8252,8212,851,2759,9021,6530,7577,1972,9201,9295,1219,5407,6493,2629,69,828,6146,9740,4086,3705,281,3975,5756,1244,5418,3704,8989,86,2926,1859,8158,3617,8800,4685,6262,1679,9940,937,5550,8102,1276,7350,1571,9969,4958,4434,8343,5607,4620,166,1456,567,6311,10542,8218,6767,9392,7219,7671,5179,2052,8174,8196,67,781,9413,5362,6840,643,4987,2612,6334,4817,5097,9712,955,2933,9275,8013,9334,10397,10535,37,1613,4085,2753,10784,6405,6631,2175,5059,1539,191,10242,6235,7228,9103,3131,6708,8213,10648,6410,5213,3334,4472,6277,2802,7632,4657,4442,2056,4153,2267,3632,1451,1476,3641,8313,2099,6923,8627,1022,6041,6387,9817,1756,379,8140,4887,6268,1917,9978,6062,10498,4029,1975,9922,10569,2665,2254,5206,3326,9282,3374,7677,8413,1855,2299,2408,7897,3778,6217,7077,2816,2473,5573,6249,9184,10079,7537,7300,3419,8664,289,7896,2468,249,4841,3267,7504,7545,4286,5552,128,6726,8577,2406,314,5839,3893,9215,5132,7875,2651,10559,8024,1497,9697,2241,8407,1964,499,6587,932,7341,2089,10213,1286,229,1433,479,7755,2581,6570,4124,969,7251,501,8260,2028,4534,6417,97,10145,8847,8816,5296,4292,1552,9161,3846,10765,1311,10296,8146,6473,7175,3693,330,4928,6052,9130,1208,3918,6785,4164,5763,4497,6397,3917,1198,8173,4090,3498,517,2969,10168,8156,1104,5622,10497,3515,8222,6551,10128,3380,3979,8882,10467,7829,3410,883,10431,9639,5791,9720,8700,4792,9247,5727,3771,5858,8546,468,548,3500,8646,6453,746,10446,459,9674,6253,2120,8472,1044,1650,3860,7778,9998,5829,10804,2498,9094,8331,10089,4561,2108,6815,667,4892,3389,6221,5341,2955,6174,10630,2451,6956,7773,10232,6549,248,528,6597,1411,60,6669,2114,8455,4920,6027,3726,6516,8474,9508,2363,9716,308,3039,4895,96,5476,10205,916,6195,5262,7131,3511,5516,1430,9217,4191,7301,4448,5598,9093,6161,7025,9710,4767,8055,5773,3843,4960,8705,7826,9457,7524,4359,4346,1102,4466,3310,9339,5282,10191,5460,7694,4042,7844,4502,9644,10333,3838,669,7428,515,7564,8854,6567,6725,177,635,367,6803,8358,10734,4586,10313,1124,9949,2453,7949,7050,5678,577,4640,44,5107,5661,4439,6409,8136,5356,1025,7595,8736,10384,7525,4728,8608,4290,2712,5538,1217,7872,5702,5635,1479,8072,6834,2928,9143,3770,9283,10200,3468,7333,6922,8663,5660,2595,9920,6428,3036,7012,5957,9315,3118,5788,10378,10659,595,6255,4779,3224,10510,4991,6175,4936,386,8901,6586,4837,4694,1324,8350,9647,3829,763,784,5687,4380,5969,10198,1732,8326,7191,1788,3282,880,2502,2461,2018,5962,5272,3776,7171,2003,1911,8239,2915,1794,5811,6839,3163,7475,2860,9085,889,1774,2958,5517,5532,10285,5775,9919,5335,7888,10738,8497,3758,5004,1498,9309,91,3677,9557,1617,9314,7597,9792,5037,8793,7644,7967,1365,5994,7079,3162,710,5926,2196,10651,5008,2852,5981,7911,9780,4811,3863,4833,4155,4298,4028,6294,5193,558,2574,7150,8423,2954,821,9669,9181,9241,4593,8702,3220,4505,10210,10705,846,1804,10746,9747,6938,2055,3611,1716,7874,6617,9745,5081,10009,1945,1152,8377,1658,929,5765,7567,4724,6374,7,10138,1007,5086,8179,2255,9614,6784,2853,2512,9010,2920,7871,6825,6697,10373,3485,8277,7100,3344,2037,782,7048,8166,9967,3614,4043,4385,8769,3204,9353,8285,10679,10219,10586,1066,6420,6537,6511,7608,2871,6819,8934,6083,1710,114,10793,4859,3472,6354,8229,9302,8873,9515,213,6472,3069,7188,7021,6505,5882,5336,8157,3067,1158,1496,8596,3936,7812,30,4850,10234,6368,2522,153,8049,7619,5241,6557,3087,10706,2166,6722,3835,2943,47,6519,9917,1866,5795,5448,6046,1439,3708,2882,3215,7291,4729,6786,2648,3608,3933,1703,7121,8625,5303,10597,814,100,3820,372,119,7357,8070,2649,10617,4856,3024,9534,7791,1436,8375,10695,5721,1105,6266,9054,10593,3512,7883,2072,8500,8309,5612,925,7466,10192,10040,5915,7078,3628,4342,1767,10377,3877,9587,461,9985,5500,8868,10111,2000,8248,2206,9409,655,8586,9321,8341,8832,1781,2078,6787,35,6558,9146,10082,2452,5000,5699,6485,3151,3011,5526,8469,6555,9335,3279,10665,6256,185,8316,277,7637,5619,4223,234,4388,8198,10435,5905,6021,10753,2847,3698,2253,1352,5624,832,6471,3662,134,1540,5657,4901,4205,715,4241,1715,7196,9765,4512,5736,10537,8573,7234,5730,10549,9968,926,9986,4413,5880,10809,5544,5892,3085,1880,10306,3259,3973,3367,7780,1839,982,9099,8581,8600,772,5302,2219,7863,3913,6085,5572,10369,4142,6134,2764,5291,9476,1551,6303,10702,4007,5317,2989,6937,1854,2201,6182,3074,4344,8782,6641,3999,5461,2150,8908,2677,5364,1965,1531,115,7549,10581,6019,8943,2749,2161,7470,7275,2151,5470,2851,1814,2445,10360,3520,5814,10055,1019,7607,3620,502,4883,10574,1389,3260,7108,2171,5411,1681,1409,8719,5755,10749,2655,9300,2511,10037,5011,9528,3508,4794,1084,3548,3125,2030,10643,6004,2534,1457,10256,8066,2232,4069,445,10766,5265,8169,9654,6113,2942,7995,6605,3403,8576,8291,9193,4036,2599,945,3244,4656,787,9645,2746,7749,10324,7299,6793,5089,8885,10348,7792,8529,3527,1888,9002,4666,7269,10622,7815,5846,3578,6406,692,3045,6369,8776,3072,6071,6746,7526,6900,5062,8011,9658,392,5817,5623,10453,8214,2722,4411,5443,1181,9498,1028,2138,66,5073,5605,3540,6440,7271,170,1188,1163,7934,1751,2224,6073,4,3052,1065,1369,1801,2499,7895,5346,10802,350,2988,2849,3619,4937,6455,10156,10041,7690,8103,9369,8619,4257,1282,5676,9827,1138,8631,5201,1099,9032,3668,9673,2146,5226,6251,895,9841,2005,6152,3927,7243,7448,4544,4231,7083,3731,3272,8293,5116,9350,8711,9137,2167,4407,4065,10047,7584,5931,8064,259,10301,9884,6692,6500,5844,1835,8021,1881,1673,6677,240,785,5465,4258,957,3742,4189,7040,1892,5083,606,9132,5506,2320,10501,3915,8530,9147,666,2187,225,10096,6074,4974,734,8852,8515,6694,7059,4056,2024,5798,10686,7659,3402,9990,9934,7364,3257,7438,165,8637,6240,4484,3126,3714,405,196,3514,3900,2804,8284,5045,1564,2754,10158,974,1173,2859,9233,10103,3839,3885,7606,9202,4044,2364,9798,7922,2318,3737,7164,8480,6543,9952,7860,1149,6778,4773,9474,6924,3407,983,9379,8728,7336,3076,8772,3110,8065,7080,10509,5148,7256,9648,9701,9832,8517,817,4202,3455,8453,681,458,8549,9836,3897,2356,10783,8974,9048,417,6302,8208,4721,5451,9620,1996,4139,7816,5421,8796,8119,3147,9465,6450,2068,6112,3167,10486,4807,532,5753,5824,9034,2304,6309,485,272,8802,416,5807,5742,8325,9739,9890,9625,7342,8585,3967,7757,843,4900,575,4814,9771,4523,6800,8942,5117,8707,10279,4430,6997,9199,1582,9292,7404,9976,3531,10093,3523,219,6155,6006,1632,9042,10422,10083,3673,9698,4546,7662,6059,1699,8880,8435,9994,922,2486,911,3114,660,29,1005,4520,1815,8202,10382,5234,8950,2503,8330,9019,5295,1860,1366,8221,614,7884,1955,174,4536,5974,2708,2307,7738,9242,8706,9039,7446,2159,10756,112,3003,9493,6167,9862,2903,9159,7519,7439,2614,6084,6072,8822,1838,3239,6675,9422,3964,3033,6899,4754,3963,4853,3710,5040,8080,4061,3597,2109,2123,1595,6211,5146,6791,4827,9996,4144,5569,7942,1159,9943,7572,1639,3672,10359,10690,4525,6196,9481,738,716,4489,2685,10147,4248,8583,6457,3387,10614,10364,993,1037,1404,10080,9258,5580,10403,3854,4116,5118,10669,9891,9737,5665,2469,10241,5489,6971,8936,6880,5977,5649,9228,3158,7784,5115,1574,9316,1797,8385,835,169,2288,8864,4121,6575,4101,407,2421,3898,5395,5616,3453,1454,5431,8122,156,1864,87,6008,7652,6427,7356,2260,1090,4823,8447,2457,10280,1631,1538,950,8406,1279,2966,7552,2347,7259,6394,10560,5868,3560,5823,3451,3046,5019,2414,7785,7929,8217,3320,4226,864,6148,6673,6490,3400,1310,9492,2348,4415,968,10323,3990,5776,9900,9336,1046,7148,8798,5151,2960,6178,6623,1204,4454,1760,7168,6117,1146,10719,10458,1649,5397,10282,10584,9794,3475,10402,10392,10762,3270,413,1759,9711,3664,3306,5274,8678,652,8665,8416,6408,8046,1624,5865,9687,8657,5888,1062,2184,1164,3280,9055,3097,5372,424,1287,5887,4647,7493,6962,10740,5382,6805,4082,8845,3590,7270,2228,2630,3021,9731,524,4700,378,7771,10656,3398,6644,990,1322,10016,6855,5261,10799,9323,6720,6162,1252,7898,487,4787,9439,5898,7813,3728,7689,8353,45,8231,2275,1665,709,2700,7044,6229,9875,3411,1846,5567,9855,6894,658,448,8410,1416,978,9570,5480,2329,290,4719,6830,7788,5473,8767,6832,2750,4270,2778,4501,1530,4417,7262,9398,3441,8026,2141,6700,4803,4545,3201,8518,9432,5734,3969,9904,8241,9760,10652,4558,5677,1535,3251,3971,10052,2199,5751,9956,7627,3299,10078,6578,4000,2567,1193,1453,2231,1398,1714,1507,2613,8977,1701,4465,5809,820,5049,9517,5595,5917,529,4815,224,3910,7054,3727,1939,7937,6884,9636,8394,5682,5405,9487,4907,9554,9743,10322,6208,9414,8843,7213,7505,10188,7604,7837,9860,10351,2087,14,7212,5999,6474,209,9842,10468,3271,741,10611,5142,4020,1870,10141,2070,238,4670,3450,3188,2479,8304,1263,9502,3330,2361,9077,160,4884,2948,2011,1032,1652,9131,2428,4917,6908,8219,6141], "time_ms": 0.194527, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 613, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "generated_query_count", "value": 1, "time_ms": 211.743233, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 428, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "generated_samples_per_query", "value": 5332800, "time_ms": 211.743233, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 429, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "generated_query_duration", "value": 123762, "time_ms": 211.743233, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 430, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "generic_message", "value": "Ending naturally: Minimum query count and test duration met.", "time_ms": 1485.557569, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/issue_query_controller.cc", "line_no": 482, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 645153.519760, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 898, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 645153.519760, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 900, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 645153.519760, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 902, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 645153.519760, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 904, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 645153.519760, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 906, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 645153.519760, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 908, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "power_begin", "value": "12-25-2024 04:20:00.565", "time_ms": 645153.520415, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 564, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "power_end", "value": "12-25-2024 04:30:45.498", "time_ms": 645153.520415, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 566, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "result_validity", "value": "VALID", "time_ms": 645189.573894, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 644, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "result_min_duration_met", "value": true, "time_ms": 645189.573894, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 649, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "result_min_queries_met", "value": true, "time_ms": 645189.573894, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 650, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "early_stopping_met", "value": true, "time_ms": 645189.573894, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 651, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "early_stopping_result", "value": "", "time_ms": 645189.573894, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 669, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "result_query_count", "value": 1, "time_ms": 645189.573894, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 679, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "result_samples_per_second", "value": 8268.76, "time_ms": 645189.573894, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 735, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "result_min_latency_ns", "value": 1430163027, "time_ms": 645189.573894, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 741, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "result_max_latency_ns", "value": 644933169947, "time_ms": 645189.573894, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 742, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "result_mean_latency_ns", "value": 391145860524, "time_ms": 645189.573894, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 743, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "result_50.00_percentile_latency_ns", "value": 416430237335, "time_ms": 645189.573894, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "result_90.00_percentile_latency_ns", "value": 615068853269, "time_ms": 645189.573894, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "result_95.00_percentile_latency_ns", "value": 632720953267, "time_ms": 645189.573894, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "result_97.00_percentile_latency_ns", "value": 638542165122, "time_ms": 645189.573894, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "result_99.00_percentile_latency_ns", "value": 643191943747, "time_ms": 645189.573894, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 18583, "tid": 18583}} -:::MLLOG {"key": "result_99.90_percentile_latency_ns", "value": 644798861050, "time_ms": 645189.573894, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 18583, "tid": 18583}} +:::MLLOG {"key": "loadgen_version", "value": "4.0 @ 486a629ea4", "time_ms": 0.003880, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 53, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "loadgen_build_date_local", "value": "2024-12-27T19:45:31.956819", "time_ms": 0.003880, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 55, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "loadgen_build_date_utc", "value": "2024-12-28T03:45:31.956827", "time_ms": 0.003880, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 56, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "loadgen_git_commit_date", "value": "2024-02-15T15:46:50-05:00", "time_ms": 0.003880, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 57, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "loadgen_git_log_message", "value": "486a629ea4d5c5150f452d0b0a196bf71fd2021e Hotfix: DLRMv2 Audit Test01 fallback failure (#1626)\nf9a643c0a0e920588da1b51a1d822e1071a9dbec Add number of tokens to offline (#1623)\nf06b920eac2170df38b5502c7443f50c9afa6045 Bugfix: equal-issue mode on offline causing accuracy run to fail (3D-UNet) (#1624)\n0ed5190e82e1ca1cf95626e50e61cb6678c7f2ff Fix submission checker and TEST06 for Llama2 (#1616)\n817dd961221d3ca284b024368130bd5dc5e4b54a Turn equal issue mode off for TEST06 (#1615)\n840435a1abe8be823e0a5a18ad3173bc520aa24f Update README.md with CM commands to download stable-diffusion, gptj and dlrmv2 models (#1604)\n3190d09a148bc1d72c9d946e4cf3b54db0d0e60a Typo fix in README.md (#1588)\nb0777f0bd3b7e5cd6a51d670d8b4df656cf6fb29 Update README.md - remove unwanted lines in CM commands (#1601)\nd7dba0826ac96a0e8bcae525086961e47c409a38 Remove loadgen warnings (#1608)\nd45a66c33bb0b6548d6172b38645bf94ffe24b39 Add upper limit to tokens per sample (#1612)\n44285d9a44041c3d4f7df0acf1e72c375e9fcbd2 Set completed samples per second as llama metric (#1613)\n357ccefca4db55ff373e029d5d1e3e6246bde5d4 Enable equal issue mode for LLM benchmarks (#1610)\n104d85531af7ae2a07ded36dd6e8c7f2c7746d06 Add sample length check to test06 (#1603)\n473053fc9f1b8c5edf60e11ec74607126938fbc4 Fix loadgen token metrics latency constrains (#1596)\ncc3daae3c7f18d7eedc5ffbc271d324e5aca6233 #1598: fix token and sample logging for Llama2 when accuracy_log_sampling_target is enabled (#1599)\n3a902e598e5a17013c26fea975c0a60e6d7feae1 Add support to dump 10 compliance images during accuracy run for SDXL (#1591)", "time_ms": 0.003880, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 58, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "loadgen_git_status_message", "value": "", "time_ms": 0.003880, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 60, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"05d4a62d355a90363847358e58afeee4a54ad838","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/bindings/c_api.cc":"32181da9e161c285f8fe46ddaa49e6cba2f9f918","/bindings/c_api.h":"91f58bd79b83b278f3240174a9af747fc38aff74","/bindings/python_api.cc":"1dc0865130371c6a8ddb97cd8039244d74d6fe46","/demos/py_demo_multi_stream.py":"6ccf7dcbe08f8382ed2d3618370c2b21e205ca13","/demos/py_demo_offline.py":"6c3075538e0a3df5e93edee917f4537099759e11","/demos/py_demo_server.py":"40e37ea121ac08b21ed6af41aca8eb5eda641b89","/demos/py_demo_single_stream.py":"4ee5f048fb0dffc415342a2bc2049fb155bd2894","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"126e952d00f4ea9efd12405fb209aa3ed585e4b2","/issue_query_controller.h":"923d9d5cdf598e3ec33d7a1110a31f7e11527ec7","/loadgen.cc":"6650091ba7a918f343b06eb7a5aa540eae87275f","/loadgen.h":"e00fdc6dbc85a8c9a8485dbcbfe2944f81251c4e","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"197efc96d178e5d33a750d07fa7b2966417506ea","/logging.h":"ddb961df7bcc145bcd7cce8c21f7cf075350dcbe","/pyproject.toml":"974adb632af4b76a1c72b914cc2cf9b3cbc66aa3","/query_dispatch_library.h":"13ad6d842200cb161d6927eb74a3fafd79c46c75","/query_sample.h":"e9187c8612bbdc972305b789feb6e15c26e96cfe","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"a170f991de69f8a5e45616e871b39a30ffed5598","/results.h":"d82500c326c2de83db411f1146882aa4692b419c","/setup.py":"e3e707e1191478332c00f1b6ff806f0ef98e0f65","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"aaf3803e700f3431a5ada72559eef5cce996ea64","/test_settings_internal.cc":"42aa46b17ac3fb2748cd5cde60fa8742e916f3c8","/test_settings_internal.h":"73b11c02228f3afb4c924db7a86fd8a2b5b9e483","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"4c9d98360b507d5673049ed70154fcfb2934f8a6"}, "time_ms": 0.003880, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 67, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "test_datetime", "value": "2024-12-27T20:14:39Z", "time_ms": 0.008939, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1198, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "sut_name", "value": "BERT SERVER", "time_ms": 0.008939, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1199, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "get_sut_name_duration_ns", "value": 467, "time_ms": 0.008939, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1200, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "qsl_name", "value": "BERT QSL", "time_ms": 0.008939, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1201, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "qsl_reported_total_count", "value": 10833, "time_ms": 0.008939, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1202, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "qsl_reported_performance_count", "value": 10833, "time_ms": 0.008939, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1203, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "requested_scenario", "value": "Offline", "time_ms": 0.013458, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 268, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "requested_test_mode", "value": "PerformanceOnly", "time_ms": 0.013458, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 269, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "requested_offline_expected_qps", "value": 8080, "time_ms": 0.013458, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 306, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "requested_min_duration_ms", "value": 600000, "time_ms": 0.013458, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 312, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "requested_max_duration_ms", "value": 0, "time_ms": 0.013458, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 313, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "requested_min_query_count", "value": 10833, "time_ms": 0.013458, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 314, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "requested_max_query_count", "value": 0, "time_ms": 0.013458, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 315, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "requested_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.013458, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 316, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.013458, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 317, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "requested_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.013458, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 319, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.013458, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 320, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.013458, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 322, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.013458, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 324, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.013458, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 326, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.013458, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 327, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.013458, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 329, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.013458, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 331, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "requested_performance_sample_count_override", "value": 10833, "time_ms": 0.013458, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 333, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "effective_scenario", "value": "Offline", "time_ms": 0.013696, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 409, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "effective_test_mode", "value": "PerformanceOnly", "time_ms": 0.013696, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 410, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "effective_samples_per_query", "value": 5332800, "time_ms": 0.013696, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 412, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "effective_target_qps", "value": 8080, "time_ms": 0.013696, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 413, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.013696, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 414, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.99, "time_ms": 0.013696, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 415, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.013696, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 417, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "effective_target_duration_ms", "value": 0, "time_ms": 0.013696, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 418, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "effective_min_duration_ms", "value": 600000, "time_ms": 0.013696, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 420, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "effective_max_duration_ms", "value": 0, "time_ms": 0.013696, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 421, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "effective_min_query_count", "value": 1, "time_ms": 0.013696, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 422, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "effective_max_query_count", "value": 0, "time_ms": 0.013696, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 423, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "effective_min_sample_count", "value": 5332800, "time_ms": 0.013696, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 424, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "effective_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.013696, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 425, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.013696, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 426, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "effective_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.013696, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 428, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.013696, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 429, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.013696, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 431, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.013696, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 433, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.013696, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 435, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.013696, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 436, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.013696, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 438, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.013696, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 440, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "effective_performance_sample_count", "value": 10833, "time_ms": 0.013696, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 442, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "generic_message", "value": "Starting performance mode", "time_ms": 0.019019, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 841, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "loaded_qsl_set", "value": [4117,9486,2947,9270,8227,4487,9799,4629,8571,8839,7101,2191,3315,6375,207,10454,3122,9490,4083,6119,1890,7657,9347,344,201,4232,5741,5454,9533,7842,1596,5330,10615,8827,8094,6714,9169,977,6619,9924,4982,4894,6544,10118,783,766,2122,7163,396,4200,9864,4254,8440,1758,10722,2829,2788,9906,2719,5613,8726,7847,440,6103,8567,7497,4793,1184,8675,10587,8270,7074,7408,8740,9265,4855,1199,220,6556,6595,1226,6695,3066,2143,6264,9806,7122,5007,10075,2370,2185,7254,77,2806,6243,6687,1156,1802,7302,391,3348,8159,9708,251,3595,3947,2302,4690,122,3116,1300,3768,6388,2986,728,5127,3904,6807,6259,2218,7555,8821,3580,7804,1321,8460,5419,6542,7833,3365,2290,6346,7952,918,707,118,10469,9023,1304,5220,1891,6878,9524,6522,7867,4037,7698,8507,9634,2145,10057,6263,7486,7557,2401,6082,3434,5927,8708,10081,8114,1349,7742,2742,7157,5705,4638,1764,3651,920,4016,2744,5859,6740,5380,9649,2,5245,5230,6421,4071,8909,6319,288,9346,9671,9646,9360,4702,5044,9157,6296,8318,8971,8184,7702,10475,7318,4024,5530,3991,1171,1909,6230,8461,8099,10563,5398,10760,10693,3688,8170,4952,6827,5893,7767,2656,10032,9178,10480,7195,9359,10608,10261,8975,5432,363,3346,3049,3360,4432,9736,473,7587,9416,1021,6596,10352,2666,2325,973,2071,7368,4267,867,1040,5796,3491,8237,9597,3489,10568,3777,7899,856,4818,2858,4662,777,10776,10026,6065,6481,2590,6780,10553,8789,6032,930,1049,10178,6187,585,3359,5832,2014,480,5866,5017,5141,1525,8271,8132,1642,6270,2265,140,1174,4641,8017,8844,6569,5575,7081,3779,3845,467,9761,3314,8653,4250,7578,8907,664,217,8419,8757,683,4565,1638,5216,2004,6679,4485,7683,5180,7422,6070,6283,2909,6281,8933,10639,4340,3079,3513,10552,2291,7487,2680,8807,8370,6127,7434,1822,908,8101,451,10316,8282,10724,885,1834,1556,6121,1468,409,1908,10539,7238,2891,10312,9435,2178,10051,4522,9650,615,7193,7796,2008,6011,4347,435,6881,1396,844,971,4216,5001,7886,6875,1348,4300,9348,2709,10580,9993,6705,8784,10821,9232,7724,6581,9764,2913,10125,6914,7693,4568,10526,10231,8502,7825,1783,795,6828,8006,4595,3892,3075,1278,10050,8235,4679,9971,75,2164,1175,3684,9869,8414,9016,765,2721,5786,10703,1326,7463,237,836,3912,5482,7835,4374,7661,4070,7754,7180,9945,10415,3638,4208,939,1097,7628,5725,3723,4547,1994,5232,5279,4049,7309,5963,10478,9834,7758,9463,3666,4663,801,4030,3010,8236,9237,2470,3382,6336,7570,190,4552,4113,2229,9018,7038,5021,8701,8268,4490,1373,6092,10181,10445,3056,4333,10394,1640,2328,3349,10069,10683,6439,3191,2544,4760,1739,8900,3,258,5416,4671,2041,9375,2661,6376,2306,5897,6373,2125,1940,5109,4256,6203,6411,9311,8029,2200,6301,1734,6176,6443,3007,9583,1841,4627,2523,4306,2801,5208,8607,4961,1467,10700,4031,2181,9958,8194,10295,9043,5366,306,9056,162,3196,10759,4349,5401,7030,6184,9216,2069,6357,5640,3015,1752,406,10049,3983,791,5610,5944,304,6380,1937,7862,9366,9365,1607,9252,2698,10350,5221,4847,10707,3023,1615,7017,5990,4045,7231,1202,7182,5172,3176,9612,6693,1013,10334,1805,8730,8841,7372,10094,6010,3216,8177,7406,9472,9608,2533,7691,3959,1623,5144,1831,751,4912,2152,8144,241,4187,10001,1924,8402,7877,8928,7294,4915,3876,3242,8996,7153,1000,10658,3946,284,5088,959,10661,1018,8931,546,4262,6489,3786,5278,4308,8178,7638,5211,7142,5771,634,8273,263,2548,4674,9910,624,3290,5925,1524,2642,7198,10345,3539,6719,10735,2006,10233,3816,9578,8624,6769,9661,6468,8952,9702,8670,5175,3449,1773,6130,636,1622,1696,1414,6523,2898,10650,4420,1280,2538,7880,3743,1227,2757,6131,3371,7445,1680,2901,36,6868,10811,7140,1900,10398,3724,8765,8278,7379,6040,7914,1580,3535,8033,7536,6480,3861,2437,5082,2881,10596,4902,8162,9637,4886,411,7189,1789,6704,9176,10045,4107,6333,4583,3923,4843,9935,9955,9297,3222,8257,6365,3095,1995,7924,5428,8640,6628,4819,7152,5016,7932,3226,1267,1225,9025,7878,6321,2373,3801,1399,9063,6506,9436,5781,7500,423,3588,7110,4607,3476,2477,3635,2425,9394,10449,4054,8148,5510,2236,9543,10499,6153,780,8692,8105,10268,172,4128,841,8925,5453,8561,2135,2976,2845,2209,7418,10817,5620,7010,9960,474,6545,8223,9713,205,10548,905,2396,2914,5591,2887,3379,4865,3304,5849,1309,10171,608,7314,6228,6961,7068,6943,4613,1317,121,8611,5902,5827,8042,1139,3121,223,2984,2897,3144,5379,10590,6433,3871,5475,43,1026,1192,1599,5729,6194,5838,1254,2054,2706,7925,6060,2177,10647,7727,5681,10000,3549,10814,4433,2100,6869,6593,9404,5534,5801,10321,9145,8363,7451,9072,4600,5096,10372,3022,679,5631,2552,723,4668,9613,10153,7423,9114,2973,3071,1413,4956,2838,5084,6777,5196,539,5036,382,7480,9327,323,4766,2273,2684,8182,10134,9203,2667,9185,2658,9087,6286,4162,6620,10472,1095,6759,7216,8191,9738,9548,5029,10326,5478,4089,8710,5439,6126,6160,4828,1006,1677,3546,6930,425,2645,4573,2770,10248,358,4975,10603,8129,8614,10180,7902,4303,2447,3427,753,8920,9449,256,6859,5711,5246,623,370,10767,2563,8300,9591,9991,5126,9568,2647,484,8644,3654,9507,8632,10076,9813,10638,4326,3862,569,7020,4213,5628,9341,839,3341,8347,5163,5563,7339,6928,1197,9158,3099,6120,1483,9545,4608,4215,2761,1259,6802,4364,4001,1606,4222,4752,3057,10022,1108,10592,4398,10376,9888,10033,4273,1238,3428,6553,6145,5731,7273,604,2436,3170,8673,5901,10074,4513,8305,8697,7476,5015,2669,3100,6344,9338,10383,4972,8727,1363,3223,10495,897,1698,9553,5662,53,9328,3733,9932,7102,2212,7382,7159,4556,10748,9294,5895,2262,10512,5913,8314,10624,2530,7560,10464,2443,8787,3800,1510,2027,9795,9670,283,2351,4260,2577,1518,906,3377,4764,2113,10699,3948,8465,9264,9267,5600,2924,3783,9209,7966,6464,720,6973,155,6227,7994,2636,535,8846,2931,2046,4953,10058,8228,5200,2211,4234,2207,8121,2007,7268,3212,7772,6496,10201,2762,7534,5539,10555,10496,628,3653,702,8683,10325,8660,8411,9387,326,8842,837,9688,3855,7053,1826,7944,1053,3193,456,361,3206,10315,2059,6612,6173,4562,9003,8034,9162,377,7551,10769,5354,6299,5997,2080,6288,6068,653,9937,10825,8833,3247,3058,4923,9104,7983,3305,9322,8339,7711,131,7328,5212,8623,10761,9718,5402,948,10640,8659,6479,2768,4616,2223,10625,4252,9980,9807,5318,6188,159,10655,3565,1478,6860,3521,4646,9880,1608,10427,9804,2092,4897,10238,6020,7642,8091,7789,4058,6681,4173,7918,5523,2444,9898,7161,7736,4667,2603,9370,7403,10797,1857,3276,2276,3445,8197,991,1863,4266,2170,4331,3296,6658,1308,7542,8458,4533,10177,6873,4190,815,2726,5589,2846,269,2634,1560,7067,7554,3102,4908,10741,10086,5286,8734,3824,1744,4111,3264,7494,3825,8834,6139,8432,147,1589,1871,941,2911,6018,7906,2874,3713,9523,4023,7522,6116,6864,5698,5464,3924,4590,7345,7395,4790,10540,1020,5309,5950,2956,4983,6163,9194,3848,3612,10525,9069,5253,7232,6876,5003,5067,2350,10375,10523,850,6399,10002,9689,7255,1016,3707,3384,2137,4242,4077,5170,8388,2387,10577,2482,3735,4313,8862,2029,8452,10528,6058,6871,5233,1195,3301,8738,3194,2638,2488,5009,6981,7935,10684,10631,1043,1962,3541,4973,9581,4285,9118,9770,8279,1316,6284,1161,877,7894,3981,2423,6872,5033,1798,6181,1255,7517,4067,3319,7457,5164,3977,1719,4436,9317,4765,4137,6515,10025,4628,3745,7739,9445,7016,1980,4481,8580,175,10704,10653,245,8163,3834,7006,9135,1950,4401,2289,1412,7325,4821,6917,1361,7569,6782,2474,8075,5757,1034,3353,8570,2678,4922,4356,6831,3401,10393,10508,5161,2635,1472,3331,3048,9526,5383,5670,7508,5113,10148,2912,8547,1183,5496,5139,1153,6897,7997,6625,9819,5766,7460,10124,6637,327,3466,1848,2455,5939,2282,6371,4812,6721,5314,4780,3112,5154,250,7103,7158,463,6920,8368,6024,6877,342,6799,437,2876,9690,4889,5129,2631,3366,8380,1528,10135,584,6292,9180,5863,10031,2432,980,7211,3117,6909,1248,3761,3601,4976,9420,10602,178,10755,1150,2009,6653,7378,4422,8445,1978,2390,5417,4353,6452,2002,10595,1306,2817,8735,8424,8976,4396,4919,9488,534,4478,8211,8264,385,8288,3063,9825,6996,9726,7239,668,2896,6212,5761,6271,618,8216,6838,195,216,465,8870,6774,8556,3392,786,2791,1096,10565,9399,7603,1723,4133,6106,5491,10443,335,8387,5006,8131,966,7321,4664,5361,8778,8085,10370,512,4408,9227,4676,927,5805,10457,5444,5333,2951,6804,7955,888,3706,8788,2895,1902,9874,7221,5222,10628,6300,3522,65,10112,7284,6826,9762,10226,8642,8030,609,3812,9914,8276,5946,4130,767,4876,3831,9441,5515,7928,4838,4550,78,6682,63,600,3798,8226,3577,8618,8919,4716,10385,5560,7695,9677,9470,6351,10585,6031,2941,2010,7623,1735,6574,1513,2774,1490,2303,5429,141,1915,7737,9086,2205,32,7165,4564,3715,8209,8559,4979,4985,7143,1656,6789,8025,7208,6845,503,3687,7028,5471,9975,5664,3404,9068,5248,2433,322,10390,3840,8058,1297,3551,3819,3342,7265,3266,8914,4394,3583,9786,9198,5609,1444,8947,3060,10252,5594,1291,7319,9559,3166,4669,9592,661,6602,4789,9415,2424,4265,7098,1876,7535,1203,10053,9809,3104,1941,2705,6635,3148,3598,1502,7383,9073,10425,6590,5162,3529,7306,10800,6393,8751,387,10172,2692,5719,2741,7855,4845,7039,2459,6901,6624,3554,4237,3867,10566,2116,531,5943,6690,792,8441,6260,682,3995,6154,4165,9066,2717,1842,5340,8261,8539,1074,4259,701,9500,8648,513,3417,2155,9041,116,4010,1218,7514,3738,10176,8592,4659,8491,291,10417,8338,7244,862,5005,4735,1362,964,9714,10691,9829,10779,351,4930,16,3561,5388,5985,5239,6821,4320,10682,3154,4367,8961,50,8733,4288,9893,824,2504,404,10529,6560,1489,8125,48,76,909,2237,2691,2416,1987,7699,4599,5585,10503,7807,3639,3994,5708,7135,4625,4516,5540,2916,5528,5975,7413,8915,1347,10065,706,5304,3436,6607,8262,5645,3625,1376,4476,2934,1592,1935,3958,5821,2456,3362,5367,4151,981,9573,10610,9207,6190,5275,4631,1708,1921,9287,699,1447,1636,696,3740,4597,3373,9444,1221,3780,8124,5381,7648,3536,10571,654,9064,9044,3503,9854,9626,3378,10675,10629,6080,4445,9605,2646,6982,5074,3618,52,6561,6185,3026,5968,9742,989,5867,9190,1178,2689,2001,6149,8334,2049,3180,5759,111,4691,2650,8777,3149,804,1868,5328,840,8687,1382,57,2419,808,10355,5588,5215,1403,5701,9,7853,8520,8669,8019,4006,9097,226,2827,1135,1901,979,819,2561,7049,504,4391,4400,6739,1402,5264,2230,10666,2566,7430,8459,10720,8794,10310,10174,443,6349,7310,533,7609,6200,4108,613,7558,7709,1553,5076,10645,300,2842,6261,2324,588,2193,2945,7510,7396,3094,1889,10781,3791,554,9284,9148,3519,9995,3209,5810,8322,9519,2560,3073,947,2781,7004,5548,7116,10152,10121,552,1887,2952,8739,9225,5819,3712,6550,5446,574,3090,8327,428,1266,8433,8383,4645,8754,6898,2652,6822,1455,2176,1519,8566,10303,6049,10068,4977,3894,7561,7488,5536,7224,7793,1989,3985,8695,3424,7035,7612,9938,6775,39,1850,7803,9174,3031,6441,2974,1603,7941,7223,7923,2831,2977,3080,2075,9236,312,8565,4634,8814,1558,10484,1272,7389,8872,10418,4755,7869,2610,10633,5046,6960,4193,7852,1754,2878,1936,3784,10314,4025,4805,4997,10007,9863,7155,10101,1342,9395,10291,9385,10218,799,1123,5843,2562,4905,1377,9226,5937,1986,7591,9962,1189,8902,619,9067,3458,7586,7210,328,8506,3000,1004,9212,4591,8020,8887,4012,592,101,3415,2527,72,7452,9400,7809,1561,7032,4840,7008,1056,8,5752,9588,3101,3106,3609,4503,8312,6536,3446,8745,1437,7547,10287,4816,1795,8004,5837,1042,9329,9008,9433,2047,1087,3421,4324,10588,6699,10747,7765,9719,10438,887,9062,6820,2946,4741,527,3043,1648,1133,2918,882,684,4939,4386,8641,8154,1323,6436,7873,10752,3686,10680,807,9886,549,3230,6147,294,7120,354,8171,8493,2663,6863,7489,6037,410,9913,124,6312,541,3159,10304,2117,735,9882,199,4315,9250,4055,3739,3826,9558,10782,3699,542,6645,7593,5123,6413,7532,8805,1517,8514,3396,9239,8883,6748,6002,5319,8690,6308,1691,4655,589,10278,5680,9489,6806,10465,4251,10142,538,5740,194,9167,9974,2107,1480,8439,5722,5674,5712,7528,3385,1906,10567,9057,340,5707,9308,6207,3078,2818,7518,6794,9555,9983,1676,1678,6079,2038,1670,2608,2335,10021,10243,7257,9405,8165,2242,1247,8620,7479,1305,8818,8790,7865,4998,5653,2097,261,5270,2140,2105,789,6984,7501,8748,9538,8167,7566,3307,8768,5195,3932,1130,3930,6535,3665,6818,870,6657,7938,8656,2975,8108,4377,1332,1792,10027,3395,5078,10380,724,6034,3423,4788,4511,9362,2462,7760,7023,2497,9802,5504,6817,4263,3361,8164,8418,2478,2287,3457,1264,9796,7218,6591,6401,1683,3534,7474,5945,9722,7233,3719,7072,9915,42,6386,1481,3426,8808,4075,9665,132,3679,1418,6327,4854,5667,1228,5621,10063,9685,9641,3792,8509,10725,1726,3510,6733,5311,6890,1127,1372,4557,9076,4493,1319,4639,6503,5373,319,6252,5205,2568,3262,5611,3692,8016,2058,10708,1990,10246,5149,6730,4745,9617,4496,8093,3764,4587,9046,8240,5654,7484,9304,9356,9122,9012,2409,2879,1728,8443,6110,2374,2695,6156,6123,7266,9525,1770,9102,10660,6745,5128,5806,10298,1081,5852,10365,5171,3316,1949,2796,5938,2888,505,9040,10137,2063,8296,703,7982,5184,5822,6713,8356,1212,10214,1643,1344,1740,6170,2765,8513,7491,313,7468,5830,5188,7420,8246,7931,5592,2300,5803,1333,1904,10524,2557,1320,9801,3887,3372,4648,8123,10207,10570,2782,9208,3493,8993,4688,9635,2367,890,4834,4750,2186,4059,9823,383,5746,6601,6995,3696,10626,6202,4034,2588,1415,3128,5342,9471,8634,6712,7283,5689,7596,5728,9160,6465,1823,8542,2381,129,3703,5297,5845,10090,9892,8109,10676,1417,6171,9518,2483,5993,6066,10657,4131,7317,4665,10546,7047,6337,6183,3537,253,7720,5316,4382,2720,8286,7478,3902,7237,3088,5066,2131,2830,10694,3644,1407,5650,5733,7970,8970,2283,8951,9288,6643,10349,4772,6849,3295,4178,1354,8522,10736,1720,4463,10409,7172,6220,1998,6367,7464,1973,6738,4238,9065,9537,2528,6911,4247,8425,183,7919,4791,5494,3274,6583,4995,7128,869,5889,7655,5199,1555,6754,10636,3874,5094,545,3805,1611,9596,2168,6048,7912,1253,3675,10190,6051,6887,5414,2919,2397,8982,2189,5410,2589,6458,7272,8280,6078,2349,9268,9229,8372,9571,2129,9070,7371,5048,8593,4925,5490,1503,6584,4713,6442,3119,687,3013,10757,8401,6102,6842,6324,7114,10127,9879,9783,4963,1626,2785,3640,9163,4835,1390,5087,8073,10203,3068,5747,1958,49,4916,620,1548,3018,9306,184,10272,3878,3053,2239,10187,1477,2902,9456,7260,5051,7639,5386,4477,80,1151,7277,5579,5072,5596,2204,10199,8267,2624,5110,6272,1330,10059,757,9947,1357,7745,6159,9580,6926,1486,10062,10488,8712,6582,7733,8905,3935,8265,7334,5495,9170,1657,4880,1542,9546,5577,3169,2880,8287,2763,1465,1041,10545,7951,2814,10729,1234,7881,6985,6670,5055,4329,9749,4744,7621,9946,665,475,6655,6976,1327,9703,33,9513,9857,3248,2158,7640,3830,6910,6599,10785,3908,9303,6403,858,2074,3941,4068,1488,4929,9954,4872,10514,7838,4677,1877,7014,8113,6883,1511,7174,381,1523,10229,9627,6966,9767,4283,1167,4168,6977,3543,6953,2737,4698,7908,8759,2429,7706,9642,7199,2064,7087,557,8591,4609,3962,2795,8283,6449,9092,9964,4970,4529,3616,6475,7523,56,2238,6431,8256,640,3416,688,10471,8348,5541,2371,994,5570,82,6166,2259,7589,10479,5713,449,2308,130,6882,6298,6517,9870,2353,4316,5452,2792,7556,3803,822,9084,798,9477,9082,657,495,8781,10619,4328,2710,460,10133,3234,2555,6069,5255,348,5511,6934,9594,5120,6729,3956,6290,830,9948,5911,1731,6291,4993,1533,9758,5693,8152,4988,514,10270,6000,9682,6946,8344,8051,5320,8247,9020,3751,7864,10163,540,5933,3650,6107,2840,9372,3701,8955,4269,8511,1024,2165,4739,5474,10239,2466,4157,6533,10780,9408,3312,2549,7113,4480,7741,3467,5710,10381,2392,4166,7485,9902,10116,1089,5281,2157,9182,412,6056,2281,697,4383,6531,7473,9231,7411,7070,1493,3570,4122,3461,4849,5119,3174,3139,8068,7162,3184,3691,2316,18,650,2382,8612,9692,6483,2442,7718,488,6642,8681,5271,9004,3942,7986,5406,6744,9014,6975,8403,7449,8721,3240,8297,752,4471,2833,10812,202,7701,10064,9164,1565,2172,3613,6589,2843,1052,8359,1932,2389,127,5508,4782,1048,2729,6216,5692,5219,4730,8987,940,7810,3907,4399,1220,9901,1549,1242,9815,673,7459,3322,8550,6865,5229,8557,2257,898,2893,8674,2446,7645,6528,2809,4711,2106,6267,1110,6415,7998,2094,9509,6269,4475,3096,1567,7066,2464,8516,1843,9885,8995,2345,4940,1077,903,6886,347,8766,3343,8534,5871,5310,5738,10217,7539,2823,9867,8238,7093,10667,2332,5535,1419,3203,7540,2417,1619,10744,4592,8817,9219,2565,4276,9380,6329,10426,8853,8106,6765,6932,3822,5864,825,8244,9566,8333,5954,5351,2978,3376,6275,10290,1604,1067,7901,3143,7979,3630,5252,5642,6098,3077,1464,325,10481,5853,10012,4658,8315,10029,2812,4123,4967,6097,4038,1755,2873,2905,8098,6941,3681,1070,8555,4875,10340,9944,9694,2180,7441,2904,8054,2531,9330,6750,6945,10067,6012,5092,10772,73,6213,1504,6425,8835,2520,1401,10768,3368,4375,4941,685,9479,453,1268,2245,1694,4325,7705,892,10775,7450,2679,842,3996,6241,1117,7713,10343,2124,5857,5617,6594,7305,8770,5928,2197,9166,5956,9299,10095,6524,10531,4335,6622,10823,802,3120,2073,3864,3490,6957,9845,3972,6935,5543,9774,154,5250,10342,8137,8040,4224,10500,7327,5779,2755,5085,8243,4632,1952,8047,7056,6304,3241,6219,5353,10320,4559,6552,4748,9475,203,4551,8552,7187,3748,4271,3232,5903,9402,1566,6381,1569,6192,5338,2965,8023,2360,10386,6383,923,10618,9172,421,3412,7582,7685,7332,4138,2820,876,1356,1325,1012,3814,9584,3799,5041,9273,5764,9757,1655,6991,8588,9423,8001,3853,6191,8755,576,6629,4115,8878,4253,3245,1885,2505,5426,2621,260,8848,3873,2060,8352,4842,7686,5359,3600,6732,857,5014,9386,9173,8150,3065,7127,10754,2987,9907,7347,1051,6114,1609,221,6666,7851,5093,2471,2465,398,2077,6459,5799,8462,7943,10731,2227,9619,3813,9031,4753,7797,6347,3836,1806,9134,6150,5643,497,3868,6915,5071,2769,3717,8575,4426,5358,1587,7752,4942,8857,742,8464,3442,3050,9660,1165,3309,5293,3171,5099,1154,1827,7728,3140,5995,7719,601,9119,2053,10056,10487,4949,5307,7866,7076,8052,4372,6045,9586,7799,4800,4575,5458,316,5425,6361,4965,1187,10816,1907,7808,10551,4531,146,3884,8978,1271,3568,10668,179,6571,163,408,1190,10175,10716,3837,9340,303,6792,5936,5979,6330,8574,6129,5555,2385,10594,4980,10728,5842,4321,10444,427,5890,1776,4598,5235,7900,602,4763,6907,6379,4804,1960,924,6133,5932,8041,98,3622,476,2517,8161,1374,10787,556,493,5804,9109,8786,1872,3138,1337,2822,6289,7562,3005,1392,6866,2518,1140,5258,133,10331,8306,5313,9575,10717,3156,2103,8495,4799,4672,838,2249,4686,2233,4281,1475,9108,206,9453,5941,8379,5103,2868,9988,9732,7417,7071,1231,2515,9371,4498,3081,9352,6529,2313,1529,4229,771,1134,3690,9005,1128,9503,9776,2571,8369,10271,4387,3408,5533,8666,9035,8390,712,9255,6634,8299,3221,7005,6850,2854,3921,4548,3501,1593,1050,1692,3685,4879,4277,2805,8468,2867,46,1258,2272,1118,816,6796,7965,2036,1722,9448,9851,6940,6862,10538,2547,6437,5343,6727,1359,6444,6573,852,7574,10336,61,7085,6247,5134,106,9192,2369,7786,9009,670,8311,638,5288,8446,5463,296,7337,5018,3103,10423,9970,10008,944,3205,5435,5332,7921,871,967,4777,271,7214,4652,6630,8199,6526,1695,5140,6215,9277,5174,8504,5181,8193,2863,126,2377,691,9662,6495,233,565,6466,1771,5348,9411,5026,5537,4801,656,1384,8396,10541,2326,6958,675,402,10098,4203,6835,6258,6969,7950,5704,4047,10642,5038,3340,1820,6525,7354,8752,5131,3937,4369,3486,586,5305,4971,9773,5948,7618,9361,6790,8743,4309,3722,7721,3944,8836,4154,700,2935,8003,7351,5521,9464,4718,4888,5124,7841,3135,5697,4530,7276,1395,9615,4825,6224,198,7805,633,8442,1840,1261,1170,879,9725,3626,3870,5050,3901,3954,9332,2906,10073,7674,722,6843,4363,9289,5077,6598,3059,3928,1114,10034,6377,1380,4378,4418,10505,4869,7246,1292,9942,7092,9139,861,7416,2747,1245,6423,9095,9653,3130,8948,6737,2404,4026,726,6101,7726,4540,10308,4243,478,2980,3227,8985,6913,2675,4504,3753,4931,2434,6753,1045,7477,834,4181,7134,10507,9514,1747,714,7790,6755,1971,8301,6009,3584,8877,4361,2927,9027,599,8160,9059,4675,8090,10091,8192,5283,719,3145,8671,8569,6824,8639,9253,5815,10361,6988,2510,2585,8904,7405,2398,1768,7989,8696,9899,9805,2439,9412,2727,7927,6257,8746,3982,5679,2013,4579,4141,7634,7553,346,603,1522,8764,7001,3603,7775,9715,7678,8482,8630,10530,7905,9211,5308,6760,5266,7308,7377,3957,7058,6340,3602,3218,4945,239,8867,9918,8130,8000,4950,8913,275,8097,1029,4630,10774,3173,2808,3754,1591,9115,3593,755,3318,6902,5762,2104,8893,3749,10401,8946,2616,8510,8937,7975,1078,6335,6238,10733,2044,7885,4233,7426,3606,5280,9756,810,9844,9695,8053,4704,3852,1484,1031,8399,5467,7672,7590,8215,5263,8028,2657,8117,6142,6273,5561,9748,672,7229,3098,84,10405,4272,2800,6889,8792,6691,4204,5646,5345,5615,4278,5315,4486,1543,9467,5355,2797,5400,1953,8917,7806,7073,3807,7312,1406,5716,7086,1738,6847,9781,10212,9631,10442,7548,10115,4678,4097,5030,2247,3002,10293,492,10429,8944,5106,4009,10269,5760,859,1201,10771,2026,6508,2957,1780,317,6445,8139,1810,1520,8983,2354,490,7375,9866,6035,5393,4751,9810,7710,5155,6576,7367,4368,332,5870,7762,7292,10020,6076,8355,1216,2119,2362,6470,6180,500,9775,298,4319,4935,7585,9563,4182,5326,1970,9266,845,6756,2256,8541,8972,7094,2748,7353,5185,10664,10743,9565,6942,7743,9912,5135,1452,1618,7592,3844,7037,2048,946,7036,5780,3146,5973,5505,8810,2892,3682,9607,931,6852,8823,10136,10440,2615,10764,9421,7909,8964,8449,3160,2153,181,10421,3357,5638,9478,9572,5797,2020,5368,10014,2415,9659,1772,9831,7340,10777,2584,1967,7190,5789,9905,3909,5499,5879,9787,2921,2683,462,2535,10730,7492,6718,10492,7124,1148,1079,3986,6891,5696,10414,5855,5737,4214,9389,3253,1273,3869,2641,8329,7323,4120,7723,4798,1387,5285,6983,7132,8731,5299,3663,4249,3438,365,5100,8684,8801,1993,4506,1671,4566,5122,1686,3195,7916,3880,1101,9672,8036,4397,2701,1786,8120,10726,3429,5114,4588,5217,1057,2735,10794,1331,8815,2261,4703,1667,9574,10803,10054,2435,8067,4393,6362,2937,985,8750,5374,770,2463,1749,5922,1063,10335,4743,2305,10396,7546,257,3297,8553,1775,10197,7279,2690,496,3454,8616,10612,7857,7126,3107,2298,4072,8906,151,1581,8061,439,6856,7903,2142,4526,5244,9029,4714,8335,881,10363,7503,6647,935,5133,7649,10561,6108,933,2551,6870,4673,2826,10129,5906,626,7717,1082,3575,1172,9417,3456,5568,5914,5856,1851,2953,4881,5851,6239,2050,4371,6477,7366,4452,9849,2246,7930,988,1951,10789,5878,9853,2323,2850,1942,1136,7421,10230,5130,6313,6929,4846,10267,7890,4964,6829,4357,8582,649,7241,6716,8258,1176,9263,5390,6994,5921,3192,1069,10796,5908,6486,6138,8562,8578,6925,1408,7917,8045,10286,6671,2352,1312,10309,8210,800,6050,2686,6214,8456,10302,8858,3443,418,8005,1445,7115,1824,5690,2569,2917,5197,6749,8643,1537,9977,5648,3997,2600,10395,8360,2861,1014,9790,9865,4095,6124,1666,9189,5160,8186,5137,10828,6025,1847,4932,1249,1299,9530,7282,6990,9223,3720,3818,7987,3137,7670,286,4731,7654,805,9668,10018,230,2467,8638,10084,8422,3481,9811,3285,4877,9281,10307,10801,102,530,1819,4918,6546,928,2930,6678,10253,10151,4094,8430,5056,7400,1313,10461,6039,7992,6664,6502,10644,6424,3383,4302,7186,8342,3559,310,10379,9446,2277,1116,5778,6538,1963,3185,1487,3294,10362,1207,6921,8181,5236,10353,5709,7359,10011,4147,910,2476,5337,9213,9916,2696,2391,7286,8797,1796,2411,2570,7990,2889,10723,10150,171,472,6246,7280,10100,8298,4517,2775,7831,10582,2365,3183,3670,6432,9074,7993,4852,646,3645,4052,5323,5556,10807,4014,5349,7109,1296,7125,8147,10043,10085,10709,6906,2251,1058,3842,1132,10328,6005,7725,1853,1862,4528,10824,2837,9129,2214,4209,3557,8758,2605,6667,8761,7620,2156,8505,4322,5269,9818,4447,3186,4709,3920,10087,568,7985,8911,2331,6494,10654,7631,2875,5363,6151,6896,5481,9705,3286,7753,5449,6013,6874,5153,1779,1494,4722,6728,4726,8008,6316,430,4403,6359,8762,6936,8704,8677,1111,8153,8680,7700,2622,582,1462,1621,4373,5430,6218,1825,10038,5597,5578,6064,7907,7892,1469,9038,6763,7197,10543,9461,3881,1536,8992,7465,235,4240,7502,1541,1616,7207,6608,4245,3470,2961,5423,9460,3993,1979,3680,8998,6844,6370,8959,3488,3217,2604,3891,3369,6467,6320,4458,9248,1793,8676,7581,10502,730,566,3694,4699,4416,2284,8110,1933,9333,10388,9873,7454,9254,8881,775,3497,1712,7090,5158,10122,7483,10818,9973,9704,1969,4606,2682,3354,8869,1338,10215,4379,9793,694,3725,7365,4431,8302,2752,3255,5392,5468,68,10751,4871,390,457,4705,3181,4081,1491,9246,5982,2524,3721,2372,727,2163,4734,6689,10005,5237,5583,280,5207,3034,4022,5934,6326,2334,7209,2702,4643,7498,10119,5825,8962,8185,5440,2169,1803,1742,7316,5183,1177,2936,5020,6959,5063,698,8234,9307,3086,8831,919,2234,3787,5714,8263,8991,6104,8554,3350,5187,4092,8071,1898,1394,4992,5371,9200,486,7787,8939,6047,2190,4210,3142,4261,1583,1121,1927,270,4948,6204,10255,6811,315,8896,3477,3895,9606,8851,4623,4244,893,7009,10344,5404,2539,3448,8682,6105,8965,9706,2862,5375,3273,242,9218,3585,7027,1112,8232,7751,8332,9015,6507,8544,4428,884,3857,1086,8115,949,4160,4268,6867,9803,8337,6903,10165,5744,3284,5854,1785,2319,3089,8448,389,2066,7236,3351,1466,123,1956,10341,8785,1741,2310,1544,10547,526,4421,8916,7915,161,2972,5424,1301,222,4661,6438,2496,3109,8488,9354,743,9345,3615,10721,9622,51,8829,9000,4890,10182,3760,8921,1867,5754,3265,2399,6089,7313,6339,9285,212,3308,1886,10441,2341,2626,1284,8138,10424,10204,3572,3966,7296,7303,6846,1875,2758,9663,7173,2959,7261,6295,8303,9088,7669,2112,1328,809,4456,3132,2346,1241,958,339,7663,7531,2519,7509,6949,8340,9959,7063,3055,4636,4650,7680,7203,6030,2264,8979,8819,2500,7559,5606,9434,3256,10689,2321,3324,523,8078,992,2736,7513,6385,7052,5970,5483,278,6350,1730,9367,729,157,7960,4446,2110,10575,3544,10795,6554,4384,6279,6659,2623,2082,145,6372,3582,4127,1878,10826,4129,1017,1837,9788,3281,1663,1929,1597,7393,10473,5167,9599,1482,5394,4906,952,2098,3465,9240,5524,3459,1724,9454,6189,5210,8609,3447,252,20,8014,560,7346,8756,9939,6463,3810,8537,7240,6703,3435,2252,8404,4582,4011,285,3744,6435,8127,6676,3492,9245,7358,4079,3555,3329,9868,10154,1435,5189,8220,1275,3926,7007,4464,8647,10250,7756,10798,10221,8891,4723,10161,6297,9083,3765,95,3530,10346,9909,1532,3700,2541,9989,6816,10311,9640,6547,6384,2081,7112,5666,3974,10374,617,5691,6310,1702,2487,2285,4644,3336,3756,1224,4343,987,369,1093,5223,1610,9981,7227,4114,915,1186,3856,583,10701,6518,7343,860,956,3165,4762,2529,10281,1429,3413,4135,483,1449,739,4878,3697,769,5841,4621,999,10557,9149,6955,1997,1073,4172,7653,5518,1485,3287,5466,9746,7146,8953,8830,7521,8289,2198,5300,4404,1659,336,5723,2857,1499,8658,5105,6122,9601,7954,7573,5243,762,3418,6234,3161,2681,4441,4649,8345,3804,1818,3177,833,9727,10225,965,7635,3006,2772,1142,3208,1098,150,3943,4761,3460,148,4784,7496,7263,612,3524,984,1983,5502,1120,7200,4104,28,9675,9544,7936,8763,9603,4304,6014,8498,10131,1367,2580,5959,2016,1205,1351,3288,4457,10673,8116,4406,1206,7046,9177,5683,7707,3547,2045,4376,1516,2865,4622,3207,3659,9175,5121,2777,2653,8524,1157,218,6418,3321,10102,2380,7980,4617,4220,8859,2550,9473,3495,5408,10677,343,7398,380,2894,5325,10235,1329,2869,7202,10437,6613,3509,2703,6314,7226,5218,10184,10627,3526,7179,6801,6853,8560,962,10511,7729,4125,4134,4392,2766,2492,5971,6144,2358,2583,1122,9632,2992,747,5145,7409,8392,543,3133,7946,10450,754,4981,5441,2618,6462,1229,8587,6498,9512,954,5848,5688,4759,5331,356,8088,7814,4350,7795,8176,3631,2773,4999,8259,7381,744,6609,2564,5940,1594,3755,6476,6540,5057,10616,9221,621,597,7887,2606,81,3980,2220,6640,518,2659,4538,9926,1733,7948,8018,3004,7363,2021,6992,2342,8483,9373,4527,4419,10224,5816,10623,2693,2160,2963,6773,74,4684,104,4898,4809,1674,7499,10696,4695,3978,2699,5214,7315,829,10410,1145,803,5951,301,3763,4196,7746,8328,3070,7407,6055,938,10522,7977,9896,2243,7033,9814,4494,9676,6036,6100,4775,12,2195,10196,1103,5862,3333,3471,6764,10222,4174,8568,8249,8362,5492,4053,5630,7000,1966,3422,1711,7682,295,865,7361,9652,5790,5626,3821,8409,9859,6363,8860,5989,4087,6395,9848,8308,4341,7520,7616,2671,6836,4860,3014,9789,9205,1918,5053,3335,9542,320,8884,5204,1442,5558,8250,9872,5952,1424,3709,8195,6140,4057,6633,659,2383,5834,1508,4246,10010,7287,8076,4554,3599,10685,1431,4050,7264,8535,5166,8086,8133,6541,4201,3545,9923,5647,4035,7740,10183,4169,1689,2664,5770,1882,1295,6978,4921,1812,5412,5387,8668,8408,4781,3579,732,3899,9510,3082,1914,7242,1557,1912,3961,3250,1039,9611,6683,9820,7647,10220,5720,10173,10330,4909,2337,2654,7133,1896,2883,2448,10727,2793,578,3199,8572,2743,10223,10662,9378,10004,4008,9331,1675,718,4682,4274,6762,1240,3261,4046,1934,79,1988,9624,5601,6621,3153,3882,10144,7429,6430,8986,5860,3225,5873,3960,9397,7687,5080,10674,10778,6193,5531,5793,88,7583,4601,2248,5748,3393,6702,6857,6950,8635,2375,1141,4236,4145,2733,5629,401,8954,5777,6783,3337,1943,5918,7821,255,10692,1434,9058,9629,3168,360,1690,1230,9458,951,9271,6469,5967,3061,7579,1003,3108,7681,8486,6265,4409,6109,8412,5618,6487,5562,6276,5907,7575,5529,10601,1725,9096,6905,7801,8610,5277,2627,5035,9569,3157,9957,7322,1600,6157,796,4460,6627,5685,10805,394,3669,7330,2426,7744,10520,8645,8932,5209,5820,8391,4769,6132,8613,1836,3841,3041,1085,521,5186,9462,7136,6429,9951,5409,1269,8128,7516,9188,8354,10167,10216,9497,9972,7399,8910,6456,2810,10339,10819,1646,9616,2174,4175,6091,748,3661,2366,6654,587,3627,10097,8367,1109,1844,8652,1364,8151,5060,2480,7817,4284,1076,5257,4861,7355,5872,9709,1298,8145,8519,806,2427,4524,3660,6293,4444,1660,3989,9374,7192,5369,8292,3914,5608,210,1002,7154,1302,2855,2412,10211,8799,4612,9744,267,5877,7137,8426,2828,4192,2378,6236,4924,9921,2440,6378,3556,9511,1897,357,8104,2400,10609,2970,3499,1830,8321,4543,7794,2971,254,6223,27,9480,7003,8856,6954,7704,318,2407,1182,4624,644,9496,10318,7961,9257,5695,6231,8894,4186,7976,812,7331,4217,7106,8092,1126,4102,8980,10591,3734,1213,2670,9679,3172,10459,8626,6492,3323,6646,9539,355,9337,8716,3808,6165,10562,5658,818,9707,8958,2134,4866,10317,7818,274,5457,3388,4282,4330,6482,109,561,4955,4874,117,8038,8866,826,3938,7069,9357,4806,4903,8741,2688,6343,2402,9895,9561,4100,10491,2597,3533,1849,3190,8737,8967,9007,4468,1068,10277,2258,4360,59,107,1036,5769,8437,9843,9700,9127,3525,1,7297,1809,5972,2484,9840,9609,431,5800,671,3009,10159,481,5025,6254,9602,2025,1001,8679,7472,8929,3164,10786,5165,7177,9987,733,4959,10276,6199,8022,9001,10576,9036,8938,10189,9238,297,3231,9080,6512,6509,1568,997,3773,9643,1030,22,2192,9407,7798,2813,10039,2088,5173,7170,6652,8536,10436,3325,10338,9437,1977,1919,9808,2430,10240,1107,6451,2784,1817,8037,9633,143,7117,6964,721,4194,4954,9730,8203,9491,1807,5659,9116,9816,6808,1160,1358,4106,1166,7206,371,3636,7656,2148,1938,7394,3634,8204,4770,4938,3896,4577,8548,7530,6016,7461,324,10646,7307,7338,8436,4381,6615,6087,5668,942,9953,1899,5546,9406,15,9684,3552,7335,7392,450,4470,6201,4040,10641,8713,953,5675,4947,8715,3092,5422,6023,5783,3950,1423,677,4710,878,7130,10822,5512,7204,9929,7326,2019,7774,3197,10810,7732,8471,5052,10750,4482,5156,334,8526,6222,9214,6614,7258,6488,6680,8320,5259,3757,9549,9196,5715,3581,5413,8031,5224,6497,7651,1100,10110,4797,7839,3091,10123,10358,8188,2208,5447,4542,373,434,3293,4904,214,8494,5450,1721,9828,9961,8747,4365,110,4910,704,9155,3504,7235,7734,873,10448,7667,8824,3796,7971,4605,10003,917,4066,705,779,1612,8861,5260,7295,3027,8487,3823,9826,4459,8941,6715,2085,3517,4808,7075,8272,4500,9656,6927,8405,1386,4737,8475,8365,1813,10088,4996,6029,6993,3364,4491,4885,4287,3667,10170,9325,2314,1129,3911,7830,5924,2676,4934,9403,7940,1982,5054,3028,9693,8633,454,1944,8599,9833,8595,41,1400,9291,25,3594,2697,8945,8840,7290,8963,2043,8431,6618,7978,4635,1669,7879,4019,3811,8168,1791,7099,6588,9120,8107,2633,7467,2490,3905,4697,3652,6742,1718,3987,6206,8467,3716,8875,9504,7822,1874,2899,2537,1515,7730,9667,9388,8007,5294,7185,9778,9451,446,1512,4152,9894,6948,6416,8189,10042,7024,232,648,1425,9531,10391,10715,6568,6353,1441,4423,470,7962,5376,2725,4610,9079,678,24,794,963,5479,2339,5023,1627,9260,19,811,1410,2506,2086,4218,7015,9600,2022,9466,7222,7410,438,2732,4654,7252,8826,1766,3483,9081,3998,2315,2767,5503,9724,4060,4507,4857,5360,8310,10600,3211,6448,7386,3790,3872,1343,5636,2280,2384,4015,5068,6328,7697,7538,1588,2760,4701,5706,2338,3925,9966,2674,5599,1985,3879,6636,10366,9877,3042,544,2724,429,1957,8924,1634,10106,6242,9852,1884,1799,3610,299,7696,1144,5923,5194,4467,5813,1925,5177,5554,9889,886,6244,8968,9883,1910,1661,4438,7766,2194,5792,7610,8874,9729,7166,5700,8477,2091,4076,4962,9797,10400,2179,6539,1243,3452,6322,7091,10164,6564,8879,1637,2222,8434,6026,9485,4235,10564,6164,10599,186,7953,1223,6484,630,2034,6225,5991,5787,2344,7376,6979,4462,204,10621,6747,13,302,7329,4339,5983,6338,3113,3397,2213,5772,6603,4140,8357,8048,9752,3875,4177,1106,4179,2789,2995,3774,625,797,4188,333,4093,7495,6043,8057,5433,9839,1038,9750,1763,8427,10470,8601,1471,7956,8118,6848,6237,3794,5899,7245,1913,8242,6565,5329,442,2644,986,2983,5684,7060,368,89,10710,8027,2481,10632,9061,3313,10698,5357,6054,2221,4615,1578,9878,2509,4810,8126,848,1303,2489,2441,764,8255,7065,1527,553,2628,6282,4518,6331,8096,3017,2039,5350,3657,6795,2357,6577,4296,510,5627,338,393,10407,6951,9090,2798,4589,8606,6434,868,4778,631,2964,231,4594,6665,3945,5427,1293,8655,8499,388,3633,1393,2877,8319,6980,736,622,8813,3797,5883,3433,10160,2786,9567,9022,5739,7920,8966,1083,4771,3532,10697,4826,137,2182,522,8984,9982,10264,4017,1974,5652,1981,2910,7029,2990,8724,3505,4651,7819,4345,5497,1928,7782,8949,433,2403,10288,6751,6342,4712,7684,9683,1035,2844,2317,10157,3858,4370,3883,1554,7061,8373,8015,5686,8795,6278,3683,8686,5493,5168,4032,1748,10790,1458,1432,7436,8032,7625,8651,10815,5998,10506,5202,8688,3678,9343,6510,1629,3182,1353,7419,3785,2472,8393,7761,1355,6332,9681,9931,364,10678,854,9098,9691,10792,5289,7764,4230,5069,7055,3955,3576,4567,10194,6414,6766,8956,3269,2997,3888,144,1777,5477,7013,5192,3642,2311,1336,849,1601,4968,7482,8135,6404,3124,8622,2949,5403,9269,1256,10521,9091,5486,6970,2815,4311,1893,7183,85,2292,9856,7984,855,1750,4279,1713,7002,7626,1705,2090,105,4687,9153,2294,10411,7939,6707,1672,10284,8545,2609,10714,8589,1757,7469,7217,2263,5027,1745,853,4429,6758,9651,7144,9286,7676,8527,8155,420,4443,1845,8082,6611,0,2794,9547,639,1222,6115,1572,9903,9495,5545,3243,9621,5182,247,5581,3289,3729,776,5095,2379,243,8744,2295,3732,4725,10544,2745,1246,10024,3345,2076,1155,627,6939,2454,2839,10259,641,5988,1808,7401,9723,6734,1294,10155,5818,1260,7904,10671,7750,1061,8224,4013,7194,9030,9482,1505,8012,10185,2293,8551,520,4831,10451,5002,1628,244,23,5459,3152,3566,2923,4389,2508,5984,9363,9138,4756,9871,3200,1729,8922,3064,5090,4626,491,1968,7768,4039,7882,6317,4822,10244,6947,6559,5,188,4424,6963,1879,10447,8269,9576,3865,2632,4003,9368,3484,8089,2790,5472,1922,6723,5553,6057,2734,4989,4706,397,8384,8002,6233,3469,1576,8470,9846,2836,9521,1829,7442,8771,7731,2751,813,9051,5637,863,329,8056,5574,9779,10558,9234,8923,366,10036,7320,8997,9522,5301,3292,3327,1113,5242,5602,5876,3817,6972,1054,7870,4310,211,1290,4005,6888,8732,7384,9564,6837,6044,6205,5655,10017,4824,2938,7281,10462,5091,6918,5488,8481,7828,5498,7613,3405,3648,6007,5634,10209,10109,10516,10466,10688,6501,2023,400,10517,10399,2297,5079,4148,6248,3482,5231,441,7759,9244,1064,8564,8973,711,10711,5525,3420,2526,4757,3759,8142,5750,1787,6232,10806,3730,4483,4255,2811,2707,8295,9699,10739,9911,7622,1784,6592,9590,9556,1687,7643,10265,1620,1119,8451,4198,10519,10607,6446,4227,4943,6810,5061,9364,4570,2738,2640,10126,8694,9666,5043,1883,8233,904,6081,4532,3553,7533,3507,7636,6788,9128,995,4867,9560,5190,1115,2771,731,9777,1653,6919,6776,10262,9881,1196,3386,6390,3574,8693,913,2714,2713,8780,674,3478,550,686,4786,8990,5885,139,10416,1684,1811,5147,5614,5178,2662,4080,3141,6885,4211,307,1232,7565,2704,7598,9383,5930,7348,9999,8415,5980,6965,3236,6685,8251,2065,1573,4893,10245,2051,1239,4581,3425,2594,2740,3394,2607,384,3646,3567,5732,10820,7380,9440,10827,7490,1194,9984,4435,5587,901,5828,3008,7045,8749,6974,1547,8225,2202,6968,1602,9124,8466,509,10066,934,5022,2542,10193,1852,5339,8180,6639,3263,4159,5632,1590,8059,3952,7225,158,3494,2723,6447,5625,10732,2864,6698,3906,7974,4002,10044,2673,1008,266,9293,1999,7714,7968,4112,7779,9610,2266,7515,7274,8558,2139,8725,6067,4642,8930,374,8485,4717,173,2575,3832,34,2449,1346,7854,7675,8378,1743,7178,7022,2495,5542,1668,40,8230,3849,3363,10006,90,1143,8636,3591,2779,2994,9997,8281,10483,3463,4899,7722,10830,8492,4136,10368,7437,5013,2783,2592,9110,10227,6638,6650,7827,5455,10583,4338,7387,4158,1318,6757,4126,5434,1931,7856,9721,8597,1209,737,4474,2718,93,4337,5102,3317,10432,83,1339,10534,572,3788,3298,8604,4176,8889,7836,9505,8400,3246,9552,3390,7599,9630,7850,6075,6854,6077,2687,866,6668,9206,3781,3587,717,2312,6600,6407,6341,264,6952,1579,2558,2532,6562,5389,3970,7034,293,7777,9506,4873,7247,9140,10493,464,4225,5639,3134,9195,2386,5603,2578,8533,9782,436,6893,5590,3596,2083,9908,5891,5064,9589,8760,1693,5571,8837,168,6382,8512,4334,3711,10474,9655,1704,7057,5724,5673,9735,5522,7289,6944,2787,1709,6198,1131,4555,2999,555,7781,8812,632,5743,6513,3414,2336,1682,8988,8087,2343,4412,9424,99,444,5874,10460,9017,6094,7614,2147,5125,10117,7840,4637,5104,1285,8602,1091,10113,9887,7576,10605,3258,4018,276,9324,7641,9276,10477,3054,9256,9450,7105,3953,5909,8717,6504,2715,2154,9197,8828,5929,3621,3016,7042,419,3847,9320,1334,3782,9312,8699,3228,7715,7370,9313,8346,4425,3047,8838,9049,6352,1340,4492,3356,7095,663,4660,4119,10146,3129,7369,2040,6674,5527,9259,2322,64,7858,7019,8417,1746,7285,3332,7129,6814,4796,8274,3083,2824,3001,8621,3746,5344,1575,10357,4758,5436,4896,1060,8371,426,5031,7991,5916,1335,1345,8444,2162,10791,4539,9850,7051,2270,961,94,4578,6521,9290,6724,3809,6306,1644,447,7084,9623,7811,8895,2410,2493,4839,6003,7889,8892,3674,1283,790,2118,1473,5070,9296,469,9447,2128,3252,3793,10408,466,9013,2776,10452,6135,9187,6662,8183,1605,3695,8077,9430,5955,2376,7511,1894,7304,8703,6063,6672,5462,4405,8742,10120,5593,2475,208,2431,2672,1288,2582,5519,2586,7843,353,8981,896,10275,647,10456,3689,3802,4317,6663,6118,4185,4708,395,4829,3538,9278,471,5010,8531,6345,3586,6688,120,847,8804,2494,6987,1125,7031,6879,2730,6717,1033,6709,1562,2274,9686,7735,5942,4863,10572,10404,71,2968,2067,5717,92,10412,3311,6833,5509,5794,4785,5042,3656,8366,3020,10494,2405,5758,10536,9541,1920,6696,6287,2756,31,10430,349,3571,7708,3037,5256,5321,10554,7529,9733,10179,5227,3702,9785,680,10140,4195,3381,9638,7385,3643,9165,10604,3355,8044,4603,580,7455,9274,6389,6169,8254,1630,4696,7633,10327,2173,8039,9220,10247,7041,6706,1492,2217,9186,9024,2127,7947,3550,7563,4143,570,5633,1816,4510,5549,10139,4149,4450,9112,4078,5910,10132,4414,3806,9204,10713,9384,7250,3444,2330,10071,4990,7959,5900,8010,695,2418,9349,2079,793,6022,8438,3569,305,8863,8888,1397,596,6355,5604,3795,4291,2731,5203,2908,7568,3939,8175,4569,1762,2017,3766,3623,8062,138,54,5884,2240,6735,4455,5961,7527,6660,9060,8855,5785,6392,7688,551,4768,10367,2546,432,6143,5703,7278,547,4440,4614,2031,760,9107,164,921,1262,1858,6701,4692,9105,4354,768,2359,2012,6566,7820,759,2144,10251,9156,6096,3370,2226,708,1514,5835,8811,10060,4184,9763,9838,2553,6095,2739,7630,4521,10515,8615,5831,2939,7650,6491,8999,311,8060,8381,9298,7748,4314,10356,6823,788,3150,5169,2525,7913,8479,2035,9235,5396,3833,6781,5501,2126,8389,8729,5767,8543,8899,149,3676,5028,9026,8523,3229,1778,2279,3562,6931,3968,2388,1586,2084,7646,331,2309,9358,9410,6419,8661,7981,2872,5966,3249,1438,1179,5514,9442,1421,7544,10463,10092,6813,7402,10433,2096,4264,3254,2188,7571,4693,9111,2183,3233,1270,9930,7201,6177,9326,6128,9272,10283,1420,482,8918,5904,1717,10166,4410,7447,10504,1500,7996,3302,62,3277,9151,7220,9469,3629,3965,3506,508,8207,7776,8825,6989,831,3439,8361,1092,1570,7605,3019,7205,3437,6454,3328,5547,6280,10300,5808,9936,4473,7111,8532,1368,10527,3775,10439,8290,3605,10169,2393,4774,6741,7802,7435,1375,4064,9210,1010,1274,662,6711,8865,3214,4099,10299,7104,571,3178,8753,2625,6461,2033,1379,5641,2095,5965,7249,6797,1947,9280,2643,8791,4864,9821,749,10294,5385,4132,8897,5726,3564,3558,6499,2032,2834,10634,10389,4163,8820,3038,6585,7673,5247,7352,9830,4171,7151,7824,2244,4848,10371,1211,8876,9133,689,7397,4352,1769,10832,6356,8081,2637,4926,7169,3409,8450,3934,1080,10061,693,3030,4740,5784,2907,5576,7288,10670,9125,8903,5886,6548,907,9950,268,4553,7424,750,5947,5377,5663,2235,3769,7615,4239,2501,6572,1559,745,1495,10273,10681,8489,4395,10550,4749,7145,2825,8650,10485,8134,10023,8463,3291,6412,9078,9224,9431,4944,3105,1422,10455,5507,4180,9075,9123,9438,6986,3671,8386,7507,7716,9927,5284,3589,1448,8503,7832,1584,8714,362,5802,5564,4604,10718,5437,4307,7506,8890,4913,4146,7823,7181,4280,7362,5384,2890,3886,9117,9728,10687,182,4183,10770,3859,10228,676,5251,690,1443,4927,2540,5520,9837,598,2513,8190,637,823,5487,4275,4862,9484,6360,8063,3762,1474,5032,6158,10419,3431,7602,9279,2841,1635,9928,8617,1959,5836,3440,4197,6061,2932,3375,7550,9791,5273,7964,10249,2579,452,2271,10518,6111,4096,1647,1753,4212,6798,5378,1700,7876,8594,4033,1869,5024,2620,9011,8095,6478,5881,9319,8205,5978,6779,8200,4074,8850,2269,4073,9222,3487,4519,2885,7160,6172,936,2458,7834,9468,7763,3430,1707,187,6770,7770,1055,352,7783,10737,9401,7167,1991,5240,1180,9121,1861,8395,2268,8957,5774,996,8605,3772,8940,4836,891,321,7988,6534,5671,9144,9301,9754,8849,2554,1546,7425,4746,10070,10742,2061,1782,6038,6250,591,1257,1391,1954,8206,6307,1903,9941,3093,4348,7666,2996,6904,9100,5694,10143,10663,1426,607,8201,10035,7107,1371,7747,2042,10206,5327,1992,8420,5391,6752,8538,9396,3479,7849,6514,6892,477,8969,1585,192,5669,516,1237,1833,2979,4488,9753,4362,176,1916,7215,5365,3213,1563,7973,5557,8685,2619,1200,3502,10104,4851,7267,8100,6858,3062,4062,9876,2821,645,9604,4742,7118,507,5987,2668,579,9459,6125,10813,10046,10202,70,227,9483,10606,10332,3198,3044,10672,774,2601,1251,5953,559,4109,6563,8803,4832,1094,4911,4514,4549,5225,9428,3931,9381,10578,8473,10305,1506,10,5012,2587,10195,10434,1162,3347,2556,976,5152,414,5445,7588,4680,1210,5566,4027,1651,10013,180,9734,4150,2216,5584,9168,6426,10533,403,6391,8317,7471,8773,6772,1072,2835,5935,4951,8382,2799,9835,8079,629,2596,8525,7123,9678,113,3992,10149,8364,761,3111,519,8672,1521,7119,1011,2057,3940,7139,1236,1450,1930,1307,1191,1147,17,8074,8428,1509,9136,7293,4515,975,1388,4537,5672,5513,2611,4156,1071,10105,4453,2998,9717,875,8084,3750,10030,9142,9784,7311,6895,2514,494,415,6325,8698,6626,2460,9933,108,282,10589,6610,4323,1427,4830,6348,26,9527,2832,2015,1383,7344,1961,1088,2149,3496,8083,4294,4105,5850,1265,4969,8187,1137,8324,1625,2413,193,5964,9355,8629,4366,7933,7456,1821,4327,9053,7859,6520,103,2420,4882,3084,4653,562,9979,1381,10099,2925,8667,10048,10482,6210,10637,5861,5958,7601,1765,5176,6366,7512,10788,4048,10513,8935,10186,1460,8398,6532,6028,3029,10258,7891,3903,4402,537,2062,1281,4813,740,7324,3040,4299,7668,3275,9071,8275,10532,9628,4580,7969,6649,5485,6042,4596,4602,125,8722,6015,758,5287,3237,3391,9664,4563,2981,7427,3951,4618,4312,2803,5268,9261,5992,8960,8376,375,1688,610,9393,2210,8111,912,4994,1214,4355,4733,3278,1168,5352,7026,5812,7390,3929,3718,9800,9249,6656,4098,7543,5415,5334,5034,2521,5228,1828,273,8886,3604,9425,7156,4574,9965,10712,2886,7298,10613,9052,4820,778,135,2132,10831,1761,6851,5039,3179,9452,2929,2982,9037,4683,9106,5150,7958,4585,5298,2395,10130,4221,4738,287,4891,2203,4572,4508,1697,5651,9191,1832,4295,10556,8454,1378,9741,4110,4301,3155,9897,713,5136,4681,9141,5138,6088,5157,10773,2485,9499,5644,1664,536,7580,8598,4088,6422,10257,6400,5551,1440,1534,4084,4633,3051,3949,3767,1946,2780,4946,2940,899,2278,10289,4063,2660,9418,9045,2250,9494,309,8245,8691,6616,642,2639,2985,1459,3235,7388,455,6579,1315,10260,9618,6197,6460,498,5290,5322,337,3025,6245,7440,3922,7141,10354,4576,4933,10292,9582,9429,2102,7462,5919,5996,4978,2333,8043,8149,2296,10028,9657,3012,6768,7692,1550,9824,3528,8871,5347,9179,10579,10297,5159,1654,7082,4167,6812,8540,11,4207,345,3210,2576,5075,10114,605,2340,7664,6736,4747,3473,3747,9183,9171,3850,5276,8994,9310,7149,7253,4103,3573,1185,1633,773,8528,246,3890,10347,9305,5312,9520,9751,8898,9154,10337,4966,9391,3889,1790,3035,2516,9318,8323,7868,1023,10077,1727,3032,10329,2922,4571,3464,9759,2545,1277,6285,4957,8397,616,7861,3542,9680,4689,7088,8521,1895,3741,8457,2856,5112,10635,7624,7665,10808,2450,4297,2950,6168,6967,4584,6179,10420,3187,943,7443,1215,2962,5896,5484,8689,4469,7147,7945,3607,2111,4479,3136,5047,4051,8050,8336,10015,2394,2573,4305,9963,4509,6916,3462,5254,4318,265,58,422,8927,279,5718,8069,6527,10620,4795,9766,4535,3189,7712,6998,10263,7096,6186,960,8809,5324,4206,4858,5143,4427,9101,4118,3352,2848,4776,10649,3175,9536,10208,2591,2944,7230,8912,7660,1614,228,2115,1385,7043,4437,8662,8349,914,10319,2130,8649,5249,8294,9152,1461,10162,1169,5198,9535,7018,399,7658,9755,4451,10107,3866,506,1526,3283,6274,7184,7926,2870,611,2884,1865,1984,10758,4449,7617,1706,2728,9443,7374,651,3592,8476,9028,9516,7453,3752,8774,6731,7011,6912,9047,10763,8654,3789,6093,2543,1009,3647,2101,3518,8563,1662,6686,2716,10072,4495,8603,10387,5306,9696,8508,8775,998,3658,21,9772,7972,10274,9585,2572,4289,564,8579,970,6209,6358,1235,1250,894,9579,5949,7248,1856,3268,3202,525,3406,6090,872,3916,9262,5442,55,7089,8429,215,4461,5749,9351,5559,8783,3624,2598,7611,8141,3432,2807,5399,1645,7458,4004,9925,7703,6684,8374,5238,9540,5565,6226,8421,5782,9113,6632,5582,1428,5438,1350,2694,3338,9419,1501,7360,1800,10573,6001,3238,9847,7769,7848,9593,7594,10254,5191,7481,7963,4161,9551,2355,6651,7846,9455,3480,6086,376,6604,9858,2286,4041,5267,6315,5875,6999,2967,9344,4336,8490,1314,3827,3736,8628,6305,152,5586,4720,8351,7064,7373,4091,3303,4390,4727,3815,900,3115,142,8009,359,590,341,10413,1075,5869,9595,2711,5920,9769,4541,4986,3219,6761,7431,1470,573,5976,5840,593,9598,10829,3984,9532,972,9550,9126,8584,2866,2593,6841,1923,7999,1463,2602,1905,4707,7600,200,10237,2819,5960,6033,10489,9243,1059,9768,9033,7097,3127,5065,2215,10745,6364,1545,10598,7391,7412,9577,4228,4783,9342,874,6,4732,725,6323,8143,8478,563,1737,7679,4868,7541,3339,10428,8035,8307,3828,5292,2491,6861,3474,9376,4984,8926,5745,9562,594,4802,1598,3637,8779,4358,5108,6053,8709,6580,7910,10266,8253,1736,9390,4560,489,5768,2368,6743,262,10490,3851,2991,1027,2993,581,3300,6809,3358,8496,3655,7176,8501,2559,236,1577,10019,1233,1926,9426,9050,4332,10236,9089,9992,2133,9230,2617,1685,2422,2900,4736,5111,8720,7062,1289,4021,3123,7415,9150,8806,2136,9501,1873,2225,5894,7800,9377,7845,9006,3649,8266,5101,5469,6318,4219,5098,5847,4170,2093,2536,6398,6137,5912,1015,8718,9812,7629,1341,292,8172,136,8723,511,6933,4351,3976,1641,5420,189,9427,827,5370,9382,9529,4914,3563,7957,6710,7138,756,6017,1047,4293,6606,3988,7432,9822,197,8112,1405,2121,4199,4715,10476,4870,5735,6099,1370,5826,2507,7893,6648,10108,8484,1360,6661,6402,7414,7349,7433,9251,38,4499,4619,3399,6136,5986,1976,5833,2301,1948,167,4844,2327,2438,7444,3919,6396,5456,8590,1446,9861,4611,5058,10406,5656,6771,3516,902,8252,8212,851,2759,9021,6530,7577,1972,9201,9295,1219,5407,6493,2629,69,828,6146,9740,4086,3705,281,3975,5756,1244,5418,3704,8989,86,2926,1859,8158,3617,8800,4685,6262,1679,9940,937,5550,8102,1276,7350,1571,9969,4958,4434,8343,5607,4620,166,1456,567,6311,10542,8218,6767,9392,7219,7671,5179,2052,8174,8196,67,781,9413,5362,6840,643,4987,2612,6334,4817,5097,9712,955,2933,9275,8013,9334,10397,10535,37,1613,4085,2753,10784,6405,6631,2175,5059,1539,191,10242,6235,7228,9103,3131,6708,8213,10648,6410,5213,3334,4472,6277,2802,7632,4657,4442,2056,4153,2267,3632,1451,1476,3641,8313,2099,6923,8627,1022,6041,6387,9817,1756,379,8140,4887,6268,1917,9978,6062,10498,4029,1975,9922,10569,2665,2254,5206,3326,9282,3374,7677,8413,1855,2299,2408,7897,3778,6217,7077,2816,2473,5573,6249,9184,10079,7537,7300,3419,8664,289,7896,2468,249,4841,3267,7504,7545,4286,5552,128,6726,8577,2406,314,5839,3893,9215,5132,7875,2651,10559,8024,1497,9697,2241,8407,1964,499,6587,932,7341,2089,10213,1286,229,1433,479,7755,2581,6570,4124,969,7251,501,8260,2028,4534,6417,97,10145,8847,8816,5296,4292,1552,9161,3846,10765,1311,10296,8146,6473,7175,3693,330,4928,6052,9130,1208,3918,6785,4164,5763,4497,6397,3917,1198,8173,4090,3498,517,2969,10168,8156,1104,5622,10497,3515,8222,6551,10128,3380,3979,8882,10467,7829,3410,883,10431,9639,5791,9720,8700,4792,9247,5727,3771,5858,8546,468,548,3500,8646,6453,746,10446,459,9674,6253,2120,8472,1044,1650,3860,7778,9998,5829,10804,2498,9094,8331,10089,4561,2108,6815,667,4892,3389,6221,5341,2955,6174,10630,2451,6956,7773,10232,6549,248,528,6597,1411,60,6669,2114,8455,4920,6027,3726,6516,8474,9508,2363,9716,308,3039,4895,96,5476,10205,916,6195,5262,7131,3511,5516,1430,9217,4191,7301,4448,5598,9093,6161,7025,9710,4767,8055,5773,3843,4960,8705,7826,9457,7524,4359,4346,1102,4466,3310,9339,5282,10191,5460,7694,4042,7844,4502,9644,10333,3838,669,7428,515,7564,8854,6567,6725,177,635,367,6803,8358,10734,4586,10313,1124,9949,2453,7949,7050,5678,577,4640,44,5107,5661,4439,6409,8136,5356,1025,7595,8736,10384,7525,4728,8608,4290,2712,5538,1217,7872,5702,5635,1479,8072,6834,2928,9143,3770,9283,10200,3468,7333,6922,8663,5660,2595,9920,6428,3036,7012,5957,9315,3118,5788,10378,10659,595,6255,4779,3224,10510,4991,6175,4936,386,8901,6586,4837,4694,1324,8350,9647,3829,763,784,5687,4380,5969,10198,1732,8326,7191,1788,3282,880,2502,2461,2018,5962,5272,3776,7171,2003,1911,8239,2915,1794,5811,6839,3163,7475,2860,9085,889,1774,2958,5517,5532,10285,5775,9919,5335,7888,10738,8497,3758,5004,1498,9309,91,3677,9557,1617,9314,7597,9792,5037,8793,7644,7967,1365,5994,7079,3162,710,5926,2196,10651,5008,2852,5981,7911,9780,4811,3863,4833,4155,4298,4028,6294,5193,558,2574,7150,8423,2954,821,9669,9181,9241,4593,8702,3220,4505,10210,10705,846,1804,10746,9747,6938,2055,3611,1716,7874,6617,9745,5081,10009,1945,1152,8377,1658,929,5765,7567,4724,6374,7,10138,1007,5086,8179,2255,9614,6784,2853,2512,9010,2920,7871,6825,6697,10373,3485,8277,7100,3344,2037,782,7048,8166,9967,3614,4043,4385,8769,3204,9353,8285,10679,10219,10586,1066,6420,6537,6511,7608,2871,6819,8934,6083,1710,114,10793,4859,3472,6354,8229,9302,8873,9515,213,6472,3069,7188,7021,6505,5882,5336,8157,3067,1158,1496,8596,3936,7812,30,4850,10234,6368,2522,153,8049,7619,5241,6557,3087,10706,2166,6722,3835,2943,47,6519,9917,1866,5795,5448,6046,1439,3708,2882,3215,7291,4729,6786,2648,3608,3933,1703,7121,8625,5303,10597,814,100,3820,372,119,7357,8070,2649,10617,4856,3024,9534,7791,1436,8375,10695,5721,1105,6266,9054,10593,3512,7883,2072,8500,8309,5612,925,7466,10192,10040,5915,7078,3628,4342,1767,10377,3877,9587,461,9985,5500,8868,10111,2000,8248,2206,9409,655,8586,9321,8341,8832,1781,2078,6787,35,6558,9146,10082,2452,5000,5699,6485,3151,3011,5526,8469,6555,9335,3279,10665,6256,185,8316,277,7637,5619,4223,234,4388,8198,10435,5905,6021,10753,2847,3698,2253,1352,5624,832,6471,3662,134,1540,5657,4901,4205,715,4241,1715,7196,9765,4512,5736,10537,8573,7234,5730,10549,9968,926,9986,4413,5880,10809,5544,5892,3085,1880,10306,3259,3973,3367,7780,1839,982,9099,8581,8600,772,5302,2219,7863,3913,6085,5572,10369,4142,6134,2764,5291,9476,1551,6303,10702,4007,5317,2989,6937,1854,2201,6182,3074,4344,8782,6641,3999,5461,2150,8908,2677,5364,1965,1531,115,7549,10581,6019,8943,2749,2161,7470,7275,2151,5470,2851,1814,2445,10360,3520,5814,10055,1019,7607,3620,502,4883,10574,1389,3260,7108,2171,5411,1681,1409,8719,5755,10749,2655,9300,2511,10037,5011,9528,3508,4794,1084,3548,3125,2030,10643,6004,2534,1457,10256,8066,2232,4069,445,10766,5265,8169,9654,6113,2942,7995,6605,3403,8576,8291,9193,4036,2599,945,3244,4656,787,9645,2746,7749,10324,7299,6793,5089,8885,10348,7792,8529,3527,1888,9002,4666,7269,10622,7815,5846,3578,6406,692,3045,6369,8776,3072,6071,6746,7526,6900,5062,8011,9658,392,5817,5623,10453,8214,2722,4411,5443,1181,9498,1028,2138,66,5073,5605,3540,6440,7271,170,1188,1163,7934,1751,2224,6073,4,3052,1065,1369,1801,2499,7895,5346,10802,350,2988,2849,3619,4937,6455,10156,10041,7690,8103,9369,8619,4257,1282,5676,9827,1138,8631,5201,1099,9032,3668,9673,2146,5226,6251,895,9841,2005,6152,3927,7243,7448,4544,4231,7083,3731,3272,8293,5116,9350,8711,9137,2167,4407,4065,10047,7584,5931,8064,259,10301,9884,6692,6500,5844,1835,8021,1881,1673,6677,240,785,5465,4258,957,3742,4189,7040,1892,5083,606,9132,5506,2320,10501,3915,8530,9147,666,2187,225,10096,6074,4974,734,8852,8515,6694,7059,4056,2024,5798,10686,7659,3402,9990,9934,7364,3257,7438,165,8637,6240,4484,3126,3714,405,196,3514,3900,2804,8284,5045,1564,2754,10158,974,1173,2859,9233,10103,3839,3885,7606,9202,4044,2364,9798,7922,2318,3737,7164,8480,6543,9952,7860,1149,6778,4773,9474,6924,3407,983,9379,8728,7336,3076,8772,3110,8065,7080,10509,5148,7256,9648,9701,9832,8517,817,4202,3455,8453,681,458,8549,9836,3897,2356,10783,8974,9048,417,6302,8208,4721,5451,9620,1996,4139,7816,5421,8796,8119,3147,9465,6450,2068,6112,3167,10486,4807,532,5753,5824,9034,2304,6309,485,272,8802,416,5807,5742,8325,9739,9890,9625,7342,8585,3967,7757,843,4900,575,4814,9771,4523,6800,8942,5117,8707,10279,4430,6997,9199,1582,9292,7404,9976,3531,10093,3523,219,6155,6006,1632,9042,10422,10083,3673,9698,4546,7662,6059,1699,8880,8435,9994,922,2486,911,3114,660,29,1005,4520,1815,8202,10382,5234,8950,2503,8330,9019,5295,1860,1366,8221,614,7884,1955,174,4536,5974,2708,2307,7738,9242,8706,9039,7446,2159,10756,112,3003,9493,6167,9862,2903,9159,7519,7439,2614,6084,6072,8822,1838,3239,6675,9422,3964,3033,6899,4754,3963,4853,3710,5040,8080,4061,3597,2109,2123,1595,6211,5146,6791,4827,9996,4144,5569,7942,1159,9943,7572,1639,3672,10359,10690,4525,6196,9481,738,716,4489,2685,10147,4248,8583,6457,3387,10614,10364,993,1037,1404,10080,9258,5580,10403,3854,4116,5118,10669,9891,9737,5665,2469,10241,5489,6971,8936,6880,5977,5649,9228,3158,7784,5115,1574,9316,1797,8385,835,169,2288,8864,4121,6575,4101,407,2421,3898,5395,5616,3453,1454,5431,8122,156,1864,87,6008,7652,6427,7356,2260,1090,4823,8447,2457,10280,1631,1538,950,8406,1279,2966,7552,2347,7259,6394,10560,5868,3560,5823,3451,3046,5019,2414,7785,7929,8217,3320,4226,864,6148,6673,6490,3400,1310,9492,2348,4415,968,10323,3990,5776,9900,9336,1046,7148,8798,5151,2960,6178,6623,1204,4454,1760,7168,6117,1146,10719,10458,1649,5397,10282,10584,9794,3475,10402,10392,10762,3270,413,1759,9711,3664,3306,5274,8678,652,8665,8416,6408,8046,1624,5865,9687,8657,5888,1062,2184,1164,3280,9055,3097,5372,424,1287,5887,4647,7493,6962,10740,5382,6805,4082,8845,3590,7270,2228,2630,3021,9731,524,4700,378,7771,10656,3398,6644,990,1322,10016,6855,5261,10799,9323,6720,6162,1252,7898,487,4787,9439,5898,7813,3728,7689,8353,45,8231,2275,1665,709,2700,7044,6229,9875,3411,1846,5567,9855,6894,658,448,8410,1416,978,9570,5480,2329,290,4719,6830,7788,5473,8767,6832,2750,4270,2778,4501,1530,4417,7262,9398,3441,8026,2141,6700,4803,4545,3201,8518,9432,5734,3969,9904,8241,9760,10652,4558,5677,1535,3251,3971,10052,2199,5751,9956,7627,3299,10078,6578,4000,2567,1193,1453,2231,1398,1714,1507,2613,8977,1701,4465,5809,820,5049,9517,5595,5917,529,4815,224,3910,7054,3727,1939,7937,6884,9636,8394,5682,5405,9487,4907,9554,9743,10322,6208,9414,8843,7213,7505,10188,7604,7837,9860,10351,2087,14,7212,5999,6474,209,9842,10468,3271,741,10611,5142,4020,1870,10141,2070,238,4670,3450,3188,2479,8304,1263,9502,3330,2361,9077,160,4884,2948,2011,1032,1652,9131,2428,4917,6908,8219,6141], "time_ms": 0.146797, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 613, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "generated_query_count", "value": 1, "time_ms": 209.095557, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 428, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "generated_samples_per_query", "value": 5332800, "time_ms": 209.095557, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 429, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "generated_query_duration", "value": 123762, "time_ms": 209.095557, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 430, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "generic_message", "value": "Ending naturally: Minimum query count and test duration met.", "time_ms": 1519.176764, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/issue_query_controller.cc", "line_no": 482, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 644450.219147, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 898, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 644450.219147, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 900, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 644450.219147, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 902, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 644450.219147, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 904, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 644450.219147, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 906, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 644450.219147, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 908, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "power_begin", "value": "12-28-2024 04:14:39.929", "time_ms": 644450.219922, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 564, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "power_end", "value": "12-28-2024 04:25:24.154", "time_ms": 644450.219922, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 566, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "result_validity", "value": "VALID", "time_ms": 644485.732540, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 644, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "result_min_duration_met", "value": true, "time_ms": 644485.732540, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 649, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "result_min_queries_met", "value": true, "time_ms": 644485.732540, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 650, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "early_stopping_met", "value": true, "time_ms": 644485.732540, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 651, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "early_stopping_result", "value": "", "time_ms": 644485.732540, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 669, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "result_query_count", "value": 1, "time_ms": 644485.732540, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 679, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "result_samples_per_second", "value": 8277.86, "time_ms": 644485.732540, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 735, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "result_min_latency_ns", "value": 1468857306, "time_ms": 644485.732540, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 741, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "result_max_latency_ns", "value": 644224807583, "time_ms": 644485.732540, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 742, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "result_mean_latency_ns", "value": 390627257098, "time_ms": 644485.732540, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 743, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "result_50.00_percentile_latency_ns", "value": 415939518318, "time_ms": 644485.732540, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "result_90.00_percentile_latency_ns", "value": 614320528108, "time_ms": 644485.732540, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "result_95.00_percentile_latency_ns", "value": 631979098623, "time_ms": 644485.732540, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "result_97.00_percentile_latency_ns", "value": 637816603814, "time_ms": 644485.732540, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "result_99.00_percentile_latency_ns", "value": 642475528110, "time_ms": 644485.732540, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 18582, "tid": 18582}} +:::MLLOG {"key": "result_99.90_percentile_latency_ns", "value": 644080347064, "time_ms": 644485.732540, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 18582, "tid": 18582}} diff --git a/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance/run_1/mlperf_log_summary.txt b/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance/run_1/mlperf_log_summary.txt index 2f7b3d6e3..36382a9ac 100644 --- a/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance/run_1/mlperf_log_summary.txt +++ b/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance/run_1/mlperf_log_summary.txt @@ -4,7 +4,7 @@ MLPerf Results Summary SUT name : BERT SERVER Scenario : Offline Mode : PerformanceOnly -Samples per second: 8268.76 +Samples per second: 8277.86 Result is : VALID Min duration satisfied : Yes Min queries satisfied : Yes @@ -13,15 +13,15 @@ Result is : VALID ================================================ Additional Stats ================================================ -Min latency (ns) : 1430163027 -Max latency (ns) : 644933169947 -Mean latency (ns) : 391145860524 -50.00 percentile latency (ns) : 416430237335 -90.00 percentile latency (ns) : 615068853269 -95.00 percentile latency (ns) : 632720953267 -97.00 percentile latency (ns) : 638542165122 -99.00 percentile latency (ns) : 643191943747 -99.90 percentile latency (ns) : 644798861050 +Min latency (ns) : 1468857306 +Max latency (ns) : 644224807583 +Mean latency (ns) : 390627257098 +50.00 percentile latency (ns) : 415939518318 +90.00 percentile latency (ns) : 614320528108 +95.00 percentile latency (ns) : 631979098623 +97.00 percentile latency (ns) : 637816603814 +99.00 percentile latency (ns) : 642475528110 +99.90 percentile latency (ns) : 644080347064 ================================================ Test Parameters Used diff --git a/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/accuracy.txt b/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/accuracy.txt index 616a336eb..a653f3c3f 100644 --- a/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/accuracy.txt +++ b/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/accuracy.txt @@ -1,11 +1,11 @@ {"exact_match": 82.81929990539263, "f1": 90.26682135974633} Reading examples... -No cached features at '/home/cmuser/CM/repos/local/cache/af0a19318f524c01/inference/language/bert/eval_features.pickle'... converting from examples... +No cached features at '/home/cmuser/CM/repos/local/cache/6c5d0d8c0f4f47c1/inference/language/bert/eval_features.pickle'... converting from examples... Creating tokenizer... Converting examples to features... -Caching features at '/home/cmuser/CM/repos/local/cache/af0a19318f524c01/inference/language/bert/eval_features.pickle'... +Caching features at '/home/cmuser/CM/repos/local/cache/6c5d0d8c0f4f47c1/inference/language/bert/eval_features.pickle'... Loading LoadGen logs... Post-processing predictions... Writing predictions to: /cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/predictions.json Evaluating predictions... -hash=0a4650a99fa666b3309b560bedc02a17a2a05928e914dfe8cae8069aa3f089a6 +hash=e621613edfbd136329e0e066da98e3e06c3f219218b498e30170f9a24e0c015d diff --git a/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/mlperf_log_accuracy.json b/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/mlperf_log_accuracy.json index 383d74223..202ca9f91 100644 --- a/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/mlperf_log_accuracy.json +++ b/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/mlperf_log_accuracy.json @@ -1,8 +1,7 @@ [ { "seq_id" : 0, "qsl_idx" : 4523, "data" : "0EC85DC842C823C96CC754C8D0C741C8DFC898C8AFC87EC8AFC853C8D6C87BC8B6C889C896C820C8A3C892C8C8C8A5C8E8C8E1C8B2C882C8B23EE1C5EC46EEC2F847E6C1E5C337C725454FC39D405648FDC3ED4311C860C77FC8ECC740C845C6A6C81EC66FC8B3C732C820C81EC8CDC6DDC851C82FC83BC866C84EC7AFC867C83AC82CC89AC76DC859C897C855C83BC8E1C883C8A1C85EC861C898C6A6C864C815C815C884C7D7C712C861C712C81BC62DC810C54EC80FC8A1C85EC878C852C841C819C8A3C81AC894C83EC8C4C863C858C82EC87EC747C657C831C76CC8E2C75EC852C842C87BC867C805C895C842C88DC857C8DFC8A2C853C87EC8B1C7E3C677C828C810C84CC861C752C746C86FC7B3C760C61DC85AC7BAC79AC77DC775C784C594C59AC7CBC65EC788C5CAC7EBC5ADC84BC81DC8A7C6E7C7E4C752C846C841C885C793C82EC824C8CFC658C8E4C730C806C816C802C8BEC789C67CC83DC7A4C820C85BC87EC777C8D9C7C1C7E4C70DC763C70FC8F7C7D5C7F7C6C8C74DC600C8FCC696C507C73BC57FC73FC633C8F0C571C7D7C63CC764C6C3C7ADC7B7C798C74BC605C81AC641C82CC8F4C711C818C828C8A4C860C85AC81BC869C818C887C851C8DFC710C82BC700C8BCC730C7BCC75AC668C884C750C816C88EC843C81DC847C8A5C848C872C817C894C829C841C824C8E9C72FC703C7BFC59AC892C8" }, -{ "seq_id" : 1, "qsl_idx" : 7827, "data" : "78C709C6E1C741C898C89CC823C864C85BC87EC84AC819C822C88AC86DC8D8C846C9E7C8FCC123C49AC6BAC69CC75EC736C808C8DBC794C793C782C7ADC681C65DC742C7E9C628C7E0C7C6C7D9C7EBC76CC761C70EC4D0C25EC703C798C734C7E0C775C7F2C6C0C689C78BC634C887C7F9C6E1C6FFC6D9C5DEC510C4B3C734C8C8C8A5C81BC82EC8D6C759C76DC83DC8E3C70BC7A5C6B0C695C888C833C85FC8D9C7F6C613C765C511C845C817C8BFC8A6C748C845C891C84BC50DC451C799C52EC82FC8AFC7EAC757C7DCC78EC667C7F6C4D8C658C666C727C692C4E1C4E6BFC4C7C5C7B9C73CC7ACC846C8C4C861C855C6BDC52AC803C83EC825C8BAC672C697C860C8E6C700C847C83FC876C5B8C624C59BC514C4DEC0A6C7F9C5CDC7FBC777C846C8AEC877C8AEC87FC87AC844C839C81DC8AFC8B2C8D3C775C794C863C898C852C88EC856C8ABC6B2C5A4C834C8A8C871C829C8D9C79CC831C8BCC87CC829C831C8D3C790C628C87EC7CFC70AC8C3C873C8DAC876C855C876C83FC792C77DC62CC53DC839C799C888C854C86BC8C2C7F2C714C85BC7CCC898C854C832C8B6C853C882C898C88BC86EC850C866C8A3C82EC87CC83DC84AC8FDC7A5C86AC8C1C878C8B2C891C8C4C70EC8C4C89AC829C84CC8BAC7BFC600C807C73AC510C7D7C77FC68BC787C62AC814C805C82DC84CC705C89DC72FC705C8DAC72CC41BC5E3C6D0C5A0C83DC899C5ECC235C823C7F1C649C64BC701C559C82AC827C747C770C71FC7CFC886C845C607C61FC6B4C4CDC73DC6B6C587C67FC7E0C6E7C6CAC771C573C647C4E0C121C8BBC7D8C6FFC676C7A8C74CC81EC893C74AC741C6D4C510C8F5C7C0C69FC7C6C79DC64BC841C8E8C712C812C84BC8D5C7A1C640C77EC663C42DC70B3A0AC6393C50C569C099C655BF23C5CABB4CC65ABB04C667C397C5973847C403BF06C11048DCBD10469F3B10C5DFC261C533C0E3C27EB393BF6F47AAC3C6430FC431C0A1C68CC6" }, -{ "seq_id" : 3, "qsl_idx" : 2491, "data" : "E1C8D0C851C8EBC813C9F9C8F0C821C90CC91BC931C92CC9E8C8FAC8A6C89EC889C86EC859C857C82AC9ADC86EC9E4C8D1C8DCC87EC16BC3A7C852C86EC806C840C8C9C7EBC8A3C8BFC894C8F1C8B2C898C888C855C8E1C7A1C852C8C8C82CC8B4C889C886C89FC893C8AFC8A0C883C8FAC8D1C87BC899C882C8E2C741C83FC828C879C82AC864C842C852C8D6C8B2C857C86CC880C8E4C70EC9B2C83AC894C869C725C889C7A1C71CC8F6C7D9C80CC8B4C44DC79CC721C855C861C867C83BC835C739C410C851C61EC874C8B8C8BDC830C87AC847C813C8CFC8CCC834C88DC853C6D3C7C7C661C710C860C884C8C2C892C881C82AC8E8C6AEC854C8DDC8F3C85DC696C717C9B0C897C8B2C814C87AC81EC8DBC7A0C853C838C8BEC87BC650C749C5FCC657C636C757C8B7C6E7C651C872C7E2C784C86CC852C538C8EAC631C5EAC283C1504845C070C48DC2BFC336C7D0C5BBC616C569C213BEABC663C6F3C6C8C657C61DC432C38CC0A9C561C541C6EDC4A7C323C08148E0C5D04080C766C8B5C677C7C1C85BC81BC79EC832C82DC368C803C0C4C8F7C6" }, -{ "seq_id" : 5, "qsl_idx" : 5332, "data" : "45C797C776C8A6C857C72DC888C87DC8EBC869C8CBC70AC89BC7DBC742C83EC82CC703C7C9C871C897C889C86BC85BC8A8C87AC8A0C85DC865C87FC645C81DC85DC928C96EC87DC83BBA33C66CC068C550C516C7F3C4A3C57CC83CC8D7C5E5C77AC58AC79FC86CC882C71AC738C68AC27BC6B0C771C31FC6D44038C41E470A47BFC71AC6E1C494C613C602C789C7FCC758C609C489C533BCB2C439C5D3B395B487C54CBD9BC502C7C2C7CAC783C73DC631C8C8C74FC795C7AEC731C88CC83BC88BC84EC80FC8A2C699C816C8F9C767C719C8D5C7E7C70FC8FAC712C7CAC861C8FCC6D4C7E4C67CC750C6E7C48FC5D0C384C676C702C8A7C74FC825C894C83DC838C85BC852C8A7C76CC817C8B9C79FC6ABC866C84FC60DC517C8ADC719C8E3C67AC64BC5F8C636C704C81FC8CAC7B2C66FC81DC827C83BC791C863C +{ "seq_id" : 2, "qsl_idx" : 8355, "data" : "9FC8C9C80DC911C9EBC75CC8AEC8D2C8C9C8A8C80EC9FDC8D7C8E5C8A8C85EC87EC89DC813C917C93EC9FAC8A0C8D1C88CC886C887C856C8FAC8A3C866C93FC9BDC8E5C872C7ABC823C6A8C554C71AC7F9C4EDC7CCC518C84CC89BC877C864C84EC849C8D1C746C864C7EDC67AC5E6C70247E7C387BFD946B6C65EB81FC86EC7B4C788C70CC86CC6C1C81EC8FDC7A2C62EC8E7C605C9BFC8A9C8FCC743C8ECC6ACC715C6E5C638C56AC645C2DEC81AC8F4C70BC8A9C7DFC78AC7E2C735C871C6CCC5D3C66CC617C3BCC77DC0B4C137C78EC62EC897C7F0C7D2C6DEBFF8C78BC512C6FDC6BFC7A4C4EAC87DC88BC83EC8FDC8AFC8F0C755C884C829C8E6C706C49DC7FCBEAFC173C77DC537C01AC804C594C84EC890C87AC872C81BC8DAC892C85BC8A7C874C8B4C865C8C0C7B9C87CC803C9ADC872C7BAC4E5C704C26CC23AC8FEC685C874C83CC85FC664C364C834C6ADC8D0C812C9E2C8B5C8E6C8C7C8E6C8C5C8A0C80CC9FCC8CDC8EBC8FDC8CDC80FC9F2C8B7C8DBC8D4C8C2C808C9E9C88AC885C682C8E6C551BFC1C7FBC7A7C869C6DCC248C838C530C89BC8AAC8B7C89AC87DC81EC9B3C8CAC825C88AC851C865C72BC8AAC89BC87DC801C815C9C5C89EC8AAC8C5C7D8C4E1C792C0C4C8FCC84BBFE2C621C8B6C83BC7BCC288C840C5A2C887C8F0C8BAC8D2C89BC8DFC87CC806C9C7C845C889C8CBC834C8FFC8BEC89AC8C6C8C4C84FC8E7C8E9C87FC8CAC8C9C8AFC870C8D0C6B6C8DBC677C8D0C82EC886C86CC762C651C8AFC5B7C763C8F2C789C7A8C82EC831C8BFC8A5C826C8F3C865C86BC896C894C80AC8D0C859C8F1C896C8BCC77DC700C401C818C85BC8ABC7D2C2CFC83CC8AAC882C89BC865C8F8C7FCC674C849C6BAC805C9" }, +{ "seq_id" : 4, "qsl_idx" : 3822, "data" : "03C820C77AC859C947C92AC8D2C86AC9E8C84BC9ECC885C882C901C932C931C91AC91FC953C992C8A9C866C9E7C865C910C90BC9F4C8F5C8E0C854C8F6C8BCC806C9C3C8F8C8A2C817C91CC880C933C939C9E9C8D7C58FC539C2D1C1FEC48CC23CC545C7EFC7AFC5D0C6EBC424C700C589C785C530C821C803C819C8C5C8A4C79EC881C73FC8B1C51DC96AC87EC889C82EC8D6C73CC82CC8DDC617C79DC650C654C81DC852C626C62CC777C521C97BC8B8C8A1C898C8EDC714C967C880C881C8A9C8EEC763C8A0C703C849C622C84FC7DAC8B3C8DEC7EBC6AFC8F1C741C8E9C751C875C69EC8D1C749C8B4C78CC855C799C75FC836C846C767C830C877C81BC757C686C574C558C653C1DBC0F4C365C09EC860C834C808C8B0C87EC8C0C8D6C771C9B8C8C3C8C2C680C80DC7DBC8E3C8E3C8AEC839C9D4C804C5B7C6B8C867C717C824C728C849C643C717C85DC9D5C8E9C7E8C5FAC5C6C75AC7A5C7ABC691C61FC712C60AC8CAC545C9B8C83AC6DEC649C8CBC739C981C87DC8D5C74FC8DFC74CC7EEC403C85DC83BC815C81BC872C70FC96FC7BCC8C0C708C8F9C8A1C83DC84FC8FBC7CEC811C822C9C8C811C9DFC80AC970C899C84CC80AC871C823C86CC851C9E9C889C894C8D1C81FC877C8ADC744C9CDC84FC9B6C81FC9B6C8AFC8E1C8B9C75DC8DCC819C86DC884C7F1C864C7ECC8A1C839C875C716C902C9D5C8C1C7C2C8B7C619C8FDC5ACC700C827C866C827C91CC87EC8B9C8E5C832C856C99AC807C9B0C8E9C836C8F7C803C909C915C9A1C876C8EAC822C8F3C88CC8ABC89BC8C4C843C837C778C63FC89AC662C82EC665C9CAC8F3C70AC89EC872C76CC883C693C8A4C6AEC852C774C7A2C3EEC831C8A7C8A1C7EBC6A0C620C8A6C770C8D1C7B1C82AC88EC84AC870C86CC814C85CC7D3C824C82DC812C8ABC819C88BC829C8D8C86EC808C8E2C761C70EC707C921C86FC8BBC6AFC822C639C9CEC844C9A0C8A3C86CC732C9C2C8B0C8ADC839C87CC8D7C879C87AC910C9EBC8F2C88BC845C829C982C897C9C9C8F5C8B4C810C84EC6D7C6B4C41CC818C6E8C812C844C809C856C8C6C8A9C842C859C828C87CC85DC820C849C874C8F8C6D4C85BC816C880C687C732C730C87EC6D1C81EC8EAC564C6CAC785C50CC984C874C61CC5 ... diff --git a/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/mlperf_log_detail.txt b/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/mlperf_log_detail.txt index ae69dcdff..e8fed302c 100644 --- a/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/mlperf_log_detail.txt +++ b/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/mlperf_log_detail.txt @@ -1,69 +1,69 @@ -:::MLLOG {"key": "loadgen_version", "value": "4.0 @ 486a629ea4", "time_ms": 0.008437, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 53, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "loadgen_build_date_local", "value": "2024-12-24T19:50:51.839855", "time_ms": 0.008437, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 55, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "loadgen_build_date_utc", "value": "2024-12-25T03:50:51.839862", "time_ms": 0.008437, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 56, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "loadgen_git_commit_date", "value": "2024-02-15T15:46:50-05:00", "time_ms": 0.008437, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 57, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "loadgen_git_log_message", "value": "486a629ea4d5c5150f452d0b0a196bf71fd2021e Hotfix: DLRMv2 Audit Test01 fallback failure (#1626)\nf9a643c0a0e920588da1b51a1d822e1071a9dbec Add number of tokens to offline (#1623)\nf06b920eac2170df38b5502c7443f50c9afa6045 Bugfix: equal-issue mode on offline causing accuracy run to fail (3D-UNet) (#1624)\n0ed5190e82e1ca1cf95626e50e61cb6678c7f2ff Fix submission checker and TEST06 for Llama2 (#1616)\n817dd961221d3ca284b024368130bd5dc5e4b54a Turn equal issue mode off for TEST06 (#1615)\n840435a1abe8be823e0a5a18ad3173bc520aa24f Update README.md with CM commands to download stable-diffusion, gptj and dlrmv2 models (#1604)\n3190d09a148bc1d72c9d946e4cf3b54db0d0e60a Typo fix in README.md (#1588)\nb0777f0bd3b7e5cd6a51d670d8b4df656cf6fb29 Update README.md - remove unwanted lines in CM commands (#1601)\nd7dba0826ac96a0e8bcae525086961e47c409a38 Remove loadgen warnings (#1608)\nd45a66c33bb0b6548d6172b38645bf94ffe24b39 Add upper limit to tokens per sample (#1612)\n44285d9a44041c3d4f7df0acf1e72c375e9fcbd2 Set completed samples per second as llama metric (#1613)\n357ccefca4db55ff373e029d5d1e3e6246bde5d4 Enable equal issue mode for LLM benchmarks (#1610)\n104d85531af7ae2a07ded36dd6e8c7f2c7746d06 Add sample length check to test06 (#1603)\n473053fc9f1b8c5edf60e11ec74607126938fbc4 Fix loadgen token metrics latency constrains (#1596)\ncc3daae3c7f18d7eedc5ffbc271d324e5aca6233 #1598: fix token and sample logging for Llama2 when accuracy_log_sampling_target is enabled (#1599)\n3a902e598e5a17013c26fea975c0a60e6d7feae1 Add support to dump 10 compliance images during accuracy run for SDXL (#1591)", "time_ms": 0.008437, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 58, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "loadgen_git_status_message", "value": "", "time_ms": 0.008437, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 60, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"05d4a62d355a90363847358e58afeee4a54ad838","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/bindings/c_api.cc":"32181da9e161c285f8fe46ddaa49e6cba2f9f918","/bindings/c_api.h":"91f58bd79b83b278f3240174a9af747fc38aff74","/bindings/python_api.cc":"1dc0865130371c6a8ddb97cd8039244d74d6fe46","/demos/py_demo_multi_stream.py":"6ccf7dcbe08f8382ed2d3618370c2b21e205ca13","/demos/py_demo_offline.py":"6c3075538e0a3df5e93edee917f4537099759e11","/demos/py_demo_server.py":"40e37ea121ac08b21ed6af41aca8eb5eda641b89","/demos/py_demo_single_stream.py":"4ee5f048fb0dffc415342a2bc2049fb155bd2894","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"126e952d00f4ea9efd12405fb209aa3ed585e4b2","/issue_query_controller.h":"923d9d5cdf598e3ec33d7a1110a31f7e11527ec7","/loadgen.cc":"6650091ba7a918f343b06eb7a5aa540eae87275f","/loadgen.h":"e00fdc6dbc85a8c9a8485dbcbfe2944f81251c4e","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"197efc96d178e5d33a750d07fa7b2966417506ea","/logging.h":"ddb961df7bcc145bcd7cce8c21f7cf075350dcbe","/pyproject.toml":"974adb632af4b76a1c72b914cc2cf9b3cbc66aa3","/query_dispatch_library.h":"13ad6d842200cb161d6927eb74a3fafd79c46c75","/query_sample.h":"e9187c8612bbdc972305b789feb6e15c26e96cfe","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"a170f991de69f8a5e45616e871b39a30ffed5598","/results.h":"d82500c326c2de83db411f1146882aa4692b419c","/setup.py":"e3e707e1191478332c00f1b6ff806f0ef98e0f65","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"aaf3803e700f3431a5ada72559eef5cce996ea64","/test_settings_internal.cc":"42aa46b17ac3fb2748cd5cde60fa8742e916f3c8","/test_settings_internal.h":"73b11c02228f3afb4c924db7a86fd8a2b5b9e483","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"4c9d98360b507d5673049ed70154fcfb2934f8a6"}, "time_ms": 0.008437, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 67, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "test_datetime", "value": "2024-12-24T20:04:35Z", "time_ms": 0.021008, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1198, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "sut_name", "value": "BERT SERVER", "time_ms": 0.021008, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1199, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "get_sut_name_duration_ns", "value": 369, "time_ms": 0.021008, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1200, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "qsl_name", "value": "BERT QSL", "time_ms": 0.021008, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1201, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "qsl_reported_total_count", "value": 10833, "time_ms": 0.021008, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1202, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "qsl_reported_performance_count", "value": 10833, "time_ms": 0.021008, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1203, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "requested_scenario", "value": "SingleStream", "time_ms": 0.027716, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 268, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "requested_test_mode", "value": "AccuracyOnly", "time_ms": 0.027716, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 269, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "requested_single_stream_expected_latency_ns", "value": 375938, "time_ms": 0.027716, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 274, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "requested_single_stream_target_latency_percentile", "value": 0.9, "time_ms": 0.027716, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 276, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "requested_min_duration_ms", "value": 600000, "time_ms": 0.027716, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 312, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "requested_max_duration_ms", "value": 660000, "time_ms": 0.027716, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 313, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "requested_min_query_count", "value": 100, "time_ms": 0.027716, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 314, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "requested_max_query_count", "value": 0, "time_ms": 0.027716, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 315, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "requested_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.027716, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 316, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.027716, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 317, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "requested_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.027716, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 319, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.027716, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 320, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.027716, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 322, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.027716, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 324, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.027716, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 326, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.027716, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 327, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.027716, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 329, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.027716, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 331, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "requested_performance_sample_count_override", "value": 10833, "time_ms": 0.027716, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 333, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "effective_scenario", "value": "SingleStream", "time_ms": 0.027851, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 409, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "effective_test_mode", "value": "AccuracyOnly", "time_ms": 0.027851, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 410, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "effective_samples_per_query", "value": 1, "time_ms": 0.027851, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 412, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "effective_target_qps", "value": 2660.01, "time_ms": 0.027851, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 413, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.027851, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 414, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.9, "time_ms": 0.027851, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 415, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.027851, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 417, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "effective_target_duration_ms", "value": 600000, "time_ms": 0.027851, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 418, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "effective_min_duration_ms", "value": 600000, "time_ms": 0.027851, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 420, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "effective_max_duration_ms", "value": 660000, "time_ms": 0.027851, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 421, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "effective_min_query_count", "value": 100, "time_ms": 0.027851, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 422, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "effective_max_query_count", "value": 0, "time_ms": 0.027851, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 423, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "effective_min_sample_count", "value": 100, "time_ms": 0.027851, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 424, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "effective_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.027851, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 425, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.027851, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 426, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "effective_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.027851, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 428, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.027851, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 429, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.027851, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 431, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.027851, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 433, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.027851, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 435, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.027851, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 436, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.027851, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 438, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.027851, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 440, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "effective_performance_sample_count", "value": 10833, "time_ms": 0.027851, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 442, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "generic_message", "value": "Starting accuracy mode", "time_ms": 0.029040, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1090, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "loaded_qsl_set", "value": [4117,9486,2947,9270,8227,4487,9799,4629,8571,8839,7101,2191,3315,6375,207,10454,3122,9490,4083,6119,1890,7657,9347,344,201,4232,5741,5454,9533,7842,1596,5330,10615,8827,8094,6714,9169,977,6619,9924,4982,4894,6544,10118,783,766,2122,7163,396,4200,9864,4254,8440,1758,10722,2829,2788,9906,2719,5613,8726,7847,440,6103,8567,7497,4793,1184,8675,10587,8270,7074,7408,8740,9265,4855,1199,220,6556,6595,1226,6695,3066,2143,6264,9806,7122,5007,10075,2370,2185,7254,77,2806,6243,6687,1156,1802,7302,391,3348,8159,9708,251,3595,3947,2302,4690,122,3116,1300,3768,6388,2986,728,5127,3904,6807,6259,2218,7555,8821,3580,7804,1321,8460,5419,6542,7833,3365,2290,6346,7952,918,707,118,10469,9023,1304,5220,1891,6878,9524,6522,7867,4037,7698,8507,9634,2145,10057,6263,7486,7557,2401,6082,3434,5927,8708,10081,8114,1349,7742,2742,7157,5705,4638,1764,3651,920,4016,2744,5859,6740,5380,9649,2,5245,5230,6421,4071,8909,6319,288,9346,9671,9646,9360,4702,5044,9157,6296,8318,8971,8184,7702,10475,7318,4024,5530,3991,1171,1909,6230,8461,8099,10563,5398,10760,10693,3688,8170,4952,6827,5893,7767,2656,10032,9178,10480,7195,9359,10608,10261,8975,5432,363,3346,3049,3360,4432,9736,473,7587,9416,1021,6596,10352,2666,2325,973,2071,7368,4267,867,1040,5796,3491,8237,9597,3489,10568,3777,7899,856,4818,2858,4662,777,10776,10026,6065,6481,2590,6780,10553,8789,6032,930,1049,10178,6187,585,3359,5832,2014,480,5866,5017,5141,1525,8271,8132,1642,6270,2265,140,1174,4641,8017,8844,6569,5575,7081,3779,3845,467,9761,3314,8653,4250,7578,8907,664,217,8419,8757,683,4565,1638,5216,2004,6679,4485,7683,5180,7422,6070,6283,2909,6281,8933,10639,4340,3079,3513,10552,2291,7487,2680,8807,8370,6127,7434,1822,908,8101,451,10316,8282,10724,885,1834,1556,6121,1468,409,1908,10539,7238,2891,10312,9435,2178,10051,4522,9650,615,7193,7796,2008,6011,4347,435,6881,1396,844,971,4216,5001,7886,6875,1348,4300,9348,2709,10580,9993,6705,8784,10821,9232,7724,6581,9764,2913,10125,6914,7693,4568,10526,10231,8502,7825,1783,795,6828,8006,4595,3892,3075,1278,10050,8235,4679,9971,75,2164,1175,3684,9869,8414,9016,765,2721,5786,10703,1326,7463,237,836,3912,5482,7835,4374,7661,4070,7754,7180,9945,10415,3638,4208,939,1097,7628,5725,3723,4547,1994,5232,5279,4049,7309,5963,10478,9834,7758,9463,3666,4663,801,4030,3010,8236,9237,2470,3382,6336,7570,190,4552,4113,2229,9018,7038,5021,8701,8268,4490,1373,6092,10181,10445,3056,4333,10394,1640,2328,3349,10069,10683,6439,3191,2544,4760,1739,8900,3,258,5416,4671,2041,9375,2661,6376,2306,5897,6373,2125,1940,5109,4256,6203,6411,9311,8029,2200,6301,1734,6176,6443,3007,9583,1841,4627,2523,4306,2801,5208,8607,4961,1467,10700,4031,2181,9958,8194,10295,9043,5366,306,9056,162,3196,10759,4349,5401,7030,6184,9216,2069,6357,5640,3015,1752,406,10049,3983,791,5610,5944,304,6380,1937,7862,9366,9365,1607,9252,2698,10350,5221,4847,10707,3023,1615,7017,5990,4045,7231,1202,7182,5172,3176,9612,6693,1013,10334,1805,8730,8841,7372,10094,6010,3216,8177,7406,9472,9608,2533,7691,3959,1623,5144,1831,751,4912,2152,8144,241,4187,10001,1924,8402,7877,8928,7294,4915,3876,3242,8996,7153,1000,10658,3946,284,5088,959,10661,1018,8931,546,4262,6489,3786,5278,4308,8178,7638,5211,7142,5771,634,8273,263,2548,4674,9910,624,3290,5925,1524,2642,7198,10345,3539,6719,10735,2006,10233,3816,9578,8624,6769,9661,6468,8952,9702,8670,5175,3449,1773,6130,636,1622,1696,1414,6523,2898,10650,4420,1280,2538,7880,3743,1227,2757,6131,3371,7445,1680,2901,36,6868,10811,7140,1900,10398,3724,8765,8278,7379,6040,7914,1580,3535,8033,7536,6480,3861,2437,5082,2881,10596,4902,8162,9637,4886,411,7189,1789,6704,9176,10045,4107,6333,4583,3923,4843,9935,9955,9297,3222,8257,6365,3095,1995,7924,5428,8640,6628,4819,7152,5016,7932,3226,1267,1225,9025,7878,6321,2373,3801,1399,9063,6506,9436,5781,7500,423,3588,7110,4607,3476,2477,3635,2425,9394,10449,4054,8148,5510,2236,9543,10499,6153,780,8692,8105,10268,172,4128,841,8925,5453,8561,2135,2976,2845,2209,7418,10817,5620,7010,9960,474,6545,8223,9713,205,10548,905,2396,2914,5591,2887,3379,4865,3304,5849,1309,10171,608,7314,6228,6961,7068,6943,4613,1317,121,8611,5902,5827,8042,1139,3121,223,2984,2897,3144,5379,10590,6433,3871,5475,43,1026,1192,1599,5729,6194,5838,1254,2054,2706,7925,6060,2177,10647,7727,5681,10000,3549,10814,4433,2100,6869,6593,9404,5534,5801,10321,9145,8363,7451,9072,4600,5096,10372,3022,679,5631,2552,723,4668,9613,10153,7423,9114,2973,3071,1413,4956,2838,5084,6777,5196,539,5036,382,7480,9327,323,4766,2273,2684,8182,10134,9203,2667,9185,2658,9087,6286,4162,6620,10472,1095,6759,7216,8191,9738,9548,5029,10326,5478,4089,8710,5439,6126,6160,4828,1006,1677,3546,6930,425,2645,4573,2770,10248,358,4975,10603,8129,8614,10180,7902,4303,2447,3427,753,8920,9449,256,6859,5711,5246,623,370,10767,2563,8300,9591,9991,5126,9568,2647,484,8644,3654,9507,8632,10076,9813,10638,4326,3862,569,7020,4213,5628,9341,839,3341,8347,5163,5563,7339,6928,1197,9158,3099,6120,1483,9545,4608,4215,2761,1259,6802,4364,4001,1606,4222,4752,3057,10022,1108,10592,4398,10376,9888,10033,4273,1238,3428,6553,6145,5731,7273,604,2436,3170,8673,5901,10074,4513,8305,8697,7476,5015,2669,3100,6344,9338,10383,4972,8727,1363,3223,10495,897,1698,9553,5662,53,9328,3733,9932,7102,2212,7382,7159,4556,10748,9294,5895,2262,10512,5913,8314,10624,2530,7560,10464,2443,8787,3800,1510,2027,9795,9670,283,2351,4260,2577,1518,906,3377,4764,2113,10699,3948,8465,9264,9267,5600,2924,3783,9209,7966,6464,720,6973,155,6227,7994,2636,535,8846,2931,2046,4953,10058,8228,5200,2211,4234,2207,8121,2007,7268,3212,7772,6496,10201,2762,7534,5539,10555,10496,628,3653,702,8683,10325,8660,8411,9387,326,8842,837,9688,3855,7053,1826,7944,1053,3193,456,361,3206,10315,2059,6612,6173,4562,9003,8034,9162,377,7551,10769,5354,6299,5997,2080,6288,6068,653,9937,10825,8833,3247,3058,4923,9104,7983,3305,9322,8339,7711,131,7328,5212,8623,10761,9718,5402,948,10640,8659,6479,2768,4616,2223,10625,4252,9980,9807,5318,6188,159,10655,3565,1478,6860,3521,4646,9880,1608,10427,9804,2092,4897,10238,6020,7642,8091,7789,4058,6681,4173,7918,5523,2444,9898,7161,7736,4667,2603,9370,7403,10797,1857,3276,2276,3445,8197,991,1863,4266,2170,4331,3296,6658,1308,7542,8458,4533,10177,6873,4190,815,2726,5589,2846,269,2634,1560,7067,7554,3102,4908,10741,10086,5286,8734,3824,1744,4111,3264,7494,3825,8834,6139,8432,147,1589,1871,941,2911,6018,7906,2874,3713,9523,4023,7522,6116,6864,5698,5464,3924,4590,7345,7395,4790,10540,1020,5309,5950,2956,4983,6163,9194,3848,3612,10525,9069,5253,7232,6876,5003,5067,2350,10375,10523,850,6399,10002,9689,7255,1016,3707,3384,2137,4242,4077,5170,8388,2387,10577,2482,3735,4313,8862,2029,8452,10528,6058,6871,5233,1195,3301,8738,3194,2638,2488,5009,6981,7935,10684,10631,1043,1962,3541,4973,9581,4285,9118,9770,8279,1316,6284,1161,877,7894,3981,2423,6872,5033,1798,6181,1255,7517,4067,3319,7457,5164,3977,1719,4436,9317,4765,4137,6515,10025,4628,3745,7739,9445,7016,1980,4481,8580,175,10704,10653,245,8163,3834,7006,9135,1950,4401,2289,1412,7325,4821,6917,1361,7569,6782,2474,8075,5757,1034,3353,8570,2678,4922,4356,6831,3401,10393,10508,5161,2635,1472,3331,3048,9526,5383,5670,7508,5113,10148,2912,8547,1183,5496,5139,1153,6897,7997,6625,9819,5766,7460,10124,6637,327,3466,1848,2455,5939,2282,6371,4812,6721,5314,4780,3112,5154,250,7103,7158,463,6920,8368,6024,6877,342,6799,437,2876,9690,4889,5129,2631,3366,8380,1528,10135,584,6292,9180,5863,10031,2432,980,7211,3117,6909,1248,3761,3601,4976,9420,10602,178,10755,1150,2009,6653,7378,4422,8445,1978,2390,5417,4353,6452,2002,10595,1306,2817,8735,8424,8976,4396,4919,9488,534,4478,8211,8264,385,8288,3063,9825,6996,9726,7239,668,2896,6212,5761,6271,618,8216,6838,195,216,465,8870,6774,8556,3392,786,2791,1096,10565,9399,7603,1723,4133,6106,5491,10443,335,8387,5006,8131,966,7321,4664,5361,8778,8085,10370,512,4408,9227,4676,927,5805,10457,5444,5333,2951,6804,7955,888,3706,8788,2895,1902,9874,7221,5222,10628,6300,3522,65,10112,7284,6826,9762,10226,8642,8030,609,3812,9914,8276,5946,4130,767,4876,3831,9441,5515,7928,4838,4550,78,6682,63,600,3798,8226,3577,8618,8919,4716,10385,5560,7695,9677,9470,6351,10585,6031,2941,2010,7623,1735,6574,1513,2774,1490,2303,5429,141,1915,7737,9086,2205,32,7165,4564,3715,8209,8559,4979,4985,7143,1656,6789,8025,7208,6845,503,3687,7028,5471,9975,5664,3404,9068,5248,2433,322,10390,3840,8058,1297,3551,3819,3342,7265,3266,8914,4394,3583,9786,9198,5609,1444,8947,3060,10252,5594,1291,7319,9559,3166,4669,9592,661,6602,4789,9415,2424,4265,7098,1876,7535,1203,10053,9809,3104,1941,2705,6635,3148,3598,1502,7383,9073,10425,6590,5162,3529,7306,10800,6393,8751,387,10172,2692,5719,2741,7855,4845,7039,2459,6901,6624,3554,4237,3867,10566,2116,531,5943,6690,792,8441,6260,682,3995,6154,4165,9066,2717,1842,5340,8261,8539,1074,4259,701,9500,8648,513,3417,2155,9041,116,4010,1218,7514,3738,10176,8592,4659,8491,291,10417,8338,7244,862,5005,4735,1362,964,9714,10691,9829,10779,351,4930,16,3561,5388,5985,5239,6821,4320,10682,3154,4367,8961,50,8733,4288,9893,824,2504,404,10529,6560,1489,8125,48,76,909,2237,2691,2416,1987,7699,4599,5585,10503,7807,3639,3994,5708,7135,4625,4516,5540,2916,5528,5975,7413,8915,1347,10065,706,5304,3436,6607,8262,5645,3625,1376,4476,2934,1592,1935,3958,5821,2456,3362,5367,4151,981,9573,10610,9207,6190,5275,4631,1708,1921,9287,699,1447,1636,696,3740,4597,3373,9444,1221,3780,8124,5381,7648,3536,10571,654,9064,9044,3503,9854,9626,3378,10675,10629,6080,4445,9605,2646,6982,5074,3618,52,6561,6185,3026,5968,9742,989,5867,9190,1178,2689,2001,6149,8334,2049,3180,5759,111,4691,2650,8777,3149,804,1868,5328,840,8687,1382,57,2419,808,10355,5588,5215,1403,5701,9,7853,8520,8669,8019,4006,9097,226,2827,1135,1901,979,819,2561,7049,504,4391,4400,6739,1402,5264,2230,10666,2566,7430,8459,10720,8794,10310,10174,443,6349,7310,533,7609,6200,4108,613,7558,7709,1553,5076,10645,300,2842,6261,2324,588,2193,2945,7510,7396,3094,1889,10781,3791,554,9284,9148,3519,9995,3209,5810,8322,9519,2560,3073,947,2781,7004,5548,7116,10152,10121,552,1887,2952,8739,9225,5819,3712,6550,5446,574,3090,8327,428,1266,8433,8383,4645,8754,6898,2652,6822,1455,2176,1519,8566,10303,6049,10068,4977,3894,7561,7488,5536,7224,7793,1989,3985,8695,3424,7035,7612,9938,6775,39,1850,7803,9174,3031,6441,2974,1603,7941,7223,7923,2831,2977,3080,2075,9236,312,8565,4634,8814,1558,10484,1272,7389,8872,10418,4755,7869,2610,10633,5046,6960,4193,7852,1754,2878,1936,3784,10314,4025,4805,4997,10007,9863,7155,10101,1342,9395,10291,9385,10218,799,1123,5843,2562,4905,1377,9226,5937,1986,7591,9962,1189,8902,619,9067,3458,7586,7210,328,8506,3000,1004,9212,4591,8020,8887,4012,592,101,3415,2527,72,7452,9400,7809,1561,7032,4840,7008,1056,8,5752,9588,3101,3106,3609,4503,8312,6536,3446,8745,1437,7547,10287,4816,1795,8004,5837,1042,9329,9008,9433,2047,1087,3421,4324,10588,6699,10747,7765,9719,10438,887,9062,6820,2946,4741,527,3043,1648,1133,2918,882,684,4939,4386,8641,8154,1323,6436,7873,10752,3686,10680,807,9886,549,3230,6147,294,7120,354,8171,8493,2663,6863,7489,6037,410,9913,124,6312,541,3159,10304,2117,735,9882,199,4315,9250,4055,3739,3826,9558,10782,3699,542,6645,7593,5123,6413,7532,8805,1517,8514,3396,9239,8883,6748,6002,5319,8690,6308,1691,4655,589,10278,5680,9489,6806,10465,4251,10142,538,5740,194,9167,9974,2107,1480,8439,5722,5674,5712,7528,3385,1906,10567,9057,340,5707,9308,6207,3078,2818,7518,6794,9555,9983,1676,1678,6079,2038,1670,2608,2335,10021,10243,7257,9405,8165,2242,1247,8620,7479,1305,8818,8790,7865,4998,5653,2097,261,5270,2140,2105,789,6984,7501,8748,9538,8167,7566,3307,8768,5195,3932,1130,3930,6535,3665,6818,870,6657,7938,8656,2975,8108,4377,1332,1792,10027,3395,5078,10380,724,6034,3423,4788,4511,9362,2462,7760,7023,2497,9802,5504,6817,4263,3361,8164,8418,2478,2287,3457,1264,9796,7218,6591,6401,1683,3534,7474,5945,9722,7233,3719,7072,9915,42,6386,1481,3426,8808,4075,9665,132,3679,1418,6327,4854,5667,1228,5621,10063,9685,9641,3792,8509,10725,1726,3510,6733,5311,6890,1127,1372,4557,9076,4493,1319,4639,6503,5373,319,6252,5205,2568,3262,5611,3692,8016,2058,10708,1990,10246,5149,6730,4745,9617,4496,8093,3764,4587,9046,8240,5654,7484,9304,9356,9122,9012,2409,2879,1728,8443,6110,2374,2695,6156,6123,7266,9525,1770,9102,10660,6745,5128,5806,10298,1081,5852,10365,5171,3316,1949,2796,5938,2888,505,9040,10137,2063,8296,703,7982,5184,5822,6713,8356,1212,10214,1643,1344,1740,6170,2765,8513,7491,313,7468,5830,5188,7420,8246,7931,5592,2300,5803,1333,1904,10524,2557,1320,9801,3887,3372,4648,8123,10207,10570,2782,9208,3493,8993,4688,9635,2367,890,4834,4750,2186,4059,9823,383,5746,6601,6995,3696,10626,6202,4034,2588,1415,3128,5342,9471,8634,6712,7283,5689,7596,5728,9160,6465,1823,8542,2381,129,3703,5297,5845,10090,9892,8109,10676,1417,6171,9518,2483,5993,6066,10657,4131,7317,4665,10546,7047,6337,6183,3537,253,7720,5316,4382,2720,8286,7478,3902,7237,3088,5066,2131,2830,10694,3644,1407,5650,5733,7970,8970,2283,8951,9288,6643,10349,4772,6849,3295,4178,1354,8522,10736,1720,4463,10409,7172,6220,1998,6367,7464,1973,6738,4238,9065,9537,2528,6911,4247,8425,183,7919,4791,5494,3274,6583,4995,7128,869,5889,7655,5199,1555,6754,10636,3874,5094,545,3805,1611,9596,2168,6048,7912,1253,3675,10190,6051,6887,5414,2919,2397,8982,2189,5410,2589,6458,7272,8280,6078,2349,9268,9229,8372,9571,2129,9070,7371,5048,8593,4925,5490,1503,6584,4713,6442,3119,687,3013,10757,8401,6102,6842,6324,7114,10127,9879,9783,4963,1626,2785,3640,9163,4835,1390,5087,8073,10203,3068,5747,1958,49,4916,620,1548,3018,9306,184,10272,3878,3053,2239,10187,1477,2902,9456,7260,5051,7639,5386,4477,80,1151,7277,5579,5072,5596,2204,10199,8267,2624,5110,6272,1330,10059,757,9947,1357,7745,6159,9580,6926,1486,10062,10488,8712,6582,7733,8905,3935,8265,7334,5495,9170,1657,4880,1542,9546,5577,3169,2880,8287,2763,1465,1041,10545,7951,2814,10729,1234,7881,6985,6670,5055,4329,9749,4744,7621,9946,665,475,6655,6976,1327,9703,33,9513,9857,3248,2158,7640,3830,6910,6599,10785,3908,9303,6403,858,2074,3941,4068,1488,4929,9954,4872,10514,7838,4677,1877,7014,8113,6883,1511,7174,381,1523,10229,9627,6966,9767,4283,1167,4168,6977,3543,6953,2737,4698,7908,8759,2429,7706,9642,7199,2064,7087,557,8591,4609,3962,2795,8283,6449,9092,9964,4970,4529,3616,6475,7523,56,2238,6431,8256,640,3416,688,10471,8348,5541,2371,994,5570,82,6166,2259,7589,10479,5713,449,2308,130,6882,6298,6517,9870,2353,4316,5452,2792,7556,3803,822,9084,798,9477,9082,657,495,8781,10619,4328,2710,460,10133,3234,2555,6069,5255,348,5511,6934,9594,5120,6729,3956,6290,830,9948,5911,1731,6291,4993,1533,9758,5693,8152,4988,514,10270,6000,9682,6946,8344,8051,5320,8247,9020,3751,7864,10163,540,5933,3650,6107,2840,9372,3701,8955,4269,8511,1024,2165,4739,5474,10239,2466,4157,6533,10780,9408,3312,2549,7113,4480,7741,3467,5710,10381,2392,4166,7485,9902,10116,1089,5281,2157,9182,412,6056,2281,697,4383,6531,7473,9231,7411,7070,1493,3570,4122,3461,4849,5119,3174,3139,8068,7162,3184,3691,2316,18,650,2382,8612,9692,6483,2442,7718,488,6642,8681,5271,9004,3942,7986,5406,6744,9014,6975,8403,7449,8721,3240,8297,752,4471,2833,10812,202,7701,10064,9164,1565,2172,3613,6589,2843,1052,8359,1932,2389,127,5508,4782,1048,2729,6216,5692,5219,4730,8987,940,7810,3907,4399,1220,9901,1549,1242,9815,673,7459,3322,8550,6865,5229,8557,2257,898,2893,8674,2446,7645,6528,2809,4711,2106,6267,1110,6415,7998,2094,9509,6269,4475,3096,1567,7066,2464,8516,1843,9885,8995,2345,4940,1077,903,6886,347,8766,3343,8534,5871,5310,5738,10217,7539,2823,9867,8238,7093,10667,2332,5535,1419,3203,7540,2417,1619,10744,4592,8817,9219,2565,4276,9380,6329,10426,8853,8106,6765,6932,3822,5864,825,8244,9566,8333,5954,5351,2978,3376,6275,10290,1604,1067,7901,3143,7979,3630,5252,5642,6098,3077,1464,325,10481,5853,10012,4658,8315,10029,2812,4123,4967,6097,4038,1755,2873,2905,8098,6941,3681,1070,8555,4875,10340,9944,9694,2180,7441,2904,8054,2531,9330,6750,6945,10067,6012,5092,10772,73,6213,1504,6425,8835,2520,1401,10768,3368,4375,4941,685,9479,453,1268,2245,1694,4325,7705,892,10775,7450,2679,842,3996,6241,1117,7713,10343,2124,5857,5617,6594,7305,8770,5928,2197,9166,5956,9299,10095,6524,10531,4335,6622,10823,802,3120,2073,3864,3490,6957,9845,3972,6935,5543,9774,154,5250,10342,8137,8040,4224,10500,7327,5779,2755,5085,8243,4632,1952,8047,7056,6304,3241,6219,5353,10320,4559,6552,4748,9475,203,4551,8552,7187,3748,4271,3232,5903,9402,1566,6381,1569,6192,5338,2965,8023,2360,10386,6383,923,10618,9172,421,3412,7582,7685,7332,4138,2820,876,1356,1325,1012,3814,9584,3799,5041,9273,5764,9757,1655,6991,8588,9423,8001,3853,6191,8755,576,6629,4115,8878,4253,3245,1885,2505,5426,2621,260,8848,3873,2060,8352,4842,7686,5359,3600,6732,857,5014,9386,9173,8150,3065,7127,10754,2987,9907,7347,1051,6114,1609,221,6666,7851,5093,2471,2465,398,2077,6459,5799,8462,7943,10731,2227,9619,3813,9031,4753,7797,6347,3836,1806,9134,6150,5643,497,3868,6915,5071,2769,3717,8575,4426,5358,1587,7752,4942,8857,742,8464,3442,3050,9660,1165,3309,5293,3171,5099,1154,1827,7728,3140,5995,7719,601,9119,2053,10056,10487,4949,5307,7866,7076,8052,4372,6045,9586,7799,4800,4575,5458,316,5425,6361,4965,1187,10816,1907,7808,10551,4531,146,3884,8978,1271,3568,10668,179,6571,163,408,1190,10175,10716,3837,9340,303,6792,5936,5979,6330,8574,6129,5555,2385,10594,4980,10728,5842,4321,10444,427,5890,1776,4598,5235,7900,602,4763,6907,6379,4804,1960,924,6133,5932,8041,98,3622,476,2517,8161,1374,10787,556,493,5804,9109,8786,1872,3138,1337,2822,6289,7562,3005,1392,6866,2518,1140,5258,133,10331,8306,5313,9575,10717,3156,2103,8495,4799,4672,838,2249,4686,2233,4281,1475,9108,206,9453,5941,8379,5103,2868,9988,9732,7417,7071,1231,2515,9371,4498,3081,9352,6529,2313,1529,4229,771,1134,3690,9005,1128,9503,9776,2571,8369,10271,4387,3408,5533,8666,9035,8390,712,9255,6634,8299,3221,7005,6850,2854,3921,4548,3501,1593,1050,1692,3685,4879,4277,2805,8468,2867,46,1258,2272,1118,816,6796,7965,2036,1722,9448,9851,6940,6862,10538,2547,6437,5343,6727,1359,6444,6573,852,7574,10336,61,7085,6247,5134,106,9192,2369,7786,9009,670,8311,638,5288,8446,5463,296,7337,5018,3103,10423,9970,10008,944,3205,5435,5332,7921,871,967,4777,271,7214,4652,6630,8199,6526,1695,5140,6215,9277,5174,8504,5181,8193,2863,126,2377,691,9662,6495,233,565,6466,1771,5348,9411,5026,5537,4801,656,1384,8396,10541,2326,6958,675,402,10098,4203,6835,6258,6969,7950,5704,4047,10642,5038,3340,1820,6525,7354,8752,5131,3937,4369,3486,586,5305,4971,9773,5948,7618,9361,6790,8743,4309,3722,7721,3944,8836,4154,700,2935,8003,7351,5521,9464,4718,4888,5124,7841,3135,5697,4530,7276,1395,9615,4825,6224,198,7805,633,8442,1840,1261,1170,879,9725,3626,3870,5050,3901,3954,9332,2906,10073,7674,722,6843,4363,9289,5077,6598,3059,3928,1114,10034,6377,1380,4378,4418,10505,4869,7246,1292,9942,7092,9139,861,7416,2747,1245,6423,9095,9653,3130,8948,6737,2404,4026,726,6101,7726,4540,10308,4243,478,2980,3227,8985,6913,2675,4504,3753,4931,2434,6753,1045,7477,834,4181,7134,10507,9514,1747,714,7790,6755,1971,8301,6009,3584,8877,4361,2927,9027,599,8160,9059,4675,8090,10091,8192,5283,719,3145,8671,8569,6824,8639,9253,5815,10361,6988,2510,2585,8904,7405,2398,1768,7989,8696,9899,9805,2439,9412,2727,7927,6257,8746,3982,5679,2013,4579,4141,7634,7553,346,603,1522,8764,7001,3603,7775,9715,7678,8482,8630,10530,7905,9211,5308,6760,5266,7308,7377,3957,7058,6340,3602,3218,4945,239,8867,9918,8130,8000,4950,8913,275,8097,1029,4630,10774,3173,2808,3754,1591,9115,3593,755,3318,6902,5762,2104,8893,3749,10401,8946,2616,8510,8937,7975,1078,6335,6238,10733,2044,7885,4233,7426,3606,5280,9756,810,9844,9695,8053,4704,3852,1484,1031,8399,5467,7672,7590,8215,5263,8028,2657,8117,6142,6273,5561,9748,672,7229,3098,84,10405,4272,2800,6889,8792,6691,4204,5646,5345,5615,4278,5315,4486,1543,9467,5355,2797,5400,1953,8917,7806,7073,3807,7312,1406,5716,7086,1738,6847,9781,10212,9631,10442,7548,10115,4678,4097,5030,2247,3002,10293,492,10429,8944,5106,4009,10269,5760,859,1201,10771,2026,6508,2957,1780,317,6445,8139,1810,1520,8983,2354,490,7375,9866,6035,5393,4751,9810,7710,5155,6576,7367,4368,332,5870,7762,7292,10020,6076,8355,1216,2119,2362,6470,6180,500,9775,298,4319,4935,7585,9563,4182,5326,1970,9266,845,6756,2256,8541,8972,7094,2748,7353,5185,10664,10743,9565,6942,7743,9912,5135,1452,1618,7592,3844,7037,2048,946,7036,5780,3146,5973,5505,8810,2892,3682,9607,931,6852,8823,10136,10440,2615,10764,9421,7909,8964,8449,3160,2153,181,10421,3357,5638,9478,9572,5797,2020,5368,10014,2415,9659,1772,9831,7340,10777,2584,1967,7190,5789,9905,3909,5499,5879,9787,2921,2683,462,2535,10730,7492,6718,10492,7124,1148,1079,3986,6891,5696,10414,5855,5737,4214,9389,3253,1273,3869,2641,8329,7323,4120,7723,4798,1387,5285,6983,7132,8731,5299,3663,4249,3438,365,5100,8684,8801,1993,4506,1671,4566,5122,1686,3195,7916,3880,1101,9672,8036,4397,2701,1786,8120,10726,3429,5114,4588,5217,1057,2735,10794,1331,8815,2261,4703,1667,9574,10803,10054,2435,8067,4393,6362,2937,985,8750,5374,770,2463,1749,5922,1063,10335,4743,2305,10396,7546,257,3297,8553,1775,10197,7279,2690,496,3454,8616,10612,7857,7126,3107,2298,4072,8906,151,1581,8061,439,6856,7903,2142,4526,5244,9029,4714,8335,881,10363,7503,6647,935,5133,7649,10561,6108,933,2551,6870,4673,2826,10129,5906,626,7717,1082,3575,1172,9417,3456,5568,5914,5856,1851,2953,4881,5851,6239,2050,4371,6477,7366,4452,9849,2246,7930,988,1951,10789,5878,9853,2323,2850,1942,1136,7421,10230,5130,6313,6929,4846,10267,7890,4964,6829,4357,8582,649,7241,6716,8258,1176,9263,5390,6994,5921,3192,1069,10796,5908,6486,6138,8562,8578,6925,1408,7917,8045,10286,6671,2352,1312,10309,8210,800,6050,2686,6214,8456,10302,8858,3443,418,8005,1445,7115,1824,5690,2569,2917,5197,6749,8643,1537,9977,5648,3997,2600,10395,8360,2861,1014,9790,9865,4095,6124,1666,9189,5160,8186,5137,10828,6025,1847,4932,1249,1299,9530,7282,6990,9223,3720,3818,7987,3137,7670,286,4731,7654,805,9668,10018,230,2467,8638,10084,8422,3481,9811,3285,4877,9281,10307,10801,102,530,1819,4918,6546,928,2930,6678,10253,10151,4094,8430,5056,7400,1313,10461,6039,7992,6664,6502,10644,6424,3383,4302,7186,8342,3559,310,10379,9446,2277,1116,5778,6538,1963,3185,1487,3294,10362,1207,6921,8181,5236,10353,5709,7359,10011,4147,910,2476,5337,9213,9916,2696,2391,7286,8797,1796,2411,2570,7990,2889,10723,10150,171,472,6246,7280,10100,8298,4517,2775,7831,10582,2365,3183,3670,6432,9074,7993,4852,646,3645,4052,5323,5556,10807,4014,5349,7109,1296,7125,8147,10043,10085,10709,6906,2251,1058,3842,1132,10328,6005,7725,1853,1862,4528,10824,2837,9129,2214,4209,3557,8758,2605,6667,8761,7620,2156,8505,4322,5269,9818,4447,3186,4709,3920,10087,568,7985,8911,2331,6494,10654,7631,2875,5363,6151,6896,5481,9705,3286,7753,5449,6013,6874,5153,1779,1494,4722,6728,4726,8008,6316,430,4403,6359,8762,6936,8704,8677,1111,8153,8680,7700,2622,582,1462,1621,4373,5430,6218,1825,10038,5597,5578,6064,7907,7892,1469,9038,6763,7197,10543,9461,3881,1536,8992,7465,235,4240,7502,1541,1616,7207,6608,4245,3470,2961,5423,9460,3993,1979,3680,8998,6844,6370,8959,3488,3217,2604,3891,3369,6467,6320,4458,9248,1793,8676,7581,10502,730,566,3694,4699,4416,2284,8110,1933,9333,10388,9873,7454,9254,8881,775,3497,1712,7090,5158,10122,7483,10818,9973,9704,1969,4606,2682,3354,8869,1338,10215,4379,9793,694,3725,7365,4431,8302,2752,3255,5392,5468,68,10751,4871,390,457,4705,3181,4081,1491,9246,5982,2524,3721,2372,727,2163,4734,6689,10005,5237,5583,280,5207,3034,4022,5934,6326,2334,7209,2702,4643,7498,10119,5825,8962,8185,5440,2169,1803,1742,7316,5183,1177,2936,5020,6959,5063,698,8234,9307,3086,8831,919,2234,3787,5714,8263,8991,6104,8554,3350,5187,4092,8071,1898,1394,4992,5371,9200,486,7787,8939,6047,2190,4210,3142,4261,1583,1121,1927,270,4948,6204,10255,6811,315,8896,3477,3895,9606,8851,4623,4244,893,7009,10344,5404,2539,3448,8682,6105,8965,9706,2862,5375,3273,242,9218,3585,7027,1112,8232,7751,8332,9015,6507,8544,4428,884,3857,1086,8115,949,4160,4268,6867,9803,8337,6903,10165,5744,3284,5854,1785,2319,3089,8448,389,2066,7236,3351,1466,123,1956,10341,8785,1741,2310,1544,10547,526,4421,8916,7915,161,2972,5424,1301,222,4661,6438,2496,3109,8488,9354,743,9345,3615,10721,9622,51,8829,9000,4890,10182,3760,8921,1867,5754,3265,2399,6089,7313,6339,9285,212,3308,1886,10441,2341,2626,1284,8138,10424,10204,3572,3966,7296,7303,6846,1875,2758,9663,7173,2959,7261,6295,8303,9088,7669,2112,1328,809,4456,3132,2346,1241,958,339,7663,7531,2519,7509,6949,8340,9959,7063,3055,4636,4650,7680,7203,6030,2264,8979,8819,2500,7559,5606,9434,3256,10689,2321,3324,523,8078,992,2736,7513,6385,7052,5970,5483,278,6350,1730,9367,729,157,7960,4446,2110,10575,3544,10795,6554,4384,6279,6659,2623,2082,145,6372,3582,4127,1878,10826,4129,1017,1837,9788,3281,1663,1929,1597,7393,10473,5167,9599,1482,5394,4906,952,2098,3465,9240,5524,3459,1724,9454,6189,5210,8609,3447,252,20,8014,560,7346,8756,9939,6463,3810,8537,7240,6703,3435,2252,8404,4582,4011,285,3744,6435,8127,6676,3492,9245,7358,4079,3555,3329,9868,10154,1435,5189,8220,1275,3926,7007,4464,8647,10250,7756,10798,10221,8891,4723,10161,6297,9083,3765,95,3530,10346,9909,1532,3700,2541,9989,6816,10311,9640,6547,6384,2081,7112,5666,3974,10374,617,5691,6310,1702,2487,2285,4644,3336,3756,1224,4343,987,369,1093,5223,1610,9981,7227,4114,915,1186,3856,583,10701,6518,7343,860,956,3165,4762,2529,10281,1429,3413,4135,483,1449,739,4878,3697,769,5841,4621,999,10557,9149,6955,1997,1073,4172,7653,5518,1485,3287,5466,9746,7146,8953,8830,7521,8289,2198,5300,4404,1659,336,5723,2857,1499,8658,5105,6122,9601,7954,7573,5243,762,3418,6234,3161,2681,4441,4649,8345,3804,1818,3177,833,9727,10225,965,7635,3006,2772,1142,3208,1098,150,3943,4761,3460,148,4784,7496,7263,612,3524,984,1983,5502,1120,7200,4104,28,9675,9544,7936,8763,9603,4304,6014,8498,10131,1367,2580,5959,2016,1205,1351,3288,4457,10673,8116,4406,1206,7046,9177,5683,7707,3547,2045,4376,1516,2865,4622,3207,3659,9175,5121,2777,2653,8524,1157,218,6418,3321,10102,2380,7980,4617,4220,8859,2550,9473,3495,5408,10677,343,7398,380,2894,5325,10235,1329,2869,7202,10437,6613,3509,2703,6314,7226,5218,10184,10627,3526,7179,6801,6853,8560,962,10511,7729,4125,4134,4392,2766,2492,5971,6144,2358,2583,1122,9632,2992,747,5145,7409,8392,543,3133,7946,10450,754,4981,5441,2618,6462,1229,8587,6498,9512,954,5848,5688,4759,5331,356,8088,7814,4350,7795,8176,3631,2773,4999,8259,7381,744,6609,2564,5940,1594,3755,6476,6540,5057,10616,9221,621,597,7887,2606,81,3980,2220,6640,518,2659,4538,9926,1733,7948,8018,3004,7363,2021,6992,2342,8483,9373,4527,4419,10224,5816,10623,2693,2160,2963,6773,74,4684,104,4898,4809,1674,7499,10696,4695,3978,2699,5214,7315,829,10410,1145,803,5951,301,3763,4196,7746,8328,3070,7407,6055,938,10522,7977,9896,2243,7033,9814,4494,9676,6036,6100,4775,12,2195,10196,1103,5862,3333,3471,6764,10222,4174,8568,8249,8362,5492,4053,5630,7000,1966,3422,1711,7682,295,865,7361,9652,5790,5626,3821,8409,9859,6363,8860,5989,4087,6395,9848,8308,4341,7520,7616,2671,6836,4860,3014,9789,9205,1918,5053,3335,9542,320,8884,5204,1442,5558,8250,9872,5952,1424,3709,8195,6140,4057,6633,659,2383,5834,1508,4246,10010,7287,8076,4554,3599,10685,1431,4050,7264,8535,5166,8086,8133,6541,4201,3545,9923,5647,4035,7740,10183,4169,1689,2664,5770,1882,1295,6978,4921,1812,5412,5387,8668,8408,4781,3579,732,3899,9510,3082,1914,7242,1557,1912,3961,3250,1039,9611,6683,9820,7647,10220,5720,10173,10330,4909,2337,2654,7133,1896,2883,2448,10727,2793,578,3199,8572,2743,10223,10662,9378,10004,4008,9331,1675,718,4682,4274,6762,1240,3261,4046,1934,79,1988,9624,5601,6621,3153,3882,10144,7429,6430,8986,5860,3225,5873,3960,9397,7687,5080,10674,10778,6193,5531,5793,88,7583,4601,2248,5748,3393,6702,6857,6950,8635,2375,1141,4236,4145,2733,5629,401,8954,5777,6783,3337,1943,5918,7821,255,10692,1434,9058,9629,3168,360,1690,1230,9458,951,9271,6469,5967,3061,7579,1003,3108,7681,8486,6265,4409,6109,8412,5618,6487,5562,6276,5907,7575,5529,10601,1725,9096,6905,7801,8610,5277,2627,5035,9569,3157,9957,7322,1600,6157,796,4460,6627,5685,10805,394,3669,7330,2426,7744,10520,8645,8932,5209,5820,8391,4769,6132,8613,1836,3841,3041,1085,521,5186,9462,7136,6429,9951,5409,1269,8128,7516,9188,8354,10167,10216,9497,9972,7399,8910,6456,2810,10339,10819,1646,9616,2174,4175,6091,748,3661,2366,6654,587,3627,10097,8367,1109,1844,8652,1364,8151,5060,2480,7817,4284,1076,5257,4861,7355,5872,9709,1298,8145,8519,806,2427,4524,3660,6293,4444,1660,3989,9374,7192,5369,8292,3914,5608,210,1002,7154,1302,2855,2412,10211,8799,4612,9744,267,5877,7137,8426,2828,4192,2378,6236,4924,9921,2440,6378,3556,9511,1897,357,8104,2400,10609,2970,3499,1830,8321,4543,7794,2971,254,6223,27,9480,7003,8856,6954,7704,318,2407,1182,4624,644,9496,10318,7961,9257,5695,6231,8894,4186,7976,812,7331,4217,7106,8092,1126,4102,8980,10591,3734,1213,2670,9679,3172,10459,8626,6492,3323,6646,9539,355,9337,8716,3808,6165,10562,5658,818,9707,8958,2134,4866,10317,7818,274,5457,3388,4282,4330,6482,109,561,4955,4874,117,8038,8866,826,3938,7069,9357,4806,4903,8741,2688,6343,2402,9895,9561,4100,10491,2597,3533,1849,3190,8737,8967,9007,4468,1068,10277,2258,4360,59,107,1036,5769,8437,9843,9700,9127,3525,1,7297,1809,5972,2484,9840,9609,431,5800,671,3009,10159,481,5025,6254,9602,2025,1001,8679,7472,8929,3164,10786,5165,7177,9987,733,4959,10276,6199,8022,9001,10576,9036,8938,10189,9238,297,3231,9080,6512,6509,1568,997,3773,9643,1030,22,2192,9407,7798,2813,10039,2088,5173,7170,6652,8536,10436,3325,10338,9437,1977,1919,9808,2430,10240,1107,6451,2784,1817,8037,9633,143,7117,6964,721,4194,4954,9730,8203,9491,1807,5659,9116,9816,6808,1160,1358,4106,1166,7206,371,3636,7656,2148,1938,7394,3634,8204,4770,4938,3896,4577,8548,7530,6016,7461,324,10646,7307,7338,8436,4381,6615,6087,5668,942,9953,1899,5546,9406,15,9684,3552,7335,7392,450,4470,6201,4040,10641,8713,953,5675,4947,8715,3092,5422,6023,5783,3950,1423,677,4710,878,7130,10822,5512,7204,9929,7326,2019,7774,3197,10810,7732,8471,5052,10750,4482,5156,334,8526,6222,9214,6614,7258,6488,6680,8320,5259,3757,9549,9196,5715,3581,5413,8031,5224,6497,7651,1100,10110,4797,7839,3091,10123,10358,8188,2208,5447,4542,373,434,3293,4904,214,8494,5450,1721,9828,9961,8747,4365,110,4910,704,9155,3504,7235,7734,873,10448,7667,8824,3796,7971,4605,10003,917,4066,705,779,1612,8861,5260,7295,3027,8487,3823,9826,4459,8941,6715,2085,3517,4808,7075,8272,4500,9656,6927,8405,1386,4737,8475,8365,1813,10088,4996,6029,6993,3364,4491,4885,4287,3667,10170,9325,2314,1129,3911,7830,5924,2676,4934,9403,7940,1982,5054,3028,9693,8633,454,1944,8599,9833,8595,41,1400,9291,25,3594,2697,8945,8840,7290,8963,2043,8431,6618,7978,4635,1669,7879,4019,3811,8168,1791,7099,6588,9120,8107,2633,7467,2490,3905,4697,3652,6742,1718,3987,6206,8467,3716,8875,9504,7822,1874,2899,2537,1515,7730,9667,9388,8007,5294,7185,9778,9451,446,1512,4152,9894,6948,6416,8189,10042,7024,232,648,1425,9531,10391,10715,6568,6353,1441,4423,470,7962,5376,2725,4610,9079,678,24,794,963,5479,2339,5023,1627,9260,19,811,1410,2506,2086,4218,7015,9600,2022,9466,7222,7410,438,2732,4654,7252,8826,1766,3483,9081,3998,2315,2767,5503,9724,4060,4507,4857,5360,8310,10600,3211,6448,7386,3790,3872,1343,5636,2280,2384,4015,5068,6328,7697,7538,1588,2760,4701,5706,2338,3925,9966,2674,5599,1985,3879,6636,10366,9877,3042,544,2724,429,1957,8924,1634,10106,6242,9852,1884,1799,3610,299,7696,1144,5923,5194,4467,5813,1925,5177,5554,9889,886,6244,8968,9883,1910,1661,4438,7766,2194,5792,7610,8874,9729,7166,5700,8477,2091,4076,4962,9797,10400,2179,6539,1243,3452,6322,7091,10164,6564,8879,1637,2222,8434,6026,9485,4235,10564,6164,10599,186,7953,1223,6484,630,2034,6225,5991,5787,2344,7376,6979,4462,204,10621,6747,13,302,7329,4339,5983,6338,3113,3397,2213,5772,6603,4140,8357,8048,9752,3875,4177,1106,4179,2789,2995,3774,625,797,4188,333,4093,7495,6043,8057,5433,9839,1038,9750,1763,8427,10470,8601,1471,7956,8118,6848,6237,3794,5899,7245,1913,8242,6565,5329,442,2644,986,2983,5684,7060,368,89,10710,8027,2481,10632,9061,3313,10698,5357,6054,2221,4615,1578,9878,2509,4810,8126,848,1303,2489,2441,764,8255,7065,1527,553,2628,6282,4518,6331,8096,3017,2039,5350,3657,6795,2357,6577,4296,510,5627,338,393,10407,6951,9090,2798,4589,8606,6434,868,4778,631,2964,231,4594,6665,3945,5427,1293,8655,8499,388,3633,1393,2877,8319,6980,736,622,8813,3797,5883,3433,10160,2786,9567,9022,5739,7920,8966,1083,4771,3532,10697,4826,137,2182,522,8984,9982,10264,4017,1974,5652,1981,2910,7029,2990,8724,3505,4651,7819,4345,5497,1928,7782,8949,433,2403,10288,6751,6342,4712,7684,9683,1035,2844,2317,10157,3858,4370,3883,1554,7061,8373,8015,5686,8795,6278,3683,8686,5493,5168,4032,1748,10790,1458,1432,7436,8032,7625,8651,10815,5998,10506,5202,8688,3678,9343,6510,1629,3182,1353,7419,3785,2472,8393,7761,1355,6332,9681,9931,364,10678,854,9098,9691,10792,5289,7764,4230,5069,7055,3955,3576,4567,10194,6414,6766,8956,3269,2997,3888,144,1777,5477,7013,5192,3642,2311,1336,849,1601,4968,7482,8135,6404,3124,8622,2949,5403,9269,1256,10521,9091,5486,6970,2815,4311,1893,7183,85,2292,9856,7984,855,1750,4279,1713,7002,7626,1705,2090,105,4687,9153,2294,10411,7939,6707,1672,10284,8545,2609,10714,8589,1757,7469,7217,2263,5027,1745,853,4429,6758,9651,7144,9286,7676,8527,8155,420,4443,1845,8082,6611,0,2794,9547,639,1222,6115,1572,9903,9495,5545,3243,9621,5182,247,5581,3289,3729,776,5095,2379,243,8744,2295,3732,4725,10544,2745,1246,10024,3345,2076,1155,627,6939,2454,2839,10259,641,5988,1808,7401,9723,6734,1294,10155,5818,1260,7904,10671,7750,1061,8224,4013,7194,9030,9482,1505,8012,10185,2293,8551,520,4831,10451,5002,1628,244,23,5459,3152,3566,2923,4389,2508,5984,9363,9138,4756,9871,3200,1729,8922,3064,5090,4626,491,1968,7768,4039,7882,6317,4822,10244,6947,6559,5,188,4424,6963,1879,10447,8269,9576,3865,2632,4003,9368,3484,8089,2790,5472,1922,6723,5553,6057,2734,4989,4706,397,8384,8002,6233,3469,1576,8470,9846,2836,9521,1829,7442,8771,7731,2751,813,9051,5637,863,329,8056,5574,9779,10558,9234,8923,366,10036,7320,8997,9522,5301,3292,3327,1113,5242,5602,5876,3817,6972,1054,7870,4310,211,1290,4005,6888,8732,7384,9564,6837,6044,6205,5655,10017,4824,2938,7281,10462,5091,6918,5488,8481,7828,5498,7613,3405,3648,6007,5634,10209,10109,10516,10466,10688,6501,2023,400,10517,10399,2297,5079,4148,6248,3482,5231,441,7759,9244,1064,8564,8973,711,10711,5525,3420,2526,4757,3759,8142,5750,1787,6232,10806,3730,4483,4255,2811,2707,8295,9699,10739,9911,7622,1784,6592,9590,9556,1687,7643,10265,1620,1119,8451,4198,10519,10607,6446,4227,4943,6810,5061,9364,4570,2738,2640,10126,8694,9666,5043,1883,8233,904,6081,4532,3553,7533,3507,7636,6788,9128,995,4867,9560,5190,1115,2771,731,9777,1653,6919,6776,10262,9881,1196,3386,6390,3574,8693,913,2714,2713,8780,674,3478,550,686,4786,8990,5885,139,10416,1684,1811,5147,5614,5178,2662,4080,3141,6885,4211,307,1232,7565,2704,7598,9383,5930,7348,9999,8415,5980,6965,3236,6685,8251,2065,1573,4893,10245,2051,1239,4581,3425,2594,2740,3394,2607,384,3646,3567,5732,10820,7380,9440,10827,7490,1194,9984,4435,5587,901,5828,3008,7045,8749,6974,1547,8225,2202,6968,1602,9124,8466,509,10066,934,5022,2542,10193,1852,5339,8180,6639,3263,4159,5632,1590,8059,3952,7225,158,3494,2723,6447,5625,10732,2864,6698,3906,7974,4002,10044,2673,1008,266,9293,1999,7714,7968,4112,7779,9610,2266,7515,7274,8558,2139,8725,6067,4642,8930,374,8485,4717,173,2575,3832,34,2449,1346,7854,7675,8378,1743,7178,7022,2495,5542,1668,40,8230,3849,3363,10006,90,1143,8636,3591,2779,2994,9997,8281,10483,3463,4899,7722,10830,8492,4136,10368,7437,5013,2783,2592,9110,10227,6638,6650,7827,5455,10583,4338,7387,4158,1318,6757,4126,5434,1931,7856,9721,8597,1209,737,4474,2718,93,4337,5102,3317,10432,83,1339,10534,572,3788,3298,8604,4176,8889,7836,9505,8400,3246,9552,3390,7599,9630,7850,6075,6854,6077,2687,866,6668,9206,3781,3587,717,2312,6600,6407,6341,264,6952,1579,2558,2532,6562,5389,3970,7034,293,7777,9506,4873,7247,9140,10493,464,4225,5639,3134,9195,2386,5603,2578,8533,9782,436,6893,5590,3596,2083,9908,5891,5064,9589,8760,1693,5571,8837,168,6382,8512,4334,3711,10474,9655,1704,7057,5724,5673,9735,5522,7289,6944,2787,1709,6198,1131,4555,2999,555,7781,8812,632,5743,6513,3414,2336,1682,8988,8087,2343,4412,9424,99,444,5874,10460,9017,6094,7614,2147,5125,10117,7840,4637,5104,1285,8602,1091,10113,9887,7576,10605,3258,4018,276,9324,7641,9276,10477,3054,9256,9450,7105,3953,5909,8717,6504,2715,2154,9197,8828,5929,3621,3016,7042,419,3847,9320,1334,3782,9312,8699,3228,7715,7370,9313,8346,4425,3047,8838,9049,6352,1340,4492,3356,7095,663,4660,4119,10146,3129,7369,2040,6674,5527,9259,2322,64,7858,7019,8417,1746,7285,3332,7129,6814,4796,8274,3083,2824,3001,8621,3746,5344,1575,10357,4758,5436,4896,1060,8371,426,5031,7991,5916,1335,1345,8444,2162,10791,4539,9850,7051,2270,961,94,4578,6521,9290,6724,3809,6306,1644,447,7084,9623,7811,8895,2410,2493,4839,6003,7889,8892,3674,1283,790,2118,1473,5070,9296,469,9447,2128,3252,3793,10408,466,9013,2776,10452,6135,9187,6662,8183,1605,3695,8077,9430,5955,2376,7511,1894,7304,8703,6063,6672,5462,4405,8742,10120,5593,2475,208,2431,2672,1288,2582,5519,2586,7843,353,8981,896,10275,647,10456,3689,3802,4317,6663,6118,4185,4708,395,4829,3538,9278,471,5010,8531,6345,3586,6688,120,847,8804,2494,6987,1125,7031,6879,2730,6717,1033,6709,1562,2274,9686,7735,5942,4863,10572,10404,71,2968,2067,5717,92,10412,3311,6833,5509,5794,4785,5042,3656,8366,3020,10494,2405,5758,10536,9541,1920,6696,6287,2756,31,10430,349,3571,7708,3037,5256,5321,10554,7529,9733,10179,5227,3702,9785,680,10140,4195,3381,9638,7385,3643,9165,10604,3355,8044,4603,580,7455,9274,6389,6169,8254,1630,4696,7633,10327,2173,8039,9220,10247,7041,6706,1492,2217,9186,9024,2127,7947,3550,7563,4143,570,5633,1816,4510,5549,10139,4149,4450,9112,4078,5910,10132,4414,3806,9204,10713,9384,7250,3444,2330,10071,4990,7959,5900,8010,695,2418,9349,2079,793,6022,8438,3569,305,8863,8888,1397,596,6355,5604,3795,4291,2731,5203,2908,7568,3939,8175,4569,1762,2017,3766,3623,8062,138,54,5884,2240,6735,4455,5961,7527,6660,9060,8855,5785,6392,7688,551,4768,10367,2546,432,6143,5703,7278,547,4440,4614,2031,760,9107,164,921,1262,1858,6701,4692,9105,4354,768,2359,2012,6566,7820,759,2144,10251,9156,6096,3370,2226,708,1514,5835,8811,10060,4184,9763,9838,2553,6095,2739,7630,4521,10515,8615,5831,2939,7650,6491,8999,311,8060,8381,9298,7748,4314,10356,6823,788,3150,5169,2525,7913,8479,2035,9235,5396,3833,6781,5501,2126,8389,8729,5767,8543,8899,149,3676,5028,9026,8523,3229,1778,2279,3562,6931,3968,2388,1586,2084,7646,331,2309,9358,9410,6419,8661,7981,2872,5966,3249,1438,1179,5514,9442,1421,7544,10463,10092,6813,7402,10433,2096,4264,3254,2188,7571,4693,9111,2183,3233,1270,9930,7201,6177,9326,6128,9272,10283,1420,482,8918,5904,1717,10166,4410,7447,10504,1500,7996,3302,62,3277,9151,7220,9469,3629,3965,3506,508,8207,7776,8825,6989,831,3439,8361,1092,1570,7605,3019,7205,3437,6454,3328,5547,6280,10300,5808,9936,4473,7111,8532,1368,10527,3775,10439,8290,3605,10169,2393,4774,6741,7802,7435,1375,4064,9210,1010,1274,662,6711,8865,3214,4099,10299,7104,571,3178,8753,2625,6461,2033,1379,5641,2095,5965,7249,6797,1947,9280,2643,8791,4864,9821,749,10294,5385,4132,8897,5726,3564,3558,6499,2032,2834,10634,10389,4163,8820,3038,6585,7673,5247,7352,9830,4171,7151,7824,2244,4848,10371,1211,8876,9133,689,7397,4352,1769,10832,6356,8081,2637,4926,7169,3409,8450,3934,1080,10061,693,3030,4740,5784,2907,5576,7288,10670,9125,8903,5886,6548,907,9950,268,4553,7424,750,5947,5377,5663,2235,3769,7615,4239,2501,6572,1559,745,1495,10273,10681,8489,4395,10550,4749,7145,2825,8650,10485,8134,10023,8463,3291,6412,9078,9224,9431,4944,3105,1422,10455,5507,4180,9075,9123,9438,6986,3671,8386,7507,7716,9927,5284,3589,1448,8503,7832,1584,8714,362,5802,5564,4604,10718,5437,4307,7506,8890,4913,4146,7823,7181,4280,7362,5384,2890,3886,9117,9728,10687,182,4183,10770,3859,10228,676,5251,690,1443,4927,2540,5520,9837,598,2513,8190,637,823,5487,4275,4862,9484,6360,8063,3762,1474,5032,6158,10419,3431,7602,9279,2841,1635,9928,8617,1959,5836,3440,4197,6061,2932,3375,7550,9791,5273,7964,10249,2579,452,2271,10518,6111,4096,1647,1753,4212,6798,5378,1700,7876,8594,4033,1869,5024,2620,9011,8095,6478,5881,9319,8205,5978,6779,8200,4074,8850,2269,4073,9222,3487,4519,2885,7160,6172,936,2458,7834,9468,7763,3430,1707,187,6770,7770,1055,352,7783,10737,9401,7167,1991,5240,1180,9121,1861,8395,2268,8957,5774,996,8605,3772,8940,4836,891,321,7988,6534,5671,9144,9301,9754,8849,2554,1546,7425,4746,10070,10742,2061,1782,6038,6250,591,1257,1391,1954,8206,6307,1903,9941,3093,4348,7666,2996,6904,9100,5694,10143,10663,1426,607,8201,10035,7107,1371,7747,2042,10206,5327,1992,8420,5391,6752,8538,9396,3479,7849,6514,6892,477,8969,1585,192,5669,516,1237,1833,2979,4488,9753,4362,176,1916,7215,5365,3213,1563,7973,5557,8685,2619,1200,3502,10104,4851,7267,8100,6858,3062,4062,9876,2821,645,9604,4742,7118,507,5987,2668,579,9459,6125,10813,10046,10202,70,227,9483,10606,10332,3198,3044,10672,774,2601,1251,5953,559,4109,6563,8803,4832,1094,4911,4514,4549,5225,9428,3931,9381,10578,8473,10305,1506,10,5012,2587,10195,10434,1162,3347,2556,976,5152,414,5445,7588,4680,1210,5566,4027,1651,10013,180,9734,4150,2216,5584,9168,6426,10533,403,6391,8317,7471,8773,6772,1072,2835,5935,4951,8382,2799,9835,8079,629,2596,8525,7123,9678,113,3992,10149,8364,761,3111,519,8672,1521,7119,1011,2057,3940,7139,1236,1450,1930,1307,1191,1147,17,8074,8428,1509,9136,7293,4515,975,1388,4537,5672,5513,2611,4156,1071,10105,4453,2998,9717,875,8084,3750,10030,9142,9784,7311,6895,2514,494,415,6325,8698,6626,2460,9933,108,282,10589,6610,4323,1427,4830,6348,26,9527,2832,2015,1383,7344,1961,1088,2149,3496,8083,4294,4105,5850,1265,4969,8187,1137,8324,1625,2413,193,5964,9355,8629,4366,7933,7456,1821,4327,9053,7859,6520,103,2420,4882,3084,4653,562,9979,1381,10099,2925,8667,10048,10482,6210,10637,5861,5958,7601,1765,5176,6366,7512,10788,4048,10513,8935,10186,1460,8398,6532,6028,3029,10258,7891,3903,4402,537,2062,1281,4813,740,7324,3040,4299,7668,3275,9071,8275,10532,9628,4580,7969,6649,5485,6042,4596,4602,125,8722,6015,758,5287,3237,3391,9664,4563,2981,7427,3951,4618,4312,2803,5268,9261,5992,8960,8376,375,1688,610,9393,2210,8111,912,4994,1214,4355,4733,3278,1168,5352,7026,5812,7390,3929,3718,9800,9249,6656,4098,7543,5415,5334,5034,2521,5228,1828,273,8886,3604,9425,7156,4574,9965,10712,2886,7298,10613,9052,4820,778,135,2132,10831,1761,6851,5039,3179,9452,2929,2982,9037,4683,9106,5150,7958,4585,5298,2395,10130,4221,4738,287,4891,2203,4572,4508,1697,5651,9191,1832,4295,10556,8454,1378,9741,4110,4301,3155,9897,713,5136,4681,9141,5138,6088,5157,10773,2485,9499,5644,1664,536,7580,8598,4088,6422,10257,6400,5551,1440,1534,4084,4633,3051,3949,3767,1946,2780,4946,2940,899,2278,10289,4063,2660,9418,9045,2250,9494,309,8245,8691,6616,642,2639,2985,1459,3235,7388,455,6579,1315,10260,9618,6197,6460,498,5290,5322,337,3025,6245,7440,3922,7141,10354,4576,4933,10292,9582,9429,2102,7462,5919,5996,4978,2333,8043,8149,2296,10028,9657,3012,6768,7692,1550,9824,3528,8871,5347,9179,10579,10297,5159,1654,7082,4167,6812,8540,11,4207,345,3210,2576,5075,10114,605,2340,7664,6736,4747,3473,3747,9183,9171,3850,5276,8994,9310,7149,7253,4103,3573,1185,1633,773,8528,246,3890,10347,9305,5312,9520,9751,8898,9154,10337,4966,9391,3889,1790,3035,2516,9318,8323,7868,1023,10077,1727,3032,10329,2922,4571,3464,9759,2545,1277,6285,4957,8397,616,7861,3542,9680,4689,7088,8521,1895,3741,8457,2856,5112,10635,7624,7665,10808,2450,4297,2950,6168,6967,4584,6179,10420,3187,943,7443,1215,2962,5896,5484,8689,4469,7147,7945,3607,2111,4479,3136,5047,4051,8050,8336,10015,2394,2573,4305,9963,4509,6916,3462,5254,4318,265,58,422,8927,279,5718,8069,6527,10620,4795,9766,4535,3189,7712,6998,10263,7096,6186,960,8809,5324,4206,4858,5143,4427,9101,4118,3352,2848,4776,10649,3175,9536,10208,2591,2944,7230,8912,7660,1614,228,2115,1385,7043,4437,8662,8349,914,10319,2130,8649,5249,8294,9152,1461,10162,1169,5198,9535,7018,399,7658,9755,4451,10107,3866,506,1526,3283,6274,7184,7926,2870,611,2884,1865,1984,10758,4449,7617,1706,2728,9443,7374,651,3592,8476,9028,9516,7453,3752,8774,6731,7011,6912,9047,10763,8654,3789,6093,2543,1009,3647,2101,3518,8563,1662,6686,2716,10072,4495,8603,10387,5306,9696,8508,8775,998,3658,21,9772,7972,10274,9585,2572,4289,564,8579,970,6209,6358,1235,1250,894,9579,5949,7248,1856,3268,3202,525,3406,6090,872,3916,9262,5442,55,7089,8429,215,4461,5749,9351,5559,8783,3624,2598,7611,8141,3432,2807,5399,1645,7458,4004,9925,7703,6684,8374,5238,9540,5565,6226,8421,5782,9113,6632,5582,1428,5438,1350,2694,3338,9419,1501,7360,1800,10573,6001,3238,9847,7769,7848,9593,7594,10254,5191,7481,7963,4161,9551,2355,6651,7846,9455,3480,6086,376,6604,9858,2286,4041,5267,6315,5875,6999,2967,9344,4336,8490,1314,3827,3736,8628,6305,152,5586,4720,8351,7064,7373,4091,3303,4390,4727,3815,900,3115,142,8009,359,590,341,10413,1075,5869,9595,2711,5920,9769,4541,4986,3219,6761,7431,1470,573,5976,5840,593,9598,10829,3984,9532,972,9550,9126,8584,2866,2593,6841,1923,7999,1463,2602,1905,4707,7600,200,10237,2819,5960,6033,10489,9243,1059,9768,9033,7097,3127,5065,2215,10745,6364,1545,10598,7391,7412,9577,4228,4783,9342,874,6,4732,725,6323,8143,8478,563,1737,7679,4868,7541,3339,10428,8035,8307,3828,5292,2491,6861,3474,9376,4984,8926,5745,9562,594,4802,1598,3637,8779,4358,5108,6053,8709,6580,7910,10266,8253,1736,9390,4560,489,5768,2368,6743,262,10490,3851,2991,1027,2993,581,3300,6809,3358,8496,3655,7176,8501,2559,236,1577,10019,1233,1926,9426,9050,4332,10236,9089,9992,2133,9230,2617,1685,2422,2900,4736,5111,8720,7062,1289,4021,3123,7415,9150,8806,2136,9501,1873,2225,5894,7800,9377,7845,9006,3649,8266,5101,5469,6318,4219,5098,5847,4170,2093,2536,6398,6137,5912,1015,8718,9812,7629,1341,292,8172,136,8723,511,6933,4351,3976,1641,5420,189,9427,827,5370,9382,9529,4914,3563,7957,6710,7138,756,6017,1047,4293,6606,3988,7432,9822,197,8112,1405,2121,4199,4715,10476,4870,5735,6099,1370,5826,2507,7893,6648,10108,8484,1360,6661,6402,7414,7349,7433,9251,38,4499,4619,3399,6136,5986,1976,5833,2301,1948,167,4844,2327,2438,7444,3919,6396,5456,8590,1446,9861,4611,5058,10406,5656,6771,3516,902,8252,8212,851,2759,9021,6530,7577,1972,9201,9295,1219,5407,6493,2629,69,828,6146,9740,4086,3705,281,3975,5756,1244,5418,3704,8989,86,2926,1859,8158,3617,8800,4685,6262,1679,9940,937,5550,8102,1276,7350,1571,9969,4958,4434,8343,5607,4620,166,1456,567,6311,10542,8218,6767,9392,7219,7671,5179,2052,8174,8196,67,781,9413,5362,6840,643,4987,2612,6334,4817,5097,9712,955,2933,9275,8013,9334,10397,10535,37,1613,4085,2753,10784,6405,6631,2175,5059,1539,191,10242,6235,7228,9103,3131,6708,8213,10648,6410,5213,3334,4472,6277,2802,7632,4657,4442,2056,4153,2267,3632,1451,1476,3641,8313,2099,6923,8627,1022,6041,6387,9817,1756,379,8140,4887,6268,1917,9978,6062,10498,4029,1975,9922,10569,2665,2254,5206,3326,9282,3374,7677,8413,1855,2299,2408,7897,3778,6217,7077,2816,2473,5573,6249,9184,10079,7537,7300,3419,8664,289,7896,2468,249,4841,3267,7504,7545,4286,5552,128,6726,8577,2406,314,5839,3893,9215,5132,7875,2651,10559,8024,1497,9697,2241,8407,1964,499,6587,932,7341,2089,10213,1286,229,1433,479,7755,2581,6570,4124,969,7251,501,8260,2028,4534,6417,97,10145,8847,8816,5296,4292,1552,9161,3846,10765,1311,10296,8146,6473,7175,3693,330,4928,6052,9130,1208,3918,6785,4164,5763,4497,6397,3917,1198,8173,4090,3498,517,2969,10168,8156,1104,5622,10497,3515,8222,6551,10128,3380,3979,8882,10467,7829,3410,883,10431,9639,5791,9720,8700,4792,9247,5727,3771,5858,8546,468,548,3500,8646,6453,746,10446,459,9674,6253,2120,8472,1044,1650,3860,7778,9998,5829,10804,2498,9094,8331,10089,4561,2108,6815,667,4892,3389,6221,5341,2955,6174,10630,2451,6956,7773,10232,6549,248,528,6597,1411,60,6669,2114,8455,4920,6027,3726,6516,8474,9508,2363,9716,308,3039,4895,96,5476,10205,916,6195,5262,7131,3511,5516,1430,9217,4191,7301,4448,5598,9093,6161,7025,9710,4767,8055,5773,3843,4960,8705,7826,9457,7524,4359,4346,1102,4466,3310,9339,5282,10191,5460,7694,4042,7844,4502,9644,10333,3838,669,7428,515,7564,8854,6567,6725,177,635,367,6803,8358,10734,4586,10313,1124,9949,2453,7949,7050,5678,577,4640,44,5107,5661,4439,6409,8136,5356,1025,7595,8736,10384,7525,4728,8608,4290,2712,5538,1217,7872,5702,5635,1479,8072,6834,2928,9143,3770,9283,10200,3468,7333,6922,8663,5660,2595,9920,6428,3036,7012,5957,9315,3118,5788,10378,10659,595,6255,4779,3224,10510,4991,6175,4936,386,8901,6586,4837,4694,1324,8350,9647,3829,763,784,5687,4380,5969,10198,1732,8326,7191,1788,3282,880,2502,2461,2018,5962,5272,3776,7171,2003,1911,8239,2915,1794,5811,6839,3163,7475,2860,9085,889,1774,2958,5517,5532,10285,5775,9919,5335,7888,10738,8497,3758,5004,1498,9309,91,3677,9557,1617,9314,7597,9792,5037,8793,7644,7967,1365,5994,7079,3162,710,5926,2196,10651,5008,2852,5981,7911,9780,4811,3863,4833,4155,4298,4028,6294,5193,558,2574,7150,8423,2954,821,9669,9181,9241,4593,8702,3220,4505,10210,10705,846,1804,10746,9747,6938,2055,3611,1716,7874,6617,9745,5081,10009,1945,1152,8377,1658,929,5765,7567,4724,6374,7,10138,1007,5086,8179,2255,9614,6784,2853,2512,9010,2920,7871,6825,6697,10373,3485,8277,7100,3344,2037,782,7048,8166,9967,3614,4043,4385,8769,3204,9353,8285,10679,10219,10586,1066,6420,6537,6511,7608,2871,6819,8934,6083,1710,114,10793,4859,3472,6354,8229,9302,8873,9515,213,6472,3069,7188,7021,6505,5882,5336,8157,3067,1158,1496,8596,3936,7812,30,4850,10234,6368,2522,153,8049,7619,5241,6557,3087,10706,2166,6722,3835,2943,47,6519,9917,1866,5795,5448,6046,1439,3708,2882,3215,7291,4729,6786,2648,3608,3933,1703,7121,8625,5303,10597,814,100,3820,372,119,7357,8070,2649,10617,4856,3024,9534,7791,1436,8375,10695,5721,1105,6266,9054,10593,3512,7883,2072,8500,8309,5612,925,7466,10192,10040,5915,7078,3628,4342,1767,10377,3877,9587,461,9985,5500,8868,10111,2000,8248,2206,9409,655,8586,9321,8341,8832,1781,2078,6787,35,6558,9146,10082,2452,5000,5699,6485,3151,3011,5526,8469,6555,9335,3279,10665,6256,185,8316,277,7637,5619,4223,234,4388,8198,10435,5905,6021,10753,2847,3698,2253,1352,5624,832,6471,3662,134,1540,5657,4901,4205,715,4241,1715,7196,9765,4512,5736,10537,8573,7234,5730,10549,9968,926,9986,4413,5880,10809,5544,5892,3085,1880,10306,3259,3973,3367,7780,1839,982,9099,8581,8600,772,5302,2219,7863,3913,6085,5572,10369,4142,6134,2764,5291,9476,1551,6303,10702,4007,5317,2989,6937,1854,2201,6182,3074,4344,8782,6641,3999,5461,2150,8908,2677,5364,1965,1531,115,7549,10581,6019,8943,2749,2161,7470,7275,2151,5470,2851,1814,2445,10360,3520,5814,10055,1019,7607,3620,502,4883,10574,1389,3260,7108,2171,5411,1681,1409,8719,5755,10749,2655,9300,2511,10037,5011,9528,3508,4794,1084,3548,3125,2030,10643,6004,2534,1457,10256,8066,2232,4069,445,10766,5265,8169,9654,6113,2942,7995,6605,3403,8576,8291,9193,4036,2599,945,3244,4656,787,9645,2746,7749,10324,7299,6793,5089,8885,10348,7792,8529,3527,1888,9002,4666,7269,10622,7815,5846,3578,6406,692,3045,6369,8776,3072,6071,6746,7526,6900,5062,8011,9658,392,5817,5623,10453,8214,2722,4411,5443,1181,9498,1028,2138,66,5073,5605,3540,6440,7271,170,1188,1163,7934,1751,2224,6073,4,3052,1065,1369,1801,2499,7895,5346,10802,350,2988,2849,3619,4937,6455,10156,10041,7690,8103,9369,8619,4257,1282,5676,9827,1138,8631,5201,1099,9032,3668,9673,2146,5226,6251,895,9841,2005,6152,3927,7243,7448,4544,4231,7083,3731,3272,8293,5116,9350,8711,9137,2167,4407,4065,10047,7584,5931,8064,259,10301,9884,6692,6500,5844,1835,8021,1881,1673,6677,240,785,5465,4258,957,3742,4189,7040,1892,5083,606,9132,5506,2320,10501,3915,8530,9147,666,2187,225,10096,6074,4974,734,8852,8515,6694,7059,4056,2024,5798,10686,7659,3402,9990,9934,7364,3257,7438,165,8637,6240,4484,3126,3714,405,196,3514,3900,2804,8284,5045,1564,2754,10158,974,1173,2859,9233,10103,3839,3885,7606,9202,4044,2364,9798,7922,2318,3737,7164,8480,6543,9952,7860,1149,6778,4773,9474,6924,3407,983,9379,8728,7336,3076,8772,3110,8065,7080,10509,5148,7256,9648,9701,9832,8517,817,4202,3455,8453,681,458,8549,9836,3897,2356,10783,8974,9048,417,6302,8208,4721,5451,9620,1996,4139,7816,5421,8796,8119,3147,9465,6450,2068,6112,3167,10486,4807,532,5753,5824,9034,2304,6309,485,272,8802,416,5807,5742,8325,9739,9890,9625,7342,8585,3967,7757,843,4900,575,4814,9771,4523,6800,8942,5117,8707,10279,4430,6997,9199,1582,9292,7404,9976,3531,10093,3523,219,6155,6006,1632,9042,10422,10083,3673,9698,4546,7662,6059,1699,8880,8435,9994,922,2486,911,3114,660,29,1005,4520,1815,8202,10382,5234,8950,2503,8330,9019,5295,1860,1366,8221,614,7884,1955,174,4536,5974,2708,2307,7738,9242,8706,9039,7446,2159,10756,112,3003,9493,6167,9862,2903,9159,7519,7439,2614,6084,6072,8822,1838,3239,6675,9422,3964,3033,6899,4754,3963,4853,3710,5040,8080,4061,3597,2109,2123,1595,6211,5146,6791,4827,9996,4144,5569,7942,1159,9943,7572,1639,3672,10359,10690,4525,6196,9481,738,716,4489,2685,10147,4248,8583,6457,3387,10614,10364,993,1037,1404,10080,9258,5580,10403,3854,4116,5118,10669,9891,9737,5665,2469,10241,5489,6971,8936,6880,5977,5649,9228,3158,7784,5115,1574,9316,1797,8385,835,169,2288,8864,4121,6575,4101,407,2421,3898,5395,5616,3453,1454,5431,8122,156,1864,87,6008,7652,6427,7356,2260,1090,4823,8447,2457,10280,1631,1538,950,8406,1279,2966,7552,2347,7259,6394,10560,5868,3560,5823,3451,3046,5019,2414,7785,7929,8217,3320,4226,864,6148,6673,6490,3400,1310,9492,2348,4415,968,10323,3990,5776,9900,9336,1046,7148,8798,5151,2960,6178,6623,1204,4454,1760,7168,6117,1146,10719,10458,1649,5397,10282,10584,9794,3475,10402,10392,10762,3270,413,1759,9711,3664,3306,5274,8678,652,8665,8416,6408,8046,1624,5865,9687,8657,5888,1062,2184,1164,3280,9055,3097,5372,424,1287,5887,4647,7493,6962,10740,5382,6805,4082,8845,3590,7270,2228,2630,3021,9731,524,4700,378,7771,10656,3398,6644,990,1322,10016,6855,5261,10799,9323,6720,6162,1252,7898,487,4787,9439,5898,7813,3728,7689,8353,45,8231,2275,1665,709,2700,7044,6229,9875,3411,1846,5567,9855,6894,658,448,8410,1416,978,9570,5480,2329,290,4719,6830,7788,5473,8767,6832,2750,4270,2778,4501,1530,4417,7262,9398,3441,8026,2141,6700,4803,4545,3201,8518,9432,5734,3969,9904,8241,9760,10652,4558,5677,1535,3251,3971,10052,2199,5751,9956,7627,3299,10078,6578,4000,2567,1193,1453,2231,1398,1714,1507,2613,8977,1701,4465,5809,820,5049,9517,5595,5917,529,4815,224,3910,7054,3727,1939,7937,6884,9636,8394,5682,5405,9487,4907,9554,9743,10322,6208,9414,8843,7213,7505,10188,7604,7837,9860,10351,2087,14,7212,5999,6474,209,9842,10468,3271,741,10611,5142,4020,1870,10141,2070,238,4670,3450,3188,2479,8304,1263,9502,3330,2361,9077,160,4884,2948,2011,1032,1652,9131,2428,4917,6908,8219,6141], "time_ms": 0.155572, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 613, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "generated_query_count", "value": 10833, "time_ms": 51.505607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 428, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "generated_samples_per_query", "value": 1, "time_ms": 51.505607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 429, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "generated_query_duration", "value": 4072536354, "time_ms": 51.505607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 430, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 10362.200849, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 898, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 10362.200849, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 900, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 10362.200849, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 902, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 10362.200849, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 904, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 10362.200849, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 906, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 10362.200849, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 908, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "power_begin", "value": "12-25-2024 04:04:35.946", "time_ms": 10362.202034, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 564, "pid": 13811, "tid": 13811}} -:::MLLOG {"key": "power_end", "value": "12-25-2024 04:04:46.255", "time_ms": 10362.202034, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 566, "pid": 13811, "tid": 13811}} +:::MLLOG {"key": "loadgen_version", "value": "4.0 @ 486a629ea4", "time_ms": 0.004992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 53, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "loadgen_build_date_local", "value": "2024-12-27T19:45:31.956819", "time_ms": 0.004992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 55, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "loadgen_build_date_utc", "value": "2024-12-28T03:45:31.956827", "time_ms": 0.004992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 56, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "loadgen_git_commit_date", "value": "2024-02-15T15:46:50-05:00", "time_ms": 0.004992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 57, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "loadgen_git_log_message", "value": "486a629ea4d5c5150f452d0b0a196bf71fd2021e Hotfix: DLRMv2 Audit Test01 fallback failure (#1626)\nf9a643c0a0e920588da1b51a1d822e1071a9dbec Add number of tokens to offline (#1623)\nf06b920eac2170df38b5502c7443f50c9afa6045 Bugfix: equal-issue mode on offline causing accuracy run to fail (3D-UNet) (#1624)\n0ed5190e82e1ca1cf95626e50e61cb6678c7f2ff Fix submission checker and TEST06 for Llama2 (#1616)\n817dd961221d3ca284b024368130bd5dc5e4b54a Turn equal issue mode off for TEST06 (#1615)\n840435a1abe8be823e0a5a18ad3173bc520aa24f Update README.md with CM commands to download stable-diffusion, gptj and dlrmv2 models (#1604)\n3190d09a148bc1d72c9d946e4cf3b54db0d0e60a Typo fix in README.md (#1588)\nb0777f0bd3b7e5cd6a51d670d8b4df656cf6fb29 Update README.md - remove unwanted lines in CM commands (#1601)\nd7dba0826ac96a0e8bcae525086961e47c409a38 Remove loadgen warnings (#1608)\nd45a66c33bb0b6548d6172b38645bf94ffe24b39 Add upper limit to tokens per sample (#1612)\n44285d9a44041c3d4f7df0acf1e72c375e9fcbd2 Set completed samples per second as llama metric (#1613)\n357ccefca4db55ff373e029d5d1e3e6246bde5d4 Enable equal issue mode for LLM benchmarks (#1610)\n104d85531af7ae2a07ded36dd6e8c7f2c7746d06 Add sample length check to test06 (#1603)\n473053fc9f1b8c5edf60e11ec74607126938fbc4 Fix loadgen token metrics latency constrains (#1596)\ncc3daae3c7f18d7eedc5ffbc271d324e5aca6233 #1598: fix token and sample logging for Llama2 when accuracy_log_sampling_target is enabled (#1599)\n3a902e598e5a17013c26fea975c0a60e6d7feae1 Add support to dump 10 compliance images during accuracy run for SDXL (#1591)", "time_ms": 0.004992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 58, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "loadgen_git_status_message", "value": "", "time_ms": 0.004992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 60, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"05d4a62d355a90363847358e58afeee4a54ad838","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/bindings/c_api.cc":"32181da9e161c285f8fe46ddaa49e6cba2f9f918","/bindings/c_api.h":"91f58bd79b83b278f3240174a9af747fc38aff74","/bindings/python_api.cc":"1dc0865130371c6a8ddb97cd8039244d74d6fe46","/demos/py_demo_multi_stream.py":"6ccf7dcbe08f8382ed2d3618370c2b21e205ca13","/demos/py_demo_offline.py":"6c3075538e0a3df5e93edee917f4537099759e11","/demos/py_demo_server.py":"40e37ea121ac08b21ed6af41aca8eb5eda641b89","/demos/py_demo_single_stream.py":"4ee5f048fb0dffc415342a2bc2049fb155bd2894","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"126e952d00f4ea9efd12405fb209aa3ed585e4b2","/issue_query_controller.h":"923d9d5cdf598e3ec33d7a1110a31f7e11527ec7","/loadgen.cc":"6650091ba7a918f343b06eb7a5aa540eae87275f","/loadgen.h":"e00fdc6dbc85a8c9a8485dbcbfe2944f81251c4e","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"197efc96d178e5d33a750d07fa7b2966417506ea","/logging.h":"ddb961df7bcc145bcd7cce8c21f7cf075350dcbe","/pyproject.toml":"974adb632af4b76a1c72b914cc2cf9b3cbc66aa3","/query_dispatch_library.h":"13ad6d842200cb161d6927eb74a3fafd79c46c75","/query_sample.h":"e9187c8612bbdc972305b789feb6e15c26e96cfe","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"a170f991de69f8a5e45616e871b39a30ffed5598","/results.h":"d82500c326c2de83db411f1146882aa4692b419c","/setup.py":"e3e707e1191478332c00f1b6ff806f0ef98e0f65","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"aaf3803e700f3431a5ada72559eef5cce996ea64","/test_settings_internal.cc":"42aa46b17ac3fb2748cd5cde60fa8742e916f3c8","/test_settings_internal.h":"73b11c02228f3afb4c924db7a86fd8a2b5b9e483","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"4c9d98360b507d5673049ed70154fcfb2934f8a6"}, "time_ms": 0.004992, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 67, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "test_datetime", "value": "2024-12-27T19:59:12Z", "time_ms": 0.013086, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1198, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "sut_name", "value": "BERT SERVER", "time_ms": 0.013086, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1199, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "get_sut_name_duration_ns", "value": 293, "time_ms": 0.013086, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1200, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "qsl_name", "value": "BERT QSL", "time_ms": 0.013086, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1201, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "qsl_reported_total_count", "value": 10833, "time_ms": 0.013086, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1202, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "qsl_reported_performance_count", "value": 10833, "time_ms": 0.013086, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1203, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "requested_scenario", "value": "SingleStream", "time_ms": 0.018273, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 268, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "requested_test_mode", "value": "AccuracyOnly", "time_ms": 0.018273, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 269, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "requested_single_stream_expected_latency_ns", "value": 374897, "time_ms": 0.018273, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 274, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "requested_single_stream_target_latency_percentile", "value": 0.9, "time_ms": 0.018273, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 276, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "requested_min_duration_ms", "value": 600000, "time_ms": 0.018273, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 312, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "requested_max_duration_ms", "value": 660000, "time_ms": 0.018273, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 313, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "requested_min_query_count", "value": 100, "time_ms": 0.018273, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 314, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "requested_max_query_count", "value": 0, "time_ms": 0.018273, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 315, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "requested_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.018273, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 316, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.018273, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 317, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "requested_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.018273, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 319, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.018273, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 320, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.018273, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 322, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.018273, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 324, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.018273, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 326, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.018273, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 327, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.018273, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 329, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.018273, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 331, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "requested_performance_sample_count_override", "value": 10833, "time_ms": 0.018273, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 333, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "effective_scenario", "value": "SingleStream", "time_ms": 0.018413, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 409, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "effective_test_mode", "value": "AccuracyOnly", "time_ms": 0.018413, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 410, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "effective_samples_per_query", "value": 1, "time_ms": 0.018413, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 412, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "effective_target_qps", "value": 2667.4, "time_ms": 0.018413, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 413, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.018413, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 414, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.9, "time_ms": 0.018413, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 415, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.018413, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 417, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "effective_target_duration_ms", "value": 600000, "time_ms": 0.018413, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 418, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "effective_min_duration_ms", "value": 600000, "time_ms": 0.018413, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 420, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "effective_max_duration_ms", "value": 660000, "time_ms": 0.018413, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 421, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "effective_min_query_count", "value": 100, "time_ms": 0.018413, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 422, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "effective_max_query_count", "value": 0, "time_ms": 0.018413, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 423, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "effective_min_sample_count", "value": 100, "time_ms": 0.018413, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 424, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "effective_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.018413, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 425, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.018413, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 426, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "effective_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.018413, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 428, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.018413, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 429, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.018413, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 431, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.018413, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 433, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.018413, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 435, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.018413, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 436, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.018413, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 438, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.018413, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 440, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "effective_performance_sample_count", "value": 10833, "time_ms": 0.018413, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 442, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "generic_message", "value": "Starting accuracy mode", "time_ms": 0.023853, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1090, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "loaded_qsl_set", "value": [4117,9486,2947,9270,8227,4487,9799,4629,8571,8839,7101,2191,3315,6375,207,10454,3122,9490,4083,6119,1890,7657,9347,344,201,4232,5741,5454,9533,7842,1596,5330,10615,8827,8094,6714,9169,977,6619,9924,4982,4894,6544,10118,783,766,2122,7163,396,4200,9864,4254,8440,1758,10722,2829,2788,9906,2719,5613,8726,7847,440,6103,8567,7497,4793,1184,8675,10587,8270,7074,7408,8740,9265,4855,1199,220,6556,6595,1226,6695,3066,2143,6264,9806,7122,5007,10075,2370,2185,7254,77,2806,6243,6687,1156,1802,7302,391,3348,8159,9708,251,3595,3947,2302,4690,122,3116,1300,3768,6388,2986,728,5127,3904,6807,6259,2218,7555,8821,3580,7804,1321,8460,5419,6542,7833,3365,2290,6346,7952,918,707,118,10469,9023,1304,5220,1891,6878,9524,6522,7867,4037,7698,8507,9634,2145,10057,6263,7486,7557,2401,6082,3434,5927,8708,10081,8114,1349,7742,2742,7157,5705,4638,1764,3651,920,4016,2744,5859,6740,5380,9649,2,5245,5230,6421,4071,8909,6319,288,9346,9671,9646,9360,4702,5044,9157,6296,8318,8971,8184,7702,10475,7318,4024,5530,3991,1171,1909,6230,8461,8099,10563,5398,10760,10693,3688,8170,4952,6827,5893,7767,2656,10032,9178,10480,7195,9359,10608,10261,8975,5432,363,3346,3049,3360,4432,9736,473,7587,9416,1021,6596,10352,2666,2325,973,2071,7368,4267,867,1040,5796,3491,8237,9597,3489,10568,3777,7899,856,4818,2858,4662,777,10776,10026,6065,6481,2590,6780,10553,8789,6032,930,1049,10178,6187,585,3359,5832,2014,480,5866,5017,5141,1525,8271,8132,1642,6270,2265,140,1174,4641,8017,8844,6569,5575,7081,3779,3845,467,9761,3314,8653,4250,7578,8907,664,217,8419,8757,683,4565,1638,5216,2004,6679,4485,7683,5180,7422,6070,6283,2909,6281,8933,10639,4340,3079,3513,10552,2291,7487,2680,8807,8370,6127,7434,1822,908,8101,451,10316,8282,10724,885,1834,1556,6121,1468,409,1908,10539,7238,2891,10312,9435,2178,10051,4522,9650,615,7193,7796,2008,6011,4347,435,6881,1396,844,971,4216,5001,7886,6875,1348,4300,9348,2709,10580,9993,6705,8784,10821,9232,7724,6581,9764,2913,10125,6914,7693,4568,10526,10231,8502,7825,1783,795,6828,8006,4595,3892,3075,1278,10050,8235,4679,9971,75,2164,1175,3684,9869,8414,9016,765,2721,5786,10703,1326,7463,237,836,3912,5482,7835,4374,7661,4070,7754,7180,9945,10415,3638,4208,939,1097,7628,5725,3723,4547,1994,5232,5279,4049,7309,5963,10478,9834,7758,9463,3666,4663,801,4030,3010,8236,9237,2470,3382,6336,7570,190,4552,4113,2229,9018,7038,5021,8701,8268,4490,1373,6092,10181,10445,3056,4333,10394,1640,2328,3349,10069,10683,6439,3191,2544,4760,1739,8900,3,258,5416,4671,2041,9375,2661,6376,2306,5897,6373,2125,1940,5109,4256,6203,6411,9311,8029,2200,6301,1734,6176,6443,3007,9583,1841,4627,2523,4306,2801,5208,8607,4961,1467,10700,4031,2181,9958,8194,10295,9043,5366,306,9056,162,3196,10759,4349,5401,7030,6184,9216,2069,6357,5640,3015,1752,406,10049,3983,791,5610,5944,304,6380,1937,7862,9366,9365,1607,9252,2698,10350,5221,4847,10707,3023,1615,7017,5990,4045,7231,1202,7182,5172,3176,9612,6693,1013,10334,1805,8730,8841,7372,10094,6010,3216,8177,7406,9472,9608,2533,7691,3959,1623,5144,1831,751,4912,2152,8144,241,4187,10001,1924,8402,7877,8928,7294,4915,3876,3242,8996,7153,1000,10658,3946,284,5088,959,10661,1018,8931,546,4262,6489,3786,5278,4308,8178,7638,5211,7142,5771,634,8273,263,2548,4674,9910,624,3290,5925,1524,2642,7198,10345,3539,6719,10735,2006,10233,3816,9578,8624,6769,9661,6468,8952,9702,8670,5175,3449,1773,6130,636,1622,1696,1414,6523,2898,10650,4420,1280,2538,7880,3743,1227,2757,6131,3371,7445,1680,2901,36,6868,10811,7140,1900,10398,3724,8765,8278,7379,6040,7914,1580,3535,8033,7536,6480,3861,2437,5082,2881,10596,4902,8162,9637,4886,411,7189,1789,6704,9176,10045,4107,6333,4583,3923,4843,9935,9955,9297,3222,8257,6365,3095,1995,7924,5428,8640,6628,4819,7152,5016,7932,3226,1267,1225,9025,7878,6321,2373,3801,1399,9063,6506,9436,5781,7500,423,3588,7110,4607,3476,2477,3635,2425,9394,10449,4054,8148,5510,2236,9543,10499,6153,780,8692,8105,10268,172,4128,841,8925,5453,8561,2135,2976,2845,2209,7418,10817,5620,7010,9960,474,6545,8223,9713,205,10548,905,2396,2914,5591,2887,3379,4865,3304,5849,1309,10171,608,7314,6228,6961,7068,6943,4613,1317,121,8611,5902,5827,8042,1139,3121,223,2984,2897,3144,5379,10590,6433,3871,5475,43,1026,1192,1599,5729,6194,5838,1254,2054,2706,7925,6060,2177,10647,7727,5681,10000,3549,10814,4433,2100,6869,6593,9404,5534,5801,10321,9145,8363,7451,9072,4600,5096,10372,3022,679,5631,2552,723,4668,9613,10153,7423,9114,2973,3071,1413,4956,2838,5084,6777,5196,539,5036,382,7480,9327,323,4766,2273,2684,8182,10134,9203,2667,9185,2658,9087,6286,4162,6620,10472,1095,6759,7216,8191,9738,9548,5029,10326,5478,4089,8710,5439,6126,6160,4828,1006,1677,3546,6930,425,2645,4573,2770,10248,358,4975,10603,8129,8614,10180,7902,4303,2447,3427,753,8920,9449,256,6859,5711,5246,623,370,10767,2563,8300,9591,9991,5126,9568,2647,484,8644,3654,9507,8632,10076,9813,10638,4326,3862,569,7020,4213,5628,9341,839,3341,8347,5163,5563,7339,6928,1197,9158,3099,6120,1483,9545,4608,4215,2761,1259,6802,4364,4001,1606,4222,4752,3057,10022,1108,10592,4398,10376,9888,10033,4273,1238,3428,6553,6145,5731,7273,604,2436,3170,8673,5901,10074,4513,8305,8697,7476,5015,2669,3100,6344,9338,10383,4972,8727,1363,3223,10495,897,1698,9553,5662,53,9328,3733,9932,7102,2212,7382,7159,4556,10748,9294,5895,2262,10512,5913,8314,10624,2530,7560,10464,2443,8787,3800,1510,2027,9795,9670,283,2351,4260,2577,1518,906,3377,4764,2113,10699,3948,8465,9264,9267,5600,2924,3783,9209,7966,6464,720,6973,155,6227,7994,2636,535,8846,2931,2046,4953,10058,8228,5200,2211,4234,2207,8121,2007,7268,3212,7772,6496,10201,2762,7534,5539,10555,10496,628,3653,702,8683,10325,8660,8411,9387,326,8842,837,9688,3855,7053,1826,7944,1053,3193,456,361,3206,10315,2059,6612,6173,4562,9003,8034,9162,377,7551,10769,5354,6299,5997,2080,6288,6068,653,9937,10825,8833,3247,3058,4923,9104,7983,3305,9322,8339,7711,131,7328,5212,8623,10761,9718,5402,948,10640,8659,6479,2768,4616,2223,10625,4252,9980,9807,5318,6188,159,10655,3565,1478,6860,3521,4646,9880,1608,10427,9804,2092,4897,10238,6020,7642,8091,7789,4058,6681,4173,7918,5523,2444,9898,7161,7736,4667,2603,9370,7403,10797,1857,3276,2276,3445,8197,991,1863,4266,2170,4331,3296,6658,1308,7542,8458,4533,10177,6873,4190,815,2726,5589,2846,269,2634,1560,7067,7554,3102,4908,10741,10086,5286,8734,3824,1744,4111,3264,7494,3825,8834,6139,8432,147,1589,1871,941,2911,6018,7906,2874,3713,9523,4023,7522,6116,6864,5698,5464,3924,4590,7345,7395,4790,10540,1020,5309,5950,2956,4983,6163,9194,3848,3612,10525,9069,5253,7232,6876,5003,5067,2350,10375,10523,850,6399,10002,9689,7255,1016,3707,3384,2137,4242,4077,5170,8388,2387,10577,2482,3735,4313,8862,2029,8452,10528,6058,6871,5233,1195,3301,8738,3194,2638,2488,5009,6981,7935,10684,10631,1043,1962,3541,4973,9581,4285,9118,9770,8279,1316,6284,1161,877,7894,3981,2423,6872,5033,1798,6181,1255,7517,4067,3319,7457,5164,3977,1719,4436,9317,4765,4137,6515,10025,4628,3745,7739,9445,7016,1980,4481,8580,175,10704,10653,245,8163,3834,7006,9135,1950,4401,2289,1412,7325,4821,6917,1361,7569,6782,2474,8075,5757,1034,3353,8570,2678,4922,4356,6831,3401,10393,10508,5161,2635,1472,3331,3048,9526,5383,5670,7508,5113,10148,2912,8547,1183,5496,5139,1153,6897,7997,6625,9819,5766,7460,10124,6637,327,3466,1848,2455,5939,2282,6371,4812,6721,5314,4780,3112,5154,250,7103,7158,463,6920,8368,6024,6877,342,6799,437,2876,9690,4889,5129,2631,3366,8380,1528,10135,584,6292,9180,5863,10031,2432,980,7211,3117,6909,1248,3761,3601,4976,9420,10602,178,10755,1150,2009,6653,7378,4422,8445,1978,2390,5417,4353,6452,2002,10595,1306,2817,8735,8424,8976,4396,4919,9488,534,4478,8211,8264,385,8288,3063,9825,6996,9726,7239,668,2896,6212,5761,6271,618,8216,6838,195,216,465,8870,6774,8556,3392,786,2791,1096,10565,9399,7603,1723,4133,6106,5491,10443,335,8387,5006,8131,966,7321,4664,5361,8778,8085,10370,512,4408,9227,4676,927,5805,10457,5444,5333,2951,6804,7955,888,3706,8788,2895,1902,9874,7221,5222,10628,6300,3522,65,10112,7284,6826,9762,10226,8642,8030,609,3812,9914,8276,5946,4130,767,4876,3831,9441,5515,7928,4838,4550,78,6682,63,600,3798,8226,3577,8618,8919,4716,10385,5560,7695,9677,9470,6351,10585,6031,2941,2010,7623,1735,6574,1513,2774,1490,2303,5429,141,1915,7737,9086,2205,32,7165,4564,3715,8209,8559,4979,4985,7143,1656,6789,8025,7208,6845,503,3687,7028,5471,9975,5664,3404,9068,5248,2433,322,10390,3840,8058,1297,3551,3819,3342,7265,3266,8914,4394,3583,9786,9198,5609,1444,8947,3060,10252,5594,1291,7319,9559,3166,4669,9592,661,6602,4789,9415,2424,4265,7098,1876,7535,1203,10053,9809,3104,1941,2705,6635,3148,3598,1502,7383,9073,10425,6590,5162,3529,7306,10800,6393,8751,387,10172,2692,5719,2741,7855,4845,7039,2459,6901,6624,3554,4237,3867,10566,2116,531,5943,6690,792,8441,6260,682,3995,6154,4165,9066,2717,1842,5340,8261,8539,1074,4259,701,9500,8648,513,3417,2155,9041,116,4010,1218,7514,3738,10176,8592,4659,8491,291,10417,8338,7244,862,5005,4735,1362,964,9714,10691,9829,10779,351,4930,16,3561,5388,5985,5239,6821,4320,10682,3154,4367,8961,50,8733,4288,9893,824,2504,404,10529,6560,1489,8125,48,76,909,2237,2691,2416,1987,7699,4599,5585,10503,7807,3639,3994,5708,7135,4625,4516,5540,2916,5528,5975,7413,8915,1347,10065,706,5304,3436,6607,8262,5645,3625,1376,4476,2934,1592,1935,3958,5821,2456,3362,5367,4151,981,9573,10610,9207,6190,5275,4631,1708,1921,9287,699,1447,1636,696,3740,4597,3373,9444,1221,3780,8124,5381,7648,3536,10571,654,9064,9044,3503,9854,9626,3378,10675,10629,6080,4445,9605,2646,6982,5074,3618,52,6561,6185,3026,5968,9742,989,5867,9190,1178,2689,2001,6149,8334,2049,3180,5759,111,4691,2650,8777,3149,804,1868,5328,840,8687,1382,57,2419,808,10355,5588,5215,1403,5701,9,7853,8520,8669,8019,4006,9097,226,2827,1135,1901,979,819,2561,7049,504,4391,4400,6739,1402,5264,2230,10666,2566,7430,8459,10720,8794,10310,10174,443,6349,7310,533,7609,6200,4108,613,7558,7709,1553,5076,10645,300,2842,6261,2324,588,2193,2945,7510,7396,3094,1889,10781,3791,554,9284,9148,3519,9995,3209,5810,8322,9519,2560,3073,947,2781,7004,5548,7116,10152,10121,552,1887,2952,8739,9225,5819,3712,6550,5446,574,3090,8327,428,1266,8433,8383,4645,8754,6898,2652,6822,1455,2176,1519,8566,10303,6049,10068,4977,3894,7561,7488,5536,7224,7793,1989,3985,8695,3424,7035,7612,9938,6775,39,1850,7803,9174,3031,6441,2974,1603,7941,7223,7923,2831,2977,3080,2075,9236,312,8565,4634,8814,1558,10484,1272,7389,8872,10418,4755,7869,2610,10633,5046,6960,4193,7852,1754,2878,1936,3784,10314,4025,4805,4997,10007,9863,7155,10101,1342,9395,10291,9385,10218,799,1123,5843,2562,4905,1377,9226,5937,1986,7591,9962,1189,8902,619,9067,3458,7586,7210,328,8506,3000,1004,9212,4591,8020,8887,4012,592,101,3415,2527,72,7452,9400,7809,1561,7032,4840,7008,1056,8,5752,9588,3101,3106,3609,4503,8312,6536,3446,8745,1437,7547,10287,4816,1795,8004,5837,1042,9329,9008,9433,2047,1087,3421,4324,10588,6699,10747,7765,9719,10438,887,9062,6820,2946,4741,527,3043,1648,1133,2918,882,684,4939,4386,8641,8154,1323,6436,7873,10752,3686,10680,807,9886,549,3230,6147,294,7120,354,8171,8493,2663,6863,7489,6037,410,9913,124,6312,541,3159,10304,2117,735,9882,199,4315,9250,4055,3739,3826,9558,10782,3699,542,6645,7593,5123,6413,7532,8805,1517,8514,3396,9239,8883,6748,6002,5319,8690,6308,1691,4655,589,10278,5680,9489,6806,10465,4251,10142,538,5740,194,9167,9974,2107,1480,8439,5722,5674,5712,7528,3385,1906,10567,9057,340,5707,9308,6207,3078,2818,7518,6794,9555,9983,1676,1678,6079,2038,1670,2608,2335,10021,10243,7257,9405,8165,2242,1247,8620,7479,1305,8818,8790,7865,4998,5653,2097,261,5270,2140,2105,789,6984,7501,8748,9538,8167,7566,3307,8768,5195,3932,1130,3930,6535,3665,6818,870,6657,7938,8656,2975,8108,4377,1332,1792,10027,3395,5078,10380,724,6034,3423,4788,4511,9362,2462,7760,7023,2497,9802,5504,6817,4263,3361,8164,8418,2478,2287,3457,1264,9796,7218,6591,6401,1683,3534,7474,5945,9722,7233,3719,7072,9915,42,6386,1481,3426,8808,4075,9665,132,3679,1418,6327,4854,5667,1228,5621,10063,9685,9641,3792,8509,10725,1726,3510,6733,5311,6890,1127,1372,4557,9076,4493,1319,4639,6503,5373,319,6252,5205,2568,3262,5611,3692,8016,2058,10708,1990,10246,5149,6730,4745,9617,4496,8093,3764,4587,9046,8240,5654,7484,9304,9356,9122,9012,2409,2879,1728,8443,6110,2374,2695,6156,6123,7266,9525,1770,9102,10660,6745,5128,5806,10298,1081,5852,10365,5171,3316,1949,2796,5938,2888,505,9040,10137,2063,8296,703,7982,5184,5822,6713,8356,1212,10214,1643,1344,1740,6170,2765,8513,7491,313,7468,5830,5188,7420,8246,7931,5592,2300,5803,1333,1904,10524,2557,1320,9801,3887,3372,4648,8123,10207,10570,2782,9208,3493,8993,4688,9635,2367,890,4834,4750,2186,4059,9823,383,5746,6601,6995,3696,10626,6202,4034,2588,1415,3128,5342,9471,8634,6712,7283,5689,7596,5728,9160,6465,1823,8542,2381,129,3703,5297,5845,10090,9892,8109,10676,1417,6171,9518,2483,5993,6066,10657,4131,7317,4665,10546,7047,6337,6183,3537,253,7720,5316,4382,2720,8286,7478,3902,7237,3088,5066,2131,2830,10694,3644,1407,5650,5733,7970,8970,2283,8951,9288,6643,10349,4772,6849,3295,4178,1354,8522,10736,1720,4463,10409,7172,6220,1998,6367,7464,1973,6738,4238,9065,9537,2528,6911,4247,8425,183,7919,4791,5494,3274,6583,4995,7128,869,5889,7655,5199,1555,6754,10636,3874,5094,545,3805,1611,9596,2168,6048,7912,1253,3675,10190,6051,6887,5414,2919,2397,8982,2189,5410,2589,6458,7272,8280,6078,2349,9268,9229,8372,9571,2129,9070,7371,5048,8593,4925,5490,1503,6584,4713,6442,3119,687,3013,10757,8401,6102,6842,6324,7114,10127,9879,9783,4963,1626,2785,3640,9163,4835,1390,5087,8073,10203,3068,5747,1958,49,4916,620,1548,3018,9306,184,10272,3878,3053,2239,10187,1477,2902,9456,7260,5051,7639,5386,4477,80,1151,7277,5579,5072,5596,2204,10199,8267,2624,5110,6272,1330,10059,757,9947,1357,7745,6159,9580,6926,1486,10062,10488,8712,6582,7733,8905,3935,8265,7334,5495,9170,1657,4880,1542,9546,5577,3169,2880,8287,2763,1465,1041,10545,7951,2814,10729,1234,7881,6985,6670,5055,4329,9749,4744,7621,9946,665,475,6655,6976,1327,9703,33,9513,9857,3248,2158,7640,3830,6910,6599,10785,3908,9303,6403,858,2074,3941,4068,1488,4929,9954,4872,10514,7838,4677,1877,7014,8113,6883,1511,7174,381,1523,10229,9627,6966,9767,4283,1167,4168,6977,3543,6953,2737,4698,7908,8759,2429,7706,9642,7199,2064,7087,557,8591,4609,3962,2795,8283,6449,9092,9964,4970,4529,3616,6475,7523,56,2238,6431,8256,640,3416,688,10471,8348,5541,2371,994,5570,82,6166,2259,7589,10479,5713,449,2308,130,6882,6298,6517,9870,2353,4316,5452,2792,7556,3803,822,9084,798,9477,9082,657,495,8781,10619,4328,2710,460,10133,3234,2555,6069,5255,348,5511,6934,9594,5120,6729,3956,6290,830,9948,5911,1731,6291,4993,1533,9758,5693,8152,4988,514,10270,6000,9682,6946,8344,8051,5320,8247,9020,3751,7864,10163,540,5933,3650,6107,2840,9372,3701,8955,4269,8511,1024,2165,4739,5474,10239,2466,4157,6533,10780,9408,3312,2549,7113,4480,7741,3467,5710,10381,2392,4166,7485,9902,10116,1089,5281,2157,9182,412,6056,2281,697,4383,6531,7473,9231,7411,7070,1493,3570,4122,3461,4849,5119,3174,3139,8068,7162,3184,3691,2316,18,650,2382,8612,9692,6483,2442,7718,488,6642,8681,5271,9004,3942,7986,5406,6744,9014,6975,8403,7449,8721,3240,8297,752,4471,2833,10812,202,7701,10064,9164,1565,2172,3613,6589,2843,1052,8359,1932,2389,127,5508,4782,1048,2729,6216,5692,5219,4730,8987,940,7810,3907,4399,1220,9901,1549,1242,9815,673,7459,3322,8550,6865,5229,8557,2257,898,2893,8674,2446,7645,6528,2809,4711,2106,6267,1110,6415,7998,2094,9509,6269,4475,3096,1567,7066,2464,8516,1843,9885,8995,2345,4940,1077,903,6886,347,8766,3343,8534,5871,5310,5738,10217,7539,2823,9867,8238,7093,10667,2332,5535,1419,3203,7540,2417,1619,10744,4592,8817,9219,2565,4276,9380,6329,10426,8853,8106,6765,6932,3822,5864,825,8244,9566,8333,5954,5351,2978,3376,6275,10290,1604,1067,7901,3143,7979,3630,5252,5642,6098,3077,1464,325,10481,5853,10012,4658,8315,10029,2812,4123,4967,6097,4038,1755,2873,2905,8098,6941,3681,1070,8555,4875,10340,9944,9694,2180,7441,2904,8054,2531,9330,6750,6945,10067,6012,5092,10772,73,6213,1504,6425,8835,2520,1401,10768,3368,4375,4941,685,9479,453,1268,2245,1694,4325,7705,892,10775,7450,2679,842,3996,6241,1117,7713,10343,2124,5857,5617,6594,7305,8770,5928,2197,9166,5956,9299,10095,6524,10531,4335,6622,10823,802,3120,2073,3864,3490,6957,9845,3972,6935,5543,9774,154,5250,10342,8137,8040,4224,10500,7327,5779,2755,5085,8243,4632,1952,8047,7056,6304,3241,6219,5353,10320,4559,6552,4748,9475,203,4551,8552,7187,3748,4271,3232,5903,9402,1566,6381,1569,6192,5338,2965,8023,2360,10386,6383,923,10618,9172,421,3412,7582,7685,7332,4138,2820,876,1356,1325,1012,3814,9584,3799,5041,9273,5764,9757,1655,6991,8588,9423,8001,3853,6191,8755,576,6629,4115,8878,4253,3245,1885,2505,5426,2621,260,8848,3873,2060,8352,4842,7686,5359,3600,6732,857,5014,9386,9173,8150,3065,7127,10754,2987,9907,7347,1051,6114,1609,221,6666,7851,5093,2471,2465,398,2077,6459,5799,8462,7943,10731,2227,9619,3813,9031,4753,7797,6347,3836,1806,9134,6150,5643,497,3868,6915,5071,2769,3717,8575,4426,5358,1587,7752,4942,8857,742,8464,3442,3050,9660,1165,3309,5293,3171,5099,1154,1827,7728,3140,5995,7719,601,9119,2053,10056,10487,4949,5307,7866,7076,8052,4372,6045,9586,7799,4800,4575,5458,316,5425,6361,4965,1187,10816,1907,7808,10551,4531,146,3884,8978,1271,3568,10668,179,6571,163,408,1190,10175,10716,3837,9340,303,6792,5936,5979,6330,8574,6129,5555,2385,10594,4980,10728,5842,4321,10444,427,5890,1776,4598,5235,7900,602,4763,6907,6379,4804,1960,924,6133,5932,8041,98,3622,476,2517,8161,1374,10787,556,493,5804,9109,8786,1872,3138,1337,2822,6289,7562,3005,1392,6866,2518,1140,5258,133,10331,8306,5313,9575,10717,3156,2103,8495,4799,4672,838,2249,4686,2233,4281,1475,9108,206,9453,5941,8379,5103,2868,9988,9732,7417,7071,1231,2515,9371,4498,3081,9352,6529,2313,1529,4229,771,1134,3690,9005,1128,9503,9776,2571,8369,10271,4387,3408,5533,8666,9035,8390,712,9255,6634,8299,3221,7005,6850,2854,3921,4548,3501,1593,1050,1692,3685,4879,4277,2805,8468,2867,46,1258,2272,1118,816,6796,7965,2036,1722,9448,9851,6940,6862,10538,2547,6437,5343,6727,1359,6444,6573,852,7574,10336,61,7085,6247,5134,106,9192,2369,7786,9009,670,8311,638,5288,8446,5463,296,7337,5018,3103,10423,9970,10008,944,3205,5435,5332,7921,871,967,4777,271,7214,4652,6630,8199,6526,1695,5140,6215,9277,5174,8504,5181,8193,2863,126,2377,691,9662,6495,233,565,6466,1771,5348,9411,5026,5537,4801,656,1384,8396,10541,2326,6958,675,402,10098,4203,6835,6258,6969,7950,5704,4047,10642,5038,3340,1820,6525,7354,8752,5131,3937,4369,3486,586,5305,4971,9773,5948,7618,9361,6790,8743,4309,3722,7721,3944,8836,4154,700,2935,8003,7351,5521,9464,4718,4888,5124,7841,3135,5697,4530,7276,1395,9615,4825,6224,198,7805,633,8442,1840,1261,1170,879,9725,3626,3870,5050,3901,3954,9332,2906,10073,7674,722,6843,4363,9289,5077,6598,3059,3928,1114,10034,6377,1380,4378,4418,10505,4869,7246,1292,9942,7092,9139,861,7416,2747,1245,6423,9095,9653,3130,8948,6737,2404,4026,726,6101,7726,4540,10308,4243,478,2980,3227,8985,6913,2675,4504,3753,4931,2434,6753,1045,7477,834,4181,7134,10507,9514,1747,714,7790,6755,1971,8301,6009,3584,8877,4361,2927,9027,599,8160,9059,4675,8090,10091,8192,5283,719,3145,8671,8569,6824,8639,9253,5815,10361,6988,2510,2585,8904,7405,2398,1768,7989,8696,9899,9805,2439,9412,2727,7927,6257,8746,3982,5679,2013,4579,4141,7634,7553,346,603,1522,8764,7001,3603,7775,9715,7678,8482,8630,10530,7905,9211,5308,6760,5266,7308,7377,3957,7058,6340,3602,3218,4945,239,8867,9918,8130,8000,4950,8913,275,8097,1029,4630,10774,3173,2808,3754,1591,9115,3593,755,3318,6902,5762,2104,8893,3749,10401,8946,2616,8510,8937,7975,1078,6335,6238,10733,2044,7885,4233,7426,3606,5280,9756,810,9844,9695,8053,4704,3852,1484,1031,8399,5467,7672,7590,8215,5263,8028,2657,8117,6142,6273,5561,9748,672,7229,3098,84,10405,4272,2800,6889,8792,6691,4204,5646,5345,5615,4278,5315,4486,1543,9467,5355,2797,5400,1953,8917,7806,7073,3807,7312,1406,5716,7086,1738,6847,9781,10212,9631,10442,7548,10115,4678,4097,5030,2247,3002,10293,492,10429,8944,5106,4009,10269,5760,859,1201,10771,2026,6508,2957,1780,317,6445,8139,1810,1520,8983,2354,490,7375,9866,6035,5393,4751,9810,7710,5155,6576,7367,4368,332,5870,7762,7292,10020,6076,8355,1216,2119,2362,6470,6180,500,9775,298,4319,4935,7585,9563,4182,5326,1970,9266,845,6756,2256,8541,8972,7094,2748,7353,5185,10664,10743,9565,6942,7743,9912,5135,1452,1618,7592,3844,7037,2048,946,7036,5780,3146,5973,5505,8810,2892,3682,9607,931,6852,8823,10136,10440,2615,10764,9421,7909,8964,8449,3160,2153,181,10421,3357,5638,9478,9572,5797,2020,5368,10014,2415,9659,1772,9831,7340,10777,2584,1967,7190,5789,9905,3909,5499,5879,9787,2921,2683,462,2535,10730,7492,6718,10492,7124,1148,1079,3986,6891,5696,10414,5855,5737,4214,9389,3253,1273,3869,2641,8329,7323,4120,7723,4798,1387,5285,6983,7132,8731,5299,3663,4249,3438,365,5100,8684,8801,1993,4506,1671,4566,5122,1686,3195,7916,3880,1101,9672,8036,4397,2701,1786,8120,10726,3429,5114,4588,5217,1057,2735,10794,1331,8815,2261,4703,1667,9574,10803,10054,2435,8067,4393,6362,2937,985,8750,5374,770,2463,1749,5922,1063,10335,4743,2305,10396,7546,257,3297,8553,1775,10197,7279,2690,496,3454,8616,10612,7857,7126,3107,2298,4072,8906,151,1581,8061,439,6856,7903,2142,4526,5244,9029,4714,8335,881,10363,7503,6647,935,5133,7649,10561,6108,933,2551,6870,4673,2826,10129,5906,626,7717,1082,3575,1172,9417,3456,5568,5914,5856,1851,2953,4881,5851,6239,2050,4371,6477,7366,4452,9849,2246,7930,988,1951,10789,5878,9853,2323,2850,1942,1136,7421,10230,5130,6313,6929,4846,10267,7890,4964,6829,4357,8582,649,7241,6716,8258,1176,9263,5390,6994,5921,3192,1069,10796,5908,6486,6138,8562,8578,6925,1408,7917,8045,10286,6671,2352,1312,10309,8210,800,6050,2686,6214,8456,10302,8858,3443,418,8005,1445,7115,1824,5690,2569,2917,5197,6749,8643,1537,9977,5648,3997,2600,10395,8360,2861,1014,9790,9865,4095,6124,1666,9189,5160,8186,5137,10828,6025,1847,4932,1249,1299,9530,7282,6990,9223,3720,3818,7987,3137,7670,286,4731,7654,805,9668,10018,230,2467,8638,10084,8422,3481,9811,3285,4877,9281,10307,10801,102,530,1819,4918,6546,928,2930,6678,10253,10151,4094,8430,5056,7400,1313,10461,6039,7992,6664,6502,10644,6424,3383,4302,7186,8342,3559,310,10379,9446,2277,1116,5778,6538,1963,3185,1487,3294,10362,1207,6921,8181,5236,10353,5709,7359,10011,4147,910,2476,5337,9213,9916,2696,2391,7286,8797,1796,2411,2570,7990,2889,10723,10150,171,472,6246,7280,10100,8298,4517,2775,7831,10582,2365,3183,3670,6432,9074,7993,4852,646,3645,4052,5323,5556,10807,4014,5349,7109,1296,7125,8147,10043,10085,10709,6906,2251,1058,3842,1132,10328,6005,7725,1853,1862,4528,10824,2837,9129,2214,4209,3557,8758,2605,6667,8761,7620,2156,8505,4322,5269,9818,4447,3186,4709,3920,10087,568,7985,8911,2331,6494,10654,7631,2875,5363,6151,6896,5481,9705,3286,7753,5449,6013,6874,5153,1779,1494,4722,6728,4726,8008,6316,430,4403,6359,8762,6936,8704,8677,1111,8153,8680,7700,2622,582,1462,1621,4373,5430,6218,1825,10038,5597,5578,6064,7907,7892,1469,9038,6763,7197,10543,9461,3881,1536,8992,7465,235,4240,7502,1541,1616,7207,6608,4245,3470,2961,5423,9460,3993,1979,3680,8998,6844,6370,8959,3488,3217,2604,3891,3369,6467,6320,4458,9248,1793,8676,7581,10502,730,566,3694,4699,4416,2284,8110,1933,9333,10388,9873,7454,9254,8881,775,3497,1712,7090,5158,10122,7483,10818,9973,9704,1969,4606,2682,3354,8869,1338,10215,4379,9793,694,3725,7365,4431,8302,2752,3255,5392,5468,68,10751,4871,390,457,4705,3181,4081,1491,9246,5982,2524,3721,2372,727,2163,4734,6689,10005,5237,5583,280,5207,3034,4022,5934,6326,2334,7209,2702,4643,7498,10119,5825,8962,8185,5440,2169,1803,1742,7316,5183,1177,2936,5020,6959,5063,698,8234,9307,3086,8831,919,2234,3787,5714,8263,8991,6104,8554,3350,5187,4092,8071,1898,1394,4992,5371,9200,486,7787,8939,6047,2190,4210,3142,4261,1583,1121,1927,270,4948,6204,10255,6811,315,8896,3477,3895,9606,8851,4623,4244,893,7009,10344,5404,2539,3448,8682,6105,8965,9706,2862,5375,3273,242,9218,3585,7027,1112,8232,7751,8332,9015,6507,8544,4428,884,3857,1086,8115,949,4160,4268,6867,9803,8337,6903,10165,5744,3284,5854,1785,2319,3089,8448,389,2066,7236,3351,1466,123,1956,10341,8785,1741,2310,1544,10547,526,4421,8916,7915,161,2972,5424,1301,222,4661,6438,2496,3109,8488,9354,743,9345,3615,10721,9622,51,8829,9000,4890,10182,3760,8921,1867,5754,3265,2399,6089,7313,6339,9285,212,3308,1886,10441,2341,2626,1284,8138,10424,10204,3572,3966,7296,7303,6846,1875,2758,9663,7173,2959,7261,6295,8303,9088,7669,2112,1328,809,4456,3132,2346,1241,958,339,7663,7531,2519,7509,6949,8340,9959,7063,3055,4636,4650,7680,7203,6030,2264,8979,8819,2500,7559,5606,9434,3256,10689,2321,3324,523,8078,992,2736,7513,6385,7052,5970,5483,278,6350,1730,9367,729,157,7960,4446,2110,10575,3544,10795,6554,4384,6279,6659,2623,2082,145,6372,3582,4127,1878,10826,4129,1017,1837,9788,3281,1663,1929,1597,7393,10473,5167,9599,1482,5394,4906,952,2098,3465,9240,5524,3459,1724,9454,6189,5210,8609,3447,252,20,8014,560,7346,8756,9939,6463,3810,8537,7240,6703,3435,2252,8404,4582,4011,285,3744,6435,8127,6676,3492,9245,7358,4079,3555,3329,9868,10154,1435,5189,8220,1275,3926,7007,4464,8647,10250,7756,10798,10221,8891,4723,10161,6297,9083,3765,95,3530,10346,9909,1532,3700,2541,9989,6816,10311,9640,6547,6384,2081,7112,5666,3974,10374,617,5691,6310,1702,2487,2285,4644,3336,3756,1224,4343,987,369,1093,5223,1610,9981,7227,4114,915,1186,3856,583,10701,6518,7343,860,956,3165,4762,2529,10281,1429,3413,4135,483,1449,739,4878,3697,769,5841,4621,999,10557,9149,6955,1997,1073,4172,7653,5518,1485,3287,5466,9746,7146,8953,8830,7521,8289,2198,5300,4404,1659,336,5723,2857,1499,8658,5105,6122,9601,7954,7573,5243,762,3418,6234,3161,2681,4441,4649,8345,3804,1818,3177,833,9727,10225,965,7635,3006,2772,1142,3208,1098,150,3943,4761,3460,148,4784,7496,7263,612,3524,984,1983,5502,1120,7200,4104,28,9675,9544,7936,8763,9603,4304,6014,8498,10131,1367,2580,5959,2016,1205,1351,3288,4457,10673,8116,4406,1206,7046,9177,5683,7707,3547,2045,4376,1516,2865,4622,3207,3659,9175,5121,2777,2653,8524,1157,218,6418,3321,10102,2380,7980,4617,4220,8859,2550,9473,3495,5408,10677,343,7398,380,2894,5325,10235,1329,2869,7202,10437,6613,3509,2703,6314,7226,5218,10184,10627,3526,7179,6801,6853,8560,962,10511,7729,4125,4134,4392,2766,2492,5971,6144,2358,2583,1122,9632,2992,747,5145,7409,8392,543,3133,7946,10450,754,4981,5441,2618,6462,1229,8587,6498,9512,954,5848,5688,4759,5331,356,8088,7814,4350,7795,8176,3631,2773,4999,8259,7381,744,6609,2564,5940,1594,3755,6476,6540,5057,10616,9221,621,597,7887,2606,81,3980,2220,6640,518,2659,4538,9926,1733,7948,8018,3004,7363,2021,6992,2342,8483,9373,4527,4419,10224,5816,10623,2693,2160,2963,6773,74,4684,104,4898,4809,1674,7499,10696,4695,3978,2699,5214,7315,829,10410,1145,803,5951,301,3763,4196,7746,8328,3070,7407,6055,938,10522,7977,9896,2243,7033,9814,4494,9676,6036,6100,4775,12,2195,10196,1103,5862,3333,3471,6764,10222,4174,8568,8249,8362,5492,4053,5630,7000,1966,3422,1711,7682,295,865,7361,9652,5790,5626,3821,8409,9859,6363,8860,5989,4087,6395,9848,8308,4341,7520,7616,2671,6836,4860,3014,9789,9205,1918,5053,3335,9542,320,8884,5204,1442,5558,8250,9872,5952,1424,3709,8195,6140,4057,6633,659,2383,5834,1508,4246,10010,7287,8076,4554,3599,10685,1431,4050,7264,8535,5166,8086,8133,6541,4201,3545,9923,5647,4035,7740,10183,4169,1689,2664,5770,1882,1295,6978,4921,1812,5412,5387,8668,8408,4781,3579,732,3899,9510,3082,1914,7242,1557,1912,3961,3250,1039,9611,6683,9820,7647,10220,5720,10173,10330,4909,2337,2654,7133,1896,2883,2448,10727,2793,578,3199,8572,2743,10223,10662,9378,10004,4008,9331,1675,718,4682,4274,6762,1240,3261,4046,1934,79,1988,9624,5601,6621,3153,3882,10144,7429,6430,8986,5860,3225,5873,3960,9397,7687,5080,10674,10778,6193,5531,5793,88,7583,4601,2248,5748,3393,6702,6857,6950,8635,2375,1141,4236,4145,2733,5629,401,8954,5777,6783,3337,1943,5918,7821,255,10692,1434,9058,9629,3168,360,1690,1230,9458,951,9271,6469,5967,3061,7579,1003,3108,7681,8486,6265,4409,6109,8412,5618,6487,5562,6276,5907,7575,5529,10601,1725,9096,6905,7801,8610,5277,2627,5035,9569,3157,9957,7322,1600,6157,796,4460,6627,5685,10805,394,3669,7330,2426,7744,10520,8645,8932,5209,5820,8391,4769,6132,8613,1836,3841,3041,1085,521,5186,9462,7136,6429,9951,5409,1269,8128,7516,9188,8354,10167,10216,9497,9972,7399,8910,6456,2810,10339,10819,1646,9616,2174,4175,6091,748,3661,2366,6654,587,3627,10097,8367,1109,1844,8652,1364,8151,5060,2480,7817,4284,1076,5257,4861,7355,5872,9709,1298,8145,8519,806,2427,4524,3660,6293,4444,1660,3989,9374,7192,5369,8292,3914,5608,210,1002,7154,1302,2855,2412,10211,8799,4612,9744,267,5877,7137,8426,2828,4192,2378,6236,4924,9921,2440,6378,3556,9511,1897,357,8104,2400,10609,2970,3499,1830,8321,4543,7794,2971,254,6223,27,9480,7003,8856,6954,7704,318,2407,1182,4624,644,9496,10318,7961,9257,5695,6231,8894,4186,7976,812,7331,4217,7106,8092,1126,4102,8980,10591,3734,1213,2670,9679,3172,10459,8626,6492,3323,6646,9539,355,9337,8716,3808,6165,10562,5658,818,9707,8958,2134,4866,10317,7818,274,5457,3388,4282,4330,6482,109,561,4955,4874,117,8038,8866,826,3938,7069,9357,4806,4903,8741,2688,6343,2402,9895,9561,4100,10491,2597,3533,1849,3190,8737,8967,9007,4468,1068,10277,2258,4360,59,107,1036,5769,8437,9843,9700,9127,3525,1,7297,1809,5972,2484,9840,9609,431,5800,671,3009,10159,481,5025,6254,9602,2025,1001,8679,7472,8929,3164,10786,5165,7177,9987,733,4959,10276,6199,8022,9001,10576,9036,8938,10189,9238,297,3231,9080,6512,6509,1568,997,3773,9643,1030,22,2192,9407,7798,2813,10039,2088,5173,7170,6652,8536,10436,3325,10338,9437,1977,1919,9808,2430,10240,1107,6451,2784,1817,8037,9633,143,7117,6964,721,4194,4954,9730,8203,9491,1807,5659,9116,9816,6808,1160,1358,4106,1166,7206,371,3636,7656,2148,1938,7394,3634,8204,4770,4938,3896,4577,8548,7530,6016,7461,324,10646,7307,7338,8436,4381,6615,6087,5668,942,9953,1899,5546,9406,15,9684,3552,7335,7392,450,4470,6201,4040,10641,8713,953,5675,4947,8715,3092,5422,6023,5783,3950,1423,677,4710,878,7130,10822,5512,7204,9929,7326,2019,7774,3197,10810,7732,8471,5052,10750,4482,5156,334,8526,6222,9214,6614,7258,6488,6680,8320,5259,3757,9549,9196,5715,3581,5413,8031,5224,6497,7651,1100,10110,4797,7839,3091,10123,10358,8188,2208,5447,4542,373,434,3293,4904,214,8494,5450,1721,9828,9961,8747,4365,110,4910,704,9155,3504,7235,7734,873,10448,7667,8824,3796,7971,4605,10003,917,4066,705,779,1612,8861,5260,7295,3027,8487,3823,9826,4459,8941,6715,2085,3517,4808,7075,8272,4500,9656,6927,8405,1386,4737,8475,8365,1813,10088,4996,6029,6993,3364,4491,4885,4287,3667,10170,9325,2314,1129,3911,7830,5924,2676,4934,9403,7940,1982,5054,3028,9693,8633,454,1944,8599,9833,8595,41,1400,9291,25,3594,2697,8945,8840,7290,8963,2043,8431,6618,7978,4635,1669,7879,4019,3811,8168,1791,7099,6588,9120,8107,2633,7467,2490,3905,4697,3652,6742,1718,3987,6206,8467,3716,8875,9504,7822,1874,2899,2537,1515,7730,9667,9388,8007,5294,7185,9778,9451,446,1512,4152,9894,6948,6416,8189,10042,7024,232,648,1425,9531,10391,10715,6568,6353,1441,4423,470,7962,5376,2725,4610,9079,678,24,794,963,5479,2339,5023,1627,9260,19,811,1410,2506,2086,4218,7015,9600,2022,9466,7222,7410,438,2732,4654,7252,8826,1766,3483,9081,3998,2315,2767,5503,9724,4060,4507,4857,5360,8310,10600,3211,6448,7386,3790,3872,1343,5636,2280,2384,4015,5068,6328,7697,7538,1588,2760,4701,5706,2338,3925,9966,2674,5599,1985,3879,6636,10366,9877,3042,544,2724,429,1957,8924,1634,10106,6242,9852,1884,1799,3610,299,7696,1144,5923,5194,4467,5813,1925,5177,5554,9889,886,6244,8968,9883,1910,1661,4438,7766,2194,5792,7610,8874,9729,7166,5700,8477,2091,4076,4962,9797,10400,2179,6539,1243,3452,6322,7091,10164,6564,8879,1637,2222,8434,6026,9485,4235,10564,6164,10599,186,7953,1223,6484,630,2034,6225,5991,5787,2344,7376,6979,4462,204,10621,6747,13,302,7329,4339,5983,6338,3113,3397,2213,5772,6603,4140,8357,8048,9752,3875,4177,1106,4179,2789,2995,3774,625,797,4188,333,4093,7495,6043,8057,5433,9839,1038,9750,1763,8427,10470,8601,1471,7956,8118,6848,6237,3794,5899,7245,1913,8242,6565,5329,442,2644,986,2983,5684,7060,368,89,10710,8027,2481,10632,9061,3313,10698,5357,6054,2221,4615,1578,9878,2509,4810,8126,848,1303,2489,2441,764,8255,7065,1527,553,2628,6282,4518,6331,8096,3017,2039,5350,3657,6795,2357,6577,4296,510,5627,338,393,10407,6951,9090,2798,4589,8606,6434,868,4778,631,2964,231,4594,6665,3945,5427,1293,8655,8499,388,3633,1393,2877,8319,6980,736,622,8813,3797,5883,3433,10160,2786,9567,9022,5739,7920,8966,1083,4771,3532,10697,4826,137,2182,522,8984,9982,10264,4017,1974,5652,1981,2910,7029,2990,8724,3505,4651,7819,4345,5497,1928,7782,8949,433,2403,10288,6751,6342,4712,7684,9683,1035,2844,2317,10157,3858,4370,3883,1554,7061,8373,8015,5686,8795,6278,3683,8686,5493,5168,4032,1748,10790,1458,1432,7436,8032,7625,8651,10815,5998,10506,5202,8688,3678,9343,6510,1629,3182,1353,7419,3785,2472,8393,7761,1355,6332,9681,9931,364,10678,854,9098,9691,10792,5289,7764,4230,5069,7055,3955,3576,4567,10194,6414,6766,8956,3269,2997,3888,144,1777,5477,7013,5192,3642,2311,1336,849,1601,4968,7482,8135,6404,3124,8622,2949,5403,9269,1256,10521,9091,5486,6970,2815,4311,1893,7183,85,2292,9856,7984,855,1750,4279,1713,7002,7626,1705,2090,105,4687,9153,2294,10411,7939,6707,1672,10284,8545,2609,10714,8589,1757,7469,7217,2263,5027,1745,853,4429,6758,9651,7144,9286,7676,8527,8155,420,4443,1845,8082,6611,0,2794,9547,639,1222,6115,1572,9903,9495,5545,3243,9621,5182,247,5581,3289,3729,776,5095,2379,243,8744,2295,3732,4725,10544,2745,1246,10024,3345,2076,1155,627,6939,2454,2839,10259,641,5988,1808,7401,9723,6734,1294,10155,5818,1260,7904,10671,7750,1061,8224,4013,7194,9030,9482,1505,8012,10185,2293,8551,520,4831,10451,5002,1628,244,23,5459,3152,3566,2923,4389,2508,5984,9363,9138,4756,9871,3200,1729,8922,3064,5090,4626,491,1968,7768,4039,7882,6317,4822,10244,6947,6559,5,188,4424,6963,1879,10447,8269,9576,3865,2632,4003,9368,3484,8089,2790,5472,1922,6723,5553,6057,2734,4989,4706,397,8384,8002,6233,3469,1576,8470,9846,2836,9521,1829,7442,8771,7731,2751,813,9051,5637,863,329,8056,5574,9779,10558,9234,8923,366,10036,7320,8997,9522,5301,3292,3327,1113,5242,5602,5876,3817,6972,1054,7870,4310,211,1290,4005,6888,8732,7384,9564,6837,6044,6205,5655,10017,4824,2938,7281,10462,5091,6918,5488,8481,7828,5498,7613,3405,3648,6007,5634,10209,10109,10516,10466,10688,6501,2023,400,10517,10399,2297,5079,4148,6248,3482,5231,441,7759,9244,1064,8564,8973,711,10711,5525,3420,2526,4757,3759,8142,5750,1787,6232,10806,3730,4483,4255,2811,2707,8295,9699,10739,9911,7622,1784,6592,9590,9556,1687,7643,10265,1620,1119,8451,4198,10519,10607,6446,4227,4943,6810,5061,9364,4570,2738,2640,10126,8694,9666,5043,1883,8233,904,6081,4532,3553,7533,3507,7636,6788,9128,995,4867,9560,5190,1115,2771,731,9777,1653,6919,6776,10262,9881,1196,3386,6390,3574,8693,913,2714,2713,8780,674,3478,550,686,4786,8990,5885,139,10416,1684,1811,5147,5614,5178,2662,4080,3141,6885,4211,307,1232,7565,2704,7598,9383,5930,7348,9999,8415,5980,6965,3236,6685,8251,2065,1573,4893,10245,2051,1239,4581,3425,2594,2740,3394,2607,384,3646,3567,5732,10820,7380,9440,10827,7490,1194,9984,4435,5587,901,5828,3008,7045,8749,6974,1547,8225,2202,6968,1602,9124,8466,509,10066,934,5022,2542,10193,1852,5339,8180,6639,3263,4159,5632,1590,8059,3952,7225,158,3494,2723,6447,5625,10732,2864,6698,3906,7974,4002,10044,2673,1008,266,9293,1999,7714,7968,4112,7779,9610,2266,7515,7274,8558,2139,8725,6067,4642,8930,374,8485,4717,173,2575,3832,34,2449,1346,7854,7675,8378,1743,7178,7022,2495,5542,1668,40,8230,3849,3363,10006,90,1143,8636,3591,2779,2994,9997,8281,10483,3463,4899,7722,10830,8492,4136,10368,7437,5013,2783,2592,9110,10227,6638,6650,7827,5455,10583,4338,7387,4158,1318,6757,4126,5434,1931,7856,9721,8597,1209,737,4474,2718,93,4337,5102,3317,10432,83,1339,10534,572,3788,3298,8604,4176,8889,7836,9505,8400,3246,9552,3390,7599,9630,7850,6075,6854,6077,2687,866,6668,9206,3781,3587,717,2312,6600,6407,6341,264,6952,1579,2558,2532,6562,5389,3970,7034,293,7777,9506,4873,7247,9140,10493,464,4225,5639,3134,9195,2386,5603,2578,8533,9782,436,6893,5590,3596,2083,9908,5891,5064,9589,8760,1693,5571,8837,168,6382,8512,4334,3711,10474,9655,1704,7057,5724,5673,9735,5522,7289,6944,2787,1709,6198,1131,4555,2999,555,7781,8812,632,5743,6513,3414,2336,1682,8988,8087,2343,4412,9424,99,444,5874,10460,9017,6094,7614,2147,5125,10117,7840,4637,5104,1285,8602,1091,10113,9887,7576,10605,3258,4018,276,9324,7641,9276,10477,3054,9256,9450,7105,3953,5909,8717,6504,2715,2154,9197,8828,5929,3621,3016,7042,419,3847,9320,1334,3782,9312,8699,3228,7715,7370,9313,8346,4425,3047,8838,9049,6352,1340,4492,3356,7095,663,4660,4119,10146,3129,7369,2040,6674,5527,9259,2322,64,7858,7019,8417,1746,7285,3332,7129,6814,4796,8274,3083,2824,3001,8621,3746,5344,1575,10357,4758,5436,4896,1060,8371,426,5031,7991,5916,1335,1345,8444,2162,10791,4539,9850,7051,2270,961,94,4578,6521,9290,6724,3809,6306,1644,447,7084,9623,7811,8895,2410,2493,4839,6003,7889,8892,3674,1283,790,2118,1473,5070,9296,469,9447,2128,3252,3793,10408,466,9013,2776,10452,6135,9187,6662,8183,1605,3695,8077,9430,5955,2376,7511,1894,7304,8703,6063,6672,5462,4405,8742,10120,5593,2475,208,2431,2672,1288,2582,5519,2586,7843,353,8981,896,10275,647,10456,3689,3802,4317,6663,6118,4185,4708,395,4829,3538,9278,471,5010,8531,6345,3586,6688,120,847,8804,2494,6987,1125,7031,6879,2730,6717,1033,6709,1562,2274,9686,7735,5942,4863,10572,10404,71,2968,2067,5717,92,10412,3311,6833,5509,5794,4785,5042,3656,8366,3020,10494,2405,5758,10536,9541,1920,6696,6287,2756,31,10430,349,3571,7708,3037,5256,5321,10554,7529,9733,10179,5227,3702,9785,680,10140,4195,3381,9638,7385,3643,9165,10604,3355,8044,4603,580,7455,9274,6389,6169,8254,1630,4696,7633,10327,2173,8039,9220,10247,7041,6706,1492,2217,9186,9024,2127,7947,3550,7563,4143,570,5633,1816,4510,5549,10139,4149,4450,9112,4078,5910,10132,4414,3806,9204,10713,9384,7250,3444,2330,10071,4990,7959,5900,8010,695,2418,9349,2079,793,6022,8438,3569,305,8863,8888,1397,596,6355,5604,3795,4291,2731,5203,2908,7568,3939,8175,4569,1762,2017,3766,3623,8062,138,54,5884,2240,6735,4455,5961,7527,6660,9060,8855,5785,6392,7688,551,4768,10367,2546,432,6143,5703,7278,547,4440,4614,2031,760,9107,164,921,1262,1858,6701,4692,9105,4354,768,2359,2012,6566,7820,759,2144,10251,9156,6096,3370,2226,708,1514,5835,8811,10060,4184,9763,9838,2553,6095,2739,7630,4521,10515,8615,5831,2939,7650,6491,8999,311,8060,8381,9298,7748,4314,10356,6823,788,3150,5169,2525,7913,8479,2035,9235,5396,3833,6781,5501,2126,8389,8729,5767,8543,8899,149,3676,5028,9026,8523,3229,1778,2279,3562,6931,3968,2388,1586,2084,7646,331,2309,9358,9410,6419,8661,7981,2872,5966,3249,1438,1179,5514,9442,1421,7544,10463,10092,6813,7402,10433,2096,4264,3254,2188,7571,4693,9111,2183,3233,1270,9930,7201,6177,9326,6128,9272,10283,1420,482,8918,5904,1717,10166,4410,7447,10504,1500,7996,3302,62,3277,9151,7220,9469,3629,3965,3506,508,8207,7776,8825,6989,831,3439,8361,1092,1570,7605,3019,7205,3437,6454,3328,5547,6280,10300,5808,9936,4473,7111,8532,1368,10527,3775,10439,8290,3605,10169,2393,4774,6741,7802,7435,1375,4064,9210,1010,1274,662,6711,8865,3214,4099,10299,7104,571,3178,8753,2625,6461,2033,1379,5641,2095,5965,7249,6797,1947,9280,2643,8791,4864,9821,749,10294,5385,4132,8897,5726,3564,3558,6499,2032,2834,10634,10389,4163,8820,3038,6585,7673,5247,7352,9830,4171,7151,7824,2244,4848,10371,1211,8876,9133,689,7397,4352,1769,10832,6356,8081,2637,4926,7169,3409,8450,3934,1080,10061,693,3030,4740,5784,2907,5576,7288,10670,9125,8903,5886,6548,907,9950,268,4553,7424,750,5947,5377,5663,2235,3769,7615,4239,2501,6572,1559,745,1495,10273,10681,8489,4395,10550,4749,7145,2825,8650,10485,8134,10023,8463,3291,6412,9078,9224,9431,4944,3105,1422,10455,5507,4180,9075,9123,9438,6986,3671,8386,7507,7716,9927,5284,3589,1448,8503,7832,1584,8714,362,5802,5564,4604,10718,5437,4307,7506,8890,4913,4146,7823,7181,4280,7362,5384,2890,3886,9117,9728,10687,182,4183,10770,3859,10228,676,5251,690,1443,4927,2540,5520,9837,598,2513,8190,637,823,5487,4275,4862,9484,6360,8063,3762,1474,5032,6158,10419,3431,7602,9279,2841,1635,9928,8617,1959,5836,3440,4197,6061,2932,3375,7550,9791,5273,7964,10249,2579,452,2271,10518,6111,4096,1647,1753,4212,6798,5378,1700,7876,8594,4033,1869,5024,2620,9011,8095,6478,5881,9319,8205,5978,6779,8200,4074,8850,2269,4073,9222,3487,4519,2885,7160,6172,936,2458,7834,9468,7763,3430,1707,187,6770,7770,1055,352,7783,10737,9401,7167,1991,5240,1180,9121,1861,8395,2268,8957,5774,996,8605,3772,8940,4836,891,321,7988,6534,5671,9144,9301,9754,8849,2554,1546,7425,4746,10070,10742,2061,1782,6038,6250,591,1257,1391,1954,8206,6307,1903,9941,3093,4348,7666,2996,6904,9100,5694,10143,10663,1426,607,8201,10035,7107,1371,7747,2042,10206,5327,1992,8420,5391,6752,8538,9396,3479,7849,6514,6892,477,8969,1585,192,5669,516,1237,1833,2979,4488,9753,4362,176,1916,7215,5365,3213,1563,7973,5557,8685,2619,1200,3502,10104,4851,7267,8100,6858,3062,4062,9876,2821,645,9604,4742,7118,507,5987,2668,579,9459,6125,10813,10046,10202,70,227,9483,10606,10332,3198,3044,10672,774,2601,1251,5953,559,4109,6563,8803,4832,1094,4911,4514,4549,5225,9428,3931,9381,10578,8473,10305,1506,10,5012,2587,10195,10434,1162,3347,2556,976,5152,414,5445,7588,4680,1210,5566,4027,1651,10013,180,9734,4150,2216,5584,9168,6426,10533,403,6391,8317,7471,8773,6772,1072,2835,5935,4951,8382,2799,9835,8079,629,2596,8525,7123,9678,113,3992,10149,8364,761,3111,519,8672,1521,7119,1011,2057,3940,7139,1236,1450,1930,1307,1191,1147,17,8074,8428,1509,9136,7293,4515,975,1388,4537,5672,5513,2611,4156,1071,10105,4453,2998,9717,875,8084,3750,10030,9142,9784,7311,6895,2514,494,415,6325,8698,6626,2460,9933,108,282,10589,6610,4323,1427,4830,6348,26,9527,2832,2015,1383,7344,1961,1088,2149,3496,8083,4294,4105,5850,1265,4969,8187,1137,8324,1625,2413,193,5964,9355,8629,4366,7933,7456,1821,4327,9053,7859,6520,103,2420,4882,3084,4653,562,9979,1381,10099,2925,8667,10048,10482,6210,10637,5861,5958,7601,1765,5176,6366,7512,10788,4048,10513,8935,10186,1460,8398,6532,6028,3029,10258,7891,3903,4402,537,2062,1281,4813,740,7324,3040,4299,7668,3275,9071,8275,10532,9628,4580,7969,6649,5485,6042,4596,4602,125,8722,6015,758,5287,3237,3391,9664,4563,2981,7427,3951,4618,4312,2803,5268,9261,5992,8960,8376,375,1688,610,9393,2210,8111,912,4994,1214,4355,4733,3278,1168,5352,7026,5812,7390,3929,3718,9800,9249,6656,4098,7543,5415,5334,5034,2521,5228,1828,273,8886,3604,9425,7156,4574,9965,10712,2886,7298,10613,9052,4820,778,135,2132,10831,1761,6851,5039,3179,9452,2929,2982,9037,4683,9106,5150,7958,4585,5298,2395,10130,4221,4738,287,4891,2203,4572,4508,1697,5651,9191,1832,4295,10556,8454,1378,9741,4110,4301,3155,9897,713,5136,4681,9141,5138,6088,5157,10773,2485,9499,5644,1664,536,7580,8598,4088,6422,10257,6400,5551,1440,1534,4084,4633,3051,3949,3767,1946,2780,4946,2940,899,2278,10289,4063,2660,9418,9045,2250,9494,309,8245,8691,6616,642,2639,2985,1459,3235,7388,455,6579,1315,10260,9618,6197,6460,498,5290,5322,337,3025,6245,7440,3922,7141,10354,4576,4933,10292,9582,9429,2102,7462,5919,5996,4978,2333,8043,8149,2296,10028,9657,3012,6768,7692,1550,9824,3528,8871,5347,9179,10579,10297,5159,1654,7082,4167,6812,8540,11,4207,345,3210,2576,5075,10114,605,2340,7664,6736,4747,3473,3747,9183,9171,3850,5276,8994,9310,7149,7253,4103,3573,1185,1633,773,8528,246,3890,10347,9305,5312,9520,9751,8898,9154,10337,4966,9391,3889,1790,3035,2516,9318,8323,7868,1023,10077,1727,3032,10329,2922,4571,3464,9759,2545,1277,6285,4957,8397,616,7861,3542,9680,4689,7088,8521,1895,3741,8457,2856,5112,10635,7624,7665,10808,2450,4297,2950,6168,6967,4584,6179,10420,3187,943,7443,1215,2962,5896,5484,8689,4469,7147,7945,3607,2111,4479,3136,5047,4051,8050,8336,10015,2394,2573,4305,9963,4509,6916,3462,5254,4318,265,58,422,8927,279,5718,8069,6527,10620,4795,9766,4535,3189,7712,6998,10263,7096,6186,960,8809,5324,4206,4858,5143,4427,9101,4118,3352,2848,4776,10649,3175,9536,10208,2591,2944,7230,8912,7660,1614,228,2115,1385,7043,4437,8662,8349,914,10319,2130,8649,5249,8294,9152,1461,10162,1169,5198,9535,7018,399,7658,9755,4451,10107,3866,506,1526,3283,6274,7184,7926,2870,611,2884,1865,1984,10758,4449,7617,1706,2728,9443,7374,651,3592,8476,9028,9516,7453,3752,8774,6731,7011,6912,9047,10763,8654,3789,6093,2543,1009,3647,2101,3518,8563,1662,6686,2716,10072,4495,8603,10387,5306,9696,8508,8775,998,3658,21,9772,7972,10274,9585,2572,4289,564,8579,970,6209,6358,1235,1250,894,9579,5949,7248,1856,3268,3202,525,3406,6090,872,3916,9262,5442,55,7089,8429,215,4461,5749,9351,5559,8783,3624,2598,7611,8141,3432,2807,5399,1645,7458,4004,9925,7703,6684,8374,5238,9540,5565,6226,8421,5782,9113,6632,5582,1428,5438,1350,2694,3338,9419,1501,7360,1800,10573,6001,3238,9847,7769,7848,9593,7594,10254,5191,7481,7963,4161,9551,2355,6651,7846,9455,3480,6086,376,6604,9858,2286,4041,5267,6315,5875,6999,2967,9344,4336,8490,1314,3827,3736,8628,6305,152,5586,4720,8351,7064,7373,4091,3303,4390,4727,3815,900,3115,142,8009,359,590,341,10413,1075,5869,9595,2711,5920,9769,4541,4986,3219,6761,7431,1470,573,5976,5840,593,9598,10829,3984,9532,972,9550,9126,8584,2866,2593,6841,1923,7999,1463,2602,1905,4707,7600,200,10237,2819,5960,6033,10489,9243,1059,9768,9033,7097,3127,5065,2215,10745,6364,1545,10598,7391,7412,9577,4228,4783,9342,874,6,4732,725,6323,8143,8478,563,1737,7679,4868,7541,3339,10428,8035,8307,3828,5292,2491,6861,3474,9376,4984,8926,5745,9562,594,4802,1598,3637,8779,4358,5108,6053,8709,6580,7910,10266,8253,1736,9390,4560,489,5768,2368,6743,262,10490,3851,2991,1027,2993,581,3300,6809,3358,8496,3655,7176,8501,2559,236,1577,10019,1233,1926,9426,9050,4332,10236,9089,9992,2133,9230,2617,1685,2422,2900,4736,5111,8720,7062,1289,4021,3123,7415,9150,8806,2136,9501,1873,2225,5894,7800,9377,7845,9006,3649,8266,5101,5469,6318,4219,5098,5847,4170,2093,2536,6398,6137,5912,1015,8718,9812,7629,1341,292,8172,136,8723,511,6933,4351,3976,1641,5420,189,9427,827,5370,9382,9529,4914,3563,7957,6710,7138,756,6017,1047,4293,6606,3988,7432,9822,197,8112,1405,2121,4199,4715,10476,4870,5735,6099,1370,5826,2507,7893,6648,10108,8484,1360,6661,6402,7414,7349,7433,9251,38,4499,4619,3399,6136,5986,1976,5833,2301,1948,167,4844,2327,2438,7444,3919,6396,5456,8590,1446,9861,4611,5058,10406,5656,6771,3516,902,8252,8212,851,2759,9021,6530,7577,1972,9201,9295,1219,5407,6493,2629,69,828,6146,9740,4086,3705,281,3975,5756,1244,5418,3704,8989,86,2926,1859,8158,3617,8800,4685,6262,1679,9940,937,5550,8102,1276,7350,1571,9969,4958,4434,8343,5607,4620,166,1456,567,6311,10542,8218,6767,9392,7219,7671,5179,2052,8174,8196,67,781,9413,5362,6840,643,4987,2612,6334,4817,5097,9712,955,2933,9275,8013,9334,10397,10535,37,1613,4085,2753,10784,6405,6631,2175,5059,1539,191,10242,6235,7228,9103,3131,6708,8213,10648,6410,5213,3334,4472,6277,2802,7632,4657,4442,2056,4153,2267,3632,1451,1476,3641,8313,2099,6923,8627,1022,6041,6387,9817,1756,379,8140,4887,6268,1917,9978,6062,10498,4029,1975,9922,10569,2665,2254,5206,3326,9282,3374,7677,8413,1855,2299,2408,7897,3778,6217,7077,2816,2473,5573,6249,9184,10079,7537,7300,3419,8664,289,7896,2468,249,4841,3267,7504,7545,4286,5552,128,6726,8577,2406,314,5839,3893,9215,5132,7875,2651,10559,8024,1497,9697,2241,8407,1964,499,6587,932,7341,2089,10213,1286,229,1433,479,7755,2581,6570,4124,969,7251,501,8260,2028,4534,6417,97,10145,8847,8816,5296,4292,1552,9161,3846,10765,1311,10296,8146,6473,7175,3693,330,4928,6052,9130,1208,3918,6785,4164,5763,4497,6397,3917,1198,8173,4090,3498,517,2969,10168,8156,1104,5622,10497,3515,8222,6551,10128,3380,3979,8882,10467,7829,3410,883,10431,9639,5791,9720,8700,4792,9247,5727,3771,5858,8546,468,548,3500,8646,6453,746,10446,459,9674,6253,2120,8472,1044,1650,3860,7778,9998,5829,10804,2498,9094,8331,10089,4561,2108,6815,667,4892,3389,6221,5341,2955,6174,10630,2451,6956,7773,10232,6549,248,528,6597,1411,60,6669,2114,8455,4920,6027,3726,6516,8474,9508,2363,9716,308,3039,4895,96,5476,10205,916,6195,5262,7131,3511,5516,1430,9217,4191,7301,4448,5598,9093,6161,7025,9710,4767,8055,5773,3843,4960,8705,7826,9457,7524,4359,4346,1102,4466,3310,9339,5282,10191,5460,7694,4042,7844,4502,9644,10333,3838,669,7428,515,7564,8854,6567,6725,177,635,367,6803,8358,10734,4586,10313,1124,9949,2453,7949,7050,5678,577,4640,44,5107,5661,4439,6409,8136,5356,1025,7595,8736,10384,7525,4728,8608,4290,2712,5538,1217,7872,5702,5635,1479,8072,6834,2928,9143,3770,9283,10200,3468,7333,6922,8663,5660,2595,9920,6428,3036,7012,5957,9315,3118,5788,10378,10659,595,6255,4779,3224,10510,4991,6175,4936,386,8901,6586,4837,4694,1324,8350,9647,3829,763,784,5687,4380,5969,10198,1732,8326,7191,1788,3282,880,2502,2461,2018,5962,5272,3776,7171,2003,1911,8239,2915,1794,5811,6839,3163,7475,2860,9085,889,1774,2958,5517,5532,10285,5775,9919,5335,7888,10738,8497,3758,5004,1498,9309,91,3677,9557,1617,9314,7597,9792,5037,8793,7644,7967,1365,5994,7079,3162,710,5926,2196,10651,5008,2852,5981,7911,9780,4811,3863,4833,4155,4298,4028,6294,5193,558,2574,7150,8423,2954,821,9669,9181,9241,4593,8702,3220,4505,10210,10705,846,1804,10746,9747,6938,2055,3611,1716,7874,6617,9745,5081,10009,1945,1152,8377,1658,929,5765,7567,4724,6374,7,10138,1007,5086,8179,2255,9614,6784,2853,2512,9010,2920,7871,6825,6697,10373,3485,8277,7100,3344,2037,782,7048,8166,9967,3614,4043,4385,8769,3204,9353,8285,10679,10219,10586,1066,6420,6537,6511,7608,2871,6819,8934,6083,1710,114,10793,4859,3472,6354,8229,9302,8873,9515,213,6472,3069,7188,7021,6505,5882,5336,8157,3067,1158,1496,8596,3936,7812,30,4850,10234,6368,2522,153,8049,7619,5241,6557,3087,10706,2166,6722,3835,2943,47,6519,9917,1866,5795,5448,6046,1439,3708,2882,3215,7291,4729,6786,2648,3608,3933,1703,7121,8625,5303,10597,814,100,3820,372,119,7357,8070,2649,10617,4856,3024,9534,7791,1436,8375,10695,5721,1105,6266,9054,10593,3512,7883,2072,8500,8309,5612,925,7466,10192,10040,5915,7078,3628,4342,1767,10377,3877,9587,461,9985,5500,8868,10111,2000,8248,2206,9409,655,8586,9321,8341,8832,1781,2078,6787,35,6558,9146,10082,2452,5000,5699,6485,3151,3011,5526,8469,6555,9335,3279,10665,6256,185,8316,277,7637,5619,4223,234,4388,8198,10435,5905,6021,10753,2847,3698,2253,1352,5624,832,6471,3662,134,1540,5657,4901,4205,715,4241,1715,7196,9765,4512,5736,10537,8573,7234,5730,10549,9968,926,9986,4413,5880,10809,5544,5892,3085,1880,10306,3259,3973,3367,7780,1839,982,9099,8581,8600,772,5302,2219,7863,3913,6085,5572,10369,4142,6134,2764,5291,9476,1551,6303,10702,4007,5317,2989,6937,1854,2201,6182,3074,4344,8782,6641,3999,5461,2150,8908,2677,5364,1965,1531,115,7549,10581,6019,8943,2749,2161,7470,7275,2151,5470,2851,1814,2445,10360,3520,5814,10055,1019,7607,3620,502,4883,10574,1389,3260,7108,2171,5411,1681,1409,8719,5755,10749,2655,9300,2511,10037,5011,9528,3508,4794,1084,3548,3125,2030,10643,6004,2534,1457,10256,8066,2232,4069,445,10766,5265,8169,9654,6113,2942,7995,6605,3403,8576,8291,9193,4036,2599,945,3244,4656,787,9645,2746,7749,10324,7299,6793,5089,8885,10348,7792,8529,3527,1888,9002,4666,7269,10622,7815,5846,3578,6406,692,3045,6369,8776,3072,6071,6746,7526,6900,5062,8011,9658,392,5817,5623,10453,8214,2722,4411,5443,1181,9498,1028,2138,66,5073,5605,3540,6440,7271,170,1188,1163,7934,1751,2224,6073,4,3052,1065,1369,1801,2499,7895,5346,10802,350,2988,2849,3619,4937,6455,10156,10041,7690,8103,9369,8619,4257,1282,5676,9827,1138,8631,5201,1099,9032,3668,9673,2146,5226,6251,895,9841,2005,6152,3927,7243,7448,4544,4231,7083,3731,3272,8293,5116,9350,8711,9137,2167,4407,4065,10047,7584,5931,8064,259,10301,9884,6692,6500,5844,1835,8021,1881,1673,6677,240,785,5465,4258,957,3742,4189,7040,1892,5083,606,9132,5506,2320,10501,3915,8530,9147,666,2187,225,10096,6074,4974,734,8852,8515,6694,7059,4056,2024,5798,10686,7659,3402,9990,9934,7364,3257,7438,165,8637,6240,4484,3126,3714,405,196,3514,3900,2804,8284,5045,1564,2754,10158,974,1173,2859,9233,10103,3839,3885,7606,9202,4044,2364,9798,7922,2318,3737,7164,8480,6543,9952,7860,1149,6778,4773,9474,6924,3407,983,9379,8728,7336,3076,8772,3110,8065,7080,10509,5148,7256,9648,9701,9832,8517,817,4202,3455,8453,681,458,8549,9836,3897,2356,10783,8974,9048,417,6302,8208,4721,5451,9620,1996,4139,7816,5421,8796,8119,3147,9465,6450,2068,6112,3167,10486,4807,532,5753,5824,9034,2304,6309,485,272,8802,416,5807,5742,8325,9739,9890,9625,7342,8585,3967,7757,843,4900,575,4814,9771,4523,6800,8942,5117,8707,10279,4430,6997,9199,1582,9292,7404,9976,3531,10093,3523,219,6155,6006,1632,9042,10422,10083,3673,9698,4546,7662,6059,1699,8880,8435,9994,922,2486,911,3114,660,29,1005,4520,1815,8202,10382,5234,8950,2503,8330,9019,5295,1860,1366,8221,614,7884,1955,174,4536,5974,2708,2307,7738,9242,8706,9039,7446,2159,10756,112,3003,9493,6167,9862,2903,9159,7519,7439,2614,6084,6072,8822,1838,3239,6675,9422,3964,3033,6899,4754,3963,4853,3710,5040,8080,4061,3597,2109,2123,1595,6211,5146,6791,4827,9996,4144,5569,7942,1159,9943,7572,1639,3672,10359,10690,4525,6196,9481,738,716,4489,2685,10147,4248,8583,6457,3387,10614,10364,993,1037,1404,10080,9258,5580,10403,3854,4116,5118,10669,9891,9737,5665,2469,10241,5489,6971,8936,6880,5977,5649,9228,3158,7784,5115,1574,9316,1797,8385,835,169,2288,8864,4121,6575,4101,407,2421,3898,5395,5616,3453,1454,5431,8122,156,1864,87,6008,7652,6427,7356,2260,1090,4823,8447,2457,10280,1631,1538,950,8406,1279,2966,7552,2347,7259,6394,10560,5868,3560,5823,3451,3046,5019,2414,7785,7929,8217,3320,4226,864,6148,6673,6490,3400,1310,9492,2348,4415,968,10323,3990,5776,9900,9336,1046,7148,8798,5151,2960,6178,6623,1204,4454,1760,7168,6117,1146,10719,10458,1649,5397,10282,10584,9794,3475,10402,10392,10762,3270,413,1759,9711,3664,3306,5274,8678,652,8665,8416,6408,8046,1624,5865,9687,8657,5888,1062,2184,1164,3280,9055,3097,5372,424,1287,5887,4647,7493,6962,10740,5382,6805,4082,8845,3590,7270,2228,2630,3021,9731,524,4700,378,7771,10656,3398,6644,990,1322,10016,6855,5261,10799,9323,6720,6162,1252,7898,487,4787,9439,5898,7813,3728,7689,8353,45,8231,2275,1665,709,2700,7044,6229,9875,3411,1846,5567,9855,6894,658,448,8410,1416,978,9570,5480,2329,290,4719,6830,7788,5473,8767,6832,2750,4270,2778,4501,1530,4417,7262,9398,3441,8026,2141,6700,4803,4545,3201,8518,9432,5734,3969,9904,8241,9760,10652,4558,5677,1535,3251,3971,10052,2199,5751,9956,7627,3299,10078,6578,4000,2567,1193,1453,2231,1398,1714,1507,2613,8977,1701,4465,5809,820,5049,9517,5595,5917,529,4815,224,3910,7054,3727,1939,7937,6884,9636,8394,5682,5405,9487,4907,9554,9743,10322,6208,9414,8843,7213,7505,10188,7604,7837,9860,10351,2087,14,7212,5999,6474,209,9842,10468,3271,741,10611,5142,4020,1870,10141,2070,238,4670,3450,3188,2479,8304,1263,9502,3330,2361,9077,160,4884,2948,2011,1032,1652,9131,2428,4917,6908,8219,6141], "time_ms": 0.158165, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 613, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "generated_query_count", "value": 10833, "time_ms": 46.596352, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 428, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "generated_samples_per_query", "value": 1, "time_ms": 46.596352, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 429, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "generated_query_duration", "value": 4061259201, "time_ms": 46.596352, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 430, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 10361.647921, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 898, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 10361.647921, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 900, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 10361.647921, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 902, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 10361.647921, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 904, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 10361.647921, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 906, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 10361.647921, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 908, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "power_begin", "value": "12-28-2024 03:59:12.055", "time_ms": 10361.649214, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 564, "pid": 13810, "tid": 13810}} +:::MLLOG {"key": "power_end", "value": "12-28-2024 03:59:22.360", "time_ms": 10361.649214, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 566, "pid": 13810, "tid": 13810}} diff --git a/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance/run_1/mlperf_log_detail.txt b/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance/run_1/mlperf_log_detail.txt index fd75592aa..a5de1b861 100644 --- a/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance/run_1/mlperf_log_detail.txt +++ b/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance/run_1/mlperf_log_detail.txt @@ -1,89 +1,89 @@ -:::MLLOG {"key": "loadgen_version", "value": "4.0 @ 486a629ea4", "time_ms": 0.004407, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 53, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "loadgen_build_date_local", "value": "2024-12-24T19:50:51.839855", "time_ms": 0.004407, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 55, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "loadgen_build_date_utc", "value": "2024-12-25T03:50:51.839862", "time_ms": 0.004407, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 56, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "loadgen_git_commit_date", "value": "2024-02-15T15:46:50-05:00", "time_ms": 0.004407, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 57, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "loadgen_git_log_message", "value": "486a629ea4d5c5150f452d0b0a196bf71fd2021e Hotfix: DLRMv2 Audit Test01 fallback failure (#1626)\nf9a643c0a0e920588da1b51a1d822e1071a9dbec Add number of tokens to offline (#1623)\nf06b920eac2170df38b5502c7443f50c9afa6045 Bugfix: equal-issue mode on offline causing accuracy run to fail (3D-UNet) (#1624)\n0ed5190e82e1ca1cf95626e50e61cb6678c7f2ff Fix submission checker and TEST06 for Llama2 (#1616)\n817dd961221d3ca284b024368130bd5dc5e4b54a Turn equal issue mode off for TEST06 (#1615)\n840435a1abe8be823e0a5a18ad3173bc520aa24f Update README.md with CM commands to download stable-diffusion, gptj and dlrmv2 models (#1604)\n3190d09a148bc1d72c9d946e4cf3b54db0d0e60a Typo fix in README.md (#1588)\nb0777f0bd3b7e5cd6a51d670d8b4df656cf6fb29 Update README.md - remove unwanted lines in CM commands (#1601)\nd7dba0826ac96a0e8bcae525086961e47c409a38 Remove loadgen warnings (#1608)\nd45a66c33bb0b6548d6172b38645bf94ffe24b39 Add upper limit to tokens per sample (#1612)\n44285d9a44041c3d4f7df0acf1e72c375e9fcbd2 Set completed samples per second as llama metric (#1613)\n357ccefca4db55ff373e029d5d1e3e6246bde5d4 Enable equal issue mode for LLM benchmarks (#1610)\n104d85531af7ae2a07ded36dd6e8c7f2c7746d06 Add sample length check to test06 (#1603)\n473053fc9f1b8c5edf60e11ec74607126938fbc4 Fix loadgen token metrics latency constrains (#1596)\ncc3daae3c7f18d7eedc5ffbc271d324e5aca6233 #1598: fix token and sample logging for Llama2 when accuracy_log_sampling_target is enabled (#1599)\n3a902e598e5a17013c26fea975c0a60e6d7feae1 Add support to dump 10 compliance images during accuracy run for SDXL (#1591)", "time_ms": 0.004407, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 58, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "loadgen_git_status_message", "value": "", "time_ms": 0.004407, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 60, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"05d4a62d355a90363847358e58afeee4a54ad838","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/bindings/c_api.cc":"32181da9e161c285f8fe46ddaa49e6cba2f9f918","/bindings/c_api.h":"91f58bd79b83b278f3240174a9af747fc38aff74","/bindings/python_api.cc":"1dc0865130371c6a8ddb97cd8039244d74d6fe46","/demos/py_demo_multi_stream.py":"6ccf7dcbe08f8382ed2d3618370c2b21e205ca13","/demos/py_demo_offline.py":"6c3075538e0a3df5e93edee917f4537099759e11","/demos/py_demo_server.py":"40e37ea121ac08b21ed6af41aca8eb5eda641b89","/demos/py_demo_single_stream.py":"4ee5f048fb0dffc415342a2bc2049fb155bd2894","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"126e952d00f4ea9efd12405fb209aa3ed585e4b2","/issue_query_controller.h":"923d9d5cdf598e3ec33d7a1110a31f7e11527ec7","/loadgen.cc":"6650091ba7a918f343b06eb7a5aa540eae87275f","/loadgen.h":"e00fdc6dbc85a8c9a8485dbcbfe2944f81251c4e","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"197efc96d178e5d33a750d07fa7b2966417506ea","/logging.h":"ddb961df7bcc145bcd7cce8c21f7cf075350dcbe","/pyproject.toml":"974adb632af4b76a1c72b914cc2cf9b3cbc66aa3","/query_dispatch_library.h":"13ad6d842200cb161d6927eb74a3fafd79c46c75","/query_sample.h":"e9187c8612bbdc972305b789feb6e15c26e96cfe","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"a170f991de69f8a5e45616e871b39a30ffed5598","/results.h":"d82500c326c2de83db411f1146882aa4692b419c","/setup.py":"e3e707e1191478332c00f1b6ff806f0ef98e0f65","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"aaf3803e700f3431a5ada72559eef5cce996ea64","/test_settings_internal.cc":"42aa46b17ac3fb2748cd5cde60fa8742e916f3c8","/test_settings_internal.h":"73b11c02228f3afb4c924db7a86fd8a2b5b9e483","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"4c9d98360b507d5673049ed70154fcfb2934f8a6"}, "time_ms": 0.004407, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 67, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "test_datetime", "value": "2024-12-24T19:53:33Z", "time_ms": 0.016785, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1198, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "sut_name", "value": "BERT SERVER", "time_ms": 0.016785, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1199, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "get_sut_name_duration_ns", "value": 486, "time_ms": 0.016785, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1200, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "qsl_name", "value": "BERT QSL", "time_ms": 0.016785, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1201, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "qsl_reported_total_count", "value": 10833, "time_ms": 0.016785, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1202, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "qsl_reported_performance_count", "value": 10833, "time_ms": 0.016785, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1203, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "requested_scenario", "value": "SingleStream", "time_ms": 0.022309, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 268, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "requested_test_mode", "value": "PerformanceOnly", "time_ms": 0.022309, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 269, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "requested_single_stream_expected_latency_ns", "value": 500000, "time_ms": 0.022309, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 274, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "requested_single_stream_target_latency_percentile", "value": 0.9, "time_ms": 0.022309, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 276, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "requested_min_duration_ms", "value": 600000, "time_ms": 0.022309, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 312, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "requested_max_duration_ms", "value": 660000, "time_ms": 0.022309, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 313, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "requested_min_query_count", "value": 100, "time_ms": 0.022309, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 314, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "requested_max_query_count", "value": 0, "time_ms": 0.022309, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 315, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "requested_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.022309, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 316, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.022309, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 317, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "requested_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.022309, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 319, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.022309, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 320, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.022309, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 322, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.022309, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 324, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.022309, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 326, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.022309, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 327, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.022309, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 329, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.022309, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 331, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "requested_performance_sample_count_override", "value": 10833, "time_ms": 0.022309, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 333, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "effective_scenario", "value": "SingleStream", "time_ms": 0.022614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 409, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "effective_test_mode", "value": "PerformanceOnly", "time_ms": 0.022614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 410, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "effective_samples_per_query", "value": 1, "time_ms": 0.022614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 412, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "effective_target_qps", "value": 2000, "time_ms": 0.022614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 413, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.022614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 414, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.9, "time_ms": 0.022614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 415, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.022614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 417, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "effective_target_duration_ms", "value": 600000, "time_ms": 0.022614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 418, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "effective_min_duration_ms", "value": 600000, "time_ms": 0.022614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 420, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "effective_max_duration_ms", "value": 660000, "time_ms": 0.022614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 421, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "effective_min_query_count", "value": 100, "time_ms": 0.022614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 422, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "effective_max_query_count", "value": 0, "time_ms": 0.022614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 423, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "effective_min_sample_count", "value": 100, "time_ms": 0.022614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 424, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "effective_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.022614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 425, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.022614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 426, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "effective_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.022614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 428, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.022614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 429, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.022614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 431, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.022614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 433, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.022614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 435, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.022614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 436, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.022614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 438, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.022614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 440, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "effective_performance_sample_count", "value": 10833, "time_ms": 0.022614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 442, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "generic_message", "value": "Starting performance mode", "time_ms": 0.027825, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 841, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "loaded_qsl_set", "value": [4117,9486,2947,9270,8227,4487,9799,4629,8571,8839,7101,2191,3315,6375,207,10454,3122,9490,4083,6119,1890,7657,9347,344,201,4232,5741,5454,9533,7842,1596,5330,10615,8827,8094,6714,9169,977,6619,9924,4982,4894,6544,10118,783,766,2122,7163,396,4200,9864,4254,8440,1758,10722,2829,2788,9906,2719,5613,8726,7847,440,6103,8567,7497,4793,1184,8675,10587,8270,7074,7408,8740,9265,4855,1199,220,6556,6595,1226,6695,3066,2143,6264,9806,7122,5007,10075,2370,2185,7254,77,2806,6243,6687,1156,1802,7302,391,3348,8159,9708,251,3595,3947,2302,4690,122,3116,1300,3768,6388,2986,728,5127,3904,6807,6259,2218,7555,8821,3580,7804,1321,8460,5419,6542,7833,3365,2290,6346,7952,918,707,118,10469,9023,1304,5220,1891,6878,9524,6522,7867,4037,7698,8507,9634,2145,10057,6263,7486,7557,2401,6082,3434,5927,8708,10081,8114,1349,7742,2742,7157,5705,4638,1764,3651,920,4016,2744,5859,6740,5380,9649,2,5245,5230,6421,4071,8909,6319,288,9346,9671,9646,9360,4702,5044,9157,6296,8318,8971,8184,7702,10475,7318,4024,5530,3991,1171,1909,6230,8461,8099,10563,5398,10760,10693,3688,8170,4952,6827,5893,7767,2656,10032,9178,10480,7195,9359,10608,10261,8975,5432,363,3346,3049,3360,4432,9736,473,7587,9416,1021,6596,10352,2666,2325,973,2071,7368,4267,867,1040,5796,3491,8237,9597,3489,10568,3777,7899,856,4818,2858,4662,777,10776,10026,6065,6481,2590,6780,10553,8789,6032,930,1049,10178,6187,585,3359,5832,2014,480,5866,5017,5141,1525,8271,8132,1642,6270,2265,140,1174,4641,8017,8844,6569,5575,7081,3779,3845,467,9761,3314,8653,4250,7578,8907,664,217,8419,8757,683,4565,1638,5216,2004,6679,4485,7683,5180,7422,6070,6283,2909,6281,8933,10639,4340,3079,3513,10552,2291,7487,2680,8807,8370,6127,7434,1822,908,8101,451,10316,8282,10724,885,1834,1556,6121,1468,409,1908,10539,7238,2891,10312,9435,2178,10051,4522,9650,615,7193,7796,2008,6011,4347,435,6881,1396,844,971,4216,5001,7886,6875,1348,4300,9348,2709,10580,9993,6705,8784,10821,9232,7724,6581,9764,2913,10125,6914,7693,4568,10526,10231,8502,7825,1783,795,6828,8006,4595,3892,3075,1278,10050,8235,4679,9971,75,2164,1175,3684,9869,8414,9016,765,2721,5786,10703,1326,7463,237,836,3912,5482,7835,4374,7661,4070,7754,7180,9945,10415,3638,4208,939,1097,7628,5725,3723,4547,1994,5232,5279,4049,7309,5963,10478,9834,7758,9463,3666,4663,801,4030,3010,8236,9237,2470,3382,6336,7570,190,4552,4113,2229,9018,7038,5021,8701,8268,4490,1373,6092,10181,10445,3056,4333,10394,1640,2328,3349,10069,10683,6439,3191,2544,4760,1739,8900,3,258,5416,4671,2041,9375,2661,6376,2306,5897,6373,2125,1940,5109,4256,6203,6411,9311,8029,2200,6301,1734,6176,6443,3007,9583,1841,4627,2523,4306,2801,5208,8607,4961,1467,10700,4031,2181,9958,8194,10295,9043,5366,306,9056,162,3196,10759,4349,5401,7030,6184,9216,2069,6357,5640,3015,1752,406,10049,3983,791,5610,5944,304,6380,1937,7862,9366,9365,1607,9252,2698,10350,5221,4847,10707,3023,1615,7017,5990,4045,7231,1202,7182,5172,3176,9612,6693,1013,10334,1805,8730,8841,7372,10094,6010,3216,8177,7406,9472,9608,2533,7691,3959,1623,5144,1831,751,4912,2152,8144,241,4187,10001,1924,8402,7877,8928,7294,4915,3876,3242,8996,7153,1000,10658,3946,284,5088,959,10661,1018,8931,546,4262,6489,3786,5278,4308,8178,7638,5211,7142,5771,634,8273,263,2548,4674,9910,624,3290,5925,1524,2642,7198,10345,3539,6719,10735,2006,10233,3816,9578,8624,6769,9661,6468,8952,9702,8670,5175,3449,1773,6130,636,1622,1696,1414,6523,2898,10650,4420,1280,2538,7880,3743,1227,2757,6131,3371,7445,1680,2901,36,6868,10811,7140,1900,10398,3724,8765,8278,7379,6040,7914,1580,3535,8033,7536,6480,3861,2437,5082,2881,10596,4902,8162,9637,4886,411,7189,1789,6704,9176,10045,4107,6333,4583,3923,4843,9935,9955,9297,3222,8257,6365,3095,1995,7924,5428,8640,6628,4819,7152,5016,7932,3226,1267,1225,9025,7878,6321,2373,3801,1399,9063,6506,9436,5781,7500,423,3588,7110,4607,3476,2477,3635,2425,9394,10449,4054,8148,5510,2236,9543,10499,6153,780,8692,8105,10268,172,4128,841,8925,5453,8561,2135,2976,2845,2209,7418,10817,5620,7010,9960,474,6545,8223,9713,205,10548,905,2396,2914,5591,2887,3379,4865,3304,5849,1309,10171,608,7314,6228,6961,7068,6943,4613,1317,121,8611,5902,5827,8042,1139,3121,223,2984,2897,3144,5379,10590,6433,3871,5475,43,1026,1192,1599,5729,6194,5838,1254,2054,2706,7925,6060,2177,10647,7727,5681,10000,3549,10814,4433,2100,6869,6593,9404,5534,5801,10321,9145,8363,7451,9072,4600,5096,10372,3022,679,5631,2552,723,4668,9613,10153,7423,9114,2973,3071,1413,4956,2838,5084,6777,5196,539,5036,382,7480,9327,323,4766,2273,2684,8182,10134,9203,2667,9185,2658,9087,6286,4162,6620,10472,1095,6759,7216,8191,9738,9548,5029,10326,5478,4089,8710,5439,6126,6160,4828,1006,1677,3546,6930,425,2645,4573,2770,10248,358,4975,10603,8129,8614,10180,7902,4303,2447,3427,753,8920,9449,256,6859,5711,5246,623,370,10767,2563,8300,9591,9991,5126,9568,2647,484,8644,3654,9507,8632,10076,9813,10638,4326,3862,569,7020,4213,5628,9341,839,3341,8347,5163,5563,7339,6928,1197,9158,3099,6120,1483,9545,4608,4215,2761,1259,6802,4364,4001,1606,4222,4752,3057,10022,1108,10592,4398,10376,9888,10033,4273,1238,3428,6553,6145,5731,7273,604,2436,3170,8673,5901,10074,4513,8305,8697,7476,5015,2669,3100,6344,9338,10383,4972,8727,1363,3223,10495,897,1698,9553,5662,53,9328,3733,9932,7102,2212,7382,7159,4556,10748,9294,5895,2262,10512,5913,8314,10624,2530,7560,10464,2443,8787,3800,1510,2027,9795,9670,283,2351,4260,2577,1518,906,3377,4764,2113,10699,3948,8465,9264,9267,5600,2924,3783,9209,7966,6464,720,6973,155,6227,7994,2636,535,8846,2931,2046,4953,10058,8228,5200,2211,4234,2207,8121,2007,7268,3212,7772,6496,10201,2762,7534,5539,10555,10496,628,3653,702,8683,10325,8660,8411,9387,326,8842,837,9688,3855,7053,1826,7944,1053,3193,456,361,3206,10315,2059,6612,6173,4562,9003,8034,9162,377,7551,10769,5354,6299,5997,2080,6288,6068,653,9937,10825,8833,3247,3058,4923,9104,7983,3305,9322,8339,7711,131,7328,5212,8623,10761,9718,5402,948,10640,8659,6479,2768,4616,2223,10625,4252,9980,9807,5318,6188,159,10655,3565,1478,6860,3521,4646,9880,1608,10427,9804,2092,4897,10238,6020,7642,8091,7789,4058,6681,4173,7918,5523,2444,9898,7161,7736,4667,2603,9370,7403,10797,1857,3276,2276,3445,8197,991,1863,4266,2170,4331,3296,6658,1308,7542,8458,4533,10177,6873,4190,815,2726,5589,2846,269,2634,1560,7067,7554,3102,4908,10741,10086,5286,8734,3824,1744,4111,3264,7494,3825,8834,6139,8432,147,1589,1871,941,2911,6018,7906,2874,3713,9523,4023,7522,6116,6864,5698,5464,3924,4590,7345,7395,4790,10540,1020,5309,5950,2956,4983,6163,9194,3848,3612,10525,9069,5253,7232,6876,5003,5067,2350,10375,10523,850,6399,10002,9689,7255,1016,3707,3384,2137,4242,4077,5170,8388,2387,10577,2482,3735,4313,8862,2029,8452,10528,6058,6871,5233,1195,3301,8738,3194,2638,2488,5009,6981,7935,10684,10631,1043,1962,3541,4973,9581,4285,9118,9770,8279,1316,6284,1161,877,7894,3981,2423,6872,5033,1798,6181,1255,7517,4067,3319,7457,5164,3977,1719,4436,9317,4765,4137,6515,10025,4628,3745,7739,9445,7016,1980,4481,8580,175,10704,10653,245,8163,3834,7006,9135,1950,4401,2289,1412,7325,4821,6917,1361,7569,6782,2474,8075,5757,1034,3353,8570,2678,4922,4356,6831,3401,10393,10508,5161,2635,1472,3331,3048,9526,5383,5670,7508,5113,10148,2912,8547,1183,5496,5139,1153,6897,7997,6625,9819,5766,7460,10124,6637,327,3466,1848,2455,5939,2282,6371,4812,6721,5314,4780,3112,5154,250,7103,7158,463,6920,8368,6024,6877,342,6799,437,2876,9690,4889,5129,2631,3366,8380,1528,10135,584,6292,9180,5863,10031,2432,980,7211,3117,6909,1248,3761,3601,4976,9420,10602,178,10755,1150,2009,6653,7378,4422,8445,1978,2390,5417,4353,6452,2002,10595,1306,2817,8735,8424,8976,4396,4919,9488,534,4478,8211,8264,385,8288,3063,9825,6996,9726,7239,668,2896,6212,5761,6271,618,8216,6838,195,216,465,8870,6774,8556,3392,786,2791,1096,10565,9399,7603,1723,4133,6106,5491,10443,335,8387,5006,8131,966,7321,4664,5361,8778,8085,10370,512,4408,9227,4676,927,5805,10457,5444,5333,2951,6804,7955,888,3706,8788,2895,1902,9874,7221,5222,10628,6300,3522,65,10112,7284,6826,9762,10226,8642,8030,609,3812,9914,8276,5946,4130,767,4876,3831,9441,5515,7928,4838,4550,78,6682,63,600,3798,8226,3577,8618,8919,4716,10385,5560,7695,9677,9470,6351,10585,6031,2941,2010,7623,1735,6574,1513,2774,1490,2303,5429,141,1915,7737,9086,2205,32,7165,4564,3715,8209,8559,4979,4985,7143,1656,6789,8025,7208,6845,503,3687,7028,5471,9975,5664,3404,9068,5248,2433,322,10390,3840,8058,1297,3551,3819,3342,7265,3266,8914,4394,3583,9786,9198,5609,1444,8947,3060,10252,5594,1291,7319,9559,3166,4669,9592,661,6602,4789,9415,2424,4265,7098,1876,7535,1203,10053,9809,3104,1941,2705,6635,3148,3598,1502,7383,9073,10425,6590,5162,3529,7306,10800,6393,8751,387,10172,2692,5719,2741,7855,4845,7039,2459,6901,6624,3554,4237,3867,10566,2116,531,5943,6690,792,8441,6260,682,3995,6154,4165,9066,2717,1842,5340,8261,8539,1074,4259,701,9500,8648,513,3417,2155,9041,116,4010,1218,7514,3738,10176,8592,4659,8491,291,10417,8338,7244,862,5005,4735,1362,964,9714,10691,9829,10779,351,4930,16,3561,5388,5985,5239,6821,4320,10682,3154,4367,8961,50,8733,4288,9893,824,2504,404,10529,6560,1489,8125,48,76,909,2237,2691,2416,1987,7699,4599,5585,10503,7807,3639,3994,5708,7135,4625,4516,5540,2916,5528,5975,7413,8915,1347,10065,706,5304,3436,6607,8262,5645,3625,1376,4476,2934,1592,1935,3958,5821,2456,3362,5367,4151,981,9573,10610,9207,6190,5275,4631,1708,1921,9287,699,1447,1636,696,3740,4597,3373,9444,1221,3780,8124,5381,7648,3536,10571,654,9064,9044,3503,9854,9626,3378,10675,10629,6080,4445,9605,2646,6982,5074,3618,52,6561,6185,3026,5968,9742,989,5867,9190,1178,2689,2001,6149,8334,2049,3180,5759,111,4691,2650,8777,3149,804,1868,5328,840,8687,1382,57,2419,808,10355,5588,5215,1403,5701,9,7853,8520,8669,8019,4006,9097,226,2827,1135,1901,979,819,2561,7049,504,4391,4400,6739,1402,5264,2230,10666,2566,7430,8459,10720,8794,10310,10174,443,6349,7310,533,7609,6200,4108,613,7558,7709,1553,5076,10645,300,2842,6261,2324,588,2193,2945,7510,7396,3094,1889,10781,3791,554,9284,9148,3519,9995,3209,5810,8322,9519,2560,3073,947,2781,7004,5548,7116,10152,10121,552,1887,2952,8739,9225,5819,3712,6550,5446,574,3090,8327,428,1266,8433,8383,4645,8754,6898,2652,6822,1455,2176,1519,8566,10303,6049,10068,4977,3894,7561,7488,5536,7224,7793,1989,3985,8695,3424,7035,7612,9938,6775,39,1850,7803,9174,3031,6441,2974,1603,7941,7223,7923,2831,2977,3080,2075,9236,312,8565,4634,8814,1558,10484,1272,7389,8872,10418,4755,7869,2610,10633,5046,6960,4193,7852,1754,2878,1936,3784,10314,4025,4805,4997,10007,9863,7155,10101,1342,9395,10291,9385,10218,799,1123,5843,2562,4905,1377,9226,5937,1986,7591,9962,1189,8902,619,9067,3458,7586,7210,328,8506,3000,1004,9212,4591,8020,8887,4012,592,101,3415,2527,72,7452,9400,7809,1561,7032,4840,7008,1056,8,5752,9588,3101,3106,3609,4503,8312,6536,3446,8745,1437,7547,10287,4816,1795,8004,5837,1042,9329,9008,9433,2047,1087,3421,4324,10588,6699,10747,7765,9719,10438,887,9062,6820,2946,4741,527,3043,1648,1133,2918,882,684,4939,4386,8641,8154,1323,6436,7873,10752,3686,10680,807,9886,549,3230,6147,294,7120,354,8171,8493,2663,6863,7489,6037,410,9913,124,6312,541,3159,10304,2117,735,9882,199,4315,9250,4055,3739,3826,9558,10782,3699,542,6645,7593,5123,6413,7532,8805,1517,8514,3396,9239,8883,6748,6002,5319,8690,6308,1691,4655,589,10278,5680,9489,6806,10465,4251,10142,538,5740,194,9167,9974,2107,1480,8439,5722,5674,5712,7528,3385,1906,10567,9057,340,5707,9308,6207,3078,2818,7518,6794,9555,9983,1676,1678,6079,2038,1670,2608,2335,10021,10243,7257,9405,8165,2242,1247,8620,7479,1305,8818,8790,7865,4998,5653,2097,261,5270,2140,2105,789,6984,7501,8748,9538,8167,7566,3307,8768,5195,3932,1130,3930,6535,3665,6818,870,6657,7938,8656,2975,8108,4377,1332,1792,10027,3395,5078,10380,724,6034,3423,4788,4511,9362,2462,7760,7023,2497,9802,5504,6817,4263,3361,8164,8418,2478,2287,3457,1264,9796,7218,6591,6401,1683,3534,7474,5945,9722,7233,3719,7072,9915,42,6386,1481,3426,8808,4075,9665,132,3679,1418,6327,4854,5667,1228,5621,10063,9685,9641,3792,8509,10725,1726,3510,6733,5311,6890,1127,1372,4557,9076,4493,1319,4639,6503,5373,319,6252,5205,2568,3262,5611,3692,8016,2058,10708,1990,10246,5149,6730,4745,9617,4496,8093,3764,4587,9046,8240,5654,7484,9304,9356,9122,9012,2409,2879,1728,8443,6110,2374,2695,6156,6123,7266,9525,1770,9102,10660,6745,5128,5806,10298,1081,5852,10365,5171,3316,1949,2796,5938,2888,505,9040,10137,2063,8296,703,7982,5184,5822,6713,8356,1212,10214,1643,1344,1740,6170,2765,8513,7491,313,7468,5830,5188,7420,8246,7931,5592,2300,5803,1333,1904,10524,2557,1320,9801,3887,3372,4648,8123,10207,10570,2782,9208,3493,8993,4688,9635,2367,890,4834,4750,2186,4059,9823,383,5746,6601,6995,3696,10626,6202,4034,2588,1415,3128,5342,9471,8634,6712,7283,5689,7596,5728,9160,6465,1823,8542,2381,129,3703,5297,5845,10090,9892,8109,10676,1417,6171,9518,2483,5993,6066,10657,4131,7317,4665,10546,7047,6337,6183,3537,253,7720,5316,4382,2720,8286,7478,3902,7237,3088,5066,2131,2830,10694,3644,1407,5650,5733,7970,8970,2283,8951,9288,6643,10349,4772,6849,3295,4178,1354,8522,10736,1720,4463,10409,7172,6220,1998,6367,7464,1973,6738,4238,9065,9537,2528,6911,4247,8425,183,7919,4791,5494,3274,6583,4995,7128,869,5889,7655,5199,1555,6754,10636,3874,5094,545,3805,1611,9596,2168,6048,7912,1253,3675,10190,6051,6887,5414,2919,2397,8982,2189,5410,2589,6458,7272,8280,6078,2349,9268,9229,8372,9571,2129,9070,7371,5048,8593,4925,5490,1503,6584,4713,6442,3119,687,3013,10757,8401,6102,6842,6324,7114,10127,9879,9783,4963,1626,2785,3640,9163,4835,1390,5087,8073,10203,3068,5747,1958,49,4916,620,1548,3018,9306,184,10272,3878,3053,2239,10187,1477,2902,9456,7260,5051,7639,5386,4477,80,1151,7277,5579,5072,5596,2204,10199,8267,2624,5110,6272,1330,10059,757,9947,1357,7745,6159,9580,6926,1486,10062,10488,8712,6582,7733,8905,3935,8265,7334,5495,9170,1657,4880,1542,9546,5577,3169,2880,8287,2763,1465,1041,10545,7951,2814,10729,1234,7881,6985,6670,5055,4329,9749,4744,7621,9946,665,475,6655,6976,1327,9703,33,9513,9857,3248,2158,7640,3830,6910,6599,10785,3908,9303,6403,858,2074,3941,4068,1488,4929,9954,4872,10514,7838,4677,1877,7014,8113,6883,1511,7174,381,1523,10229,9627,6966,9767,4283,1167,4168,6977,3543,6953,2737,4698,7908,8759,2429,7706,9642,7199,2064,7087,557,8591,4609,3962,2795,8283,6449,9092,9964,4970,4529,3616,6475,7523,56,2238,6431,8256,640,3416,688,10471,8348,5541,2371,994,5570,82,6166,2259,7589,10479,5713,449,2308,130,6882,6298,6517,9870,2353,4316,5452,2792,7556,3803,822,9084,798,9477,9082,657,495,8781,10619,4328,2710,460,10133,3234,2555,6069,5255,348,5511,6934,9594,5120,6729,3956,6290,830,9948,5911,1731,6291,4993,1533,9758,5693,8152,4988,514,10270,6000,9682,6946,8344,8051,5320,8247,9020,3751,7864,10163,540,5933,3650,6107,2840,9372,3701,8955,4269,8511,1024,2165,4739,5474,10239,2466,4157,6533,10780,9408,3312,2549,7113,4480,7741,3467,5710,10381,2392,4166,7485,9902,10116,1089,5281,2157,9182,412,6056,2281,697,4383,6531,7473,9231,7411,7070,1493,3570,4122,3461,4849,5119,3174,3139,8068,7162,3184,3691,2316,18,650,2382,8612,9692,6483,2442,7718,488,6642,8681,5271,9004,3942,7986,5406,6744,9014,6975,8403,7449,8721,3240,8297,752,4471,2833,10812,202,7701,10064,9164,1565,2172,3613,6589,2843,1052,8359,1932,2389,127,5508,4782,1048,2729,6216,5692,5219,4730,8987,940,7810,3907,4399,1220,9901,1549,1242,9815,673,7459,3322,8550,6865,5229,8557,2257,898,2893,8674,2446,7645,6528,2809,4711,2106,6267,1110,6415,7998,2094,9509,6269,4475,3096,1567,7066,2464,8516,1843,9885,8995,2345,4940,1077,903,6886,347,8766,3343,8534,5871,5310,5738,10217,7539,2823,9867,8238,7093,10667,2332,5535,1419,3203,7540,2417,1619,10744,4592,8817,9219,2565,4276,9380,6329,10426,8853,8106,6765,6932,3822,5864,825,8244,9566,8333,5954,5351,2978,3376,6275,10290,1604,1067,7901,3143,7979,3630,5252,5642,6098,3077,1464,325,10481,5853,10012,4658,8315,10029,2812,4123,4967,6097,4038,1755,2873,2905,8098,6941,3681,1070,8555,4875,10340,9944,9694,2180,7441,2904,8054,2531,9330,6750,6945,10067,6012,5092,10772,73,6213,1504,6425,8835,2520,1401,10768,3368,4375,4941,685,9479,453,1268,2245,1694,4325,7705,892,10775,7450,2679,842,3996,6241,1117,7713,10343,2124,5857,5617,6594,7305,8770,5928,2197,9166,5956,9299,10095,6524,10531,4335,6622,10823,802,3120,2073,3864,3490,6957,9845,3972,6935,5543,9774,154,5250,10342,8137,8040,4224,10500,7327,5779,2755,5085,8243,4632,1952,8047,7056,6304,3241,6219,5353,10320,4559,6552,4748,9475,203,4551,8552,7187,3748,4271,3232,5903,9402,1566,6381,1569,6192,5338,2965,8023,2360,10386,6383,923,10618,9172,421,3412,7582,7685,7332,4138,2820,876,1356,1325,1012,3814,9584,3799,5041,9273,5764,9757,1655,6991,8588,9423,8001,3853,6191,8755,576,6629,4115,8878,4253,3245,1885,2505,5426,2621,260,8848,3873,2060,8352,4842,7686,5359,3600,6732,857,5014,9386,9173,8150,3065,7127,10754,2987,9907,7347,1051,6114,1609,221,6666,7851,5093,2471,2465,398,2077,6459,5799,8462,7943,10731,2227,9619,3813,9031,4753,7797,6347,3836,1806,9134,6150,5643,497,3868,6915,5071,2769,3717,8575,4426,5358,1587,7752,4942,8857,742,8464,3442,3050,9660,1165,3309,5293,3171,5099,1154,1827,7728,3140,5995,7719,601,9119,2053,10056,10487,4949,5307,7866,7076,8052,4372,6045,9586,7799,4800,4575,5458,316,5425,6361,4965,1187,10816,1907,7808,10551,4531,146,3884,8978,1271,3568,10668,179,6571,163,408,1190,10175,10716,3837,9340,303,6792,5936,5979,6330,8574,6129,5555,2385,10594,4980,10728,5842,4321,10444,427,5890,1776,4598,5235,7900,602,4763,6907,6379,4804,1960,924,6133,5932,8041,98,3622,476,2517,8161,1374,10787,556,493,5804,9109,8786,1872,3138,1337,2822,6289,7562,3005,1392,6866,2518,1140,5258,133,10331,8306,5313,9575,10717,3156,2103,8495,4799,4672,838,2249,4686,2233,4281,1475,9108,206,9453,5941,8379,5103,2868,9988,9732,7417,7071,1231,2515,9371,4498,3081,9352,6529,2313,1529,4229,771,1134,3690,9005,1128,9503,9776,2571,8369,10271,4387,3408,5533,8666,9035,8390,712,9255,6634,8299,3221,7005,6850,2854,3921,4548,3501,1593,1050,1692,3685,4879,4277,2805,8468,2867,46,1258,2272,1118,816,6796,7965,2036,1722,9448,9851,6940,6862,10538,2547,6437,5343,6727,1359,6444,6573,852,7574,10336,61,7085,6247,5134,106,9192,2369,7786,9009,670,8311,638,5288,8446,5463,296,7337,5018,3103,10423,9970,10008,944,3205,5435,5332,7921,871,967,4777,271,7214,4652,6630,8199,6526,1695,5140,6215,9277,5174,8504,5181,8193,2863,126,2377,691,9662,6495,233,565,6466,1771,5348,9411,5026,5537,4801,656,1384,8396,10541,2326,6958,675,402,10098,4203,6835,6258,6969,7950,5704,4047,10642,5038,3340,1820,6525,7354,8752,5131,3937,4369,3486,586,5305,4971,9773,5948,7618,9361,6790,8743,4309,3722,7721,3944,8836,4154,700,2935,8003,7351,5521,9464,4718,4888,5124,7841,3135,5697,4530,7276,1395,9615,4825,6224,198,7805,633,8442,1840,1261,1170,879,9725,3626,3870,5050,3901,3954,9332,2906,10073,7674,722,6843,4363,9289,5077,6598,3059,3928,1114,10034,6377,1380,4378,4418,10505,4869,7246,1292,9942,7092,9139,861,7416,2747,1245,6423,9095,9653,3130,8948,6737,2404,4026,726,6101,7726,4540,10308,4243,478,2980,3227,8985,6913,2675,4504,3753,4931,2434,6753,1045,7477,834,4181,7134,10507,9514,1747,714,7790,6755,1971,8301,6009,3584,8877,4361,2927,9027,599,8160,9059,4675,8090,10091,8192,5283,719,3145,8671,8569,6824,8639,9253,5815,10361,6988,2510,2585,8904,7405,2398,1768,7989,8696,9899,9805,2439,9412,2727,7927,6257,8746,3982,5679,2013,4579,4141,7634,7553,346,603,1522,8764,7001,3603,7775,9715,7678,8482,8630,10530,7905,9211,5308,6760,5266,7308,7377,3957,7058,6340,3602,3218,4945,239,8867,9918,8130,8000,4950,8913,275,8097,1029,4630,10774,3173,2808,3754,1591,9115,3593,755,3318,6902,5762,2104,8893,3749,10401,8946,2616,8510,8937,7975,1078,6335,6238,10733,2044,7885,4233,7426,3606,5280,9756,810,9844,9695,8053,4704,3852,1484,1031,8399,5467,7672,7590,8215,5263,8028,2657,8117,6142,6273,5561,9748,672,7229,3098,84,10405,4272,2800,6889,8792,6691,4204,5646,5345,5615,4278,5315,4486,1543,9467,5355,2797,5400,1953,8917,7806,7073,3807,7312,1406,5716,7086,1738,6847,9781,10212,9631,10442,7548,10115,4678,4097,5030,2247,3002,10293,492,10429,8944,5106,4009,10269,5760,859,1201,10771,2026,6508,2957,1780,317,6445,8139,1810,1520,8983,2354,490,7375,9866,6035,5393,4751,9810,7710,5155,6576,7367,4368,332,5870,7762,7292,10020,6076,8355,1216,2119,2362,6470,6180,500,9775,298,4319,4935,7585,9563,4182,5326,1970,9266,845,6756,2256,8541,8972,7094,2748,7353,5185,10664,10743,9565,6942,7743,9912,5135,1452,1618,7592,3844,7037,2048,946,7036,5780,3146,5973,5505,8810,2892,3682,9607,931,6852,8823,10136,10440,2615,10764,9421,7909,8964,8449,3160,2153,181,10421,3357,5638,9478,9572,5797,2020,5368,10014,2415,9659,1772,9831,7340,10777,2584,1967,7190,5789,9905,3909,5499,5879,9787,2921,2683,462,2535,10730,7492,6718,10492,7124,1148,1079,3986,6891,5696,10414,5855,5737,4214,9389,3253,1273,3869,2641,8329,7323,4120,7723,4798,1387,5285,6983,7132,8731,5299,3663,4249,3438,365,5100,8684,8801,1993,4506,1671,4566,5122,1686,3195,7916,3880,1101,9672,8036,4397,2701,1786,8120,10726,3429,5114,4588,5217,1057,2735,10794,1331,8815,2261,4703,1667,9574,10803,10054,2435,8067,4393,6362,2937,985,8750,5374,770,2463,1749,5922,1063,10335,4743,2305,10396,7546,257,3297,8553,1775,10197,7279,2690,496,3454,8616,10612,7857,7126,3107,2298,4072,8906,151,1581,8061,439,6856,7903,2142,4526,5244,9029,4714,8335,881,10363,7503,6647,935,5133,7649,10561,6108,933,2551,6870,4673,2826,10129,5906,626,7717,1082,3575,1172,9417,3456,5568,5914,5856,1851,2953,4881,5851,6239,2050,4371,6477,7366,4452,9849,2246,7930,988,1951,10789,5878,9853,2323,2850,1942,1136,7421,10230,5130,6313,6929,4846,10267,7890,4964,6829,4357,8582,649,7241,6716,8258,1176,9263,5390,6994,5921,3192,1069,10796,5908,6486,6138,8562,8578,6925,1408,7917,8045,10286,6671,2352,1312,10309,8210,800,6050,2686,6214,8456,10302,8858,3443,418,8005,1445,7115,1824,5690,2569,2917,5197,6749,8643,1537,9977,5648,3997,2600,10395,8360,2861,1014,9790,9865,4095,6124,1666,9189,5160,8186,5137,10828,6025,1847,4932,1249,1299,9530,7282,6990,9223,3720,3818,7987,3137,7670,286,4731,7654,805,9668,10018,230,2467,8638,10084,8422,3481,9811,3285,4877,9281,10307,10801,102,530,1819,4918,6546,928,2930,6678,10253,10151,4094,8430,5056,7400,1313,10461,6039,7992,6664,6502,10644,6424,3383,4302,7186,8342,3559,310,10379,9446,2277,1116,5778,6538,1963,3185,1487,3294,10362,1207,6921,8181,5236,10353,5709,7359,10011,4147,910,2476,5337,9213,9916,2696,2391,7286,8797,1796,2411,2570,7990,2889,10723,10150,171,472,6246,7280,10100,8298,4517,2775,7831,10582,2365,3183,3670,6432,9074,7993,4852,646,3645,4052,5323,5556,10807,4014,5349,7109,1296,7125,8147,10043,10085,10709,6906,2251,1058,3842,1132,10328,6005,7725,1853,1862,4528,10824,2837,9129,2214,4209,3557,8758,2605,6667,8761,7620,2156,8505,4322,5269,9818,4447,3186,4709,3920,10087,568,7985,8911,2331,6494,10654,7631,2875,5363,6151,6896,5481,9705,3286,7753,5449,6013,6874,5153,1779,1494,4722,6728,4726,8008,6316,430,4403,6359,8762,6936,8704,8677,1111,8153,8680,7700,2622,582,1462,1621,4373,5430,6218,1825,10038,5597,5578,6064,7907,7892,1469,9038,6763,7197,10543,9461,3881,1536,8992,7465,235,4240,7502,1541,1616,7207,6608,4245,3470,2961,5423,9460,3993,1979,3680,8998,6844,6370,8959,3488,3217,2604,3891,3369,6467,6320,4458,9248,1793,8676,7581,10502,730,566,3694,4699,4416,2284,8110,1933,9333,10388,9873,7454,9254,8881,775,3497,1712,7090,5158,10122,7483,10818,9973,9704,1969,4606,2682,3354,8869,1338,10215,4379,9793,694,3725,7365,4431,8302,2752,3255,5392,5468,68,10751,4871,390,457,4705,3181,4081,1491,9246,5982,2524,3721,2372,727,2163,4734,6689,10005,5237,5583,280,5207,3034,4022,5934,6326,2334,7209,2702,4643,7498,10119,5825,8962,8185,5440,2169,1803,1742,7316,5183,1177,2936,5020,6959,5063,698,8234,9307,3086,8831,919,2234,3787,5714,8263,8991,6104,8554,3350,5187,4092,8071,1898,1394,4992,5371,9200,486,7787,8939,6047,2190,4210,3142,4261,1583,1121,1927,270,4948,6204,10255,6811,315,8896,3477,3895,9606,8851,4623,4244,893,7009,10344,5404,2539,3448,8682,6105,8965,9706,2862,5375,3273,242,9218,3585,7027,1112,8232,7751,8332,9015,6507,8544,4428,884,3857,1086,8115,949,4160,4268,6867,9803,8337,6903,10165,5744,3284,5854,1785,2319,3089,8448,389,2066,7236,3351,1466,123,1956,10341,8785,1741,2310,1544,10547,526,4421,8916,7915,161,2972,5424,1301,222,4661,6438,2496,3109,8488,9354,743,9345,3615,10721,9622,51,8829,9000,4890,10182,3760,8921,1867,5754,3265,2399,6089,7313,6339,9285,212,3308,1886,10441,2341,2626,1284,8138,10424,10204,3572,3966,7296,7303,6846,1875,2758,9663,7173,2959,7261,6295,8303,9088,7669,2112,1328,809,4456,3132,2346,1241,958,339,7663,7531,2519,7509,6949,8340,9959,7063,3055,4636,4650,7680,7203,6030,2264,8979,8819,2500,7559,5606,9434,3256,10689,2321,3324,523,8078,992,2736,7513,6385,7052,5970,5483,278,6350,1730,9367,729,157,7960,4446,2110,10575,3544,10795,6554,4384,6279,6659,2623,2082,145,6372,3582,4127,1878,10826,4129,1017,1837,9788,3281,1663,1929,1597,7393,10473,5167,9599,1482,5394,4906,952,2098,3465,9240,5524,3459,1724,9454,6189,5210,8609,3447,252,20,8014,560,7346,8756,9939,6463,3810,8537,7240,6703,3435,2252,8404,4582,4011,285,3744,6435,8127,6676,3492,9245,7358,4079,3555,3329,9868,10154,1435,5189,8220,1275,3926,7007,4464,8647,10250,7756,10798,10221,8891,4723,10161,6297,9083,3765,95,3530,10346,9909,1532,3700,2541,9989,6816,10311,9640,6547,6384,2081,7112,5666,3974,10374,617,5691,6310,1702,2487,2285,4644,3336,3756,1224,4343,987,369,1093,5223,1610,9981,7227,4114,915,1186,3856,583,10701,6518,7343,860,956,3165,4762,2529,10281,1429,3413,4135,483,1449,739,4878,3697,769,5841,4621,999,10557,9149,6955,1997,1073,4172,7653,5518,1485,3287,5466,9746,7146,8953,8830,7521,8289,2198,5300,4404,1659,336,5723,2857,1499,8658,5105,6122,9601,7954,7573,5243,762,3418,6234,3161,2681,4441,4649,8345,3804,1818,3177,833,9727,10225,965,7635,3006,2772,1142,3208,1098,150,3943,4761,3460,148,4784,7496,7263,612,3524,984,1983,5502,1120,7200,4104,28,9675,9544,7936,8763,9603,4304,6014,8498,10131,1367,2580,5959,2016,1205,1351,3288,4457,10673,8116,4406,1206,7046,9177,5683,7707,3547,2045,4376,1516,2865,4622,3207,3659,9175,5121,2777,2653,8524,1157,218,6418,3321,10102,2380,7980,4617,4220,8859,2550,9473,3495,5408,10677,343,7398,380,2894,5325,10235,1329,2869,7202,10437,6613,3509,2703,6314,7226,5218,10184,10627,3526,7179,6801,6853,8560,962,10511,7729,4125,4134,4392,2766,2492,5971,6144,2358,2583,1122,9632,2992,747,5145,7409,8392,543,3133,7946,10450,754,4981,5441,2618,6462,1229,8587,6498,9512,954,5848,5688,4759,5331,356,8088,7814,4350,7795,8176,3631,2773,4999,8259,7381,744,6609,2564,5940,1594,3755,6476,6540,5057,10616,9221,621,597,7887,2606,81,3980,2220,6640,518,2659,4538,9926,1733,7948,8018,3004,7363,2021,6992,2342,8483,9373,4527,4419,10224,5816,10623,2693,2160,2963,6773,74,4684,104,4898,4809,1674,7499,10696,4695,3978,2699,5214,7315,829,10410,1145,803,5951,301,3763,4196,7746,8328,3070,7407,6055,938,10522,7977,9896,2243,7033,9814,4494,9676,6036,6100,4775,12,2195,10196,1103,5862,3333,3471,6764,10222,4174,8568,8249,8362,5492,4053,5630,7000,1966,3422,1711,7682,295,865,7361,9652,5790,5626,3821,8409,9859,6363,8860,5989,4087,6395,9848,8308,4341,7520,7616,2671,6836,4860,3014,9789,9205,1918,5053,3335,9542,320,8884,5204,1442,5558,8250,9872,5952,1424,3709,8195,6140,4057,6633,659,2383,5834,1508,4246,10010,7287,8076,4554,3599,10685,1431,4050,7264,8535,5166,8086,8133,6541,4201,3545,9923,5647,4035,7740,10183,4169,1689,2664,5770,1882,1295,6978,4921,1812,5412,5387,8668,8408,4781,3579,732,3899,9510,3082,1914,7242,1557,1912,3961,3250,1039,9611,6683,9820,7647,10220,5720,10173,10330,4909,2337,2654,7133,1896,2883,2448,10727,2793,578,3199,8572,2743,10223,10662,9378,10004,4008,9331,1675,718,4682,4274,6762,1240,3261,4046,1934,79,1988,9624,5601,6621,3153,3882,10144,7429,6430,8986,5860,3225,5873,3960,9397,7687,5080,10674,10778,6193,5531,5793,88,7583,4601,2248,5748,3393,6702,6857,6950,8635,2375,1141,4236,4145,2733,5629,401,8954,5777,6783,3337,1943,5918,7821,255,10692,1434,9058,9629,3168,360,1690,1230,9458,951,9271,6469,5967,3061,7579,1003,3108,7681,8486,6265,4409,6109,8412,5618,6487,5562,6276,5907,7575,5529,10601,1725,9096,6905,7801,8610,5277,2627,5035,9569,3157,9957,7322,1600,6157,796,4460,6627,5685,10805,394,3669,7330,2426,7744,10520,8645,8932,5209,5820,8391,4769,6132,8613,1836,3841,3041,1085,521,5186,9462,7136,6429,9951,5409,1269,8128,7516,9188,8354,10167,10216,9497,9972,7399,8910,6456,2810,10339,10819,1646,9616,2174,4175,6091,748,3661,2366,6654,587,3627,10097,8367,1109,1844,8652,1364,8151,5060,2480,7817,4284,1076,5257,4861,7355,5872,9709,1298,8145,8519,806,2427,4524,3660,6293,4444,1660,3989,9374,7192,5369,8292,3914,5608,210,1002,7154,1302,2855,2412,10211,8799,4612,9744,267,5877,7137,8426,2828,4192,2378,6236,4924,9921,2440,6378,3556,9511,1897,357,8104,2400,10609,2970,3499,1830,8321,4543,7794,2971,254,6223,27,9480,7003,8856,6954,7704,318,2407,1182,4624,644,9496,10318,7961,9257,5695,6231,8894,4186,7976,812,7331,4217,7106,8092,1126,4102,8980,10591,3734,1213,2670,9679,3172,10459,8626,6492,3323,6646,9539,355,9337,8716,3808,6165,10562,5658,818,9707,8958,2134,4866,10317,7818,274,5457,3388,4282,4330,6482,109,561,4955,4874,117,8038,8866,826,3938,7069,9357,4806,4903,8741,2688,6343,2402,9895,9561,4100,10491,2597,3533,1849,3190,8737,8967,9007,4468,1068,10277,2258,4360,59,107,1036,5769,8437,9843,9700,9127,3525,1,7297,1809,5972,2484,9840,9609,431,5800,671,3009,10159,481,5025,6254,9602,2025,1001,8679,7472,8929,3164,10786,5165,7177,9987,733,4959,10276,6199,8022,9001,10576,9036,8938,10189,9238,297,3231,9080,6512,6509,1568,997,3773,9643,1030,22,2192,9407,7798,2813,10039,2088,5173,7170,6652,8536,10436,3325,10338,9437,1977,1919,9808,2430,10240,1107,6451,2784,1817,8037,9633,143,7117,6964,721,4194,4954,9730,8203,9491,1807,5659,9116,9816,6808,1160,1358,4106,1166,7206,371,3636,7656,2148,1938,7394,3634,8204,4770,4938,3896,4577,8548,7530,6016,7461,324,10646,7307,7338,8436,4381,6615,6087,5668,942,9953,1899,5546,9406,15,9684,3552,7335,7392,450,4470,6201,4040,10641,8713,953,5675,4947,8715,3092,5422,6023,5783,3950,1423,677,4710,878,7130,10822,5512,7204,9929,7326,2019,7774,3197,10810,7732,8471,5052,10750,4482,5156,334,8526,6222,9214,6614,7258,6488,6680,8320,5259,3757,9549,9196,5715,3581,5413,8031,5224,6497,7651,1100,10110,4797,7839,3091,10123,10358,8188,2208,5447,4542,373,434,3293,4904,214,8494,5450,1721,9828,9961,8747,4365,110,4910,704,9155,3504,7235,7734,873,10448,7667,8824,3796,7971,4605,10003,917,4066,705,779,1612,8861,5260,7295,3027,8487,3823,9826,4459,8941,6715,2085,3517,4808,7075,8272,4500,9656,6927,8405,1386,4737,8475,8365,1813,10088,4996,6029,6993,3364,4491,4885,4287,3667,10170,9325,2314,1129,3911,7830,5924,2676,4934,9403,7940,1982,5054,3028,9693,8633,454,1944,8599,9833,8595,41,1400,9291,25,3594,2697,8945,8840,7290,8963,2043,8431,6618,7978,4635,1669,7879,4019,3811,8168,1791,7099,6588,9120,8107,2633,7467,2490,3905,4697,3652,6742,1718,3987,6206,8467,3716,8875,9504,7822,1874,2899,2537,1515,7730,9667,9388,8007,5294,7185,9778,9451,446,1512,4152,9894,6948,6416,8189,10042,7024,232,648,1425,9531,10391,10715,6568,6353,1441,4423,470,7962,5376,2725,4610,9079,678,24,794,963,5479,2339,5023,1627,9260,19,811,1410,2506,2086,4218,7015,9600,2022,9466,7222,7410,438,2732,4654,7252,8826,1766,3483,9081,3998,2315,2767,5503,9724,4060,4507,4857,5360,8310,10600,3211,6448,7386,3790,3872,1343,5636,2280,2384,4015,5068,6328,7697,7538,1588,2760,4701,5706,2338,3925,9966,2674,5599,1985,3879,6636,10366,9877,3042,544,2724,429,1957,8924,1634,10106,6242,9852,1884,1799,3610,299,7696,1144,5923,5194,4467,5813,1925,5177,5554,9889,886,6244,8968,9883,1910,1661,4438,7766,2194,5792,7610,8874,9729,7166,5700,8477,2091,4076,4962,9797,10400,2179,6539,1243,3452,6322,7091,10164,6564,8879,1637,2222,8434,6026,9485,4235,10564,6164,10599,186,7953,1223,6484,630,2034,6225,5991,5787,2344,7376,6979,4462,204,10621,6747,13,302,7329,4339,5983,6338,3113,3397,2213,5772,6603,4140,8357,8048,9752,3875,4177,1106,4179,2789,2995,3774,625,797,4188,333,4093,7495,6043,8057,5433,9839,1038,9750,1763,8427,10470,8601,1471,7956,8118,6848,6237,3794,5899,7245,1913,8242,6565,5329,442,2644,986,2983,5684,7060,368,89,10710,8027,2481,10632,9061,3313,10698,5357,6054,2221,4615,1578,9878,2509,4810,8126,848,1303,2489,2441,764,8255,7065,1527,553,2628,6282,4518,6331,8096,3017,2039,5350,3657,6795,2357,6577,4296,510,5627,338,393,10407,6951,9090,2798,4589,8606,6434,868,4778,631,2964,231,4594,6665,3945,5427,1293,8655,8499,388,3633,1393,2877,8319,6980,736,622,8813,3797,5883,3433,10160,2786,9567,9022,5739,7920,8966,1083,4771,3532,10697,4826,137,2182,522,8984,9982,10264,4017,1974,5652,1981,2910,7029,2990,8724,3505,4651,7819,4345,5497,1928,7782,8949,433,2403,10288,6751,6342,4712,7684,9683,1035,2844,2317,10157,3858,4370,3883,1554,7061,8373,8015,5686,8795,6278,3683,8686,5493,5168,4032,1748,10790,1458,1432,7436,8032,7625,8651,10815,5998,10506,5202,8688,3678,9343,6510,1629,3182,1353,7419,3785,2472,8393,7761,1355,6332,9681,9931,364,10678,854,9098,9691,10792,5289,7764,4230,5069,7055,3955,3576,4567,10194,6414,6766,8956,3269,2997,3888,144,1777,5477,7013,5192,3642,2311,1336,849,1601,4968,7482,8135,6404,3124,8622,2949,5403,9269,1256,10521,9091,5486,6970,2815,4311,1893,7183,85,2292,9856,7984,855,1750,4279,1713,7002,7626,1705,2090,105,4687,9153,2294,10411,7939,6707,1672,10284,8545,2609,10714,8589,1757,7469,7217,2263,5027,1745,853,4429,6758,9651,7144,9286,7676,8527,8155,420,4443,1845,8082,6611,0,2794,9547,639,1222,6115,1572,9903,9495,5545,3243,9621,5182,247,5581,3289,3729,776,5095,2379,243,8744,2295,3732,4725,10544,2745,1246,10024,3345,2076,1155,627,6939,2454,2839,10259,641,5988,1808,7401,9723,6734,1294,10155,5818,1260,7904,10671,7750,1061,8224,4013,7194,9030,9482,1505,8012,10185,2293,8551,520,4831,10451,5002,1628,244,23,5459,3152,3566,2923,4389,2508,5984,9363,9138,4756,9871,3200,1729,8922,3064,5090,4626,491,1968,7768,4039,7882,6317,4822,10244,6947,6559,5,188,4424,6963,1879,10447,8269,9576,3865,2632,4003,9368,3484,8089,2790,5472,1922,6723,5553,6057,2734,4989,4706,397,8384,8002,6233,3469,1576,8470,9846,2836,9521,1829,7442,8771,7731,2751,813,9051,5637,863,329,8056,5574,9779,10558,9234,8923,366,10036,7320,8997,9522,5301,3292,3327,1113,5242,5602,5876,3817,6972,1054,7870,4310,211,1290,4005,6888,8732,7384,9564,6837,6044,6205,5655,10017,4824,2938,7281,10462,5091,6918,5488,8481,7828,5498,7613,3405,3648,6007,5634,10209,10109,10516,10466,10688,6501,2023,400,10517,10399,2297,5079,4148,6248,3482,5231,441,7759,9244,1064,8564,8973,711,10711,5525,3420,2526,4757,3759,8142,5750,1787,6232,10806,3730,4483,4255,2811,2707,8295,9699,10739,9911,7622,1784,6592,9590,9556,1687,7643,10265,1620,1119,8451,4198,10519,10607,6446,4227,4943,6810,5061,9364,4570,2738,2640,10126,8694,9666,5043,1883,8233,904,6081,4532,3553,7533,3507,7636,6788,9128,995,4867,9560,5190,1115,2771,731,9777,1653,6919,6776,10262,9881,1196,3386,6390,3574,8693,913,2714,2713,8780,674,3478,550,686,4786,8990,5885,139,10416,1684,1811,5147,5614,5178,2662,4080,3141,6885,4211,307,1232,7565,2704,7598,9383,5930,7348,9999,8415,5980,6965,3236,6685,8251,2065,1573,4893,10245,2051,1239,4581,3425,2594,2740,3394,2607,384,3646,3567,5732,10820,7380,9440,10827,7490,1194,9984,4435,5587,901,5828,3008,7045,8749,6974,1547,8225,2202,6968,1602,9124,8466,509,10066,934,5022,2542,10193,1852,5339,8180,6639,3263,4159,5632,1590,8059,3952,7225,158,3494,2723,6447,5625,10732,2864,6698,3906,7974,4002,10044,2673,1008,266,9293,1999,7714,7968,4112,7779,9610,2266,7515,7274,8558,2139,8725,6067,4642,8930,374,8485,4717,173,2575,3832,34,2449,1346,7854,7675,8378,1743,7178,7022,2495,5542,1668,40,8230,3849,3363,10006,90,1143,8636,3591,2779,2994,9997,8281,10483,3463,4899,7722,10830,8492,4136,10368,7437,5013,2783,2592,9110,10227,6638,6650,7827,5455,10583,4338,7387,4158,1318,6757,4126,5434,1931,7856,9721,8597,1209,737,4474,2718,93,4337,5102,3317,10432,83,1339,10534,572,3788,3298,8604,4176,8889,7836,9505,8400,3246,9552,3390,7599,9630,7850,6075,6854,6077,2687,866,6668,9206,3781,3587,717,2312,6600,6407,6341,264,6952,1579,2558,2532,6562,5389,3970,7034,293,7777,9506,4873,7247,9140,10493,464,4225,5639,3134,9195,2386,5603,2578,8533,9782,436,6893,5590,3596,2083,9908,5891,5064,9589,8760,1693,5571,8837,168,6382,8512,4334,3711,10474,9655,1704,7057,5724,5673,9735,5522,7289,6944,2787,1709,6198,1131,4555,2999,555,7781,8812,632,5743,6513,3414,2336,1682,8988,8087,2343,4412,9424,99,444,5874,10460,9017,6094,7614,2147,5125,10117,7840,4637,5104,1285,8602,1091,10113,9887,7576,10605,3258,4018,276,9324,7641,9276,10477,3054,9256,9450,7105,3953,5909,8717,6504,2715,2154,9197,8828,5929,3621,3016,7042,419,3847,9320,1334,3782,9312,8699,3228,7715,7370,9313,8346,4425,3047,8838,9049,6352,1340,4492,3356,7095,663,4660,4119,10146,3129,7369,2040,6674,5527,9259,2322,64,7858,7019,8417,1746,7285,3332,7129,6814,4796,8274,3083,2824,3001,8621,3746,5344,1575,10357,4758,5436,4896,1060,8371,426,5031,7991,5916,1335,1345,8444,2162,10791,4539,9850,7051,2270,961,94,4578,6521,9290,6724,3809,6306,1644,447,7084,9623,7811,8895,2410,2493,4839,6003,7889,8892,3674,1283,790,2118,1473,5070,9296,469,9447,2128,3252,3793,10408,466,9013,2776,10452,6135,9187,6662,8183,1605,3695,8077,9430,5955,2376,7511,1894,7304,8703,6063,6672,5462,4405,8742,10120,5593,2475,208,2431,2672,1288,2582,5519,2586,7843,353,8981,896,10275,647,10456,3689,3802,4317,6663,6118,4185,4708,395,4829,3538,9278,471,5010,8531,6345,3586,6688,120,847,8804,2494,6987,1125,7031,6879,2730,6717,1033,6709,1562,2274,9686,7735,5942,4863,10572,10404,71,2968,2067,5717,92,10412,3311,6833,5509,5794,4785,5042,3656,8366,3020,10494,2405,5758,10536,9541,1920,6696,6287,2756,31,10430,349,3571,7708,3037,5256,5321,10554,7529,9733,10179,5227,3702,9785,680,10140,4195,3381,9638,7385,3643,9165,10604,3355,8044,4603,580,7455,9274,6389,6169,8254,1630,4696,7633,10327,2173,8039,9220,10247,7041,6706,1492,2217,9186,9024,2127,7947,3550,7563,4143,570,5633,1816,4510,5549,10139,4149,4450,9112,4078,5910,10132,4414,3806,9204,10713,9384,7250,3444,2330,10071,4990,7959,5900,8010,695,2418,9349,2079,793,6022,8438,3569,305,8863,8888,1397,596,6355,5604,3795,4291,2731,5203,2908,7568,3939,8175,4569,1762,2017,3766,3623,8062,138,54,5884,2240,6735,4455,5961,7527,6660,9060,8855,5785,6392,7688,551,4768,10367,2546,432,6143,5703,7278,547,4440,4614,2031,760,9107,164,921,1262,1858,6701,4692,9105,4354,768,2359,2012,6566,7820,759,2144,10251,9156,6096,3370,2226,708,1514,5835,8811,10060,4184,9763,9838,2553,6095,2739,7630,4521,10515,8615,5831,2939,7650,6491,8999,311,8060,8381,9298,7748,4314,10356,6823,788,3150,5169,2525,7913,8479,2035,9235,5396,3833,6781,5501,2126,8389,8729,5767,8543,8899,149,3676,5028,9026,8523,3229,1778,2279,3562,6931,3968,2388,1586,2084,7646,331,2309,9358,9410,6419,8661,7981,2872,5966,3249,1438,1179,5514,9442,1421,7544,10463,10092,6813,7402,10433,2096,4264,3254,2188,7571,4693,9111,2183,3233,1270,9930,7201,6177,9326,6128,9272,10283,1420,482,8918,5904,1717,10166,4410,7447,10504,1500,7996,3302,62,3277,9151,7220,9469,3629,3965,3506,508,8207,7776,8825,6989,831,3439,8361,1092,1570,7605,3019,7205,3437,6454,3328,5547,6280,10300,5808,9936,4473,7111,8532,1368,10527,3775,10439,8290,3605,10169,2393,4774,6741,7802,7435,1375,4064,9210,1010,1274,662,6711,8865,3214,4099,10299,7104,571,3178,8753,2625,6461,2033,1379,5641,2095,5965,7249,6797,1947,9280,2643,8791,4864,9821,749,10294,5385,4132,8897,5726,3564,3558,6499,2032,2834,10634,10389,4163,8820,3038,6585,7673,5247,7352,9830,4171,7151,7824,2244,4848,10371,1211,8876,9133,689,7397,4352,1769,10832,6356,8081,2637,4926,7169,3409,8450,3934,1080,10061,693,3030,4740,5784,2907,5576,7288,10670,9125,8903,5886,6548,907,9950,268,4553,7424,750,5947,5377,5663,2235,3769,7615,4239,2501,6572,1559,745,1495,10273,10681,8489,4395,10550,4749,7145,2825,8650,10485,8134,10023,8463,3291,6412,9078,9224,9431,4944,3105,1422,10455,5507,4180,9075,9123,9438,6986,3671,8386,7507,7716,9927,5284,3589,1448,8503,7832,1584,8714,362,5802,5564,4604,10718,5437,4307,7506,8890,4913,4146,7823,7181,4280,7362,5384,2890,3886,9117,9728,10687,182,4183,10770,3859,10228,676,5251,690,1443,4927,2540,5520,9837,598,2513,8190,637,823,5487,4275,4862,9484,6360,8063,3762,1474,5032,6158,10419,3431,7602,9279,2841,1635,9928,8617,1959,5836,3440,4197,6061,2932,3375,7550,9791,5273,7964,10249,2579,452,2271,10518,6111,4096,1647,1753,4212,6798,5378,1700,7876,8594,4033,1869,5024,2620,9011,8095,6478,5881,9319,8205,5978,6779,8200,4074,8850,2269,4073,9222,3487,4519,2885,7160,6172,936,2458,7834,9468,7763,3430,1707,187,6770,7770,1055,352,7783,10737,9401,7167,1991,5240,1180,9121,1861,8395,2268,8957,5774,996,8605,3772,8940,4836,891,321,7988,6534,5671,9144,9301,9754,8849,2554,1546,7425,4746,10070,10742,2061,1782,6038,6250,591,1257,1391,1954,8206,6307,1903,9941,3093,4348,7666,2996,6904,9100,5694,10143,10663,1426,607,8201,10035,7107,1371,7747,2042,10206,5327,1992,8420,5391,6752,8538,9396,3479,7849,6514,6892,477,8969,1585,192,5669,516,1237,1833,2979,4488,9753,4362,176,1916,7215,5365,3213,1563,7973,5557,8685,2619,1200,3502,10104,4851,7267,8100,6858,3062,4062,9876,2821,645,9604,4742,7118,507,5987,2668,579,9459,6125,10813,10046,10202,70,227,9483,10606,10332,3198,3044,10672,774,2601,1251,5953,559,4109,6563,8803,4832,1094,4911,4514,4549,5225,9428,3931,9381,10578,8473,10305,1506,10,5012,2587,10195,10434,1162,3347,2556,976,5152,414,5445,7588,4680,1210,5566,4027,1651,10013,180,9734,4150,2216,5584,9168,6426,10533,403,6391,8317,7471,8773,6772,1072,2835,5935,4951,8382,2799,9835,8079,629,2596,8525,7123,9678,113,3992,10149,8364,761,3111,519,8672,1521,7119,1011,2057,3940,7139,1236,1450,1930,1307,1191,1147,17,8074,8428,1509,9136,7293,4515,975,1388,4537,5672,5513,2611,4156,1071,10105,4453,2998,9717,875,8084,3750,10030,9142,9784,7311,6895,2514,494,415,6325,8698,6626,2460,9933,108,282,10589,6610,4323,1427,4830,6348,26,9527,2832,2015,1383,7344,1961,1088,2149,3496,8083,4294,4105,5850,1265,4969,8187,1137,8324,1625,2413,193,5964,9355,8629,4366,7933,7456,1821,4327,9053,7859,6520,103,2420,4882,3084,4653,562,9979,1381,10099,2925,8667,10048,10482,6210,10637,5861,5958,7601,1765,5176,6366,7512,10788,4048,10513,8935,10186,1460,8398,6532,6028,3029,10258,7891,3903,4402,537,2062,1281,4813,740,7324,3040,4299,7668,3275,9071,8275,10532,9628,4580,7969,6649,5485,6042,4596,4602,125,8722,6015,758,5287,3237,3391,9664,4563,2981,7427,3951,4618,4312,2803,5268,9261,5992,8960,8376,375,1688,610,9393,2210,8111,912,4994,1214,4355,4733,3278,1168,5352,7026,5812,7390,3929,3718,9800,9249,6656,4098,7543,5415,5334,5034,2521,5228,1828,273,8886,3604,9425,7156,4574,9965,10712,2886,7298,10613,9052,4820,778,135,2132,10831,1761,6851,5039,3179,9452,2929,2982,9037,4683,9106,5150,7958,4585,5298,2395,10130,4221,4738,287,4891,2203,4572,4508,1697,5651,9191,1832,4295,10556,8454,1378,9741,4110,4301,3155,9897,713,5136,4681,9141,5138,6088,5157,10773,2485,9499,5644,1664,536,7580,8598,4088,6422,10257,6400,5551,1440,1534,4084,4633,3051,3949,3767,1946,2780,4946,2940,899,2278,10289,4063,2660,9418,9045,2250,9494,309,8245,8691,6616,642,2639,2985,1459,3235,7388,455,6579,1315,10260,9618,6197,6460,498,5290,5322,337,3025,6245,7440,3922,7141,10354,4576,4933,10292,9582,9429,2102,7462,5919,5996,4978,2333,8043,8149,2296,10028,9657,3012,6768,7692,1550,9824,3528,8871,5347,9179,10579,10297,5159,1654,7082,4167,6812,8540,11,4207,345,3210,2576,5075,10114,605,2340,7664,6736,4747,3473,3747,9183,9171,3850,5276,8994,9310,7149,7253,4103,3573,1185,1633,773,8528,246,3890,10347,9305,5312,9520,9751,8898,9154,10337,4966,9391,3889,1790,3035,2516,9318,8323,7868,1023,10077,1727,3032,10329,2922,4571,3464,9759,2545,1277,6285,4957,8397,616,7861,3542,9680,4689,7088,8521,1895,3741,8457,2856,5112,10635,7624,7665,10808,2450,4297,2950,6168,6967,4584,6179,10420,3187,943,7443,1215,2962,5896,5484,8689,4469,7147,7945,3607,2111,4479,3136,5047,4051,8050,8336,10015,2394,2573,4305,9963,4509,6916,3462,5254,4318,265,58,422,8927,279,5718,8069,6527,10620,4795,9766,4535,3189,7712,6998,10263,7096,6186,960,8809,5324,4206,4858,5143,4427,9101,4118,3352,2848,4776,10649,3175,9536,10208,2591,2944,7230,8912,7660,1614,228,2115,1385,7043,4437,8662,8349,914,10319,2130,8649,5249,8294,9152,1461,10162,1169,5198,9535,7018,399,7658,9755,4451,10107,3866,506,1526,3283,6274,7184,7926,2870,611,2884,1865,1984,10758,4449,7617,1706,2728,9443,7374,651,3592,8476,9028,9516,7453,3752,8774,6731,7011,6912,9047,10763,8654,3789,6093,2543,1009,3647,2101,3518,8563,1662,6686,2716,10072,4495,8603,10387,5306,9696,8508,8775,998,3658,21,9772,7972,10274,9585,2572,4289,564,8579,970,6209,6358,1235,1250,894,9579,5949,7248,1856,3268,3202,525,3406,6090,872,3916,9262,5442,55,7089,8429,215,4461,5749,9351,5559,8783,3624,2598,7611,8141,3432,2807,5399,1645,7458,4004,9925,7703,6684,8374,5238,9540,5565,6226,8421,5782,9113,6632,5582,1428,5438,1350,2694,3338,9419,1501,7360,1800,10573,6001,3238,9847,7769,7848,9593,7594,10254,5191,7481,7963,4161,9551,2355,6651,7846,9455,3480,6086,376,6604,9858,2286,4041,5267,6315,5875,6999,2967,9344,4336,8490,1314,3827,3736,8628,6305,152,5586,4720,8351,7064,7373,4091,3303,4390,4727,3815,900,3115,142,8009,359,590,341,10413,1075,5869,9595,2711,5920,9769,4541,4986,3219,6761,7431,1470,573,5976,5840,593,9598,10829,3984,9532,972,9550,9126,8584,2866,2593,6841,1923,7999,1463,2602,1905,4707,7600,200,10237,2819,5960,6033,10489,9243,1059,9768,9033,7097,3127,5065,2215,10745,6364,1545,10598,7391,7412,9577,4228,4783,9342,874,6,4732,725,6323,8143,8478,563,1737,7679,4868,7541,3339,10428,8035,8307,3828,5292,2491,6861,3474,9376,4984,8926,5745,9562,594,4802,1598,3637,8779,4358,5108,6053,8709,6580,7910,10266,8253,1736,9390,4560,489,5768,2368,6743,262,10490,3851,2991,1027,2993,581,3300,6809,3358,8496,3655,7176,8501,2559,236,1577,10019,1233,1926,9426,9050,4332,10236,9089,9992,2133,9230,2617,1685,2422,2900,4736,5111,8720,7062,1289,4021,3123,7415,9150,8806,2136,9501,1873,2225,5894,7800,9377,7845,9006,3649,8266,5101,5469,6318,4219,5098,5847,4170,2093,2536,6398,6137,5912,1015,8718,9812,7629,1341,292,8172,136,8723,511,6933,4351,3976,1641,5420,189,9427,827,5370,9382,9529,4914,3563,7957,6710,7138,756,6017,1047,4293,6606,3988,7432,9822,197,8112,1405,2121,4199,4715,10476,4870,5735,6099,1370,5826,2507,7893,6648,10108,8484,1360,6661,6402,7414,7349,7433,9251,38,4499,4619,3399,6136,5986,1976,5833,2301,1948,167,4844,2327,2438,7444,3919,6396,5456,8590,1446,9861,4611,5058,10406,5656,6771,3516,902,8252,8212,851,2759,9021,6530,7577,1972,9201,9295,1219,5407,6493,2629,69,828,6146,9740,4086,3705,281,3975,5756,1244,5418,3704,8989,86,2926,1859,8158,3617,8800,4685,6262,1679,9940,937,5550,8102,1276,7350,1571,9969,4958,4434,8343,5607,4620,166,1456,567,6311,10542,8218,6767,9392,7219,7671,5179,2052,8174,8196,67,781,9413,5362,6840,643,4987,2612,6334,4817,5097,9712,955,2933,9275,8013,9334,10397,10535,37,1613,4085,2753,10784,6405,6631,2175,5059,1539,191,10242,6235,7228,9103,3131,6708,8213,10648,6410,5213,3334,4472,6277,2802,7632,4657,4442,2056,4153,2267,3632,1451,1476,3641,8313,2099,6923,8627,1022,6041,6387,9817,1756,379,8140,4887,6268,1917,9978,6062,10498,4029,1975,9922,10569,2665,2254,5206,3326,9282,3374,7677,8413,1855,2299,2408,7897,3778,6217,7077,2816,2473,5573,6249,9184,10079,7537,7300,3419,8664,289,7896,2468,249,4841,3267,7504,7545,4286,5552,128,6726,8577,2406,314,5839,3893,9215,5132,7875,2651,10559,8024,1497,9697,2241,8407,1964,499,6587,932,7341,2089,10213,1286,229,1433,479,7755,2581,6570,4124,969,7251,501,8260,2028,4534,6417,97,10145,8847,8816,5296,4292,1552,9161,3846,10765,1311,10296,8146,6473,7175,3693,330,4928,6052,9130,1208,3918,6785,4164,5763,4497,6397,3917,1198,8173,4090,3498,517,2969,10168,8156,1104,5622,10497,3515,8222,6551,10128,3380,3979,8882,10467,7829,3410,883,10431,9639,5791,9720,8700,4792,9247,5727,3771,5858,8546,468,548,3500,8646,6453,746,10446,459,9674,6253,2120,8472,1044,1650,3860,7778,9998,5829,10804,2498,9094,8331,10089,4561,2108,6815,667,4892,3389,6221,5341,2955,6174,10630,2451,6956,7773,10232,6549,248,528,6597,1411,60,6669,2114,8455,4920,6027,3726,6516,8474,9508,2363,9716,308,3039,4895,96,5476,10205,916,6195,5262,7131,3511,5516,1430,9217,4191,7301,4448,5598,9093,6161,7025,9710,4767,8055,5773,3843,4960,8705,7826,9457,7524,4359,4346,1102,4466,3310,9339,5282,10191,5460,7694,4042,7844,4502,9644,10333,3838,669,7428,515,7564,8854,6567,6725,177,635,367,6803,8358,10734,4586,10313,1124,9949,2453,7949,7050,5678,577,4640,44,5107,5661,4439,6409,8136,5356,1025,7595,8736,10384,7525,4728,8608,4290,2712,5538,1217,7872,5702,5635,1479,8072,6834,2928,9143,3770,9283,10200,3468,7333,6922,8663,5660,2595,9920,6428,3036,7012,5957,9315,3118,5788,10378,10659,595,6255,4779,3224,10510,4991,6175,4936,386,8901,6586,4837,4694,1324,8350,9647,3829,763,784,5687,4380,5969,10198,1732,8326,7191,1788,3282,880,2502,2461,2018,5962,5272,3776,7171,2003,1911,8239,2915,1794,5811,6839,3163,7475,2860,9085,889,1774,2958,5517,5532,10285,5775,9919,5335,7888,10738,8497,3758,5004,1498,9309,91,3677,9557,1617,9314,7597,9792,5037,8793,7644,7967,1365,5994,7079,3162,710,5926,2196,10651,5008,2852,5981,7911,9780,4811,3863,4833,4155,4298,4028,6294,5193,558,2574,7150,8423,2954,821,9669,9181,9241,4593,8702,3220,4505,10210,10705,846,1804,10746,9747,6938,2055,3611,1716,7874,6617,9745,5081,10009,1945,1152,8377,1658,929,5765,7567,4724,6374,7,10138,1007,5086,8179,2255,9614,6784,2853,2512,9010,2920,7871,6825,6697,10373,3485,8277,7100,3344,2037,782,7048,8166,9967,3614,4043,4385,8769,3204,9353,8285,10679,10219,10586,1066,6420,6537,6511,7608,2871,6819,8934,6083,1710,114,10793,4859,3472,6354,8229,9302,8873,9515,213,6472,3069,7188,7021,6505,5882,5336,8157,3067,1158,1496,8596,3936,7812,30,4850,10234,6368,2522,153,8049,7619,5241,6557,3087,10706,2166,6722,3835,2943,47,6519,9917,1866,5795,5448,6046,1439,3708,2882,3215,7291,4729,6786,2648,3608,3933,1703,7121,8625,5303,10597,814,100,3820,372,119,7357,8070,2649,10617,4856,3024,9534,7791,1436,8375,10695,5721,1105,6266,9054,10593,3512,7883,2072,8500,8309,5612,925,7466,10192,10040,5915,7078,3628,4342,1767,10377,3877,9587,461,9985,5500,8868,10111,2000,8248,2206,9409,655,8586,9321,8341,8832,1781,2078,6787,35,6558,9146,10082,2452,5000,5699,6485,3151,3011,5526,8469,6555,9335,3279,10665,6256,185,8316,277,7637,5619,4223,234,4388,8198,10435,5905,6021,10753,2847,3698,2253,1352,5624,832,6471,3662,134,1540,5657,4901,4205,715,4241,1715,7196,9765,4512,5736,10537,8573,7234,5730,10549,9968,926,9986,4413,5880,10809,5544,5892,3085,1880,10306,3259,3973,3367,7780,1839,982,9099,8581,8600,772,5302,2219,7863,3913,6085,5572,10369,4142,6134,2764,5291,9476,1551,6303,10702,4007,5317,2989,6937,1854,2201,6182,3074,4344,8782,6641,3999,5461,2150,8908,2677,5364,1965,1531,115,7549,10581,6019,8943,2749,2161,7470,7275,2151,5470,2851,1814,2445,10360,3520,5814,10055,1019,7607,3620,502,4883,10574,1389,3260,7108,2171,5411,1681,1409,8719,5755,10749,2655,9300,2511,10037,5011,9528,3508,4794,1084,3548,3125,2030,10643,6004,2534,1457,10256,8066,2232,4069,445,10766,5265,8169,9654,6113,2942,7995,6605,3403,8576,8291,9193,4036,2599,945,3244,4656,787,9645,2746,7749,10324,7299,6793,5089,8885,10348,7792,8529,3527,1888,9002,4666,7269,10622,7815,5846,3578,6406,692,3045,6369,8776,3072,6071,6746,7526,6900,5062,8011,9658,392,5817,5623,10453,8214,2722,4411,5443,1181,9498,1028,2138,66,5073,5605,3540,6440,7271,170,1188,1163,7934,1751,2224,6073,4,3052,1065,1369,1801,2499,7895,5346,10802,350,2988,2849,3619,4937,6455,10156,10041,7690,8103,9369,8619,4257,1282,5676,9827,1138,8631,5201,1099,9032,3668,9673,2146,5226,6251,895,9841,2005,6152,3927,7243,7448,4544,4231,7083,3731,3272,8293,5116,9350,8711,9137,2167,4407,4065,10047,7584,5931,8064,259,10301,9884,6692,6500,5844,1835,8021,1881,1673,6677,240,785,5465,4258,957,3742,4189,7040,1892,5083,606,9132,5506,2320,10501,3915,8530,9147,666,2187,225,10096,6074,4974,734,8852,8515,6694,7059,4056,2024,5798,10686,7659,3402,9990,9934,7364,3257,7438,165,8637,6240,4484,3126,3714,405,196,3514,3900,2804,8284,5045,1564,2754,10158,974,1173,2859,9233,10103,3839,3885,7606,9202,4044,2364,9798,7922,2318,3737,7164,8480,6543,9952,7860,1149,6778,4773,9474,6924,3407,983,9379,8728,7336,3076,8772,3110,8065,7080,10509,5148,7256,9648,9701,9832,8517,817,4202,3455,8453,681,458,8549,9836,3897,2356,10783,8974,9048,417,6302,8208,4721,5451,9620,1996,4139,7816,5421,8796,8119,3147,9465,6450,2068,6112,3167,10486,4807,532,5753,5824,9034,2304,6309,485,272,8802,416,5807,5742,8325,9739,9890,9625,7342,8585,3967,7757,843,4900,575,4814,9771,4523,6800,8942,5117,8707,10279,4430,6997,9199,1582,9292,7404,9976,3531,10093,3523,219,6155,6006,1632,9042,10422,10083,3673,9698,4546,7662,6059,1699,8880,8435,9994,922,2486,911,3114,660,29,1005,4520,1815,8202,10382,5234,8950,2503,8330,9019,5295,1860,1366,8221,614,7884,1955,174,4536,5974,2708,2307,7738,9242,8706,9039,7446,2159,10756,112,3003,9493,6167,9862,2903,9159,7519,7439,2614,6084,6072,8822,1838,3239,6675,9422,3964,3033,6899,4754,3963,4853,3710,5040,8080,4061,3597,2109,2123,1595,6211,5146,6791,4827,9996,4144,5569,7942,1159,9943,7572,1639,3672,10359,10690,4525,6196,9481,738,716,4489,2685,10147,4248,8583,6457,3387,10614,10364,993,1037,1404,10080,9258,5580,10403,3854,4116,5118,10669,9891,9737,5665,2469,10241,5489,6971,8936,6880,5977,5649,9228,3158,7784,5115,1574,9316,1797,8385,835,169,2288,8864,4121,6575,4101,407,2421,3898,5395,5616,3453,1454,5431,8122,156,1864,87,6008,7652,6427,7356,2260,1090,4823,8447,2457,10280,1631,1538,950,8406,1279,2966,7552,2347,7259,6394,10560,5868,3560,5823,3451,3046,5019,2414,7785,7929,8217,3320,4226,864,6148,6673,6490,3400,1310,9492,2348,4415,968,10323,3990,5776,9900,9336,1046,7148,8798,5151,2960,6178,6623,1204,4454,1760,7168,6117,1146,10719,10458,1649,5397,10282,10584,9794,3475,10402,10392,10762,3270,413,1759,9711,3664,3306,5274,8678,652,8665,8416,6408,8046,1624,5865,9687,8657,5888,1062,2184,1164,3280,9055,3097,5372,424,1287,5887,4647,7493,6962,10740,5382,6805,4082,8845,3590,7270,2228,2630,3021,9731,524,4700,378,7771,10656,3398,6644,990,1322,10016,6855,5261,10799,9323,6720,6162,1252,7898,487,4787,9439,5898,7813,3728,7689,8353,45,8231,2275,1665,709,2700,7044,6229,9875,3411,1846,5567,9855,6894,658,448,8410,1416,978,9570,5480,2329,290,4719,6830,7788,5473,8767,6832,2750,4270,2778,4501,1530,4417,7262,9398,3441,8026,2141,6700,4803,4545,3201,8518,9432,5734,3969,9904,8241,9760,10652,4558,5677,1535,3251,3971,10052,2199,5751,9956,7627,3299,10078,6578,4000,2567,1193,1453,2231,1398,1714,1507,2613,8977,1701,4465,5809,820,5049,9517,5595,5917,529,4815,224,3910,7054,3727,1939,7937,6884,9636,8394,5682,5405,9487,4907,9554,9743,10322,6208,9414,8843,7213,7505,10188,7604,7837,9860,10351,2087,14,7212,5999,6474,209,9842,10468,3271,741,10611,5142,4020,1870,10141,2070,238,4670,3450,3188,2479,8304,1263,9502,3330,2361,9077,160,4884,2948,2011,1032,1652,9131,2428,4917,6908,8219,6141], "time_ms": 0.160283, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 613, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "generated_query_count", "value": 2400001, "time_ms": 1300.699204, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 428, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "generated_samples_per_query", "value": 1, "time_ms": 1300.699204, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 429, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "generated_query_duration", "value": 1200000500000, "time_ms": 1300.699204, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 430, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "generic_message", "value": "Ending naturally: Minimum query count and test duration met.", "time_ms": 601301.117155, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/issue_query_controller.cc", "line_no": 482, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 601310.374918, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 898, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 601310.374918, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 900, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 601310.374918, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 902, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 601310.374918, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 904, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 601310.374918, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 906, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 601310.374918, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 908, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "power_begin", "value": "12-25-2024 03:53:34.764", "time_ms": 601310.375815, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 564, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "power_end", "value": "12-25-2024 04:03:34.765", "time_ms": 601310.375815, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 566, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "result_validity", "value": "VALID", "time_ms": 601754.779231, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 644, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "result_min_duration_met", "value": true, "time_ms": 601754.779231, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 649, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "result_min_queries_met", "value": true, "time_ms": 601754.779231, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 650, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "early_stopping_met", "value": true, "time_ms": 601754.779231, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 651, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "early_stopping_result", "value": " * Processed at least 64 queries (634470). * Would discard 62890 highest latency queries. * Early stopping 90th percentile estimate: 1029890 * Early stopping 99th percentile estimate: 1211301", "time_ms": 601754.779231, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 669, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "result_query_count", "value": 634470, "time_ms": 601754.779231, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 679, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "result_qps_with_loadgen_overhead", "value": 1057.45, "time_ms": 601754.779231, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 702, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "result_qps_without_loadgen_overhead", "value": 1064, "time_ms": 601754.779231, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 703, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "early_stopping_latency_ss", "value": 1029890, "time_ms": 601754.779231, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 704, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "early_stopping_latency_ms", "value": 1211301, "time_ms": 601754.779231, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 706, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "result_min_latency_ns", "value": 860498, "time_ms": 601754.779231, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 741, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "result_max_latency_ns", "value": 1485171, "time_ms": 601754.779231, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 742, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "result_mean_latency_ns", "value": 939846, "time_ms": 601754.779231, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 743, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "result_50.00_percentile_latency_ns", "value": 920898, "time_ms": 601754.779231, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "result_90.00_percentile_latency_ns", "value": 1029406, "time_ms": 601754.779231, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "result_95.00_percentile_latency_ns", "value": 1133784, "time_ms": 601754.779231, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "result_97.00_percentile_latency_ns", "value": 1189113, "time_ms": 601754.779231, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "result_99.00_percentile_latency_ns", "value": 1210866, "time_ms": 601754.779231, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 10578, "tid": 10578}} -:::MLLOG {"key": "result_99.90_percentile_latency_ns", "value": 1222247, "time_ms": 601754.779231, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 10578, "tid": 10578}} +:::MLLOG {"key": "loadgen_version", "value": "4.0 @ 486a629ea4", "time_ms": 0.008190, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 53, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "loadgen_build_date_local", "value": "2024-12-27T19:45:31.956819", "time_ms": 0.008190, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 55, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "loadgen_build_date_utc", "value": "2024-12-28T03:45:31.956827", "time_ms": 0.008190, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 56, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "loadgen_git_commit_date", "value": "2024-02-15T15:46:50-05:00", "time_ms": 0.008190, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 57, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "loadgen_git_log_message", "value": "486a629ea4d5c5150f452d0b0a196bf71fd2021e Hotfix: DLRMv2 Audit Test01 fallback failure (#1626)\nf9a643c0a0e920588da1b51a1d822e1071a9dbec Add number of tokens to offline (#1623)\nf06b920eac2170df38b5502c7443f50c9afa6045 Bugfix: equal-issue mode on offline causing accuracy run to fail (3D-UNet) (#1624)\n0ed5190e82e1ca1cf95626e50e61cb6678c7f2ff Fix submission checker and TEST06 for Llama2 (#1616)\n817dd961221d3ca284b024368130bd5dc5e4b54a Turn equal issue mode off for TEST06 (#1615)\n840435a1abe8be823e0a5a18ad3173bc520aa24f Update README.md with CM commands to download stable-diffusion, gptj and dlrmv2 models (#1604)\n3190d09a148bc1d72c9d946e4cf3b54db0d0e60a Typo fix in README.md (#1588)\nb0777f0bd3b7e5cd6a51d670d8b4df656cf6fb29 Update README.md - remove unwanted lines in CM commands (#1601)\nd7dba0826ac96a0e8bcae525086961e47c409a38 Remove loadgen warnings (#1608)\nd45a66c33bb0b6548d6172b38645bf94ffe24b39 Add upper limit to tokens per sample (#1612)\n44285d9a44041c3d4f7df0acf1e72c375e9fcbd2 Set completed samples per second as llama metric (#1613)\n357ccefca4db55ff373e029d5d1e3e6246bde5d4 Enable equal issue mode for LLM benchmarks (#1610)\n104d85531af7ae2a07ded36dd6e8c7f2c7746d06 Add sample length check to test06 (#1603)\n473053fc9f1b8c5edf60e11ec74607126938fbc4 Fix loadgen token metrics latency constrains (#1596)\ncc3daae3c7f18d7eedc5ffbc271d324e5aca6233 #1598: fix token and sample logging for Llama2 when accuracy_log_sampling_target is enabled (#1599)\n3a902e598e5a17013c26fea975c0a60e6d7feae1 Add support to dump 10 compliance images during accuracy run for SDXL (#1591)", "time_ms": 0.008190, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 58, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "loadgen_git_status_message", "value": "", "time_ms": 0.008190, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 60, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"05d4a62d355a90363847358e58afeee4a54ad838","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/bindings/c_api.cc":"32181da9e161c285f8fe46ddaa49e6cba2f9f918","/bindings/c_api.h":"91f58bd79b83b278f3240174a9af747fc38aff74","/bindings/python_api.cc":"1dc0865130371c6a8ddb97cd8039244d74d6fe46","/demos/py_demo_multi_stream.py":"6ccf7dcbe08f8382ed2d3618370c2b21e205ca13","/demos/py_demo_offline.py":"6c3075538e0a3df5e93edee917f4537099759e11","/demos/py_demo_server.py":"40e37ea121ac08b21ed6af41aca8eb5eda641b89","/demos/py_demo_single_stream.py":"4ee5f048fb0dffc415342a2bc2049fb155bd2894","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"126e952d00f4ea9efd12405fb209aa3ed585e4b2","/issue_query_controller.h":"923d9d5cdf598e3ec33d7a1110a31f7e11527ec7","/loadgen.cc":"6650091ba7a918f343b06eb7a5aa540eae87275f","/loadgen.h":"e00fdc6dbc85a8c9a8485dbcbfe2944f81251c4e","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"197efc96d178e5d33a750d07fa7b2966417506ea","/logging.h":"ddb961df7bcc145bcd7cce8c21f7cf075350dcbe","/pyproject.toml":"974adb632af4b76a1c72b914cc2cf9b3cbc66aa3","/query_dispatch_library.h":"13ad6d842200cb161d6927eb74a3fafd79c46c75","/query_sample.h":"e9187c8612bbdc972305b789feb6e15c26e96cfe","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"a170f991de69f8a5e45616e871b39a30ffed5598","/results.h":"d82500c326c2de83db411f1146882aa4692b419c","/setup.py":"e3e707e1191478332c00f1b6ff806f0ef98e0f65","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"aaf3803e700f3431a5ada72559eef5cce996ea64","/test_settings_internal.cc":"42aa46b17ac3fb2748cd5cde60fa8742e916f3c8","/test_settings_internal.h":"73b11c02228f3afb4c924db7a86fd8a2b5b9e483","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"4c9d98360b507d5673049ed70154fcfb2934f8a6"}, "time_ms": 0.008190, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 67, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "test_datetime", "value": "2024-12-27T19:48:10Z", "time_ms": 0.015944, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1198, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "sut_name", "value": "BERT SERVER", "time_ms": 0.015944, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1199, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "get_sut_name_duration_ns", "value": 295, "time_ms": 0.015944, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1200, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "qsl_name", "value": "BERT QSL", "time_ms": 0.015944, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1201, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "qsl_reported_total_count", "value": 10833, "time_ms": 0.015944, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1202, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "qsl_reported_performance_count", "value": 10833, "time_ms": 0.015944, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1203, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "requested_scenario", "value": "SingleStream", "time_ms": 0.021090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 268, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "requested_test_mode", "value": "PerformanceOnly", "time_ms": 0.021090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 269, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "requested_single_stream_expected_latency_ns", "value": 500000, "time_ms": 0.021090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 274, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "requested_single_stream_target_latency_percentile", "value": 0.9, "time_ms": 0.021090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 276, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "requested_min_duration_ms", "value": 600000, "time_ms": 0.021090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 312, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "requested_max_duration_ms", "value": 660000, "time_ms": 0.021090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 313, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "requested_min_query_count", "value": 100, "time_ms": 0.021090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 314, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "requested_max_query_count", "value": 0, "time_ms": 0.021090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 315, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "requested_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.021090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 316, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.021090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 317, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "requested_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.021090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 319, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.021090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 320, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.021090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 322, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.021090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 324, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.021090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 326, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.021090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 327, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.021090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 329, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.021090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 331, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "requested_performance_sample_count_override", "value": 10833, "time_ms": 0.021090, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 333, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "effective_scenario", "value": "SingleStream", "time_ms": 0.021322, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 409, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "effective_test_mode", "value": "PerformanceOnly", "time_ms": 0.021322, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 410, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "effective_samples_per_query", "value": 1, "time_ms": 0.021322, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 412, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "effective_target_qps", "value": 2000, "time_ms": 0.021322, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 413, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.021322, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 414, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.9, "time_ms": 0.021322, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 415, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.021322, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 417, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "effective_target_duration_ms", "value": 600000, "time_ms": 0.021322, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 418, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "effective_min_duration_ms", "value": 600000, "time_ms": 0.021322, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 420, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "effective_max_duration_ms", "value": 660000, "time_ms": 0.021322, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 421, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "effective_min_query_count", "value": 100, "time_ms": 0.021322, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 422, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "effective_max_query_count", "value": 0, "time_ms": 0.021322, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 423, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "effective_min_sample_count", "value": 100, "time_ms": 0.021322, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 424, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "effective_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.021322, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 425, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.021322, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 426, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "effective_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.021322, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 428, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.021322, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 429, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.021322, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 431, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.021322, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 433, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.021322, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 435, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.021322, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 436, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.021322, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 438, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.021322, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 440, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "effective_performance_sample_count", "value": 10833, "time_ms": 0.021322, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 442, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "generic_message", "value": "Starting performance mode", "time_ms": 0.022770, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 841, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "loaded_qsl_set", "value": [4117,9486,2947,9270,8227,4487,9799,4629,8571,8839,7101,2191,3315,6375,207,10454,3122,9490,4083,6119,1890,7657,9347,344,201,4232,5741,5454,9533,7842,1596,5330,10615,8827,8094,6714,9169,977,6619,9924,4982,4894,6544,10118,783,766,2122,7163,396,4200,9864,4254,8440,1758,10722,2829,2788,9906,2719,5613,8726,7847,440,6103,8567,7497,4793,1184,8675,10587,8270,7074,7408,8740,9265,4855,1199,220,6556,6595,1226,6695,3066,2143,6264,9806,7122,5007,10075,2370,2185,7254,77,2806,6243,6687,1156,1802,7302,391,3348,8159,9708,251,3595,3947,2302,4690,122,3116,1300,3768,6388,2986,728,5127,3904,6807,6259,2218,7555,8821,3580,7804,1321,8460,5419,6542,7833,3365,2290,6346,7952,918,707,118,10469,9023,1304,5220,1891,6878,9524,6522,7867,4037,7698,8507,9634,2145,10057,6263,7486,7557,2401,6082,3434,5927,8708,10081,8114,1349,7742,2742,7157,5705,4638,1764,3651,920,4016,2744,5859,6740,5380,9649,2,5245,5230,6421,4071,8909,6319,288,9346,9671,9646,9360,4702,5044,9157,6296,8318,8971,8184,7702,10475,7318,4024,5530,3991,1171,1909,6230,8461,8099,10563,5398,10760,10693,3688,8170,4952,6827,5893,7767,2656,10032,9178,10480,7195,9359,10608,10261,8975,5432,363,3346,3049,3360,4432,9736,473,7587,9416,1021,6596,10352,2666,2325,973,2071,7368,4267,867,1040,5796,3491,8237,9597,3489,10568,3777,7899,856,4818,2858,4662,777,10776,10026,6065,6481,2590,6780,10553,8789,6032,930,1049,10178,6187,585,3359,5832,2014,480,5866,5017,5141,1525,8271,8132,1642,6270,2265,140,1174,4641,8017,8844,6569,5575,7081,3779,3845,467,9761,3314,8653,4250,7578,8907,664,217,8419,8757,683,4565,1638,5216,2004,6679,4485,7683,5180,7422,6070,6283,2909,6281,8933,10639,4340,3079,3513,10552,2291,7487,2680,8807,8370,6127,7434,1822,908,8101,451,10316,8282,10724,885,1834,1556,6121,1468,409,1908,10539,7238,2891,10312,9435,2178,10051,4522,9650,615,7193,7796,2008,6011,4347,435,6881,1396,844,971,4216,5001,7886,6875,1348,4300,9348,2709,10580,9993,6705,8784,10821,9232,7724,6581,9764,2913,10125,6914,7693,4568,10526,10231,8502,7825,1783,795,6828,8006,4595,3892,3075,1278,10050,8235,4679,9971,75,2164,1175,3684,9869,8414,9016,765,2721,5786,10703,1326,7463,237,836,3912,5482,7835,4374,7661,4070,7754,7180,9945,10415,3638,4208,939,1097,7628,5725,3723,4547,1994,5232,5279,4049,7309,5963,10478,9834,7758,9463,3666,4663,801,4030,3010,8236,9237,2470,3382,6336,7570,190,4552,4113,2229,9018,7038,5021,8701,8268,4490,1373,6092,10181,10445,3056,4333,10394,1640,2328,3349,10069,10683,6439,3191,2544,4760,1739,8900,3,258,5416,4671,2041,9375,2661,6376,2306,5897,6373,2125,1940,5109,4256,6203,6411,9311,8029,2200,6301,1734,6176,6443,3007,9583,1841,4627,2523,4306,2801,5208,8607,4961,1467,10700,4031,2181,9958,8194,10295,9043,5366,306,9056,162,3196,10759,4349,5401,7030,6184,9216,2069,6357,5640,3015,1752,406,10049,3983,791,5610,5944,304,6380,1937,7862,9366,9365,1607,9252,2698,10350,5221,4847,10707,3023,1615,7017,5990,4045,7231,1202,7182,5172,3176,9612,6693,1013,10334,1805,8730,8841,7372,10094,6010,3216,8177,7406,9472,9608,2533,7691,3959,1623,5144,1831,751,4912,2152,8144,241,4187,10001,1924,8402,7877,8928,7294,4915,3876,3242,8996,7153,1000,10658,3946,284,5088,959,10661,1018,8931,546,4262,6489,3786,5278,4308,8178,7638,5211,7142,5771,634,8273,263,2548,4674,9910,624,3290,5925,1524,2642,7198,10345,3539,6719,10735,2006,10233,3816,9578,8624,6769,9661,6468,8952,9702,8670,5175,3449,1773,6130,636,1622,1696,1414,6523,2898,10650,4420,1280,2538,7880,3743,1227,2757,6131,3371,7445,1680,2901,36,6868,10811,7140,1900,10398,3724,8765,8278,7379,6040,7914,1580,3535,8033,7536,6480,3861,2437,5082,2881,10596,4902,8162,9637,4886,411,7189,1789,6704,9176,10045,4107,6333,4583,3923,4843,9935,9955,9297,3222,8257,6365,3095,1995,7924,5428,8640,6628,4819,7152,5016,7932,3226,1267,1225,9025,7878,6321,2373,3801,1399,9063,6506,9436,5781,7500,423,3588,7110,4607,3476,2477,3635,2425,9394,10449,4054,8148,5510,2236,9543,10499,6153,780,8692,8105,10268,172,4128,841,8925,5453,8561,2135,2976,2845,2209,7418,10817,5620,7010,9960,474,6545,8223,9713,205,10548,905,2396,2914,5591,2887,3379,4865,3304,5849,1309,10171,608,7314,6228,6961,7068,6943,4613,1317,121,8611,5902,5827,8042,1139,3121,223,2984,2897,3144,5379,10590,6433,3871,5475,43,1026,1192,1599,5729,6194,5838,1254,2054,2706,7925,6060,2177,10647,7727,5681,10000,3549,10814,4433,2100,6869,6593,9404,5534,5801,10321,9145,8363,7451,9072,4600,5096,10372,3022,679,5631,2552,723,4668,9613,10153,7423,9114,2973,3071,1413,4956,2838,5084,6777,5196,539,5036,382,7480,9327,323,4766,2273,2684,8182,10134,9203,2667,9185,2658,9087,6286,4162,6620,10472,1095,6759,7216,8191,9738,9548,5029,10326,5478,4089,8710,5439,6126,6160,4828,1006,1677,3546,6930,425,2645,4573,2770,10248,358,4975,10603,8129,8614,10180,7902,4303,2447,3427,753,8920,9449,256,6859,5711,5246,623,370,10767,2563,8300,9591,9991,5126,9568,2647,484,8644,3654,9507,8632,10076,9813,10638,4326,3862,569,7020,4213,5628,9341,839,3341,8347,5163,5563,7339,6928,1197,9158,3099,6120,1483,9545,4608,4215,2761,1259,6802,4364,4001,1606,4222,4752,3057,10022,1108,10592,4398,10376,9888,10033,4273,1238,3428,6553,6145,5731,7273,604,2436,3170,8673,5901,10074,4513,8305,8697,7476,5015,2669,3100,6344,9338,10383,4972,8727,1363,3223,10495,897,1698,9553,5662,53,9328,3733,9932,7102,2212,7382,7159,4556,10748,9294,5895,2262,10512,5913,8314,10624,2530,7560,10464,2443,8787,3800,1510,2027,9795,9670,283,2351,4260,2577,1518,906,3377,4764,2113,10699,3948,8465,9264,9267,5600,2924,3783,9209,7966,6464,720,6973,155,6227,7994,2636,535,8846,2931,2046,4953,10058,8228,5200,2211,4234,2207,8121,2007,7268,3212,7772,6496,10201,2762,7534,5539,10555,10496,628,3653,702,8683,10325,8660,8411,9387,326,8842,837,9688,3855,7053,1826,7944,1053,3193,456,361,3206,10315,2059,6612,6173,4562,9003,8034,9162,377,7551,10769,5354,6299,5997,2080,6288,6068,653,9937,10825,8833,3247,3058,4923,9104,7983,3305,9322,8339,7711,131,7328,5212,8623,10761,9718,5402,948,10640,8659,6479,2768,4616,2223,10625,4252,9980,9807,5318,6188,159,10655,3565,1478,6860,3521,4646,9880,1608,10427,9804,2092,4897,10238,6020,7642,8091,7789,4058,6681,4173,7918,5523,2444,9898,7161,7736,4667,2603,9370,7403,10797,1857,3276,2276,3445,8197,991,1863,4266,2170,4331,3296,6658,1308,7542,8458,4533,10177,6873,4190,815,2726,5589,2846,269,2634,1560,7067,7554,3102,4908,10741,10086,5286,8734,3824,1744,4111,3264,7494,3825,8834,6139,8432,147,1589,1871,941,2911,6018,7906,2874,3713,9523,4023,7522,6116,6864,5698,5464,3924,4590,7345,7395,4790,10540,1020,5309,5950,2956,4983,6163,9194,3848,3612,10525,9069,5253,7232,6876,5003,5067,2350,10375,10523,850,6399,10002,9689,7255,1016,3707,3384,2137,4242,4077,5170,8388,2387,10577,2482,3735,4313,8862,2029,8452,10528,6058,6871,5233,1195,3301,8738,3194,2638,2488,5009,6981,7935,10684,10631,1043,1962,3541,4973,9581,4285,9118,9770,8279,1316,6284,1161,877,7894,3981,2423,6872,5033,1798,6181,1255,7517,4067,3319,7457,5164,3977,1719,4436,9317,4765,4137,6515,10025,4628,3745,7739,9445,7016,1980,4481,8580,175,10704,10653,245,8163,3834,7006,9135,1950,4401,2289,1412,7325,4821,6917,1361,7569,6782,2474,8075,5757,1034,3353,8570,2678,4922,4356,6831,3401,10393,10508,5161,2635,1472,3331,3048,9526,5383,5670,7508,5113,10148,2912,8547,1183,5496,5139,1153,6897,7997,6625,9819,5766,7460,10124,6637,327,3466,1848,2455,5939,2282,6371,4812,6721,5314,4780,3112,5154,250,7103,7158,463,6920,8368,6024,6877,342,6799,437,2876,9690,4889,5129,2631,3366,8380,1528,10135,584,6292,9180,5863,10031,2432,980,7211,3117,6909,1248,3761,3601,4976,9420,10602,178,10755,1150,2009,6653,7378,4422,8445,1978,2390,5417,4353,6452,2002,10595,1306,2817,8735,8424,8976,4396,4919,9488,534,4478,8211,8264,385,8288,3063,9825,6996,9726,7239,668,2896,6212,5761,6271,618,8216,6838,195,216,465,8870,6774,8556,3392,786,2791,1096,10565,9399,7603,1723,4133,6106,5491,10443,335,8387,5006,8131,966,7321,4664,5361,8778,8085,10370,512,4408,9227,4676,927,5805,10457,5444,5333,2951,6804,7955,888,3706,8788,2895,1902,9874,7221,5222,10628,6300,3522,65,10112,7284,6826,9762,10226,8642,8030,609,3812,9914,8276,5946,4130,767,4876,3831,9441,5515,7928,4838,4550,78,6682,63,600,3798,8226,3577,8618,8919,4716,10385,5560,7695,9677,9470,6351,10585,6031,2941,2010,7623,1735,6574,1513,2774,1490,2303,5429,141,1915,7737,9086,2205,32,7165,4564,3715,8209,8559,4979,4985,7143,1656,6789,8025,7208,6845,503,3687,7028,5471,9975,5664,3404,9068,5248,2433,322,10390,3840,8058,1297,3551,3819,3342,7265,3266,8914,4394,3583,9786,9198,5609,1444,8947,3060,10252,5594,1291,7319,9559,3166,4669,9592,661,6602,4789,9415,2424,4265,7098,1876,7535,1203,10053,9809,3104,1941,2705,6635,3148,3598,1502,7383,9073,10425,6590,5162,3529,7306,10800,6393,8751,387,10172,2692,5719,2741,7855,4845,7039,2459,6901,6624,3554,4237,3867,10566,2116,531,5943,6690,792,8441,6260,682,3995,6154,4165,9066,2717,1842,5340,8261,8539,1074,4259,701,9500,8648,513,3417,2155,9041,116,4010,1218,7514,3738,10176,8592,4659,8491,291,10417,8338,7244,862,5005,4735,1362,964,9714,10691,9829,10779,351,4930,16,3561,5388,5985,5239,6821,4320,10682,3154,4367,8961,50,8733,4288,9893,824,2504,404,10529,6560,1489,8125,48,76,909,2237,2691,2416,1987,7699,4599,5585,10503,7807,3639,3994,5708,7135,4625,4516,5540,2916,5528,5975,7413,8915,1347,10065,706,5304,3436,6607,8262,5645,3625,1376,4476,2934,1592,1935,3958,5821,2456,3362,5367,4151,981,9573,10610,9207,6190,5275,4631,1708,1921,9287,699,1447,1636,696,3740,4597,3373,9444,1221,3780,8124,5381,7648,3536,10571,654,9064,9044,3503,9854,9626,3378,10675,10629,6080,4445,9605,2646,6982,5074,3618,52,6561,6185,3026,5968,9742,989,5867,9190,1178,2689,2001,6149,8334,2049,3180,5759,111,4691,2650,8777,3149,804,1868,5328,840,8687,1382,57,2419,808,10355,5588,5215,1403,5701,9,7853,8520,8669,8019,4006,9097,226,2827,1135,1901,979,819,2561,7049,504,4391,4400,6739,1402,5264,2230,10666,2566,7430,8459,10720,8794,10310,10174,443,6349,7310,533,7609,6200,4108,613,7558,7709,1553,5076,10645,300,2842,6261,2324,588,2193,2945,7510,7396,3094,1889,10781,3791,554,9284,9148,3519,9995,3209,5810,8322,9519,2560,3073,947,2781,7004,5548,7116,10152,10121,552,1887,2952,8739,9225,5819,3712,6550,5446,574,3090,8327,428,1266,8433,8383,4645,8754,6898,2652,6822,1455,2176,1519,8566,10303,6049,10068,4977,3894,7561,7488,5536,7224,7793,1989,3985,8695,3424,7035,7612,9938,6775,39,1850,7803,9174,3031,6441,2974,1603,7941,7223,7923,2831,2977,3080,2075,9236,312,8565,4634,8814,1558,10484,1272,7389,8872,10418,4755,7869,2610,10633,5046,6960,4193,7852,1754,2878,1936,3784,10314,4025,4805,4997,10007,9863,7155,10101,1342,9395,10291,9385,10218,799,1123,5843,2562,4905,1377,9226,5937,1986,7591,9962,1189,8902,619,9067,3458,7586,7210,328,8506,3000,1004,9212,4591,8020,8887,4012,592,101,3415,2527,72,7452,9400,7809,1561,7032,4840,7008,1056,8,5752,9588,3101,3106,3609,4503,8312,6536,3446,8745,1437,7547,10287,4816,1795,8004,5837,1042,9329,9008,9433,2047,1087,3421,4324,10588,6699,10747,7765,9719,10438,887,9062,6820,2946,4741,527,3043,1648,1133,2918,882,684,4939,4386,8641,8154,1323,6436,7873,10752,3686,10680,807,9886,549,3230,6147,294,7120,354,8171,8493,2663,6863,7489,6037,410,9913,124,6312,541,3159,10304,2117,735,9882,199,4315,9250,4055,3739,3826,9558,10782,3699,542,6645,7593,5123,6413,7532,8805,1517,8514,3396,9239,8883,6748,6002,5319,8690,6308,1691,4655,589,10278,5680,9489,6806,10465,4251,10142,538,5740,194,9167,9974,2107,1480,8439,5722,5674,5712,7528,3385,1906,10567,9057,340,5707,9308,6207,3078,2818,7518,6794,9555,9983,1676,1678,6079,2038,1670,2608,2335,10021,10243,7257,9405,8165,2242,1247,8620,7479,1305,8818,8790,7865,4998,5653,2097,261,5270,2140,2105,789,6984,7501,8748,9538,8167,7566,3307,8768,5195,3932,1130,3930,6535,3665,6818,870,6657,7938,8656,2975,8108,4377,1332,1792,10027,3395,5078,10380,724,6034,3423,4788,4511,9362,2462,7760,7023,2497,9802,5504,6817,4263,3361,8164,8418,2478,2287,3457,1264,9796,7218,6591,6401,1683,3534,7474,5945,9722,7233,3719,7072,9915,42,6386,1481,3426,8808,4075,9665,132,3679,1418,6327,4854,5667,1228,5621,10063,9685,9641,3792,8509,10725,1726,3510,6733,5311,6890,1127,1372,4557,9076,4493,1319,4639,6503,5373,319,6252,5205,2568,3262,5611,3692,8016,2058,10708,1990,10246,5149,6730,4745,9617,4496,8093,3764,4587,9046,8240,5654,7484,9304,9356,9122,9012,2409,2879,1728,8443,6110,2374,2695,6156,6123,7266,9525,1770,9102,10660,6745,5128,5806,10298,1081,5852,10365,5171,3316,1949,2796,5938,2888,505,9040,10137,2063,8296,703,7982,5184,5822,6713,8356,1212,10214,1643,1344,1740,6170,2765,8513,7491,313,7468,5830,5188,7420,8246,7931,5592,2300,5803,1333,1904,10524,2557,1320,9801,3887,3372,4648,8123,10207,10570,2782,9208,3493,8993,4688,9635,2367,890,4834,4750,2186,4059,9823,383,5746,6601,6995,3696,10626,6202,4034,2588,1415,3128,5342,9471,8634,6712,7283,5689,7596,5728,9160,6465,1823,8542,2381,129,3703,5297,5845,10090,9892,8109,10676,1417,6171,9518,2483,5993,6066,10657,4131,7317,4665,10546,7047,6337,6183,3537,253,7720,5316,4382,2720,8286,7478,3902,7237,3088,5066,2131,2830,10694,3644,1407,5650,5733,7970,8970,2283,8951,9288,6643,10349,4772,6849,3295,4178,1354,8522,10736,1720,4463,10409,7172,6220,1998,6367,7464,1973,6738,4238,9065,9537,2528,6911,4247,8425,183,7919,4791,5494,3274,6583,4995,7128,869,5889,7655,5199,1555,6754,10636,3874,5094,545,3805,1611,9596,2168,6048,7912,1253,3675,10190,6051,6887,5414,2919,2397,8982,2189,5410,2589,6458,7272,8280,6078,2349,9268,9229,8372,9571,2129,9070,7371,5048,8593,4925,5490,1503,6584,4713,6442,3119,687,3013,10757,8401,6102,6842,6324,7114,10127,9879,9783,4963,1626,2785,3640,9163,4835,1390,5087,8073,10203,3068,5747,1958,49,4916,620,1548,3018,9306,184,10272,3878,3053,2239,10187,1477,2902,9456,7260,5051,7639,5386,4477,80,1151,7277,5579,5072,5596,2204,10199,8267,2624,5110,6272,1330,10059,757,9947,1357,7745,6159,9580,6926,1486,10062,10488,8712,6582,7733,8905,3935,8265,7334,5495,9170,1657,4880,1542,9546,5577,3169,2880,8287,2763,1465,1041,10545,7951,2814,10729,1234,7881,6985,6670,5055,4329,9749,4744,7621,9946,665,475,6655,6976,1327,9703,33,9513,9857,3248,2158,7640,3830,6910,6599,10785,3908,9303,6403,858,2074,3941,4068,1488,4929,9954,4872,10514,7838,4677,1877,7014,8113,6883,1511,7174,381,1523,10229,9627,6966,9767,4283,1167,4168,6977,3543,6953,2737,4698,7908,8759,2429,7706,9642,7199,2064,7087,557,8591,4609,3962,2795,8283,6449,9092,9964,4970,4529,3616,6475,7523,56,2238,6431,8256,640,3416,688,10471,8348,5541,2371,994,5570,82,6166,2259,7589,10479,5713,449,2308,130,6882,6298,6517,9870,2353,4316,5452,2792,7556,3803,822,9084,798,9477,9082,657,495,8781,10619,4328,2710,460,10133,3234,2555,6069,5255,348,5511,6934,9594,5120,6729,3956,6290,830,9948,5911,1731,6291,4993,1533,9758,5693,8152,4988,514,10270,6000,9682,6946,8344,8051,5320,8247,9020,3751,7864,10163,540,5933,3650,6107,2840,9372,3701,8955,4269,8511,1024,2165,4739,5474,10239,2466,4157,6533,10780,9408,3312,2549,7113,4480,7741,3467,5710,10381,2392,4166,7485,9902,10116,1089,5281,2157,9182,412,6056,2281,697,4383,6531,7473,9231,7411,7070,1493,3570,4122,3461,4849,5119,3174,3139,8068,7162,3184,3691,2316,18,650,2382,8612,9692,6483,2442,7718,488,6642,8681,5271,9004,3942,7986,5406,6744,9014,6975,8403,7449,8721,3240,8297,752,4471,2833,10812,202,7701,10064,9164,1565,2172,3613,6589,2843,1052,8359,1932,2389,127,5508,4782,1048,2729,6216,5692,5219,4730,8987,940,7810,3907,4399,1220,9901,1549,1242,9815,673,7459,3322,8550,6865,5229,8557,2257,898,2893,8674,2446,7645,6528,2809,4711,2106,6267,1110,6415,7998,2094,9509,6269,4475,3096,1567,7066,2464,8516,1843,9885,8995,2345,4940,1077,903,6886,347,8766,3343,8534,5871,5310,5738,10217,7539,2823,9867,8238,7093,10667,2332,5535,1419,3203,7540,2417,1619,10744,4592,8817,9219,2565,4276,9380,6329,10426,8853,8106,6765,6932,3822,5864,825,8244,9566,8333,5954,5351,2978,3376,6275,10290,1604,1067,7901,3143,7979,3630,5252,5642,6098,3077,1464,325,10481,5853,10012,4658,8315,10029,2812,4123,4967,6097,4038,1755,2873,2905,8098,6941,3681,1070,8555,4875,10340,9944,9694,2180,7441,2904,8054,2531,9330,6750,6945,10067,6012,5092,10772,73,6213,1504,6425,8835,2520,1401,10768,3368,4375,4941,685,9479,453,1268,2245,1694,4325,7705,892,10775,7450,2679,842,3996,6241,1117,7713,10343,2124,5857,5617,6594,7305,8770,5928,2197,9166,5956,9299,10095,6524,10531,4335,6622,10823,802,3120,2073,3864,3490,6957,9845,3972,6935,5543,9774,154,5250,10342,8137,8040,4224,10500,7327,5779,2755,5085,8243,4632,1952,8047,7056,6304,3241,6219,5353,10320,4559,6552,4748,9475,203,4551,8552,7187,3748,4271,3232,5903,9402,1566,6381,1569,6192,5338,2965,8023,2360,10386,6383,923,10618,9172,421,3412,7582,7685,7332,4138,2820,876,1356,1325,1012,3814,9584,3799,5041,9273,5764,9757,1655,6991,8588,9423,8001,3853,6191,8755,576,6629,4115,8878,4253,3245,1885,2505,5426,2621,260,8848,3873,2060,8352,4842,7686,5359,3600,6732,857,5014,9386,9173,8150,3065,7127,10754,2987,9907,7347,1051,6114,1609,221,6666,7851,5093,2471,2465,398,2077,6459,5799,8462,7943,10731,2227,9619,3813,9031,4753,7797,6347,3836,1806,9134,6150,5643,497,3868,6915,5071,2769,3717,8575,4426,5358,1587,7752,4942,8857,742,8464,3442,3050,9660,1165,3309,5293,3171,5099,1154,1827,7728,3140,5995,7719,601,9119,2053,10056,10487,4949,5307,7866,7076,8052,4372,6045,9586,7799,4800,4575,5458,316,5425,6361,4965,1187,10816,1907,7808,10551,4531,146,3884,8978,1271,3568,10668,179,6571,163,408,1190,10175,10716,3837,9340,303,6792,5936,5979,6330,8574,6129,5555,2385,10594,4980,10728,5842,4321,10444,427,5890,1776,4598,5235,7900,602,4763,6907,6379,4804,1960,924,6133,5932,8041,98,3622,476,2517,8161,1374,10787,556,493,5804,9109,8786,1872,3138,1337,2822,6289,7562,3005,1392,6866,2518,1140,5258,133,10331,8306,5313,9575,10717,3156,2103,8495,4799,4672,838,2249,4686,2233,4281,1475,9108,206,9453,5941,8379,5103,2868,9988,9732,7417,7071,1231,2515,9371,4498,3081,9352,6529,2313,1529,4229,771,1134,3690,9005,1128,9503,9776,2571,8369,10271,4387,3408,5533,8666,9035,8390,712,9255,6634,8299,3221,7005,6850,2854,3921,4548,3501,1593,1050,1692,3685,4879,4277,2805,8468,2867,46,1258,2272,1118,816,6796,7965,2036,1722,9448,9851,6940,6862,10538,2547,6437,5343,6727,1359,6444,6573,852,7574,10336,61,7085,6247,5134,106,9192,2369,7786,9009,670,8311,638,5288,8446,5463,296,7337,5018,3103,10423,9970,10008,944,3205,5435,5332,7921,871,967,4777,271,7214,4652,6630,8199,6526,1695,5140,6215,9277,5174,8504,5181,8193,2863,126,2377,691,9662,6495,233,565,6466,1771,5348,9411,5026,5537,4801,656,1384,8396,10541,2326,6958,675,402,10098,4203,6835,6258,6969,7950,5704,4047,10642,5038,3340,1820,6525,7354,8752,5131,3937,4369,3486,586,5305,4971,9773,5948,7618,9361,6790,8743,4309,3722,7721,3944,8836,4154,700,2935,8003,7351,5521,9464,4718,4888,5124,7841,3135,5697,4530,7276,1395,9615,4825,6224,198,7805,633,8442,1840,1261,1170,879,9725,3626,3870,5050,3901,3954,9332,2906,10073,7674,722,6843,4363,9289,5077,6598,3059,3928,1114,10034,6377,1380,4378,4418,10505,4869,7246,1292,9942,7092,9139,861,7416,2747,1245,6423,9095,9653,3130,8948,6737,2404,4026,726,6101,7726,4540,10308,4243,478,2980,3227,8985,6913,2675,4504,3753,4931,2434,6753,1045,7477,834,4181,7134,10507,9514,1747,714,7790,6755,1971,8301,6009,3584,8877,4361,2927,9027,599,8160,9059,4675,8090,10091,8192,5283,719,3145,8671,8569,6824,8639,9253,5815,10361,6988,2510,2585,8904,7405,2398,1768,7989,8696,9899,9805,2439,9412,2727,7927,6257,8746,3982,5679,2013,4579,4141,7634,7553,346,603,1522,8764,7001,3603,7775,9715,7678,8482,8630,10530,7905,9211,5308,6760,5266,7308,7377,3957,7058,6340,3602,3218,4945,239,8867,9918,8130,8000,4950,8913,275,8097,1029,4630,10774,3173,2808,3754,1591,9115,3593,755,3318,6902,5762,2104,8893,3749,10401,8946,2616,8510,8937,7975,1078,6335,6238,10733,2044,7885,4233,7426,3606,5280,9756,810,9844,9695,8053,4704,3852,1484,1031,8399,5467,7672,7590,8215,5263,8028,2657,8117,6142,6273,5561,9748,672,7229,3098,84,10405,4272,2800,6889,8792,6691,4204,5646,5345,5615,4278,5315,4486,1543,9467,5355,2797,5400,1953,8917,7806,7073,3807,7312,1406,5716,7086,1738,6847,9781,10212,9631,10442,7548,10115,4678,4097,5030,2247,3002,10293,492,10429,8944,5106,4009,10269,5760,859,1201,10771,2026,6508,2957,1780,317,6445,8139,1810,1520,8983,2354,490,7375,9866,6035,5393,4751,9810,7710,5155,6576,7367,4368,332,5870,7762,7292,10020,6076,8355,1216,2119,2362,6470,6180,500,9775,298,4319,4935,7585,9563,4182,5326,1970,9266,845,6756,2256,8541,8972,7094,2748,7353,5185,10664,10743,9565,6942,7743,9912,5135,1452,1618,7592,3844,7037,2048,946,7036,5780,3146,5973,5505,8810,2892,3682,9607,931,6852,8823,10136,10440,2615,10764,9421,7909,8964,8449,3160,2153,181,10421,3357,5638,9478,9572,5797,2020,5368,10014,2415,9659,1772,9831,7340,10777,2584,1967,7190,5789,9905,3909,5499,5879,9787,2921,2683,462,2535,10730,7492,6718,10492,7124,1148,1079,3986,6891,5696,10414,5855,5737,4214,9389,3253,1273,3869,2641,8329,7323,4120,7723,4798,1387,5285,6983,7132,8731,5299,3663,4249,3438,365,5100,8684,8801,1993,4506,1671,4566,5122,1686,3195,7916,3880,1101,9672,8036,4397,2701,1786,8120,10726,3429,5114,4588,5217,1057,2735,10794,1331,8815,2261,4703,1667,9574,10803,10054,2435,8067,4393,6362,2937,985,8750,5374,770,2463,1749,5922,1063,10335,4743,2305,10396,7546,257,3297,8553,1775,10197,7279,2690,496,3454,8616,10612,7857,7126,3107,2298,4072,8906,151,1581,8061,439,6856,7903,2142,4526,5244,9029,4714,8335,881,10363,7503,6647,935,5133,7649,10561,6108,933,2551,6870,4673,2826,10129,5906,626,7717,1082,3575,1172,9417,3456,5568,5914,5856,1851,2953,4881,5851,6239,2050,4371,6477,7366,4452,9849,2246,7930,988,1951,10789,5878,9853,2323,2850,1942,1136,7421,10230,5130,6313,6929,4846,10267,7890,4964,6829,4357,8582,649,7241,6716,8258,1176,9263,5390,6994,5921,3192,1069,10796,5908,6486,6138,8562,8578,6925,1408,7917,8045,10286,6671,2352,1312,10309,8210,800,6050,2686,6214,8456,10302,8858,3443,418,8005,1445,7115,1824,5690,2569,2917,5197,6749,8643,1537,9977,5648,3997,2600,10395,8360,2861,1014,9790,9865,4095,6124,1666,9189,5160,8186,5137,10828,6025,1847,4932,1249,1299,9530,7282,6990,9223,3720,3818,7987,3137,7670,286,4731,7654,805,9668,10018,230,2467,8638,10084,8422,3481,9811,3285,4877,9281,10307,10801,102,530,1819,4918,6546,928,2930,6678,10253,10151,4094,8430,5056,7400,1313,10461,6039,7992,6664,6502,10644,6424,3383,4302,7186,8342,3559,310,10379,9446,2277,1116,5778,6538,1963,3185,1487,3294,10362,1207,6921,8181,5236,10353,5709,7359,10011,4147,910,2476,5337,9213,9916,2696,2391,7286,8797,1796,2411,2570,7990,2889,10723,10150,171,472,6246,7280,10100,8298,4517,2775,7831,10582,2365,3183,3670,6432,9074,7993,4852,646,3645,4052,5323,5556,10807,4014,5349,7109,1296,7125,8147,10043,10085,10709,6906,2251,1058,3842,1132,10328,6005,7725,1853,1862,4528,10824,2837,9129,2214,4209,3557,8758,2605,6667,8761,7620,2156,8505,4322,5269,9818,4447,3186,4709,3920,10087,568,7985,8911,2331,6494,10654,7631,2875,5363,6151,6896,5481,9705,3286,7753,5449,6013,6874,5153,1779,1494,4722,6728,4726,8008,6316,430,4403,6359,8762,6936,8704,8677,1111,8153,8680,7700,2622,582,1462,1621,4373,5430,6218,1825,10038,5597,5578,6064,7907,7892,1469,9038,6763,7197,10543,9461,3881,1536,8992,7465,235,4240,7502,1541,1616,7207,6608,4245,3470,2961,5423,9460,3993,1979,3680,8998,6844,6370,8959,3488,3217,2604,3891,3369,6467,6320,4458,9248,1793,8676,7581,10502,730,566,3694,4699,4416,2284,8110,1933,9333,10388,9873,7454,9254,8881,775,3497,1712,7090,5158,10122,7483,10818,9973,9704,1969,4606,2682,3354,8869,1338,10215,4379,9793,694,3725,7365,4431,8302,2752,3255,5392,5468,68,10751,4871,390,457,4705,3181,4081,1491,9246,5982,2524,3721,2372,727,2163,4734,6689,10005,5237,5583,280,5207,3034,4022,5934,6326,2334,7209,2702,4643,7498,10119,5825,8962,8185,5440,2169,1803,1742,7316,5183,1177,2936,5020,6959,5063,698,8234,9307,3086,8831,919,2234,3787,5714,8263,8991,6104,8554,3350,5187,4092,8071,1898,1394,4992,5371,9200,486,7787,8939,6047,2190,4210,3142,4261,1583,1121,1927,270,4948,6204,10255,6811,315,8896,3477,3895,9606,8851,4623,4244,893,7009,10344,5404,2539,3448,8682,6105,8965,9706,2862,5375,3273,242,9218,3585,7027,1112,8232,7751,8332,9015,6507,8544,4428,884,3857,1086,8115,949,4160,4268,6867,9803,8337,6903,10165,5744,3284,5854,1785,2319,3089,8448,389,2066,7236,3351,1466,123,1956,10341,8785,1741,2310,1544,10547,526,4421,8916,7915,161,2972,5424,1301,222,4661,6438,2496,3109,8488,9354,743,9345,3615,10721,9622,51,8829,9000,4890,10182,3760,8921,1867,5754,3265,2399,6089,7313,6339,9285,212,3308,1886,10441,2341,2626,1284,8138,10424,10204,3572,3966,7296,7303,6846,1875,2758,9663,7173,2959,7261,6295,8303,9088,7669,2112,1328,809,4456,3132,2346,1241,958,339,7663,7531,2519,7509,6949,8340,9959,7063,3055,4636,4650,7680,7203,6030,2264,8979,8819,2500,7559,5606,9434,3256,10689,2321,3324,523,8078,992,2736,7513,6385,7052,5970,5483,278,6350,1730,9367,729,157,7960,4446,2110,10575,3544,10795,6554,4384,6279,6659,2623,2082,145,6372,3582,4127,1878,10826,4129,1017,1837,9788,3281,1663,1929,1597,7393,10473,5167,9599,1482,5394,4906,952,2098,3465,9240,5524,3459,1724,9454,6189,5210,8609,3447,252,20,8014,560,7346,8756,9939,6463,3810,8537,7240,6703,3435,2252,8404,4582,4011,285,3744,6435,8127,6676,3492,9245,7358,4079,3555,3329,9868,10154,1435,5189,8220,1275,3926,7007,4464,8647,10250,7756,10798,10221,8891,4723,10161,6297,9083,3765,95,3530,10346,9909,1532,3700,2541,9989,6816,10311,9640,6547,6384,2081,7112,5666,3974,10374,617,5691,6310,1702,2487,2285,4644,3336,3756,1224,4343,987,369,1093,5223,1610,9981,7227,4114,915,1186,3856,583,10701,6518,7343,860,956,3165,4762,2529,10281,1429,3413,4135,483,1449,739,4878,3697,769,5841,4621,999,10557,9149,6955,1997,1073,4172,7653,5518,1485,3287,5466,9746,7146,8953,8830,7521,8289,2198,5300,4404,1659,336,5723,2857,1499,8658,5105,6122,9601,7954,7573,5243,762,3418,6234,3161,2681,4441,4649,8345,3804,1818,3177,833,9727,10225,965,7635,3006,2772,1142,3208,1098,150,3943,4761,3460,148,4784,7496,7263,612,3524,984,1983,5502,1120,7200,4104,28,9675,9544,7936,8763,9603,4304,6014,8498,10131,1367,2580,5959,2016,1205,1351,3288,4457,10673,8116,4406,1206,7046,9177,5683,7707,3547,2045,4376,1516,2865,4622,3207,3659,9175,5121,2777,2653,8524,1157,218,6418,3321,10102,2380,7980,4617,4220,8859,2550,9473,3495,5408,10677,343,7398,380,2894,5325,10235,1329,2869,7202,10437,6613,3509,2703,6314,7226,5218,10184,10627,3526,7179,6801,6853,8560,962,10511,7729,4125,4134,4392,2766,2492,5971,6144,2358,2583,1122,9632,2992,747,5145,7409,8392,543,3133,7946,10450,754,4981,5441,2618,6462,1229,8587,6498,9512,954,5848,5688,4759,5331,356,8088,7814,4350,7795,8176,3631,2773,4999,8259,7381,744,6609,2564,5940,1594,3755,6476,6540,5057,10616,9221,621,597,7887,2606,81,3980,2220,6640,518,2659,4538,9926,1733,7948,8018,3004,7363,2021,6992,2342,8483,9373,4527,4419,10224,5816,10623,2693,2160,2963,6773,74,4684,104,4898,4809,1674,7499,10696,4695,3978,2699,5214,7315,829,10410,1145,803,5951,301,3763,4196,7746,8328,3070,7407,6055,938,10522,7977,9896,2243,7033,9814,4494,9676,6036,6100,4775,12,2195,10196,1103,5862,3333,3471,6764,10222,4174,8568,8249,8362,5492,4053,5630,7000,1966,3422,1711,7682,295,865,7361,9652,5790,5626,3821,8409,9859,6363,8860,5989,4087,6395,9848,8308,4341,7520,7616,2671,6836,4860,3014,9789,9205,1918,5053,3335,9542,320,8884,5204,1442,5558,8250,9872,5952,1424,3709,8195,6140,4057,6633,659,2383,5834,1508,4246,10010,7287,8076,4554,3599,10685,1431,4050,7264,8535,5166,8086,8133,6541,4201,3545,9923,5647,4035,7740,10183,4169,1689,2664,5770,1882,1295,6978,4921,1812,5412,5387,8668,8408,4781,3579,732,3899,9510,3082,1914,7242,1557,1912,3961,3250,1039,9611,6683,9820,7647,10220,5720,10173,10330,4909,2337,2654,7133,1896,2883,2448,10727,2793,578,3199,8572,2743,10223,10662,9378,10004,4008,9331,1675,718,4682,4274,6762,1240,3261,4046,1934,79,1988,9624,5601,6621,3153,3882,10144,7429,6430,8986,5860,3225,5873,3960,9397,7687,5080,10674,10778,6193,5531,5793,88,7583,4601,2248,5748,3393,6702,6857,6950,8635,2375,1141,4236,4145,2733,5629,401,8954,5777,6783,3337,1943,5918,7821,255,10692,1434,9058,9629,3168,360,1690,1230,9458,951,9271,6469,5967,3061,7579,1003,3108,7681,8486,6265,4409,6109,8412,5618,6487,5562,6276,5907,7575,5529,10601,1725,9096,6905,7801,8610,5277,2627,5035,9569,3157,9957,7322,1600,6157,796,4460,6627,5685,10805,394,3669,7330,2426,7744,10520,8645,8932,5209,5820,8391,4769,6132,8613,1836,3841,3041,1085,521,5186,9462,7136,6429,9951,5409,1269,8128,7516,9188,8354,10167,10216,9497,9972,7399,8910,6456,2810,10339,10819,1646,9616,2174,4175,6091,748,3661,2366,6654,587,3627,10097,8367,1109,1844,8652,1364,8151,5060,2480,7817,4284,1076,5257,4861,7355,5872,9709,1298,8145,8519,806,2427,4524,3660,6293,4444,1660,3989,9374,7192,5369,8292,3914,5608,210,1002,7154,1302,2855,2412,10211,8799,4612,9744,267,5877,7137,8426,2828,4192,2378,6236,4924,9921,2440,6378,3556,9511,1897,357,8104,2400,10609,2970,3499,1830,8321,4543,7794,2971,254,6223,27,9480,7003,8856,6954,7704,318,2407,1182,4624,644,9496,10318,7961,9257,5695,6231,8894,4186,7976,812,7331,4217,7106,8092,1126,4102,8980,10591,3734,1213,2670,9679,3172,10459,8626,6492,3323,6646,9539,355,9337,8716,3808,6165,10562,5658,818,9707,8958,2134,4866,10317,7818,274,5457,3388,4282,4330,6482,109,561,4955,4874,117,8038,8866,826,3938,7069,9357,4806,4903,8741,2688,6343,2402,9895,9561,4100,10491,2597,3533,1849,3190,8737,8967,9007,4468,1068,10277,2258,4360,59,107,1036,5769,8437,9843,9700,9127,3525,1,7297,1809,5972,2484,9840,9609,431,5800,671,3009,10159,481,5025,6254,9602,2025,1001,8679,7472,8929,3164,10786,5165,7177,9987,733,4959,10276,6199,8022,9001,10576,9036,8938,10189,9238,297,3231,9080,6512,6509,1568,997,3773,9643,1030,22,2192,9407,7798,2813,10039,2088,5173,7170,6652,8536,10436,3325,10338,9437,1977,1919,9808,2430,10240,1107,6451,2784,1817,8037,9633,143,7117,6964,721,4194,4954,9730,8203,9491,1807,5659,9116,9816,6808,1160,1358,4106,1166,7206,371,3636,7656,2148,1938,7394,3634,8204,4770,4938,3896,4577,8548,7530,6016,7461,324,10646,7307,7338,8436,4381,6615,6087,5668,942,9953,1899,5546,9406,15,9684,3552,7335,7392,450,4470,6201,4040,10641,8713,953,5675,4947,8715,3092,5422,6023,5783,3950,1423,677,4710,878,7130,10822,5512,7204,9929,7326,2019,7774,3197,10810,7732,8471,5052,10750,4482,5156,334,8526,6222,9214,6614,7258,6488,6680,8320,5259,3757,9549,9196,5715,3581,5413,8031,5224,6497,7651,1100,10110,4797,7839,3091,10123,10358,8188,2208,5447,4542,373,434,3293,4904,214,8494,5450,1721,9828,9961,8747,4365,110,4910,704,9155,3504,7235,7734,873,10448,7667,8824,3796,7971,4605,10003,917,4066,705,779,1612,8861,5260,7295,3027,8487,3823,9826,4459,8941,6715,2085,3517,4808,7075,8272,4500,9656,6927,8405,1386,4737,8475,8365,1813,10088,4996,6029,6993,3364,4491,4885,4287,3667,10170,9325,2314,1129,3911,7830,5924,2676,4934,9403,7940,1982,5054,3028,9693,8633,454,1944,8599,9833,8595,41,1400,9291,25,3594,2697,8945,8840,7290,8963,2043,8431,6618,7978,4635,1669,7879,4019,3811,8168,1791,7099,6588,9120,8107,2633,7467,2490,3905,4697,3652,6742,1718,3987,6206,8467,3716,8875,9504,7822,1874,2899,2537,1515,7730,9667,9388,8007,5294,7185,9778,9451,446,1512,4152,9894,6948,6416,8189,10042,7024,232,648,1425,9531,10391,10715,6568,6353,1441,4423,470,7962,5376,2725,4610,9079,678,24,794,963,5479,2339,5023,1627,9260,19,811,1410,2506,2086,4218,7015,9600,2022,9466,7222,7410,438,2732,4654,7252,8826,1766,3483,9081,3998,2315,2767,5503,9724,4060,4507,4857,5360,8310,10600,3211,6448,7386,3790,3872,1343,5636,2280,2384,4015,5068,6328,7697,7538,1588,2760,4701,5706,2338,3925,9966,2674,5599,1985,3879,6636,10366,9877,3042,544,2724,429,1957,8924,1634,10106,6242,9852,1884,1799,3610,299,7696,1144,5923,5194,4467,5813,1925,5177,5554,9889,886,6244,8968,9883,1910,1661,4438,7766,2194,5792,7610,8874,9729,7166,5700,8477,2091,4076,4962,9797,10400,2179,6539,1243,3452,6322,7091,10164,6564,8879,1637,2222,8434,6026,9485,4235,10564,6164,10599,186,7953,1223,6484,630,2034,6225,5991,5787,2344,7376,6979,4462,204,10621,6747,13,302,7329,4339,5983,6338,3113,3397,2213,5772,6603,4140,8357,8048,9752,3875,4177,1106,4179,2789,2995,3774,625,797,4188,333,4093,7495,6043,8057,5433,9839,1038,9750,1763,8427,10470,8601,1471,7956,8118,6848,6237,3794,5899,7245,1913,8242,6565,5329,442,2644,986,2983,5684,7060,368,89,10710,8027,2481,10632,9061,3313,10698,5357,6054,2221,4615,1578,9878,2509,4810,8126,848,1303,2489,2441,764,8255,7065,1527,553,2628,6282,4518,6331,8096,3017,2039,5350,3657,6795,2357,6577,4296,510,5627,338,393,10407,6951,9090,2798,4589,8606,6434,868,4778,631,2964,231,4594,6665,3945,5427,1293,8655,8499,388,3633,1393,2877,8319,6980,736,622,8813,3797,5883,3433,10160,2786,9567,9022,5739,7920,8966,1083,4771,3532,10697,4826,137,2182,522,8984,9982,10264,4017,1974,5652,1981,2910,7029,2990,8724,3505,4651,7819,4345,5497,1928,7782,8949,433,2403,10288,6751,6342,4712,7684,9683,1035,2844,2317,10157,3858,4370,3883,1554,7061,8373,8015,5686,8795,6278,3683,8686,5493,5168,4032,1748,10790,1458,1432,7436,8032,7625,8651,10815,5998,10506,5202,8688,3678,9343,6510,1629,3182,1353,7419,3785,2472,8393,7761,1355,6332,9681,9931,364,10678,854,9098,9691,10792,5289,7764,4230,5069,7055,3955,3576,4567,10194,6414,6766,8956,3269,2997,3888,144,1777,5477,7013,5192,3642,2311,1336,849,1601,4968,7482,8135,6404,3124,8622,2949,5403,9269,1256,10521,9091,5486,6970,2815,4311,1893,7183,85,2292,9856,7984,855,1750,4279,1713,7002,7626,1705,2090,105,4687,9153,2294,10411,7939,6707,1672,10284,8545,2609,10714,8589,1757,7469,7217,2263,5027,1745,853,4429,6758,9651,7144,9286,7676,8527,8155,420,4443,1845,8082,6611,0,2794,9547,639,1222,6115,1572,9903,9495,5545,3243,9621,5182,247,5581,3289,3729,776,5095,2379,243,8744,2295,3732,4725,10544,2745,1246,10024,3345,2076,1155,627,6939,2454,2839,10259,641,5988,1808,7401,9723,6734,1294,10155,5818,1260,7904,10671,7750,1061,8224,4013,7194,9030,9482,1505,8012,10185,2293,8551,520,4831,10451,5002,1628,244,23,5459,3152,3566,2923,4389,2508,5984,9363,9138,4756,9871,3200,1729,8922,3064,5090,4626,491,1968,7768,4039,7882,6317,4822,10244,6947,6559,5,188,4424,6963,1879,10447,8269,9576,3865,2632,4003,9368,3484,8089,2790,5472,1922,6723,5553,6057,2734,4989,4706,397,8384,8002,6233,3469,1576,8470,9846,2836,9521,1829,7442,8771,7731,2751,813,9051,5637,863,329,8056,5574,9779,10558,9234,8923,366,10036,7320,8997,9522,5301,3292,3327,1113,5242,5602,5876,3817,6972,1054,7870,4310,211,1290,4005,6888,8732,7384,9564,6837,6044,6205,5655,10017,4824,2938,7281,10462,5091,6918,5488,8481,7828,5498,7613,3405,3648,6007,5634,10209,10109,10516,10466,10688,6501,2023,400,10517,10399,2297,5079,4148,6248,3482,5231,441,7759,9244,1064,8564,8973,711,10711,5525,3420,2526,4757,3759,8142,5750,1787,6232,10806,3730,4483,4255,2811,2707,8295,9699,10739,9911,7622,1784,6592,9590,9556,1687,7643,10265,1620,1119,8451,4198,10519,10607,6446,4227,4943,6810,5061,9364,4570,2738,2640,10126,8694,9666,5043,1883,8233,904,6081,4532,3553,7533,3507,7636,6788,9128,995,4867,9560,5190,1115,2771,731,9777,1653,6919,6776,10262,9881,1196,3386,6390,3574,8693,913,2714,2713,8780,674,3478,550,686,4786,8990,5885,139,10416,1684,1811,5147,5614,5178,2662,4080,3141,6885,4211,307,1232,7565,2704,7598,9383,5930,7348,9999,8415,5980,6965,3236,6685,8251,2065,1573,4893,10245,2051,1239,4581,3425,2594,2740,3394,2607,384,3646,3567,5732,10820,7380,9440,10827,7490,1194,9984,4435,5587,901,5828,3008,7045,8749,6974,1547,8225,2202,6968,1602,9124,8466,509,10066,934,5022,2542,10193,1852,5339,8180,6639,3263,4159,5632,1590,8059,3952,7225,158,3494,2723,6447,5625,10732,2864,6698,3906,7974,4002,10044,2673,1008,266,9293,1999,7714,7968,4112,7779,9610,2266,7515,7274,8558,2139,8725,6067,4642,8930,374,8485,4717,173,2575,3832,34,2449,1346,7854,7675,8378,1743,7178,7022,2495,5542,1668,40,8230,3849,3363,10006,90,1143,8636,3591,2779,2994,9997,8281,10483,3463,4899,7722,10830,8492,4136,10368,7437,5013,2783,2592,9110,10227,6638,6650,7827,5455,10583,4338,7387,4158,1318,6757,4126,5434,1931,7856,9721,8597,1209,737,4474,2718,93,4337,5102,3317,10432,83,1339,10534,572,3788,3298,8604,4176,8889,7836,9505,8400,3246,9552,3390,7599,9630,7850,6075,6854,6077,2687,866,6668,9206,3781,3587,717,2312,6600,6407,6341,264,6952,1579,2558,2532,6562,5389,3970,7034,293,7777,9506,4873,7247,9140,10493,464,4225,5639,3134,9195,2386,5603,2578,8533,9782,436,6893,5590,3596,2083,9908,5891,5064,9589,8760,1693,5571,8837,168,6382,8512,4334,3711,10474,9655,1704,7057,5724,5673,9735,5522,7289,6944,2787,1709,6198,1131,4555,2999,555,7781,8812,632,5743,6513,3414,2336,1682,8988,8087,2343,4412,9424,99,444,5874,10460,9017,6094,7614,2147,5125,10117,7840,4637,5104,1285,8602,1091,10113,9887,7576,10605,3258,4018,276,9324,7641,9276,10477,3054,9256,9450,7105,3953,5909,8717,6504,2715,2154,9197,8828,5929,3621,3016,7042,419,3847,9320,1334,3782,9312,8699,3228,7715,7370,9313,8346,4425,3047,8838,9049,6352,1340,4492,3356,7095,663,4660,4119,10146,3129,7369,2040,6674,5527,9259,2322,64,7858,7019,8417,1746,7285,3332,7129,6814,4796,8274,3083,2824,3001,8621,3746,5344,1575,10357,4758,5436,4896,1060,8371,426,5031,7991,5916,1335,1345,8444,2162,10791,4539,9850,7051,2270,961,94,4578,6521,9290,6724,3809,6306,1644,447,7084,9623,7811,8895,2410,2493,4839,6003,7889,8892,3674,1283,790,2118,1473,5070,9296,469,9447,2128,3252,3793,10408,466,9013,2776,10452,6135,9187,6662,8183,1605,3695,8077,9430,5955,2376,7511,1894,7304,8703,6063,6672,5462,4405,8742,10120,5593,2475,208,2431,2672,1288,2582,5519,2586,7843,353,8981,896,10275,647,10456,3689,3802,4317,6663,6118,4185,4708,395,4829,3538,9278,471,5010,8531,6345,3586,6688,120,847,8804,2494,6987,1125,7031,6879,2730,6717,1033,6709,1562,2274,9686,7735,5942,4863,10572,10404,71,2968,2067,5717,92,10412,3311,6833,5509,5794,4785,5042,3656,8366,3020,10494,2405,5758,10536,9541,1920,6696,6287,2756,31,10430,349,3571,7708,3037,5256,5321,10554,7529,9733,10179,5227,3702,9785,680,10140,4195,3381,9638,7385,3643,9165,10604,3355,8044,4603,580,7455,9274,6389,6169,8254,1630,4696,7633,10327,2173,8039,9220,10247,7041,6706,1492,2217,9186,9024,2127,7947,3550,7563,4143,570,5633,1816,4510,5549,10139,4149,4450,9112,4078,5910,10132,4414,3806,9204,10713,9384,7250,3444,2330,10071,4990,7959,5900,8010,695,2418,9349,2079,793,6022,8438,3569,305,8863,8888,1397,596,6355,5604,3795,4291,2731,5203,2908,7568,3939,8175,4569,1762,2017,3766,3623,8062,138,54,5884,2240,6735,4455,5961,7527,6660,9060,8855,5785,6392,7688,551,4768,10367,2546,432,6143,5703,7278,547,4440,4614,2031,760,9107,164,921,1262,1858,6701,4692,9105,4354,768,2359,2012,6566,7820,759,2144,10251,9156,6096,3370,2226,708,1514,5835,8811,10060,4184,9763,9838,2553,6095,2739,7630,4521,10515,8615,5831,2939,7650,6491,8999,311,8060,8381,9298,7748,4314,10356,6823,788,3150,5169,2525,7913,8479,2035,9235,5396,3833,6781,5501,2126,8389,8729,5767,8543,8899,149,3676,5028,9026,8523,3229,1778,2279,3562,6931,3968,2388,1586,2084,7646,331,2309,9358,9410,6419,8661,7981,2872,5966,3249,1438,1179,5514,9442,1421,7544,10463,10092,6813,7402,10433,2096,4264,3254,2188,7571,4693,9111,2183,3233,1270,9930,7201,6177,9326,6128,9272,10283,1420,482,8918,5904,1717,10166,4410,7447,10504,1500,7996,3302,62,3277,9151,7220,9469,3629,3965,3506,508,8207,7776,8825,6989,831,3439,8361,1092,1570,7605,3019,7205,3437,6454,3328,5547,6280,10300,5808,9936,4473,7111,8532,1368,10527,3775,10439,8290,3605,10169,2393,4774,6741,7802,7435,1375,4064,9210,1010,1274,662,6711,8865,3214,4099,10299,7104,571,3178,8753,2625,6461,2033,1379,5641,2095,5965,7249,6797,1947,9280,2643,8791,4864,9821,749,10294,5385,4132,8897,5726,3564,3558,6499,2032,2834,10634,10389,4163,8820,3038,6585,7673,5247,7352,9830,4171,7151,7824,2244,4848,10371,1211,8876,9133,689,7397,4352,1769,10832,6356,8081,2637,4926,7169,3409,8450,3934,1080,10061,693,3030,4740,5784,2907,5576,7288,10670,9125,8903,5886,6548,907,9950,268,4553,7424,750,5947,5377,5663,2235,3769,7615,4239,2501,6572,1559,745,1495,10273,10681,8489,4395,10550,4749,7145,2825,8650,10485,8134,10023,8463,3291,6412,9078,9224,9431,4944,3105,1422,10455,5507,4180,9075,9123,9438,6986,3671,8386,7507,7716,9927,5284,3589,1448,8503,7832,1584,8714,362,5802,5564,4604,10718,5437,4307,7506,8890,4913,4146,7823,7181,4280,7362,5384,2890,3886,9117,9728,10687,182,4183,10770,3859,10228,676,5251,690,1443,4927,2540,5520,9837,598,2513,8190,637,823,5487,4275,4862,9484,6360,8063,3762,1474,5032,6158,10419,3431,7602,9279,2841,1635,9928,8617,1959,5836,3440,4197,6061,2932,3375,7550,9791,5273,7964,10249,2579,452,2271,10518,6111,4096,1647,1753,4212,6798,5378,1700,7876,8594,4033,1869,5024,2620,9011,8095,6478,5881,9319,8205,5978,6779,8200,4074,8850,2269,4073,9222,3487,4519,2885,7160,6172,936,2458,7834,9468,7763,3430,1707,187,6770,7770,1055,352,7783,10737,9401,7167,1991,5240,1180,9121,1861,8395,2268,8957,5774,996,8605,3772,8940,4836,891,321,7988,6534,5671,9144,9301,9754,8849,2554,1546,7425,4746,10070,10742,2061,1782,6038,6250,591,1257,1391,1954,8206,6307,1903,9941,3093,4348,7666,2996,6904,9100,5694,10143,10663,1426,607,8201,10035,7107,1371,7747,2042,10206,5327,1992,8420,5391,6752,8538,9396,3479,7849,6514,6892,477,8969,1585,192,5669,516,1237,1833,2979,4488,9753,4362,176,1916,7215,5365,3213,1563,7973,5557,8685,2619,1200,3502,10104,4851,7267,8100,6858,3062,4062,9876,2821,645,9604,4742,7118,507,5987,2668,579,9459,6125,10813,10046,10202,70,227,9483,10606,10332,3198,3044,10672,774,2601,1251,5953,559,4109,6563,8803,4832,1094,4911,4514,4549,5225,9428,3931,9381,10578,8473,10305,1506,10,5012,2587,10195,10434,1162,3347,2556,976,5152,414,5445,7588,4680,1210,5566,4027,1651,10013,180,9734,4150,2216,5584,9168,6426,10533,403,6391,8317,7471,8773,6772,1072,2835,5935,4951,8382,2799,9835,8079,629,2596,8525,7123,9678,113,3992,10149,8364,761,3111,519,8672,1521,7119,1011,2057,3940,7139,1236,1450,1930,1307,1191,1147,17,8074,8428,1509,9136,7293,4515,975,1388,4537,5672,5513,2611,4156,1071,10105,4453,2998,9717,875,8084,3750,10030,9142,9784,7311,6895,2514,494,415,6325,8698,6626,2460,9933,108,282,10589,6610,4323,1427,4830,6348,26,9527,2832,2015,1383,7344,1961,1088,2149,3496,8083,4294,4105,5850,1265,4969,8187,1137,8324,1625,2413,193,5964,9355,8629,4366,7933,7456,1821,4327,9053,7859,6520,103,2420,4882,3084,4653,562,9979,1381,10099,2925,8667,10048,10482,6210,10637,5861,5958,7601,1765,5176,6366,7512,10788,4048,10513,8935,10186,1460,8398,6532,6028,3029,10258,7891,3903,4402,537,2062,1281,4813,740,7324,3040,4299,7668,3275,9071,8275,10532,9628,4580,7969,6649,5485,6042,4596,4602,125,8722,6015,758,5287,3237,3391,9664,4563,2981,7427,3951,4618,4312,2803,5268,9261,5992,8960,8376,375,1688,610,9393,2210,8111,912,4994,1214,4355,4733,3278,1168,5352,7026,5812,7390,3929,3718,9800,9249,6656,4098,7543,5415,5334,5034,2521,5228,1828,273,8886,3604,9425,7156,4574,9965,10712,2886,7298,10613,9052,4820,778,135,2132,10831,1761,6851,5039,3179,9452,2929,2982,9037,4683,9106,5150,7958,4585,5298,2395,10130,4221,4738,287,4891,2203,4572,4508,1697,5651,9191,1832,4295,10556,8454,1378,9741,4110,4301,3155,9897,713,5136,4681,9141,5138,6088,5157,10773,2485,9499,5644,1664,536,7580,8598,4088,6422,10257,6400,5551,1440,1534,4084,4633,3051,3949,3767,1946,2780,4946,2940,899,2278,10289,4063,2660,9418,9045,2250,9494,309,8245,8691,6616,642,2639,2985,1459,3235,7388,455,6579,1315,10260,9618,6197,6460,498,5290,5322,337,3025,6245,7440,3922,7141,10354,4576,4933,10292,9582,9429,2102,7462,5919,5996,4978,2333,8043,8149,2296,10028,9657,3012,6768,7692,1550,9824,3528,8871,5347,9179,10579,10297,5159,1654,7082,4167,6812,8540,11,4207,345,3210,2576,5075,10114,605,2340,7664,6736,4747,3473,3747,9183,9171,3850,5276,8994,9310,7149,7253,4103,3573,1185,1633,773,8528,246,3890,10347,9305,5312,9520,9751,8898,9154,10337,4966,9391,3889,1790,3035,2516,9318,8323,7868,1023,10077,1727,3032,10329,2922,4571,3464,9759,2545,1277,6285,4957,8397,616,7861,3542,9680,4689,7088,8521,1895,3741,8457,2856,5112,10635,7624,7665,10808,2450,4297,2950,6168,6967,4584,6179,10420,3187,943,7443,1215,2962,5896,5484,8689,4469,7147,7945,3607,2111,4479,3136,5047,4051,8050,8336,10015,2394,2573,4305,9963,4509,6916,3462,5254,4318,265,58,422,8927,279,5718,8069,6527,10620,4795,9766,4535,3189,7712,6998,10263,7096,6186,960,8809,5324,4206,4858,5143,4427,9101,4118,3352,2848,4776,10649,3175,9536,10208,2591,2944,7230,8912,7660,1614,228,2115,1385,7043,4437,8662,8349,914,10319,2130,8649,5249,8294,9152,1461,10162,1169,5198,9535,7018,399,7658,9755,4451,10107,3866,506,1526,3283,6274,7184,7926,2870,611,2884,1865,1984,10758,4449,7617,1706,2728,9443,7374,651,3592,8476,9028,9516,7453,3752,8774,6731,7011,6912,9047,10763,8654,3789,6093,2543,1009,3647,2101,3518,8563,1662,6686,2716,10072,4495,8603,10387,5306,9696,8508,8775,998,3658,21,9772,7972,10274,9585,2572,4289,564,8579,970,6209,6358,1235,1250,894,9579,5949,7248,1856,3268,3202,525,3406,6090,872,3916,9262,5442,55,7089,8429,215,4461,5749,9351,5559,8783,3624,2598,7611,8141,3432,2807,5399,1645,7458,4004,9925,7703,6684,8374,5238,9540,5565,6226,8421,5782,9113,6632,5582,1428,5438,1350,2694,3338,9419,1501,7360,1800,10573,6001,3238,9847,7769,7848,9593,7594,10254,5191,7481,7963,4161,9551,2355,6651,7846,9455,3480,6086,376,6604,9858,2286,4041,5267,6315,5875,6999,2967,9344,4336,8490,1314,3827,3736,8628,6305,152,5586,4720,8351,7064,7373,4091,3303,4390,4727,3815,900,3115,142,8009,359,590,341,10413,1075,5869,9595,2711,5920,9769,4541,4986,3219,6761,7431,1470,573,5976,5840,593,9598,10829,3984,9532,972,9550,9126,8584,2866,2593,6841,1923,7999,1463,2602,1905,4707,7600,200,10237,2819,5960,6033,10489,9243,1059,9768,9033,7097,3127,5065,2215,10745,6364,1545,10598,7391,7412,9577,4228,4783,9342,874,6,4732,725,6323,8143,8478,563,1737,7679,4868,7541,3339,10428,8035,8307,3828,5292,2491,6861,3474,9376,4984,8926,5745,9562,594,4802,1598,3637,8779,4358,5108,6053,8709,6580,7910,10266,8253,1736,9390,4560,489,5768,2368,6743,262,10490,3851,2991,1027,2993,581,3300,6809,3358,8496,3655,7176,8501,2559,236,1577,10019,1233,1926,9426,9050,4332,10236,9089,9992,2133,9230,2617,1685,2422,2900,4736,5111,8720,7062,1289,4021,3123,7415,9150,8806,2136,9501,1873,2225,5894,7800,9377,7845,9006,3649,8266,5101,5469,6318,4219,5098,5847,4170,2093,2536,6398,6137,5912,1015,8718,9812,7629,1341,292,8172,136,8723,511,6933,4351,3976,1641,5420,189,9427,827,5370,9382,9529,4914,3563,7957,6710,7138,756,6017,1047,4293,6606,3988,7432,9822,197,8112,1405,2121,4199,4715,10476,4870,5735,6099,1370,5826,2507,7893,6648,10108,8484,1360,6661,6402,7414,7349,7433,9251,38,4499,4619,3399,6136,5986,1976,5833,2301,1948,167,4844,2327,2438,7444,3919,6396,5456,8590,1446,9861,4611,5058,10406,5656,6771,3516,902,8252,8212,851,2759,9021,6530,7577,1972,9201,9295,1219,5407,6493,2629,69,828,6146,9740,4086,3705,281,3975,5756,1244,5418,3704,8989,86,2926,1859,8158,3617,8800,4685,6262,1679,9940,937,5550,8102,1276,7350,1571,9969,4958,4434,8343,5607,4620,166,1456,567,6311,10542,8218,6767,9392,7219,7671,5179,2052,8174,8196,67,781,9413,5362,6840,643,4987,2612,6334,4817,5097,9712,955,2933,9275,8013,9334,10397,10535,37,1613,4085,2753,10784,6405,6631,2175,5059,1539,191,10242,6235,7228,9103,3131,6708,8213,10648,6410,5213,3334,4472,6277,2802,7632,4657,4442,2056,4153,2267,3632,1451,1476,3641,8313,2099,6923,8627,1022,6041,6387,9817,1756,379,8140,4887,6268,1917,9978,6062,10498,4029,1975,9922,10569,2665,2254,5206,3326,9282,3374,7677,8413,1855,2299,2408,7897,3778,6217,7077,2816,2473,5573,6249,9184,10079,7537,7300,3419,8664,289,7896,2468,249,4841,3267,7504,7545,4286,5552,128,6726,8577,2406,314,5839,3893,9215,5132,7875,2651,10559,8024,1497,9697,2241,8407,1964,499,6587,932,7341,2089,10213,1286,229,1433,479,7755,2581,6570,4124,969,7251,501,8260,2028,4534,6417,97,10145,8847,8816,5296,4292,1552,9161,3846,10765,1311,10296,8146,6473,7175,3693,330,4928,6052,9130,1208,3918,6785,4164,5763,4497,6397,3917,1198,8173,4090,3498,517,2969,10168,8156,1104,5622,10497,3515,8222,6551,10128,3380,3979,8882,10467,7829,3410,883,10431,9639,5791,9720,8700,4792,9247,5727,3771,5858,8546,468,548,3500,8646,6453,746,10446,459,9674,6253,2120,8472,1044,1650,3860,7778,9998,5829,10804,2498,9094,8331,10089,4561,2108,6815,667,4892,3389,6221,5341,2955,6174,10630,2451,6956,7773,10232,6549,248,528,6597,1411,60,6669,2114,8455,4920,6027,3726,6516,8474,9508,2363,9716,308,3039,4895,96,5476,10205,916,6195,5262,7131,3511,5516,1430,9217,4191,7301,4448,5598,9093,6161,7025,9710,4767,8055,5773,3843,4960,8705,7826,9457,7524,4359,4346,1102,4466,3310,9339,5282,10191,5460,7694,4042,7844,4502,9644,10333,3838,669,7428,515,7564,8854,6567,6725,177,635,367,6803,8358,10734,4586,10313,1124,9949,2453,7949,7050,5678,577,4640,44,5107,5661,4439,6409,8136,5356,1025,7595,8736,10384,7525,4728,8608,4290,2712,5538,1217,7872,5702,5635,1479,8072,6834,2928,9143,3770,9283,10200,3468,7333,6922,8663,5660,2595,9920,6428,3036,7012,5957,9315,3118,5788,10378,10659,595,6255,4779,3224,10510,4991,6175,4936,386,8901,6586,4837,4694,1324,8350,9647,3829,763,784,5687,4380,5969,10198,1732,8326,7191,1788,3282,880,2502,2461,2018,5962,5272,3776,7171,2003,1911,8239,2915,1794,5811,6839,3163,7475,2860,9085,889,1774,2958,5517,5532,10285,5775,9919,5335,7888,10738,8497,3758,5004,1498,9309,91,3677,9557,1617,9314,7597,9792,5037,8793,7644,7967,1365,5994,7079,3162,710,5926,2196,10651,5008,2852,5981,7911,9780,4811,3863,4833,4155,4298,4028,6294,5193,558,2574,7150,8423,2954,821,9669,9181,9241,4593,8702,3220,4505,10210,10705,846,1804,10746,9747,6938,2055,3611,1716,7874,6617,9745,5081,10009,1945,1152,8377,1658,929,5765,7567,4724,6374,7,10138,1007,5086,8179,2255,9614,6784,2853,2512,9010,2920,7871,6825,6697,10373,3485,8277,7100,3344,2037,782,7048,8166,9967,3614,4043,4385,8769,3204,9353,8285,10679,10219,10586,1066,6420,6537,6511,7608,2871,6819,8934,6083,1710,114,10793,4859,3472,6354,8229,9302,8873,9515,213,6472,3069,7188,7021,6505,5882,5336,8157,3067,1158,1496,8596,3936,7812,30,4850,10234,6368,2522,153,8049,7619,5241,6557,3087,10706,2166,6722,3835,2943,47,6519,9917,1866,5795,5448,6046,1439,3708,2882,3215,7291,4729,6786,2648,3608,3933,1703,7121,8625,5303,10597,814,100,3820,372,119,7357,8070,2649,10617,4856,3024,9534,7791,1436,8375,10695,5721,1105,6266,9054,10593,3512,7883,2072,8500,8309,5612,925,7466,10192,10040,5915,7078,3628,4342,1767,10377,3877,9587,461,9985,5500,8868,10111,2000,8248,2206,9409,655,8586,9321,8341,8832,1781,2078,6787,35,6558,9146,10082,2452,5000,5699,6485,3151,3011,5526,8469,6555,9335,3279,10665,6256,185,8316,277,7637,5619,4223,234,4388,8198,10435,5905,6021,10753,2847,3698,2253,1352,5624,832,6471,3662,134,1540,5657,4901,4205,715,4241,1715,7196,9765,4512,5736,10537,8573,7234,5730,10549,9968,926,9986,4413,5880,10809,5544,5892,3085,1880,10306,3259,3973,3367,7780,1839,982,9099,8581,8600,772,5302,2219,7863,3913,6085,5572,10369,4142,6134,2764,5291,9476,1551,6303,10702,4007,5317,2989,6937,1854,2201,6182,3074,4344,8782,6641,3999,5461,2150,8908,2677,5364,1965,1531,115,7549,10581,6019,8943,2749,2161,7470,7275,2151,5470,2851,1814,2445,10360,3520,5814,10055,1019,7607,3620,502,4883,10574,1389,3260,7108,2171,5411,1681,1409,8719,5755,10749,2655,9300,2511,10037,5011,9528,3508,4794,1084,3548,3125,2030,10643,6004,2534,1457,10256,8066,2232,4069,445,10766,5265,8169,9654,6113,2942,7995,6605,3403,8576,8291,9193,4036,2599,945,3244,4656,787,9645,2746,7749,10324,7299,6793,5089,8885,10348,7792,8529,3527,1888,9002,4666,7269,10622,7815,5846,3578,6406,692,3045,6369,8776,3072,6071,6746,7526,6900,5062,8011,9658,392,5817,5623,10453,8214,2722,4411,5443,1181,9498,1028,2138,66,5073,5605,3540,6440,7271,170,1188,1163,7934,1751,2224,6073,4,3052,1065,1369,1801,2499,7895,5346,10802,350,2988,2849,3619,4937,6455,10156,10041,7690,8103,9369,8619,4257,1282,5676,9827,1138,8631,5201,1099,9032,3668,9673,2146,5226,6251,895,9841,2005,6152,3927,7243,7448,4544,4231,7083,3731,3272,8293,5116,9350,8711,9137,2167,4407,4065,10047,7584,5931,8064,259,10301,9884,6692,6500,5844,1835,8021,1881,1673,6677,240,785,5465,4258,957,3742,4189,7040,1892,5083,606,9132,5506,2320,10501,3915,8530,9147,666,2187,225,10096,6074,4974,734,8852,8515,6694,7059,4056,2024,5798,10686,7659,3402,9990,9934,7364,3257,7438,165,8637,6240,4484,3126,3714,405,196,3514,3900,2804,8284,5045,1564,2754,10158,974,1173,2859,9233,10103,3839,3885,7606,9202,4044,2364,9798,7922,2318,3737,7164,8480,6543,9952,7860,1149,6778,4773,9474,6924,3407,983,9379,8728,7336,3076,8772,3110,8065,7080,10509,5148,7256,9648,9701,9832,8517,817,4202,3455,8453,681,458,8549,9836,3897,2356,10783,8974,9048,417,6302,8208,4721,5451,9620,1996,4139,7816,5421,8796,8119,3147,9465,6450,2068,6112,3167,10486,4807,532,5753,5824,9034,2304,6309,485,272,8802,416,5807,5742,8325,9739,9890,9625,7342,8585,3967,7757,843,4900,575,4814,9771,4523,6800,8942,5117,8707,10279,4430,6997,9199,1582,9292,7404,9976,3531,10093,3523,219,6155,6006,1632,9042,10422,10083,3673,9698,4546,7662,6059,1699,8880,8435,9994,922,2486,911,3114,660,29,1005,4520,1815,8202,10382,5234,8950,2503,8330,9019,5295,1860,1366,8221,614,7884,1955,174,4536,5974,2708,2307,7738,9242,8706,9039,7446,2159,10756,112,3003,9493,6167,9862,2903,9159,7519,7439,2614,6084,6072,8822,1838,3239,6675,9422,3964,3033,6899,4754,3963,4853,3710,5040,8080,4061,3597,2109,2123,1595,6211,5146,6791,4827,9996,4144,5569,7942,1159,9943,7572,1639,3672,10359,10690,4525,6196,9481,738,716,4489,2685,10147,4248,8583,6457,3387,10614,10364,993,1037,1404,10080,9258,5580,10403,3854,4116,5118,10669,9891,9737,5665,2469,10241,5489,6971,8936,6880,5977,5649,9228,3158,7784,5115,1574,9316,1797,8385,835,169,2288,8864,4121,6575,4101,407,2421,3898,5395,5616,3453,1454,5431,8122,156,1864,87,6008,7652,6427,7356,2260,1090,4823,8447,2457,10280,1631,1538,950,8406,1279,2966,7552,2347,7259,6394,10560,5868,3560,5823,3451,3046,5019,2414,7785,7929,8217,3320,4226,864,6148,6673,6490,3400,1310,9492,2348,4415,968,10323,3990,5776,9900,9336,1046,7148,8798,5151,2960,6178,6623,1204,4454,1760,7168,6117,1146,10719,10458,1649,5397,10282,10584,9794,3475,10402,10392,10762,3270,413,1759,9711,3664,3306,5274,8678,652,8665,8416,6408,8046,1624,5865,9687,8657,5888,1062,2184,1164,3280,9055,3097,5372,424,1287,5887,4647,7493,6962,10740,5382,6805,4082,8845,3590,7270,2228,2630,3021,9731,524,4700,378,7771,10656,3398,6644,990,1322,10016,6855,5261,10799,9323,6720,6162,1252,7898,487,4787,9439,5898,7813,3728,7689,8353,45,8231,2275,1665,709,2700,7044,6229,9875,3411,1846,5567,9855,6894,658,448,8410,1416,978,9570,5480,2329,290,4719,6830,7788,5473,8767,6832,2750,4270,2778,4501,1530,4417,7262,9398,3441,8026,2141,6700,4803,4545,3201,8518,9432,5734,3969,9904,8241,9760,10652,4558,5677,1535,3251,3971,10052,2199,5751,9956,7627,3299,10078,6578,4000,2567,1193,1453,2231,1398,1714,1507,2613,8977,1701,4465,5809,820,5049,9517,5595,5917,529,4815,224,3910,7054,3727,1939,7937,6884,9636,8394,5682,5405,9487,4907,9554,9743,10322,6208,9414,8843,7213,7505,10188,7604,7837,9860,10351,2087,14,7212,5999,6474,209,9842,10468,3271,741,10611,5142,4020,1870,10141,2070,238,4670,3450,3188,2479,8304,1263,9502,3330,2361,9077,160,4884,2948,2011,1032,1652,9131,2428,4917,6908,8219,6141], "time_ms": 0.159014, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 613, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "generated_query_count", "value": 2400001, "time_ms": 1093.894222, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 428, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "generated_samples_per_query", "value": 1, "time_ms": 1093.894222, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 429, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "generated_query_duration", "value": 1200000500000, "time_ms": 1093.894222, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 430, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "generic_message", "value": "Ending naturally: Minimum query count and test duration met.", "time_ms": 601094.012405, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/issue_query_controller.cc", "line_no": 482, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 601097.676138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 898, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 601097.676138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 900, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 601097.676138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 902, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 601097.676138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 904, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 601097.676138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 906, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 601097.676138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 908, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "power_begin", "value": "12-28-2024 03:48:11.659", "time_ms": 601097.676664, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 564, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "power_end", "value": "12-28-2024 03:58:11.660", "time_ms": 601097.676664, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 566, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "result_validity", "value": "VALID", "time_ms": 601444.389443, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 644, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "result_min_duration_met", "value": true, "time_ms": 601444.389443, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 649, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "result_min_queries_met", "value": true, "time_ms": 601444.389443, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 650, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "early_stopping_met", "value": true, "time_ms": 601444.389443, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 651, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "early_stopping_result", "value": " * Processed at least 64 queries (635928). * Would discard 63035 highest latency queries. * Early stopping 90th percentile estimate: 1032023 * Early stopping 99th percentile estimate: 1208727", "time_ms": 601444.389443, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 669, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "result_query_count", "value": 635928, "time_ms": 601444.389443, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 679, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "result_qps_with_loadgen_overhead", "value": 1059.88, "time_ms": 601444.389443, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 702, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "result_qps_without_loadgen_overhead", "value": 1066.96, "time_ms": 601444.389443, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 703, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "early_stopping_latency_ss", "value": 1032023, "time_ms": 601444.389443, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 704, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "early_stopping_latency_ms", "value": 1208727, "time_ms": 601444.389443, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 706, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "result_min_latency_ns", "value": 840130, "time_ms": 601444.389443, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 741, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "result_max_latency_ns", "value": 1457764, "time_ms": 601444.389443, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 742, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "result_mean_latency_ns", "value": 937243, "time_ms": 601444.389443, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 743, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "result_50.00_percentile_latency_ns", "value": 918945, "time_ms": 601444.389443, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "result_90.00_percentile_latency_ns", "value": 1031143, "time_ms": 601444.389443, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "result_95.00_percentile_latency_ns", "value": 1127281, "time_ms": 601444.389443, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "result_97.00_percentile_latency_ns", "value": 1184293, "time_ms": 601444.389443, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "result_99.00_percentile_latency_ns", "value": 1208398, "time_ms": 601444.389443, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 10576, "tid": 10576}} +:::MLLOG {"key": "result_99.90_percentile_latency_ns", "value": 1222522, "time_ms": 601444.389443, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/94a57f78972843c6/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 10576, "tid": 10576}} diff --git a/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance/run_1/mlperf_log_summary.txt b/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance/run_1/mlperf_log_summary.txt index daeeda6a9..85b761abe 100644 --- a/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance/run_1/mlperf_log_summary.txt +++ b/open/MLCommons/results/RTX4090x2-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance/run_1/mlperf_log_summary.txt @@ -4,32 +4,32 @@ MLPerf Results Summary SUT name : BERT SERVER Scenario : SingleStream Mode : PerformanceOnly -90th percentile latency (ns) : 1029406 +90th percentile latency (ns) : 1031143 Result is : VALID Min duration satisfied : Yes Min queries satisfied : Yes Early stopping satisfied: Yes Early Stopping Result: - * Processed at least 64 queries (634470). - * Would discard 62890 highest latency queries. - * Early stopping 90th percentile estimate: 1029890 - * Early stopping 99th percentile estimate: 1211301 + * Processed at least 64 queries (635928). + * Would discard 63035 highest latency queries. + * Early stopping 90th percentile estimate: 1032023 + * Early stopping 99th percentile estimate: 1208727 ================================================ Additional Stats ================================================ -QPS w/ loadgen overhead : 1057.45 -QPS w/o loadgen overhead : 1064.00 +QPS w/ loadgen overhead : 1059.88 +QPS w/o loadgen overhead : 1066.96 -Min latency (ns) : 860498 -Max latency (ns) : 1485171 -Mean latency (ns) : 939846 -50.00 percentile latency (ns) : 920898 -90.00 percentile latency (ns) : 1029406 -95.00 percentile latency (ns) : 1133784 -97.00 percentile latency (ns) : 1189113 -99.00 percentile latency (ns) : 1210866 -99.90 percentile latency (ns) : 1222247 +Min latency (ns) : 840130 +Max latency (ns) : 1457764 +Mean latency (ns) : 937243 +50.00 percentile latency (ns) : 918945 +90.00 percentile latency (ns) : 1031143 +95.00 percentile latency (ns) : 1127281 +97.00 percentile latency (ns) : 1184293 +99.00 percentile latency (ns) : 1208398 +99.90 percentile latency (ns) : 1222522 ================================================ Test Parameters Used