From 3a7e125da681877c7d70c941029303f7c7555926 Mon Sep 17 00:00:00 2001 From: Nikhil Verma Date: Fri, 1 Nov 2024 02:01:55 -0400 Subject: [PATCH 1/5] Add args for wandb entity --- etalon/metrics/metric_store.py | 3 +++ etalon/metrics/service_metrics.py | 2 ++ etalon/prefill_profiler.py | 1 + etalon/run_benchmark.py | 14 +++++++++++++- 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/etalon/metrics/metric_store.py b/etalon/metrics/metric_store.py index 79a1990..d37c595 100644 --- a/etalon/metrics/metric_store.py +++ b/etalon/metrics/metric_store.py @@ -35,6 +35,7 @@ def __init__( target_deadline_miss_rate: float = 0.1, should_write_metrics: bool = True, wandb_project: str = None, + wandb_entity: str = None, wandb_group: str = None, wandb_run_name: str = None, ) -> None: @@ -54,6 +55,7 @@ def __init__( self.service_level_total_deadlines = 0 self.should_write_metrics = should_write_metrics self.wandb_project = wandb_project + self.wandb_entity = wandb_entity self.wandb_group = wandb_group self.wandb_run_name = wandb_run_name @@ -104,6 +106,7 @@ def _init_wandb(self): wandb.init( project=self.wandb_project, + entity=self.wandb_entity, group=self.wandb_group, name=self.wandb_run_name, config={ diff --git a/etalon/metrics/service_metrics.py b/etalon/metrics/service_metrics.py index 2002589..b00546e 100644 --- a/etalon/metrics/service_metrics.py +++ b/etalon/metrics/service_metrics.py @@ -15,6 +15,7 @@ def __init__( target_deadline_miss_rate: float = 0.1, should_write_metrics: bool = True, wandb_project: str = None, + wandb_entity: str = None, wandb_group: str = None, wandb_run_name: str = None, ) -> None: @@ -31,6 +32,7 @@ def __init__( target_deadline_miss_rate=target_deadline_miss_rate, should_write_metrics=should_write_metrics, wandb_project=wandb_project, + wandb_entity=wandb_entity, wandb_group=wandb_group, wandb_run_name=wandb_run_name, ) diff --git a/etalon/prefill_profiler.py b/etalon/prefill_profiler.py index fc6aa47..47a26b0 100644 --- a/etalon/prefill_profiler.py +++ b/etalon/prefill_profiler.py @@ -92,6 +92,7 @@ def run(self): request_generator_config=request_generator_config, should_write_metrics=False, wandb_project=self.args.wandb_project, + wandb_entity=self.args.wandb_entity, wandb_group=self.args.wandb_group, wandb_run_name=f"prefill_p{prefill_value}_{self.args.model}", ) diff --git a/etalon/run_benchmark.py b/etalon/run_benchmark.py index 0f7aab5..38a8cef 100644 --- a/etalon/run_benchmark.py +++ b/etalon/run_benchmark.py @@ -201,6 +201,7 @@ def run_benchmark( target_deadline_miss_rate: float = 0.1, should_write_metrics: bool = True, wandb_project: str = None, + wandb_entity: str = None, wandb_group: str = None, wandb_run_name: str = None, address_append_value: Optional[str] = "chat/completions", @@ -237,6 +238,7 @@ def run_benchmark( target_deadline_miss_rate=target_deadline_miss_rate, should_write_metrics=should_write_metrics, wandb_project=wandb_project, + wandb_entity=wandb_entity, wandb_group=wandb_group, wandb_run_name=wandb_run_name, ) @@ -558,6 +560,12 @@ def parse_args(): default=None, help=("The wandb project name. (default: %(default)s)"), ) + args.add_argument( + "--wandb-entity", + type=str, + default=None, + help=("The wandb entity name. (default: %(default)s)"), + ) args.add_argument( "--wandb-group", type=str, @@ -629,7 +637,7 @@ def parse_args(): return args -if __name__ == "__main__": +def entrypoint(): random.seed(11111) ray.init(runtime_env={"env_vars": dict(os.environ)}) @@ -654,8 +662,12 @@ def parse_args(): target_deadline_miss_rate=args.target_deadline_miss_rate, should_write_metrics=args.should_write_metrics, wandb_project=args.wandb_project, + wandb_entity=args.wandb_entity, wandb_group=args.wandb_group, wandb_run_name=args.wandb_run_name, address_append_value=args.address_append_value, request_every_minute=args.request_every_minute, ) + +if __name__ == "__main__": + entrypoint() \ No newline at end of file From 4728cfe29f19c51d77d6680a424d4def5ea9d10e Mon Sep 17 00:00:00 2001 From: Nikhil Verma Date: Fri, 1 Nov 2024 02:04:11 -0400 Subject: [PATCH 2/5] Add EOL --- etalon/run_benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etalon/run_benchmark.py b/etalon/run_benchmark.py index 38a8cef..edb5135 100644 --- a/etalon/run_benchmark.py +++ b/etalon/run_benchmark.py @@ -670,4 +670,4 @@ def entrypoint(): ) if __name__ == "__main__": - entrypoint() \ No newline at end of file + entrypoint() From c18d28a8f458b4aa9c3c8a35176205e25ca996d5 Mon Sep 17 00:00:00 2001 From: Nikhil Verma Date: Fri, 1 Nov 2024 02:11:06 -0400 Subject: [PATCH 3/5] ignore_reinit_error --- etalon/run_benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etalon/run_benchmark.py b/etalon/run_benchmark.py index edb5135..a0d8d78 100644 --- a/etalon/run_benchmark.py +++ b/etalon/run_benchmark.py @@ -640,7 +640,7 @@ def parse_args(): def entrypoint(): random.seed(11111) - ray.init(runtime_env={"env_vars": dict(os.environ)}) + ray.init(runtime_env={"env_vars": dict(os.environ)}, ignore_reinit_error=True) args = parse_args() From 79b8f8d891f73b7aff3cd52a97a6a9d8c7807a7a Mon Sep 17 00:00:00 2001 From: Nikhil Verma Date: Wed, 6 Nov 2024 20:33:25 -0500 Subject: [PATCH 4/5] format using black --- etalon/run_benchmark.py | 1 + 1 file changed, 1 insertion(+) diff --git a/etalon/run_benchmark.py b/etalon/run_benchmark.py index a0d8d78..4ee1d31 100644 --- a/etalon/run_benchmark.py +++ b/etalon/run_benchmark.py @@ -669,5 +669,6 @@ def entrypoint(): request_every_minute=args.request_every_minute, ) + if __name__ == "__main__": entrypoint() From fe67b4068874e53ede49f1747b07833a52873f9c Mon Sep 17 00:00:00 2001 From: Nikhil Verma Date: Mon, 25 Nov 2024 20:47:10 -0500 Subject: [PATCH 5/5] remove entrypoint --- etalon/run_benchmark.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/etalon/run_benchmark.py b/etalon/run_benchmark.py index 4ee1d31..42090b8 100644 --- a/etalon/run_benchmark.py +++ b/etalon/run_benchmark.py @@ -637,10 +637,10 @@ def parse_args(): return args -def entrypoint(): +if __name__ == "__main__": random.seed(11111) - ray.init(runtime_env={"env_vars": dict(os.environ)}, ignore_reinit_error=True) + ray.init(runtime_env={"env_vars": dict(os.environ)}) args = parse_args() @@ -668,7 +668,3 @@ def entrypoint(): address_append_value=args.address_append_value, request_every_minute=args.request_every_minute, ) - - -if __name__ == "__main__": - entrypoint()