From 42b3ad5287f8e0833e21fbf43e0972f2439aa1ca Mon Sep 17 00:00:00 2001 From: Weilin Xu Date: Thu, 24 Oct 2024 22:12:48 -0700 Subject: [PATCH] Export the flattened config in benchmark CSV. (#2391) * Export the flattened config in benchmark CSV. Signed-off-by: Weilin Xu * Update CHANGELOG Signed-off-by: Weilin Xu * Reuse the existing flatten_dict(). Signed-off-by: Weilin Xu --------- Signed-off-by: Weilin Xu Co-authored-by: Samet Akcay --- CHANGELOG.md | 1 + src/anomalib/pipelines/benchmark/generator.py | 4 ++++ src/anomalib/pipelines/benchmark/job.py | 16 +++++++++++----- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c6f07555a..8152e202b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - Add duration of experiments in seconds in the benchmark CSV result by [mzweilin](https://github.com/mzweilin) in https://github.com/openvinotoolkit/anomalib/pull/2392 +- Export flat configurations in benchmark CSV results by [mzweilin](https://github.com/mzweilin) in https://github.com/openvinotoolkit/anomalib/pull/2391 ### Deprecated diff --git a/src/anomalib/pipelines/benchmark/generator.py b/src/anomalib/pipelines/benchmark/generator.py index 922dfa06cb..988e0111b7 100644 --- a/src/anomalib/pipelines/benchmark/generator.py +++ b/src/anomalib/pipelines/benchmark/generator.py @@ -10,6 +10,7 @@ from anomalib.pipelines.components import JobGenerator from anomalib.pipelines.components.utils import get_iterator_from_grid_dict from anomalib.pipelines.types import PREV_STAGE_RESULT +from anomalib.utils.config import flatten_dict from anomalib.utils.logging import hide_output from .job import BenchmarkJob @@ -39,9 +40,12 @@ def generate_jobs( """Return iterator based on the arguments.""" del previous_stage_result # Not needed for this job for _container in get_iterator_from_grid_dict(args): + # Pass experimental configs as a flatten dictionary to the job runner. + flat_cfg = flatten_dict(_container) yield BenchmarkJob( accelerator=self.accelerator, seed=_container["seed"], model=get_model(_container["model"]), datamodule=get_datamodule(_container["data"]), + flat_cfg=flat_cfg, ) diff --git a/src/anomalib/pipelines/benchmark/job.py b/src/anomalib/pipelines/benchmark/job.py index 01822d5f29..f56899ac5d 100644 --- a/src/anomalib/pipelines/benchmark/job.py +++ b/src/anomalib/pipelines/benchmark/job.py @@ -32,16 +32,25 @@ class BenchmarkJob(Job): model (AnomalyModule): The model to use. datamodule (AnomalibDataModule): The data module to use. seed (int): The seed to use. + flat_cfg (dict): The flat dictionary of configs with dotted keys. """ name = "benchmark" - def __init__(self, accelerator: str, model: AnomalyModule, datamodule: AnomalibDataModule, seed: int) -> None: + def __init__( + self, + accelerator: str, + model: AnomalyModule, + datamodule: AnomalibDataModule, + seed: int, + flat_cfg: dict, + ) -> None: super().__init__() self.accelerator = accelerator self.model = model self.datamodule = datamodule self.seed = seed + self.flat_cfg = flat_cfg @hide_output def run( @@ -74,12 +83,9 @@ def run( # TODO(ashwinvaidya17): Restore throughput # https://github.com/openvinotoolkit/anomalib/issues/2054 output = { - "seed": self.seed, "accelerator": self.accelerator, - "model": self.model.__class__.__name__, - "data": self.datamodule.__class__.__name__, - "category": self.datamodule.category, **durations, + **self.flat_cfg, **test_results[0], } logger.info(f"Completed with result {output}")