From fd4db1973685a0f15f2babc9bbafeab382b80f15 Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Mon, 14 Oct 2024 14:57:03 -0600 Subject: [PATCH 01/22] really adding the script this time (??) --- helper_scripts/generate_adf_config_file.py | 68 ++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 helper_scripts/generate_adf_config_file.py diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py new file mode 100644 index 0000000..561a83a --- /dev/null +++ b/helper_scripts/generate_adf_config_file.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +import argparse +import yaml +import os + +def _parse_args(): + """Parse command line arguments""" + + parser = argparse.ArgumentParser( + description="Generate cupid_adf_config.yml based on an existing CUPID YAML file", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument("--cupid_file", + action="store", + required=True, + help="CUPID YAML file") + parser.add_argument("--adf_template", + action='store', + required=True, + help="an adf config file to use as a base") + parser.add_argument("--out_file", action='store', required=True, help='the output file to save') + return parser.parse_args() + + +def generate_adf_config(cupid_file, adf_file, out_file): + """Use cupid_file (YAML) and adf_file (YAML) to produce out_file + by modifying adf_file with data from cupid_file. + """ + with open(cupid_file, encoding='UTF-8') as c: + c_dict = yaml.load(c, Loader=yaml.SafeLoader) + with open(adf_file, encoding='UTF-8') as a: + a_dict = yaml.load(a, Loader=yaml.SafeLoader) + + # Mapping from CUPID's global_params: + a_dict['diag_cam_climo']['cam_case_name'] = c_dict['global_params']['case_name'] + a_dict['diag_cam_baseline_climo']['cam_case_name'] = c_dict['global_params']['base_case_name'] + + # QUESTION: how to specify locations for model output? + # - separate history files, time series files, climo files ? + # - separate for 'base case' and 'test case(s)'? + a_dict['diag_cam_climo']['cam_hist_loc'] = c_dict['global_params']['CESM_output_dir'] + a_dict['diag_cam_baseline_climo']['cam_hist_loc'] = c_dict['global_params']['CESM_output_dir'] + + # QUESTION: how to specify different start/end dates for 'base case' and 'test case(s)'? + a_dict['diag_cam_climo']['start_year'] = int(c_dict['global_params']['start_date'].split('-')[0]) + a_dict['diag_cam_climo']['end_year'] = int(c_dict['global_params']['end_date'].split('-')[0]) + a_dict['diag_cam_baseline_climo']['start_year'] = int(c_dict['global_params']['start_date'].split('-')[0]) + a_dict['diag_cam_baseline_climo']['end_year'] = int(c_dict['global_params']['end_date'].split('-')[0]) + + a_dict['diag_basic_info']['num_procs'] = c_dict['timeseries'].get('num_procs', 1) + + a_dict['user'] = os.getenv("USER") + + with open(out_file, "w") as f: + # Header of file is a comment logging provenance + f.write(f"# This file has been auto-generated using generate_adf_config_file.py\n") + f.write("# Arguments:\n") + f.write(f"# {cupid_file = }\n") + f.write(f"# {adf_file = }\n") + f.write(f"# Output: {out_file = }\n") + # enter in each element of the dictionary into the new file + yaml.dump(a_dict, f, sort_keys=False) + + +if __name__ == "__main__": + args = vars(_parse_args()) + print(args) + generate_adf_config(args['cupid_file'], args['adf_template'], args['out_file']) \ No newline at end of file From 97565d07b19d031e50744d61e8f5b724d50f89af Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Mon, 14 Oct 2024 15:17:05 -0600 Subject: [PATCH 02/22] added newline at end of file per linting suggestion --- helper_scripts/generate_adf_config_file.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py index 561a83a..bc750ae 100644 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -65,4 +65,5 @@ def generate_adf_config(cupid_file, adf_file, out_file): if __name__ == "__main__": args = vars(_parse_args()) print(args) - generate_adf_config(args['cupid_file'], args['adf_template'], args['out_file']) \ No newline at end of file + generate_adf_config(args['cupid_file'], args['adf_template'], args['out_file']) + \ No newline at end of file From bd92dc1f34713dd6a76dd8cb26c3ec20c07504f9 Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Mon, 14 Oct 2024 15:20:07 -0600 Subject: [PATCH 03/22] just more linting/formatting on my end --- helper_scripts/generate_adf_config_file.py | 71 ++++++++++++++-------- 1 file changed, 45 insertions(+), 26 deletions(-) diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py index bc750ae..ac35cdc 100644 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -3,6 +3,7 @@ import yaml import os + def _parse_args(): """Parse command line arguments""" @@ -10,50 +11,69 @@ def _parse_args(): description="Generate cupid_adf_config.yml based on an existing CUPID YAML file", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - parser.add_argument("--cupid_file", - action="store", - required=True, - help="CUPID YAML file") - parser.add_argument("--adf_template", - action='store', - required=True, - help="an adf config file to use as a base") - parser.add_argument("--out_file", action='store', required=True, help='the output file to save') + parser.add_argument( + "--cupid_file", action="store", required=True, help="CUPID YAML file" + ) + parser.add_argument( + "--adf_template", + action="store", + required=True, + help="an adf config file to use as a base", + ) + parser.add_argument( + "--out_file", action="store", required=True, help="the output file to save" + ) return parser.parse_args() def generate_adf_config(cupid_file, adf_file, out_file): """Use cupid_file (YAML) and adf_file (YAML) to produce out_file - by modifying adf_file with data from cupid_file. + by modifying adf_file with data from cupid_file. """ - with open(cupid_file, encoding='UTF-8') as c: + with open(cupid_file, encoding="UTF-8") as c: c_dict = yaml.load(c, Loader=yaml.SafeLoader) - with open(adf_file, encoding='UTF-8') as a: + with open(adf_file, encoding="UTF-8") as a: a_dict = yaml.load(a, Loader=yaml.SafeLoader) # Mapping from CUPID's global_params: - a_dict['diag_cam_climo']['cam_case_name'] = c_dict['global_params']['case_name'] - a_dict['diag_cam_baseline_climo']['cam_case_name'] = c_dict['global_params']['base_case_name'] + a_dict["diag_cam_climo"]["cam_case_name"] = c_dict["global_params"]["case_name"] + a_dict["diag_cam_baseline_climo"]["cam_case_name"] = c_dict["global_params"][ + "base_case_name" + ] # QUESTION: how to specify locations for model output? - # - separate history files, time series files, climo files ? + # - separate history files, time series files, climo files ? # - separate for 'base case' and 'test case(s)'? - a_dict['diag_cam_climo']['cam_hist_loc'] = c_dict['global_params']['CESM_output_dir'] - a_dict['diag_cam_baseline_climo']['cam_hist_loc'] = c_dict['global_params']['CESM_output_dir'] + a_dict["diag_cam_climo"]["cam_hist_loc"] = c_dict["global_params"][ + "CESM_output_dir" + ] + a_dict["diag_cam_baseline_climo"]["cam_hist_loc"] = c_dict["global_params"][ + "CESM_output_dir" + ] # QUESTION: how to specify different start/end dates for 'base case' and 'test case(s)'? - a_dict['diag_cam_climo']['start_year'] = int(c_dict['global_params']['start_date'].split('-')[0]) - a_dict['diag_cam_climo']['end_year'] = int(c_dict['global_params']['end_date'].split('-')[0]) - a_dict['diag_cam_baseline_climo']['start_year'] = int(c_dict['global_params']['start_date'].split('-')[0]) - a_dict['diag_cam_baseline_climo']['end_year'] = int(c_dict['global_params']['end_date'].split('-')[0]) + a_dict["diag_cam_climo"]["start_year"] = int( + c_dict["global_params"]["start_date"].split("-")[0] + ) + a_dict["diag_cam_climo"]["end_year"] = int( + c_dict["global_params"]["end_date"].split("-")[0] + ) + a_dict["diag_cam_baseline_climo"]["start_year"] = int( + c_dict["global_params"]["start_date"].split("-")[0] + ) + a_dict["diag_cam_baseline_climo"]["end_year"] = int( + c_dict["global_params"]["end_date"].split("-")[0] + ) - a_dict['diag_basic_info']['num_procs'] = c_dict['timeseries'].get('num_procs', 1) + a_dict["diag_basic_info"]["num_procs"] = c_dict["timeseries"].get("num_procs", 1) - a_dict['user'] = os.getenv("USER") + a_dict["user"] = os.getenv("USER") with open(out_file, "w") as f: # Header of file is a comment logging provenance - f.write(f"# This file has been auto-generated using generate_adf_config_file.py\n") + f.write( + f"# This file has been auto-generated using generate_adf_config_file.py\n" + ) f.write("# Arguments:\n") f.write(f"# {cupid_file = }\n") f.write(f"# {adf_file = }\n") @@ -65,5 +85,4 @@ def generate_adf_config(cupid_file, adf_file, out_file): if __name__ == "__main__": args = vars(_parse_args()) print(args) - generate_adf_config(args['cupid_file'], args['adf_template'], args['out_file']) - \ No newline at end of file + generate_adf_config(args["cupid_file"], args["adf_template"], args["out_file"]) From 49a0f9933841de4305f8cfb0e961cf56f9ba1cf8 Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Wed, 23 Oct 2024 15:05:21 -0600 Subject: [PATCH 04/22] add more details for ADF config --- helper_scripts/generate_adf_config_file.py | 90 ++++++++++++++-------- 1 file changed, 60 insertions(+), 30 deletions(-) diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py index ac35cdc..c29b5e9 100644 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -35,38 +35,54 @@ def generate_adf_config(cupid_file, adf_file, out_file): with open(adf_file, encoding="UTF-8") as a: a_dict = yaml.load(a, Loader=yaml.SafeLoader) - # Mapping from CUPID's global_params: - a_dict["diag_cam_climo"]["cam_case_name"] = c_dict["global_params"]["case_name"] - a_dict["diag_cam_baseline_climo"]["cam_case_name"] = c_dict["global_params"][ - "base_case_name" - ] - - # QUESTION: how to specify locations for model output? - # - separate history files, time series files, climo files ? - # - separate for 'base case' and 'test case(s)'? - a_dict["diag_cam_climo"]["cam_hist_loc"] = c_dict["global_params"][ - "CESM_output_dir" - ] - a_dict["diag_cam_baseline_climo"]["cam_hist_loc"] = c_dict["global_params"][ - "CESM_output_dir" - ] - - # QUESTION: how to specify different start/end dates for 'base case' and 'test case(s)'? - a_dict["diag_cam_climo"]["start_year"] = int( - c_dict["global_params"]["start_date"].split("-")[0] - ) - a_dict["diag_cam_climo"]["end_year"] = int( - c_dict["global_params"]["end_date"].split("-")[0] - ) - a_dict["diag_cam_baseline_climo"]["start_year"] = int( - c_dict["global_params"]["start_date"].split("-")[0] - ) - a_dict["diag_cam_baseline_climo"]["end_year"] = int( - c_dict["global_params"]["end_date"].split("-")[0] - ) + # read parameters from CUPID + # use `get` to default to None + DOUT = c_dict["global_params"]["CESM_output_dir"] + base_case_name = c_dict["global_params"].get("base_case_name") + test_case_name = c_dict["global_params"]["case_name"] + c_ts = c_dict["timeseries"] + ts_case_names = c_ts.get("case_name") + if not ts_case_names: + raise ValueError("CUPiD file does not have timeseries case_name array.") - a_dict["diag_basic_info"]["num_procs"] = c_dict["timeseries"].get("num_procs", 1) + # Set case names for ADF config + a_dict["diag_cam_climo"]["cam_case_name"] = test_case_name + a_dict["diag_cam_baseline_climo"]["cam_case_name"] = base_case_name + + + # TEST CASE HISTORY FILE PATH + a_dict["diag_cam_climo"]["cam_hist_loc"] = "/".join([DOUT, "atm", "hist"]) + # TEST CASE TIME SERIES FILE PATH + a_dict["diag_cam_climo"]["cam_ts_loc"] = "/".join([DOUT, "proc", "tseries"]) + # TEST CASE CLIMO FILE PATH + a_dict["diag_cam_climo"]["cam_climo_loc"] = "/".join([DOUT, "proc", "climo"]) + # TEST CASE START / END YEARS + test_case_cupid_ts_index = ts_case_names.index(test_case_name) if test_case_name in ts_case_names else None + start_date = get_date_from_ts(c_ts["atm"], "start_years", test_case_cupid_ts_index) + end_date = get_date_from_ts(c_ts["atm"], "end_years", test_case_cupid_ts_index) + a_dict["diag_cam_climo"]["start_year"] = start_date + a_dict["diag_cam_climo"]["end_year"] = end_date + # Set values for BASELINE + base_case_cupid_ts_index = ts_case_names.index(base_case_name) if base_case_name in ts_case_names else None + if base_case_name is not None: + base_case_output_dir = c_dict["global_params"].get("base_case_output_dir", DOUT) + base_start_date = get_date_from_ts(c_ts["atm"], "start_years", base_case_cupid_ts_index) + base_end_date = get_date_from_ts(c_ts["atm"], "end_years", base_case_cupid_ts_index) + if base_start_date is None: + base_start_date = start_date + if base_end_date is None: + base_end_date = end_date + + a_dict["diag_cam_baseline_climo"]["cam_hist_loc"] = "/".join([base_case_output_dir, "atm", "hist"]) + a_dict["diag_cam_baseline_climo"]["cam_ts_loc"] = "/".join([base_case_output_dir, "proc", "tseries"]) + a_dict["diag_cam_baseline_climo"]["cam_climo_loc"] = "/".join([base_case_output_dir, "proc", "climo"]) + a_dict["diag_cam_baseline_climo"]["start_year"] = base_start_date + a_dict["diag_cam_baseline_climo"]["end_year"] = base_end_date + + a_dict["diag_basic_info"]["num_procs"] = c_dict["timeseries"].get("num_procs", 1) + a_dict["diag_basic_info"]["cam_regrid_loc"] = "/".join([DOUT, "proc", "regrid"]) # This is where ADF will make "regrid" files + a_dict["diag_basic_info"]["cam_diag_plot_loc"] = "/".join([c_dict["sname"], "ADF"]) # this is where ADF will put plots, and "website" directory a_dict["user"] = os.getenv("USER") with open(out_file, "w") as f: @@ -82,6 +98,20 @@ def generate_adf_config(cupid_file, adf_file, out_file): yaml.dump(a_dict, f, sort_keys=False) +def get_date_from_ts(data: dict, keyname: str, listindex: int, default=None): + if type(data) is not dict: + raise TypeError(f"first argument needs to be dict, got {type(data)}") + if keyname not in data: + raise KeyError(f"no entry {keyname} in the dict") + x = data[keyname] + if isinstance(x, list): + return x[listindex] + elif isinstance(x, int): + return x + else: + return default + + if __name__ == "__main__": args = vars(_parse_args()) print(args) From 13c0620e85d310b15337c0ee56f35a27094c12b3 Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Wed, 23 Oct 2024 15:14:43 -0600 Subject: [PATCH 05/22] fix typo for sname --- helper_scripts/generate_adf_config_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py index c29b5e9..35e5ef1 100644 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -82,7 +82,7 @@ def generate_adf_config(cupid_file, adf_file, out_file): a_dict["diag_basic_info"]["num_procs"] = c_dict["timeseries"].get("num_procs", 1) a_dict["diag_basic_info"]["cam_regrid_loc"] = "/".join([DOUT, "proc", "regrid"]) # This is where ADF will make "regrid" files - a_dict["diag_basic_info"]["cam_diag_plot_loc"] = "/".join([c_dict["sname"], "ADF"]) # this is where ADF will put plots, and "website" directory + a_dict["diag_basic_info"]["cam_diag_plot_loc"] = "/".join([c_dict['data_sources']["sname"], "ADF"]) # this is where ADF will put plots, and "website" directory a_dict["user"] = os.getenv("USER") with open(out_file, "w") as f: From eb25a31d61d5930be09d15d885d78dc4a713b867 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Mon, 4 Nov 2024 12:55:37 -0700 Subject: [PATCH 06/22] ran pre-commit --- helper_scripts/generate_adf_config_file.py | 60 ++++++++++++++++------ 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py index 35e5ef1..e0c20f3 100644 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -1,8 +1,11 @@ #!/usr/bin/env python3 +from __future__ import annotations + import argparse -import yaml import os +import yaml + def _parse_args(): """Parse command line arguments""" @@ -12,7 +15,10 @@ def _parse_args(): formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) parser.add_argument( - "--cupid_file", action="store", required=True, help="CUPID YAML file" + "--cupid_file", + action="store", + required=True, + help="CUPID YAML file", ) parser.add_argument( "--adf_template", @@ -21,7 +27,10 @@ def _parse_args(): help="an adf config file to use as a base", ) parser.add_argument( - "--out_file", action="store", required=True, help="the output file to save" + "--out_file", + action="store", + required=True, + help="the output file to save", ) return parser.parse_args() @@ -49,7 +58,6 @@ def generate_adf_config(cupid_file, adf_file, out_file): a_dict["diag_cam_climo"]["cam_case_name"] = test_case_name a_dict["diag_cam_baseline_climo"]["cam_case_name"] = base_case_name - # TEST CASE HISTORY FILE PATH a_dict["diag_cam_climo"]["cam_hist_loc"] = "/".join([DOUT, "atm", "hist"]) # TEST CASE TIME SERIES FILE PATH @@ -57,38 +65,60 @@ def generate_adf_config(cupid_file, adf_file, out_file): # TEST CASE CLIMO FILE PATH a_dict["diag_cam_climo"]["cam_climo_loc"] = "/".join([DOUT, "proc", "climo"]) # TEST CASE START / END YEARS - test_case_cupid_ts_index = ts_case_names.index(test_case_name) if test_case_name in ts_case_names else None + test_case_cupid_ts_index = ( + ts_case_names.index(test_case_name) if test_case_name in ts_case_names else None + ) start_date = get_date_from_ts(c_ts["atm"], "start_years", test_case_cupid_ts_index) end_date = get_date_from_ts(c_ts["atm"], "end_years", test_case_cupid_ts_index) a_dict["diag_cam_climo"]["start_year"] = start_date a_dict["diag_cam_climo"]["end_year"] = end_date - # Set values for BASELINE - base_case_cupid_ts_index = ts_case_names.index(base_case_name) if base_case_name in ts_case_names else None + # Set values for BASELINE + base_case_cupid_ts_index = ( + ts_case_names.index(base_case_name) if base_case_name in ts_case_names else None + ) if base_case_name is not None: base_case_output_dir = c_dict["global_params"].get("base_case_output_dir", DOUT) - base_start_date = get_date_from_ts(c_ts["atm"], "start_years", base_case_cupid_ts_index) - base_end_date = get_date_from_ts(c_ts["atm"], "end_years", base_case_cupid_ts_index) + base_start_date = get_date_from_ts( + c_ts["atm"], + "start_years", + base_case_cupid_ts_index, + ) + base_end_date = get_date_from_ts( + c_ts["atm"], + "end_years", + base_case_cupid_ts_index, + ) if base_start_date is None: base_start_date = start_date if base_end_date is None: base_end_date = end_date - a_dict["diag_cam_baseline_climo"]["cam_hist_loc"] = "/".join([base_case_output_dir, "atm", "hist"]) - a_dict["diag_cam_baseline_climo"]["cam_ts_loc"] = "/".join([base_case_output_dir, "proc", "tseries"]) - a_dict["diag_cam_baseline_climo"]["cam_climo_loc"] = "/".join([base_case_output_dir, "proc", "climo"]) + a_dict["diag_cam_baseline_climo"]["cam_hist_loc"] = "/".join( + [base_case_output_dir, "atm", "hist"], + ) + a_dict["diag_cam_baseline_climo"]["cam_ts_loc"] = "/".join( + [base_case_output_dir, "proc", "tseries"], + ) + a_dict["diag_cam_baseline_climo"]["cam_climo_loc"] = "/".join( + [base_case_output_dir, "proc", "climo"], + ) a_dict["diag_cam_baseline_climo"]["start_year"] = base_start_date a_dict["diag_cam_baseline_climo"]["end_year"] = base_end_date a_dict["diag_basic_info"]["num_procs"] = c_dict["timeseries"].get("num_procs", 1) - a_dict["diag_basic_info"]["cam_regrid_loc"] = "/".join([DOUT, "proc", "regrid"]) # This is where ADF will make "regrid" files - a_dict["diag_basic_info"]["cam_diag_plot_loc"] = "/".join([c_dict['data_sources']["sname"], "ADF"]) # this is where ADF will put plots, and "website" directory + a_dict["diag_basic_info"]["cam_regrid_loc"] = "/".join( + [DOUT, "proc", "regrid"], + ) # This is where ADF will make "regrid" files + a_dict["diag_basic_info"]["cam_diag_plot_loc"] = "/".join( + [c_dict["data_sources"]["sname"], "ADF"], + ) # this is where ADF will put plots, and "website" directory a_dict["user"] = os.getenv("USER") with open(out_file, "w") as f: # Header of file is a comment logging provenance f.write( - f"# This file has been auto-generated using generate_adf_config_file.py\n" + "# This file has been auto-generated using generate_adf_config_file.py\n", ) f.write("# Arguments:\n") f.write(f"# {cupid_file = }\n") From 9fd4ca989902e7a3307ced9371fedf9df6418b27 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Mon, 4 Nov 2024 13:18:08 -0700 Subject: [PATCH 07/22] minor change & note sname portion --- helper_scripts/generate_adf_config_file.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py index e0c20f3..0fd7c69 100644 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -111,7 +111,7 @@ def generate_adf_config(cupid_file, adf_file, out_file): [DOUT, "proc", "regrid"], ) # This is where ADF will make "regrid" files a_dict["diag_basic_info"]["cam_diag_plot_loc"] = "/".join( - [c_dict["data_sources"]["sname"], "ADF"], + [c_dict["data_sources"]["sname"], "ADF"], # TODO: update this ) # this is where ADF will put plots, and "website" directory a_dict["user"] = os.getenv("USER") @@ -121,9 +121,9 @@ def generate_adf_config(cupid_file, adf_file, out_file): "# This file has been auto-generated using generate_adf_config_file.py\n", ) f.write("# Arguments:\n") - f.write(f"# {cupid_file = }\n") - f.write(f"# {adf_file = }\n") - f.write(f"# Output: {out_file = }\n") + f.write(f"# {cupid_file =}\n") + f.write(f"# {adf_file =}\n") + f.write(f"# Output: {out_file =}\n") # enter in each element of the dictionary into the new file yaml.dump(a_dict, f, sort_keys=False) From 9ed14fdeead783add7178573ebd10a1e786ae139 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Mon, 4 Nov 2024 13:20:57 -0700 Subject: [PATCH 08/22] linting suggestion from after commit --- helper_scripts/generate_adf_config_file.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py index 0fd7c69..76e00b3 100644 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -121,9 +121,9 @@ def generate_adf_config(cupid_file, adf_file, out_file): "# This file has been auto-generated using generate_adf_config_file.py\n", ) f.write("# Arguments:\n") - f.write(f"# {cupid_file =}\n") - f.write(f"# {adf_file =}\n") - f.write(f"# Output: {out_file =}\n") + f.write(f"# {cupid_file=}\n") + f.write(f"# {adf_file=}\n") + f.write(f"# Output: {out_file=}\n") # enter in each element of the dictionary into the new file yaml.dump(a_dict, f, sort_keys=False) From 9ba9d6d0ef4fe045f3b6d936b7ba5cf819a76b91 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Mon, 4 Nov 2024 13:30:58 -0700 Subject: [PATCH 09/22] update path to adf --- helper_scripts/generate_adf_config_file.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py index 76e00b3..3988f88 100644 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -111,7 +111,12 @@ def generate_adf_config(cupid_file, adf_file, out_file): [DOUT, "proc", "regrid"], ) # This is where ADF will make "regrid" files a_dict["diag_basic_info"]["cam_diag_plot_loc"] = "/".join( - [c_dict["data_sources"]["sname"], "ADF"], # TODO: update this + [ + c_dict["data_sources"]["sname"], + "computed_notebooks", + c_dict["data_sources"]["sname"], + "ADF", + ], ) # this is where ADF will put plots, and "website" directory a_dict["user"] = os.getenv("USER") From ab8d539e5f2772db83360a22932dd756caee277c Mon Sep 17 00:00:00 2001 From: Teagan King Date: Mon, 4 Nov 2024 13:57:37 -0700 Subject: [PATCH 10/22] include case name and not just CESM output dir --- helper_scripts/generate_adf_config_file.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py index 3988f88..a823be9 100644 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -59,11 +59,17 @@ def generate_adf_config(cupid_file, adf_file, out_file): a_dict["diag_cam_baseline_climo"]["cam_case_name"] = base_case_name # TEST CASE HISTORY FILE PATH - a_dict["diag_cam_climo"]["cam_hist_loc"] = "/".join([DOUT, "atm", "hist"]) + a_dict["diag_cam_climo"]["cam_hist_loc"] = "/".join( + [DOUT, test_case_name, "atm", "hist"], + ) # TEST CASE TIME SERIES FILE PATH - a_dict["diag_cam_climo"]["cam_ts_loc"] = "/".join([DOUT, "proc", "tseries"]) + a_dict["diag_cam_climo"]["cam_ts_loc"] = "/".join( + [DOUT, test_case_name, "proc", "tseries"], + ) # TEST CASE CLIMO FILE PATH - a_dict["diag_cam_climo"]["cam_climo_loc"] = "/".join([DOUT, "proc", "climo"]) + a_dict["diag_cam_climo"]["cam_climo_loc"] = "/".join( + [DOUT, test_case_name, "proc", "climo"], + ) # TEST CASE START / END YEARS test_case_cupid_ts_index = ( ts_case_names.index(test_case_name) if test_case_name in ts_case_names else None @@ -78,7 +84,10 @@ def generate_adf_config(cupid_file, adf_file, out_file): ts_case_names.index(base_case_name) if base_case_name in ts_case_names else None ) if base_case_name is not None: - base_case_output_dir = c_dict["global_params"].get("base_case_output_dir", DOUT) + base_case_output_dir = c_dict["global_params"].get( + "base_case_output_dir", + DOUT + "/" + base_case_name, + ) base_start_date = get_date_from_ts( c_ts["atm"], "start_years", @@ -108,7 +117,7 @@ def generate_adf_config(cupid_file, adf_file, out_file): a_dict["diag_basic_info"]["num_procs"] = c_dict["timeseries"].get("num_procs", 1) a_dict["diag_basic_info"]["cam_regrid_loc"] = "/".join( - [DOUT, "proc", "regrid"], + [DOUT, base_case_name, "proc", "regrid"], ) # This is where ADF will make "regrid" files a_dict["diag_basic_info"]["cam_diag_plot_loc"] = "/".join( [ From db5a6f9bb2ed3afcac2c7fecedf799dc738faaa6 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Mon, 4 Nov 2024 14:41:36 -0700 Subject: [PATCH 11/22] included hist_str for top level rather than case-specific --- helper_scripts/generate_adf_config_file.py | 1 + 1 file changed, 1 insertion(+) diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py index a823be9..57eaecf 100644 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -115,6 +115,7 @@ def generate_adf_config(cupid_file, adf_file, out_file): a_dict["diag_cam_baseline_climo"]["start_year"] = base_start_date a_dict["diag_cam_baseline_climo"]["end_year"] = base_end_date + a_dict["diag_basic_info"]["hist_str"] = c_dict["timeseries"]["atm"].get("hist_str") a_dict["diag_basic_info"]["num_procs"] = c_dict["timeseries"].get("num_procs", 1) a_dict["diag_basic_info"]["cam_regrid_loc"] = "/".join( [DOUT, base_case_name, "proc", "regrid"], From db47343cc203348f1181d2220e2f10155ae59e06 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Wed, 6 Nov 2024 15:11:14 -0700 Subject: [PATCH 12/22] updates to config file generation to include full path to ADF output and add atm in proc/tseries paths --- helper_scripts/generate_adf_config_file.py | 25 +++++++++++++++------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py index 57eaecf..55ff711 100644 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -2,10 +2,11 @@ from __future__ import annotations import argparse -import os import yaml +# import os + def _parse_args(): """Parse command line arguments""" @@ -64,11 +65,11 @@ def generate_adf_config(cupid_file, adf_file, out_file): ) # TEST CASE TIME SERIES FILE PATH a_dict["diag_cam_climo"]["cam_ts_loc"] = "/".join( - [DOUT, test_case_name, "proc", "tseries"], + [DOUT, test_case_name, "atm", "proc", "tseries"], ) # TEST CASE CLIMO FILE PATH a_dict["diag_cam_climo"]["cam_climo_loc"] = "/".join( - [DOUT, test_case_name, "proc", "climo"], + [DOUT, test_case_name, "atm", "proc", "climo"], ) # TEST CASE START / END YEARS test_case_cupid_ts_index = ( @@ -107,10 +108,10 @@ def generate_adf_config(cupid_file, adf_file, out_file): [base_case_output_dir, "atm", "hist"], ) a_dict["diag_cam_baseline_climo"]["cam_ts_loc"] = "/".join( - [base_case_output_dir, "proc", "tseries"], + [base_case_output_dir, "atm", "proc", "tseries"], ) a_dict["diag_cam_baseline_climo"]["cam_climo_loc"] = "/".join( - [base_case_output_dir, "proc", "climo"], + [base_case_output_dir, "atm", "proc", "climo"], ) a_dict["diag_cam_baseline_climo"]["start_year"] = base_start_date a_dict["diag_cam_baseline_climo"]["end_year"] = base_end_date @@ -118,17 +119,25 @@ def generate_adf_config(cupid_file, adf_file, out_file): a_dict["diag_basic_info"]["hist_str"] = c_dict["timeseries"]["atm"].get("hist_str") a_dict["diag_basic_info"]["num_procs"] = c_dict["timeseries"].get("num_procs", 1) a_dict["diag_basic_info"]["cam_regrid_loc"] = "/".join( - [DOUT, base_case_name, "proc", "regrid"], + [DOUT, base_case_name, "atm", "proc", "regrid"], ) # This is where ADF will make "regrid" files a_dict["diag_basic_info"]["cam_diag_plot_loc"] = "/".join( [ - c_dict["data_sources"]["sname"], + "/".join(cupid_file.split("/")[:-1]), "computed_notebooks", c_dict["data_sources"]["sname"], "ADF", ], ) # this is where ADF will put plots, and "website" directory - a_dict["user"] = os.getenv("USER") + a_dict["user"] = "/".join( + [ + "/".join(cupid_file.split("/")[:-1]), + "computed_notebooks", + c_dict["data_sources"]["sname"], + "ADF", + ], + ) + # os.getenv("USER") with open(out_file, "w") as f: # Header of file is a comment logging provenance From 35a9ebbad7f5bc46dfa280037bd4ade844083406 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Fri, 8 Nov 2024 13:55:30 -0700 Subject: [PATCH 13/22] include _build/html in path --- helper_scripts/generate_adf_config_file.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py index 55ff711..0f70c5f 100644 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -126,6 +126,8 @@ def generate_adf_config(cupid_file, adf_file, out_file): "/".join(cupid_file.split("/")[:-1]), "computed_notebooks", c_dict["data_sources"]["sname"], + "_build", + "html", "ADF", ], ) # this is where ADF will put plots, and "website" directory @@ -134,6 +136,8 @@ def generate_adf_config(cupid_file, adf_file, out_file): "/".join(cupid_file.split("/")[:-1]), "computed_notebooks", c_dict["data_sources"]["sname"], + "_build", + "html", "ADF", ], ) From 8f79704d3311126a83dbca6b7f34637a40aae35e Mon Sep 17 00:00:00 2001 From: Teagan King Date: Wed, 13 Nov 2024 11:29:06 -0700 Subject: [PATCH 14/22] include option for overwriting vars in adf config file --- helper_scripts/generate_adf_config_file.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py index 0f70c5f..6f9c33d 100644 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -141,6 +141,17 @@ def generate_adf_config(cupid_file, adf_file, out_file): "ADF", ], ) + + try: + cupid_config_vars = c_dict["compute_notebooks"]["atm"]["link_to_ADF"][ + "parameter_groups" + ]["none"]["adf_vars"] + # if added in cupid config file, overwrite adf default diag_var_list + a_dict["diag_var_list"] = cupid_config_vars + except KeyError: + # use default adf diag_var_list + pass + # os.getenv("USER") with open(out_file, "w") as f: From 0388b5467d866ea6792be603e67db8e243adb231 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Fri, 15 Nov 2024 12:26:39 -0700 Subject: [PATCH 15/22] initial response to review comments --- helper_scripts/generate_adf_config_file.py | 68 +++++++++++++++++----- 1 file changed, 52 insertions(+), 16 deletions(-) mode change 100644 => 100755 helper_scripts/generate_adf_config_file.py diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py old mode 100644 new mode 100755 index 6f9c33d..3456647 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -2,11 +2,11 @@ from __future__ import annotations import argparse +import os +import sys import yaml -# import os - def _parse_args(): """Parse command line arguments""" @@ -15,20 +15,30 @@ def _parse_args(): description="Generate cupid_adf_config.yml based on an existing CUPID YAML file", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) + # Command line argument for location of CESM source code (required) parser.add_argument( - "--cupid_file", + "--cesm-root", action="store", + dest="cesm_root", required=True, - help="CUPID YAML file", + help="Location of CESM source code", + ) + # Command line argument for CUPiD example from which to get config.yml + parser.add_argument( + "--cupid-example", + action="store", + dest="cupid_example", + default="external_diag_packages", + help="CUPiD example to use as template for config.yml", ) parser.add_argument( - "--adf_template", + "--adf-template", action="store", required=True, help="an adf config file to use as a base", ) parser.add_argument( - "--out_file", + "--out-file", action="store", required=True, help="the output file to save", @@ -36,14 +46,30 @@ def _parse_args(): return parser.parse_args() -def generate_adf_config(cupid_file, adf_file, out_file): - """Use cupid_file (YAML) and adf_file (YAML) to produce out_file - by modifying adf_file with data from cupid_file. +def generate_adf_config(cesm_root, cupid_example, adf_file, out_file): + """Use cupid config file (YAML) from cupid_example and adf_file (YAML) + to produce out_file by modifying adf_file with data from cupid config file. """ - with open(cupid_file, encoding="UTF-8") as c: - c_dict = yaml.load(c, Loader=yaml.SafeLoader) + sys.path.append(os.path.join(cesm_root, "cime")) + + # Is cupid_example a valid value? + cupid_root = os.path.join(cesm_root, "tools", "CUPiD") + cupid_examples = os.path.join(cupid_root, "examples") + valid_examples = [ + example + for example in next(os.walk(cupid_examples))[1] + if example not in ["ilamb", "nblibrary"] + ] + if cupid_example not in valid_examples: + error_msg = f"argument --cupid-example: invalid choice '{cupid_example}'" + raise KeyError( + f"{error_msg} (choose from subdirectories of {cupid_examples}: {valid_examples})", + ) + + with open(os.path.join(cupid_root, "examples", cupid_example, "config.yml")) as c: + c_dict = yaml.safe_load(c) with open(adf_file, encoding="UTF-8") as a: - a_dict = yaml.load(a, Loader=yaml.SafeLoader) + a_dict = yaml.safe_load(a) # read parameters from CUPID # use `get` to default to None @@ -123,7 +149,9 @@ def generate_adf_config(cupid_file, adf_file, out_file): ) # This is where ADF will make "regrid" files a_dict["diag_basic_info"]["cam_diag_plot_loc"] = "/".join( [ - "/".join(cupid_file.split("/")[:-1]), + cupid_root, + "examples", + cupid_example, "computed_notebooks", c_dict["data_sources"]["sname"], "_build", @@ -133,7 +161,9 @@ def generate_adf_config(cupid_file, adf_file, out_file): ) # this is where ADF will put plots, and "website" directory a_dict["user"] = "/".join( [ - "/".join(cupid_file.split("/")[:-1]), + cupid_root, + "examples", + cupid_example, "computed_notebooks", c_dict["data_sources"]["sname"], "_build", @@ -160,7 +190,8 @@ def generate_adf_config(cupid_file, adf_file, out_file): "# This file has been auto-generated using generate_adf_config_file.py\n", ) f.write("# Arguments:\n") - f.write(f"# {cupid_file=}\n") + f.write(f"# {cesm_root=}\n") + f.write(f"# {cupid_example=}\n") f.write(f"# {adf_file=}\n") f.write(f"# Output: {out_file=}\n") # enter in each element of the dictionary into the new file @@ -184,4 +215,9 @@ def get_date_from_ts(data: dict, keyname: str, listindex: int, default=None): if __name__ == "__main__": args = vars(_parse_args()) print(args) - generate_adf_config(args["cupid_file"], args["adf_template"], args["out_file"]) + generate_adf_config( + args["cesm_root"], + args["cupid_example"], + args["adf_template"], + args["out_file"], + ) From d8a563f161afd6e64f64e70d1f9b388c1188466b Mon Sep 17 00:00:00 2001 From: Teagan King Date: Fri, 15 Nov 2024 13:10:03 -0700 Subject: [PATCH 16/22] additional review updates --- helper_scripts/generate_adf_config_file.py | 107 +++++++++++++-------- 1 file changed, 67 insertions(+), 40 deletions(-) diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py index 3456647..f504d32 100755 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -83,19 +83,34 @@ def generate_adf_config(cesm_root, cupid_example, adf_file, out_file): # Set case names for ADF config a_dict["diag_cam_climo"]["cam_case_name"] = test_case_name - a_dict["diag_cam_baseline_climo"]["cam_case_name"] = base_case_name + if base_case_name: + a_dict["diag_cam_baseline_climo"]["cam_case_name"] = base_case_name + + elif "cam_case_name" in "cam_case_name": + del a_dict["diag_cam_baseline_climo"]["cam_case_name"] # TEST CASE HISTORY FILE PATH - a_dict["diag_cam_climo"]["cam_hist_loc"] = "/".join( - [DOUT, test_case_name, "atm", "hist"], + a_dict["diag_cam_climo"]["cam_hist_loc"] = os.path.join( + DOUT, + test_case_name, + "atm", + "hist", ) # TEST CASE TIME SERIES FILE PATH - a_dict["diag_cam_climo"]["cam_ts_loc"] = "/".join( - [DOUT, test_case_name, "atm", "proc", "tseries"], + a_dict["diag_cam_climo"]["cam_ts_loc"] = os.path.join( + DOUT, + test_case_name, + "atm", + "proc", + "tseries", ) # TEST CASE CLIMO FILE PATH - a_dict["diag_cam_climo"]["cam_climo_loc"] = "/".join( - [DOUT, test_case_name, "atm", "proc", "climo"], + a_dict["diag_cam_climo"]["cam_climo_loc"] = os.path.join( + DOUT, + test_case_name, + "atm", + "proc", + "climo", ) # TEST CASE START / END YEARS test_case_cupid_ts_index = ( @@ -107,10 +122,13 @@ def generate_adf_config(cesm_root, cupid_example, adf_file, out_file): a_dict["diag_cam_climo"]["end_year"] = end_date # Set values for BASELINE - base_case_cupid_ts_index = ( - ts_case_names.index(base_case_name) if base_case_name in ts_case_names else None - ) if base_case_name is not None: + base_case_cupid_ts_index = ( + ts_case_names.index(base_case_name) + if base_case_name in ts_case_names + else None + ) + base_case_output_dir = c_dict["global_params"].get( "base_case_output_dir", DOUT + "/" + base_case_name, @@ -130,46 +148,54 @@ def generate_adf_config(cesm_root, cupid_example, adf_file, out_file): if base_end_date is None: base_end_date = end_date - a_dict["diag_cam_baseline_climo"]["cam_hist_loc"] = "/".join( - [base_case_output_dir, "atm", "hist"], + a_dict["diag_cam_baseline_climo"]["cam_hist_loc"] = os.path.join( + base_case_output_dir, + "atm", + "hist", ) - a_dict["diag_cam_baseline_climo"]["cam_ts_loc"] = "/".join( - [base_case_output_dir, "atm", "proc", "tseries"], + a_dict["diag_cam_baseline_climo"]["cam_ts_loc"] = os.path.join( + base_case_output_dir, + "atm", + "proc", + "tseries", ) - a_dict["diag_cam_baseline_climo"]["cam_climo_loc"] = "/".join( - [base_case_output_dir, "atm", "proc", "climo"], + a_dict["diag_cam_baseline_climo"]["cam_climo_loc"] = os.path.join( + base_case_output_dir, + "atm", + "proc", + "climo", ) a_dict["diag_cam_baseline_climo"]["start_year"] = base_start_date a_dict["diag_cam_baseline_climo"]["end_year"] = base_end_date a_dict["diag_basic_info"]["hist_str"] = c_dict["timeseries"]["atm"].get("hist_str") a_dict["diag_basic_info"]["num_procs"] = c_dict["timeseries"].get("num_procs", 1) - a_dict["diag_basic_info"]["cam_regrid_loc"] = "/".join( - [DOUT, base_case_name, "atm", "proc", "regrid"], + a_dict["diag_basic_info"]["cam_regrid_loc"] = os.path.join( + DOUT, + base_case_name, + "atm", + "proc", + "regrid", ) # This is where ADF will make "regrid" files - a_dict["diag_basic_info"]["cam_diag_plot_loc"] = "/".join( - [ - cupid_root, - "examples", - cupid_example, - "computed_notebooks", - c_dict["data_sources"]["sname"], - "_build", - "html", - "ADF", - ], + a_dict["diag_basic_info"]["cam_diag_plot_loc"] = os.path.join( + cupid_root, + "examples", + cupid_example, + "computed_notebooks", + c_dict["data_sources"]["sname"], + "_build", + "html", + "ADF", ) # this is where ADF will put plots, and "website" directory - a_dict["user"] = "/".join( - [ - cupid_root, - "examples", - cupid_example, - "computed_notebooks", - c_dict["data_sources"]["sname"], - "_build", - "html", - "ADF", - ], + a_dict["user"] = os.path.join( + cupid_root, + "examples", + cupid_example, + "computed_notebooks", + c_dict["data_sources"]["sname"], + "_build", + "html", + "ADF", ) try: @@ -189,6 +215,7 @@ def generate_adf_config(cesm_root, cupid_example, adf_file, out_file): f.write( "# This file has been auto-generated using generate_adf_config_file.py\n", ) + f.write(f"# It is based off of examples/{cupid_example}/config.yml\n") f.write("# Arguments:\n") f.write(f"# {cesm_root=}\n") f.write(f"# {cupid_example=}\n") From 634c2a20f9085d32362bddbf345167cced014fe5 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Fri, 15 Nov 2024 16:02:22 -0700 Subject: [PATCH 17/22] Improve how we get diag_var_list Instead of looking for a variable list in compute_notebooks.atm.link_to_ADF.parameter_groups.none.adf_vars Loop through all compute_notebooks.${component}.${notebook} and look for compute_notebooks.${component}.${notebook}.external_tool if external_tool.tool_name is 'ADF', append everything in external_tool.vars to diag_var_list. Also append everything in external_tool.plotting_scripts to plotting_scripts (to make it easier to control what plots get made). for both cases, if variable isn't found in any external_tool then the default list values from ADF will be used. --- helper_scripts/generate_adf_config_file.py | 24 ++++++++++++++-------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py index f504d32..28c0d70 100755 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -198,15 +198,21 @@ def generate_adf_config(cesm_root, cupid_example, adf_file, out_file): "ADF", ) - try: - cupid_config_vars = c_dict["compute_notebooks"]["atm"]["link_to_ADF"][ - "parameter_groups" - ]["none"]["adf_vars"] - # if added in cupid config file, overwrite adf default diag_var_list - a_dict["diag_var_list"] = cupid_config_vars - except KeyError: - # use default adf diag_var_list - pass + diag_var_list = [] + plotting_scripts = [] + for component in c_dict["compute_notebooks"]: + for nb in c_dict["compute_notebooks"][component]: + if c_dict["compute_notebooks"][component][nb].get("external_tool",{}).get("tool_name") == "ADF": + for var in c_dict["compute_notebooks"][component][nb]["external_tool"].get("vars", []): + if var not in diag_var_list: + diag_var_list.append(var) + for script in c_dict["compute_notebooks"][component][nb]["external_tool"].get("plotting_scripts", []): + if script not in plotting_scripts: + plotting_scripts.append(script) + if diag_var_list: + a_dict["diag_var_list"] = diag_var_list + if plotting_scripts: + a_dict["plotting_scripts"] = plotting_scripts # os.getenv("USER") From 605a59c88335997d46238b695ea9dfce4137ba3d Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Sat, 16 Nov 2024 09:12:48 -0700 Subject: [PATCH 18/22] Pass CI tests Didn't realize pre-commit hadn't run on my last commit --- helper_scripts/generate_adf_config_file.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py index 28c0d70..79045da 100755 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -202,11 +202,20 @@ def generate_adf_config(cesm_root, cupid_example, adf_file, out_file): plotting_scripts = [] for component in c_dict["compute_notebooks"]: for nb in c_dict["compute_notebooks"][component]: - if c_dict["compute_notebooks"][component][nb].get("external_tool",{}).get("tool_name") == "ADF": - for var in c_dict["compute_notebooks"][component][nb]["external_tool"].get("vars", []): + if ( + c_dict["compute_notebooks"][component][nb] + .get("external_tool", {}) + .get("tool_name") + == "ADF" + ): + for var in c_dict["compute_notebooks"][component][nb][ + "external_tool" + ].get("vars", []): if var not in diag_var_list: diag_var_list.append(var) - for script in c_dict["compute_notebooks"][component][nb]["external_tool"].get("plotting_scripts", []): + for script in c_dict["compute_notebooks"][component][nb][ + "external_tool" + ].get("plotting_scripts", []): if script not in plotting_scripts: plotting_scripts.append(script) if diag_var_list: From a2308d7d3f1f27cb987ca232674ebe82c1b77222 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Mon, 18 Nov 2024 08:31:17 -0700 Subject: [PATCH 19/22] update hist_str line --- helper_scripts/generate_adf_config_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py index 79045da..65c8898 100755 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -168,7 +168,7 @@ def generate_adf_config(cesm_root, cupid_example, adf_file, out_file): a_dict["diag_cam_baseline_climo"]["start_year"] = base_start_date a_dict["diag_cam_baseline_climo"]["end_year"] = base_end_date - a_dict["diag_basic_info"]["hist_str"] = c_dict["timeseries"]["atm"].get("hist_str") + a_dict["diag_basic_info"]["hist_str"] = c_dict["timeseries"]["atm"]["hist_str"] a_dict["diag_basic_info"]["num_procs"] = c_dict["timeseries"].get("num_procs", 1) a_dict["diag_basic_info"]["cam_regrid_loc"] = os.path.join( DOUT, From 677a6683641ad5265e4c7a4925669031b83b2448 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Mon, 18 Nov 2024 08:52:46 -0700 Subject: [PATCH 20/22] remove .get from base_case_name --- helper_scripts/generate_adf_config_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py index 65c8898..dd5da70 100755 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -74,7 +74,7 @@ def generate_adf_config(cesm_root, cupid_example, adf_file, out_file): # read parameters from CUPID # use `get` to default to None DOUT = c_dict["global_params"]["CESM_output_dir"] - base_case_name = c_dict["global_params"].get("base_case_name") + base_case_name = c_dict["global_params"]["base_case_name"] test_case_name = c_dict["global_params"]["case_name"] c_ts = c_dict["timeseries"] ts_case_names = c_ts.get("case_name") From 101e1acf6028494779b5964041c22d60f69bf01c Mon Sep 17 00:00:00 2001 From: Teagan King Date: Mon, 18 Nov 2024 09:38:18 -0700 Subject: [PATCH 21/22] update if statements for base case name --- helper_scripts/generate_adf_config_file.py | 50 ++++++++++------------ 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py index dd5da70..76148f3 100755 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -83,10 +83,9 @@ def generate_adf_config(cesm_root, cupid_example, adf_file, out_file): # Set case names for ADF config a_dict["diag_cam_climo"]["cam_case_name"] = test_case_name - if base_case_name: - a_dict["diag_cam_baseline_climo"]["cam_case_name"] = base_case_name + a_dict["diag_cam_baseline_climo"]["cam_case_name"] = base_case_name - elif "cam_case_name" in "cam_case_name": + if "cam_case_name" in "cam_case_name": del a_dict["diag_cam_baseline_climo"]["cam_case_name"] # TEST CASE HISTORY FILE PATH @@ -122,31 +121,28 @@ def generate_adf_config(cesm_root, cupid_example, adf_file, out_file): a_dict["diag_cam_climo"]["end_year"] = end_date # Set values for BASELINE - if base_case_name is not None: - base_case_cupid_ts_index = ( - ts_case_names.index(base_case_name) - if base_case_name in ts_case_names - else None - ) + base_case_cupid_ts_index = ( + ts_case_names.index(base_case_name) if base_case_name in ts_case_names else None + ) - base_case_output_dir = c_dict["global_params"].get( - "base_case_output_dir", - DOUT + "/" + base_case_name, - ) - base_start_date = get_date_from_ts( - c_ts["atm"], - "start_years", - base_case_cupid_ts_index, - ) - base_end_date = get_date_from_ts( - c_ts["atm"], - "end_years", - base_case_cupid_ts_index, - ) - if base_start_date is None: - base_start_date = start_date - if base_end_date is None: - base_end_date = end_date + base_case_output_dir = c_dict["global_params"].get( + "base_case_output_dir", + DOUT + "/" + base_case_name, + ) + base_start_date = get_date_from_ts( + c_ts["atm"], + "start_years", + base_case_cupid_ts_index, + ) + base_end_date = get_date_from_ts( + c_ts["atm"], + "end_years", + base_case_cupid_ts_index, + ) + if base_start_date is None: + base_start_date = start_date + if base_end_date is None: + base_end_date = end_date a_dict["diag_cam_baseline_climo"]["cam_hist_loc"] = os.path.join( base_case_output_dir, From 750e005690298b97d52def175dc81755171df889 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Mon, 18 Nov 2024 10:30:52 -0700 Subject: [PATCH 22/22] remove if statement cam_case_name --- helper_scripts/generate_adf_config_file.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/helper_scripts/generate_adf_config_file.py b/helper_scripts/generate_adf_config_file.py index 76148f3..1a0a6cb 100755 --- a/helper_scripts/generate_adf_config_file.py +++ b/helper_scripts/generate_adf_config_file.py @@ -85,9 +85,6 @@ def generate_adf_config(cesm_root, cupid_example, adf_file, out_file): a_dict["diag_cam_climo"]["cam_case_name"] = test_case_name a_dict["diag_cam_baseline_climo"]["cam_case_name"] = base_case_name - if "cam_case_name" in "cam_case_name": - del a_dict["diag_cam_baseline_climo"]["cam_case_name"] - # TEST CASE HISTORY FILE PATH a_dict["diag_cam_climo"]["cam_hist_loc"] = os.path.join( DOUT,