From 79c728a474b3a28d414ce2d5555a9d579758b819 Mon Sep 17 00:00:00 2001 From: jmills-ncar Date: Wed, 1 Aug 2018 16:26:27 -0600 Subject: [PATCH 1/7] updated setup.py requires --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0f8b712a2..a89704221 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ license='MIT', install_requires=['pandas', 'f90nml', - 'netcdf4', + 'netCDF4', 'deepdiff', 'pathlib', 'xarray', From d0708f2fe20a44a6d8b8b278452b6414fdced9ad Mon Sep 17 00:00:00 2001 From: jmills-ncar Date: Wed, 1 Aug 2018 16:29:42 -0600 Subject: [PATCH 2/7] updated setup.py requires --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a89704221..6ca546a4f 100644 --- a/setup.py +++ b/setup.py @@ -20,4 +20,4 @@ author='Joe Mills', author_email='jmills@ucar.edu', description='Crude API for the WRF-Hydro model', -) +) \ No newline at end of file From deca731c352db5322a3fc8873143875b90281ced Mon Sep 17 00:00:00 2001 From: jmills-ncar Date: Thu, 2 Aug 2018 06:44:54 -0600 Subject: [PATCH 3/7] moved diff namelists to namelist module --- setup.py | 2 +- wrfhydropy/core/namelist.py | 28 ++++++++++++++++++++++++++++ wrfhydropy/core/outputdiffs.py | 21 --------------------- 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/setup.py b/setup.py index 6ca546a4f..cff714da7 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='wrfhydropy', - version='0.0.5', + version='0.0.6.dev0', packages=find_packages(), package_data={'wrfhydropy': ['core/data/*']}, url='https://github.com/NCAR/wrf_hydro_py', diff --git a/wrfhydropy/core/namelist.py b/wrfhydropy/core/namelist.py index de4960369..e92a136e7 100644 --- a/wrfhydropy/core/namelist.py +++ b/wrfhydropy/core/namelist.py @@ -1,6 +1,7 @@ import f90nml import json import copy +import deepdiff def dict_merge(dct: dict, merge_dct: dict) -> dict: """ Recursive dict merge. Inspired by :meth:``dict.update()``, instead of @@ -22,6 +23,33 @@ def dict_merge(dct: dict, merge_dct: dict) -> dict: return(dct) +def diff_namelist(namelist1: str, namelist2: str, **kwargs) -> dict: + """Diff two fortran namelist files and return a dictionary of differences. + + Args: + old_namelist: String containing path to the first namelist file, referred to as 'old' in + outputs. + new_namelist: String containing path to the second namelist file, referred to as 'new' in + outputs. + **kwargs: Additional arguments passed onto deepdiff.DeepDiff method + Returns: + The differences between the two namelists + """ + + # If supplied as strings try and read in from file path + if type(namelist1) == str: + namelist1 = f90nml.read(namelist1) + namelist1 = Namelist(json.loads(json.dumps(namelist1))) + if type(namelist2) == str: + namelist1 = f90nml.read(namelist2) + namelist1 = Namelist(json.loads(json.dumps(namelist2))) + + + # Diff the namelists + differences = deepdiff.DeepDiff(namelist1, namelist2, ignore_order=True, **kwargs) + differences_dict = dict(differences) + return (differences_dict) + class JSONNamelist(object): """Class for a WRF-Hydro JSON namelist containing one more configurations""" def __init__( diff --git a/wrfhydropy/core/outputdiffs.py b/wrfhydropy/core/outputdiffs.py index ed471ae48..b3cdd5641 100644 --- a/wrfhydropy/core/outputdiffs.py +++ b/wrfhydropy/core/outputdiffs.py @@ -8,27 +8,6 @@ from .simulation import SimulationOutput -def diff_namelist(namelist1: str, namelist2: str, **kwargs) -> dict: - """Diff two fortran namelist files and return a dictionary of differences. - - Args: - old_namelist: String containing path to the first namelist file, referred to as 'old' in - outputs. - new_namelist: String containing path to the second namelist file, referred to as 'new' in - outputs. - **kwargs: Additional arguments passed onto deepdiff.DeepDiff method - Returns: - The differences between the two namelists - """ - - # Read namelists into dicts - namelist1 = f90nml.read(namelist1) - namelist2 = f90nml.read(namelist2) - # Diff the namelists - differences = deepdiff.DeepDiff(namelist1, namelist2, ignore_order=True, **kwargs) - differences_dict = dict(differences) - return (differences_dict) - def compare_ncfiles(candidate_files: list, reference_files: list, nccmp_options: list = ['--data', '--metadata', '--force'], From a8aebf5cce24ea2702697b82bc1aaabb61b2430f Mon Sep 17 00:00:00 2001 From: jmills-ncar Date: Thu, 2 Aug 2018 08:50:01 -0600 Subject: [PATCH 4/7] updated diff_namelist to handle f90nml files or Namelist objects --- wrfhydropy/core/namelist.py | 100 +++++++++++++++++++----------------- 1 file changed, 52 insertions(+), 48 deletions(-) diff --git a/wrfhydropy/core/namelist.py b/wrfhydropy/core/namelist.py index e92a136e7..fad5c4074 100644 --- a/wrfhydropy/core/namelist.py +++ b/wrfhydropy/core/namelist.py @@ -2,53 +2,7 @@ import json import copy import deepdiff - -def dict_merge(dct: dict, merge_dct: dict) -> dict: - """ Recursive dict merge. Inspired by :meth:``dict.update()``, instead of - updating only top-level keys, dict_merge recurses down into dicts nested - to an arbitrary depth, updating keys. The ``merge_dct`` is merged into - ``dct``. - Args: - dct: dict onto which the merge is executed - merge_dct: dct merged into dct - Returns: - The merged dict - """ - - for key, value in merge_dct.items(): - if key in dct.keys() and type(value) is dict: - dict_merge(dct[key], merge_dct[key]) - else: - dct[key] = merge_dct[key] - - return(dct) - -def diff_namelist(namelist1: str, namelist2: str, **kwargs) -> dict: - """Diff two fortran namelist files and return a dictionary of differences. - - Args: - old_namelist: String containing path to the first namelist file, referred to as 'old' in - outputs. - new_namelist: String containing path to the second namelist file, referred to as 'new' in - outputs. - **kwargs: Additional arguments passed onto deepdiff.DeepDiff method - Returns: - The differences between the two namelists - """ - - # If supplied as strings try and read in from file path - if type(namelist1) == str: - namelist1 = f90nml.read(namelist1) - namelist1 = Namelist(json.loads(json.dumps(namelist1))) - if type(namelist2) == str: - namelist1 = f90nml.read(namelist2) - namelist1 = Namelist(json.loads(json.dumps(namelist2))) - - - # Diff the namelists - differences = deepdiff.DeepDiff(namelist1, namelist2, ignore_order=True, **kwargs) - differences_dict = dict(differences) - return (differences_dict) +from typing import Union class JSONNamelist(object): """Class for a WRF-Hydro JSON namelist containing one more configurations""" @@ -95,4 +49,54 @@ def patch(self,patch: dict): patched_namelist = dict_merge(copy.deepcopy(self),copy.deepcopy(patch)) - return patched_namelist \ No newline at end of file + return patched_namelist + +def dict_merge(dct: dict, merge_dct: dict) -> dict: + """ Recursive dict merge. Inspired by :meth:``dict.update()``, instead of + updating only top-level keys, dict_merge recurses down into dicts nested + to an arbitrary depth, updating keys. The ``merge_dct`` is merged into + ``dct``. + Args: + dct: dict onto which the merge is executed + merge_dct: dct merged into dct + Returns: + The merged dict + """ + + for key, value in merge_dct.items(): + if key in dct.keys() and type(value) is dict: + dict_merge(dct[key], merge_dct[key]) + else: + dct[key] = merge_dct[key] + + return(dct) + +def diff_namelist(old_namelist: Union[Namelist,str], new_namelist: Union[Namelist,str], **kwargs) \ + -> \ + dict: + """Diff two Namelist objects or fortran 90 namelist files and return a dictionary of + differences. + + Args: + old_namelist: String containing path to the first namelist file, referred to as 'old' in + outputs. + new_namelist: String containing path to the second namelist file, referred to as 'new' in + outputs. + **kwargs: Additional arguments passed onto deepdiff.DeepDiff method + Returns: + The differences between the two namelists + """ + + # If supplied as strings try and read in from file path + if type(old_namelist) == str: + namelist1 = f90nml.read(old_namelist) + namelist1 = Namelist(json.loads(json.dumps(namelist1,sort_keys=True))) + if type(new_namelist) == str: + namelist1 = f90nml.read(new_namelist) + namelist1 = Namelist(json.loads(json.dumps(new_namelist,sort_keys=True))) + + + # Diff the namelists + differences = deepdiff.DeepDiff(old_namelist, new_namelist, ignore_order=True, **kwargs) + differences_dict = dict(differences) + return (differences_dict) From 6dd009e4316bef35d3c628d24f03942236c417b4 Mon Sep 17 00:00:00 2001 From: jmills-ncar Date: Fri, 3 Aug 2018 16:06:46 -0600 Subject: [PATCH 5/7] Added git hash to model object --- wrfhydropy/core/model.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/wrfhydropy/core/model.py b/wrfhydropy/core/model.py index 045d36520..13a0e4500 100644 --- a/wrfhydropy/core/model.py +++ b/wrfhydropy/core/model.py @@ -10,26 +10,29 @@ from .namelist import JSONNamelist -def get_git_revision_hash(the_dir): +def get_git_revision_hash(the_dir: str) -> str: + """Get the last git revision hash from a directory if directory is a git repository + Args: + the_dir: String for the directory path + Returns: + String with the git hash if a git repo or message if not + """ + + the_dir = pathlib.Path(the_dir) # First test if this is even a git repo. (Have to allow for this unless the wrfhydropy # testing brings in the wrf_hydro_code as a repo with a .git file.) - dir_is_repo = subprocess.call( - ["git", "branch"], + dir_is_repo = subprocess.run(["git", "branch"], stderr=subprocess.STDOUT, stdout=open(os.devnull, 'w'), - cwd=str(the_dir.absolute()) - ) - if dir_is_repo != 0: - warnings.warn('The source directory is NOT a git repo: ' + str(the_dir)) - return 'not-a-repo' - - dirty = subprocess.run( - ['git', 'diff-index', 'HEAD'], # --quiet seems to give the wrong result. + cwd=str(the_dir.absolute())) + if dir_is_repo.returncode != 0: + return 'could_not_get_hash' + + dirty = subprocess.run(['git', 'diff-index', 'HEAD'], # --quiet seems to give the wrong result. stdout=subprocess.PIPE, stderr=subprocess.PIPE, - cwd=str(the_dir.absolute()) - ).returncode + cwd=str(the_dir.absolute())).returncode the_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=str(the_dir.absolute())) the_hash = the_hash.decode('utf-8').split()[0] if dirty: @@ -95,7 +98,9 @@ def __init__( self.compile_dir = None """pathlib.Path: pathlib.Path object pointing to the compile directory.""" - self.git_hash = None + self.git_hash = self._get_githash() + """str: The git revision hash if seld.source_dir is a git repository""" + self.version = None """str: Source code version from .version file stored with the source code.""" From 2703558f653e1927886bff28551cca4cd74d671d Mon Sep 17 00:00:00 2001 From: jmills-ncar Date: Mon, 6 Aug 2018 08:34:25 -0600 Subject: [PATCH 6/7] Added restarrt option to job --- wrfhydropy/core/job.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/wrfhydropy/core/job.py b/wrfhydropy/core/job.py index ad5f9aa4e..3532dc33a 100644 --- a/wrfhydropy/core/job.py +++ b/wrfhydropy/core/job.py @@ -24,6 +24,7 @@ def __init__( job_id: str, model_start_time: Union[str,pd.datetime] = None, model_end_time: Union[str,pd.datetime] = None, + restart: bool = True, exe_cmd: str = None, entry_cmd: str = None, exit_cmd: str = None): @@ -34,6 +35,7 @@ def __init__( a pandas.to_datetime compatible string or a pandas datetime object. model_end_time: The model end time to use for the WRF-Hydro model run. Can be a pandas.to_datetime compatible string or a pandas datetime object. + restart: Job is starting from a restart file. Use False for a cold start. exe_cmd: The system-specific command to execute WRF-Hydro, for example 'mpirun -np 36 ./wrf_hydro.exe'. Can be left as None if jobs is added to a scheduler or if a scheduler is used in a simulation. @@ -56,6 +58,9 @@ def __init__( self.job_id = job_id """str: The job id.""" + self.restart = restart + """bool: Start model from a restart.""" + self._model_start_time = pd.to_datetime(model_start_time) """np.datetime64: The model time at the start of the execution.""" @@ -241,20 +246,21 @@ def _set_hrldas_times(self): self._hrldas_times['noahlsm_offline']['start_hour'] = int(self._model_start_time.hour) self._hrldas_times['noahlsm_offline']['start_min'] = int(self._model_start_time.minute) - lsm_restart_dirname = '.' # os.path.dirname(noah_nlst['restart_filename_requested']) + if self.restart: + lsm_restart_dirname = '.' # os.path.dirname(noah_nlst['restart_filename_requested']) - # Format - 2011082600 - no minutes - lsm_restart_basename = 'RESTART.' + \ - self._model_start_time.strftime('%Y%m%d%H') + '_DOMAIN1' + # Format - 2011082600 - no minutes + lsm_restart_basename = 'RESTART.' + \ + self._model_start_time.strftime('%Y%m%d%H') + '_DOMAIN1' - lsm_restart_file = lsm_restart_dirname + '/' + lsm_restart_basename + lsm_restart_file = lsm_restart_dirname + '/' + lsm_restart_basename - self._hrldas_times['noahlsm_offline']['restart_filename_requested'] = lsm_restart_file + self._hrldas_times['noahlsm_offline']['restart_filename_requested'] = lsm_restart_file def _set_hydro_times(self): """Private method to set model run times in the hydro namelist""" - if self._model_start_time is not None: + if self._model_start_time is not None and self.restart: # Format - 2011-08-26_00_00 - minutes hydro_restart_basename = 'HYDRO_RST.' + \ self._model_start_time.strftime('%Y-%m-%d_%H:%M') + '_DOMAIN1' From e359b9127f79179a686e7bfb3aeec46b030727bd Mon Sep 17 00:00:00 2001 From: jmills-ncar Date: Mon, 6 Aug 2018 08:39:46 -0600 Subject: [PATCH 7/7] incremented dev version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cff714da7..261684a4c 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='wrfhydropy', - version='0.0.6.dev0', + version='0.0.6.dev1', packages=find_packages(), package_data={'wrfhydropy': ['core/data/*']}, url='https://github.com/NCAR/wrf_hydro_py',