Merge pull request NCAR#103 from jmills-ncar/master

Updates to job class to more easily handle cold start jobs

setup.py

@@ -2,14 +2,14 @@
 
 setup(
     name='wrfhydropy',
-    version='0.0.5',
+    version='0.0.6.dev1',
     packages=find_packages(),
     package_data={'wrfhydropy': ['core/data/*']},
     url='https://github.com/NCAR/wrf_hydro_py',
     license='MIT',
     install_requires=['pandas',
                       'f90nml',
-                      'netcdf4',
+                      'netCDF4',
                       'deepdiff',
                       'pathlib',
                       'xarray',
@@ -20,4 +20,4 @@
     author='Joe Mills',
     author_email='jmills@ucar.edu',
     description='Crude API for the WRF-Hydro model',
-)
+)

wrfhydropy/core/job.py

@@ -24,6 +24,7 @@ def __init__(
             job_id: str,
             model_start_time: Union[str,pd.datetime] = None,
             model_end_time: Union[str,pd.datetime] = None,
+            restart: bool = True,
             exe_cmd: str = None,
             entry_cmd: str = None,
             exit_cmd: str = None):
@@ -34,6 +35,7 @@ def __init__(
             a pandas.to_datetime compatible string or a pandas datetime object.
             model_end_time: The model end time to use for the WRF-Hydro model run. Can be
             a pandas.to_datetime compatible string or a pandas datetime object.
+            restart: Job is starting from a restart file. Use False for a cold start.
             exe_cmd: The system-specific command to execute WRF-Hydro, for example 'mpirun -np
             36 ./wrf_hydro.exe'. Can be left as None if jobs is added to a scheduler or if a
             scheduler is used in a simulation.
@@ -56,6 +58,9 @@ def __init__(
         self.job_id = job_id
         """str: The job id."""
 
+        self.restart = restart
+        """bool: Start model from a restart."""
+
         self._model_start_time = pd.to_datetime(model_start_time)
         """np.datetime64: The model time at the start of the execution."""
 
@@ -241,20 +246,21 @@ def _set_hrldas_times(self):
             self._hrldas_times['noahlsm_offline']['start_hour'] = int(self._model_start_time.hour)
             self._hrldas_times['noahlsm_offline']['start_min'] = int(self._model_start_time.minute)
 
-            lsm_restart_dirname = '.'  # os.path.dirname(noah_nlst['restart_filename_requested'])
+            if self.restart:
+                lsm_restart_dirname = '.'  # os.path.dirname(noah_nlst['restart_filename_requested'])
 
-            # Format - 2011082600 - no minutes
-            lsm_restart_basename = 'RESTART.' + \
-                                   self._model_start_time.strftime('%Y%m%d%H') + '_DOMAIN1'
+                # Format - 2011082600 - no minutes
+                lsm_restart_basename = 'RESTART.' + \
+                                       self._model_start_time.strftime('%Y%m%d%H') + '_DOMAIN1'
 
-            lsm_restart_file = lsm_restart_dirname + '/' + lsm_restart_basename
+                lsm_restart_file = lsm_restart_dirname + '/' + lsm_restart_basename
 
-            self._hrldas_times['noahlsm_offline']['restart_filename_requested'] = lsm_restart_file
+                self._hrldas_times['noahlsm_offline']['restart_filename_requested'] = lsm_restart_file
 
     def _set_hydro_times(self):
         """Private method to set model run times in the hydro namelist"""
 
-        if self._model_start_time is not None:
+        if self._model_start_time is not None and self.restart:
             # Format - 2011-08-26_00_00 - minutes
             hydro_restart_basename = 'HYDRO_RST.' + \
                                      self._model_start_time.strftime('%Y-%m-%d_%H:%M') + '_DOMAIN1'

wrfhydropy/core/model.py

@@ -10,26 +10,29 @@
 
 from .namelist import JSONNamelist
 
-def get_git_revision_hash(the_dir):
+def get_git_revision_hash(the_dir: str) -> str:
+    """Get the last git revision hash from a directory if directory is a git repository
+    Args:
+        the_dir: String for the directory path
+    Returns:
+         String with the git hash if a git repo or message if not
+    """
+
+    the_dir = pathlib.Path(the_dir)
 
     # First test if this is even a git repo. (Have to allow for this unless the wrfhydropy
     # testing brings in the wrf_hydro_code as a repo with a .git file.)
-    dir_is_repo = subprocess.call(
-        ["git", "branch"],
+    dir_is_repo = subprocess.run(["git", "branch"],
         stderr=subprocess.STDOUT,
         stdout=open(os.devnull, 'w'),
-        cwd=str(the_dir.absolute())
-    )
-    if dir_is_repo != 0:
-        warnings.warn('The source directory is NOT a git repo: ' + str(the_dir))
-        return 'not-a-repo'
-
-    dirty = subprocess.run(
-        ['git', 'diff-index', 'HEAD'],  # --quiet seems to give the wrong result.
+        cwd=str(the_dir.absolute()))
+    if dir_is_repo.returncode != 0:
+        return 'could_not_get_hash'
+
+    dirty = subprocess.run(['git', 'diff-index', 'HEAD'],  # --quiet seems to give the wrong result.
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
-        cwd=str(the_dir.absolute())
-    ).returncode
+        cwd=str(the_dir.absolute())).returncode
     the_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=str(the_dir.absolute()))
     the_hash = the_hash.decode('utf-8').split()[0]
     if dirty:
@@ -95,7 +98,9 @@ def __init__(
         self.compile_dir = None
         """pathlib.Path: pathlib.Path object pointing to the compile directory."""
 
-        self.git_hash = None
+        self.git_hash = self._get_githash()
+        """str: The git revision hash if seld.source_dir is a git repository"""
+
         self.version = None
         """str: Source code version from .version file stored with the source code."""
 

wrfhydropy/core/namelist.py

@@ -1,26 +1,8 @@
 import f90nml
 import json
 import copy
-
-def dict_merge(dct: dict, merge_dct: dict) -> dict:
-    """ Recursive dict merge. Inspired by :meth:``dict.update()``, instead of
-    updating only top-level keys, dict_merge recurses down into dicts nested
-    to an arbitrary depth, updating keys. The ``merge_dct`` is merged into
-    ``dct``.
-    Args:
-     dct: dict onto which the merge is executed
-     merge_dct: dct merged into dct
-    Returns:
-        The merged dict
-    """
-
-    for key, value in merge_dct.items():
-        if key in dct.keys() and type(value) is dict:
-            dict_merge(dct[key], merge_dct[key])
-        else:
-            dct[key] = merge_dct[key]
-
-    return(dct)
+import deepdiff
+from typing import Union
 
 class JSONNamelist(object):
     """Class for a WRF-Hydro JSON namelist containing one more configurations"""
@@ -67,4 +49,54 @@ def patch(self,patch: dict):
 
         patched_namelist = dict_merge(copy.deepcopy(self),copy.deepcopy(patch))
 
-        return patched_namelist
+        return patched_namelist
+
+def dict_merge(dct: dict, merge_dct: dict) -> dict:
+    """ Recursive dict merge. Inspired by :meth:``dict.update()``, instead of
+    updating only top-level keys, dict_merge recurses down into dicts nested
+    to an arbitrary depth, updating keys. The ``merge_dct`` is merged into
+    ``dct``.
+    Args:
+     dct: dict onto which the merge is executed
+     merge_dct: dct merged into dct
+    Returns:
+        The merged dict
+    """
+
+    for key, value in merge_dct.items():
+        if key in dct.keys() and type(value) is dict:
+            dict_merge(dct[key], merge_dct[key])
+        else:
+            dct[key] = merge_dct[key]
+
+    return(dct)
+
+def diff_namelist(old_namelist: Union[Namelist,str], new_namelist: Union[Namelist,str], **kwargs) \
+        -> \
+        dict:
+    """Diff two Namelist objects or fortran 90 namelist files and return a dictionary of
+    differences.
+
+    Args:
+        old_namelist: String containing path to the first namelist file, referred to as 'old' in
+        outputs.
+        new_namelist: String containing path to the second namelist file, referred to as 'new' in
+        outputs.
+        **kwargs: Additional arguments passed onto deepdiff.DeepDiff method
+    Returns:
+        The differences between the two namelists
+    """
+
+    # If supplied as strings try and read in from file path
+    if type(old_namelist) == str:
+        namelist1 = f90nml.read(old_namelist)
+        namelist1 = Namelist(json.loads(json.dumps(namelist1,sort_keys=True)))
+    if type(new_namelist) == str:
+        namelist1 = f90nml.read(new_namelist)
+        namelist1 = Namelist(json.loads(json.dumps(new_namelist,sort_keys=True)))
+
+
+    # Diff the namelists
+    differences = deepdiff.DeepDiff(old_namelist, new_namelist, ignore_order=True, **kwargs)
+    differences_dict = dict(differences)
+    return (differences_dict)

wrfhydropy/core/outputdiffs.py

@@ -8,27 +8,6 @@
 
 from .simulation import SimulationOutput
 
-def diff_namelist(namelist1: str, namelist2: str, **kwargs) -> dict:
-    """Diff two fortran namelist files and return a dictionary of differences.
-
-    Args:
-        old_namelist: String containing path to the first namelist file, referred to as 'old' in
-        outputs.
-        new_namelist: String containing path to the second namelist file, referred to as 'new' in
-        outputs.
-        **kwargs: Additional arguments passed onto deepdiff.DeepDiff method
-    Returns:
-        The differences between the two namelists
-    """
-
-    # Read namelists into dicts
-    namelist1 = f90nml.read(namelist1)
-    namelist2 = f90nml.read(namelist2)
-    # Diff the namelists
-    differences = deepdiff.DeepDiff(namelist1, namelist2, ignore_order=True, **kwargs)
-    differences_dict = dict(differences)
-    return (differences_dict)
-
 def compare_ncfiles(candidate_files: list,
                     reference_files: list,
                     nccmp_options: list = ['--data', '--metadata', '--force'],