Merge pull request #165 from noaa-ocs-modeling/depend/pyver

Update python version limitation

.github/workflows/quick_test.yml

@@ -44,7 +44,7 @@ jobs:
       - name: install Python
         uses: actions/setup-python@v2
         with:
-          python-version: '3.10'  # due to numba
+          python-version: '3.10'
       - name: load cached Python installation
         id: cache
         uses: actions/cache@v2

.github/workflows/tests.yml

@@ -44,37 +44,27 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest, macos-latest ]
-        python-version: [ '3.8', '3.9', '3.10' ]
+        python-version: [ '3.8', '3.9', '3.10', '3.11' ]
     steps:
       - name: clone repository
         uses: actions/checkout@v2
-      - name: install Python
-        uses: actions/setup-python@v2
+      - name: conda virtual environment
+        uses: mamba-org/setup-micromamba@v1
         with:
-          python-version: ${{ matrix.python-version }}
-      - name: load cached Python installation
-        id: cache
-        uses: actions/cache@v2
-        with:
-          path: ${{ env.pythonLocation }}
-          key: test-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}
-      - name: install non-python dependencies
-        if: ${{ matrix.os == 'ubuntu-latest' }}
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y libhdf5-dev
-          sudo apt-get install -y libnetcdf-dev
-          sudo apt-get install -y udunits-bin
-      - name: install non-python dependencies
-        if: ${{ matrix.os == 'macos-latest' }}
-        run: |
-          brew install hdf5
-          brew install netcdf
-          brew install udunits
+          init-shell: bash
+          environment-name: ci-env
+          create-args: >-
+            python=${{ matrix.python-version }}
+            libnetcdf
+            hdf5
+            udunits2
+        # TODO: cache?
       - name: install dependencies
         run: pip install ".[testing]"
+        shell: micromamba-shell {0}
       - name: run tests
         run: pytest --numprocesses auto
+        shell: micromamba-shell {0}
   test_with_coverage:
     needs: [ lint, test ]
     name: test with coverage

coupledmodeldriver/_depend.py

@@ -0,0 +1,43 @@
+import importlib.util
+import logging
+import sys
+
+
+_logger = logging.getLogger(__file__)
+# TODO: Forward proper types?
+
+
+def optional_import(name):
+    if name in sys.modules:
+        _logger.warning(f'{name!r} already in sys.modules')
+        return sys.modules[name]
+    elif (spec := importlib.util.find_spec(name)) is not None:
+        # If you chose to perform the actual import ...
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[name] = module
+        spec.loader.exec_module(module)
+        return module
+
+    _logger.warning(f"can't find the {name!r} module")
+    return None
+
+
+def can_import(name):
+    if name in sys.modules:
+        return True
+    elif (spec := importlib.util.find_spec(name)) is not None:
+        return True
+    else:
+        return False
+
+
+HAS_ADCIRCPY = can_import('adcircpy')
+
+
+def requires_adcircpy(func):
+    def wrapper(*args, **kwargs):
+        if not HAS_ADCIRCPY:
+            raise ImportError(f"{func.__name__} requires `adcircpy`, but it's not available!")
+        return func(*args, **kwargs)
+
+    return wrapper

coupledmodeldriver/client/__init__.py

@@ -1,3 +1,17 @@
 import logging
 
+from coupledmodeldriver._depend import optional_import
+
 logging.basicConfig(format='[%(asctime)s] %(name)-9s %(levelname)-8s: %(message)s')
+
+__all__ = [
+    'check_completion',
+    'generate_schism',
+    'initialize_schism',
+    'unqueued_runs',
+]
+
+if optional_import('adcircpy') is not None:
+    __all__.extend(
+        ['generate_adcirc', 'initialize_adcirc',]
+    )

coupledmodeldriver/client/check_completion.py

@@ -7,22 +7,20 @@
 
 from typepigeon import convert_value
 
-from coupledmodeldriver.configure import ModelJSON
-from coupledmodeldriver.generate.adcirc.base import ADCIRCJSON
-from coupledmodeldriver.generate.adcirc.check import (
-    check_adcirc_completion,
-    CompletionStatus,
-    is_adcirc_run_directory,
-)
-from coupledmodeldriver.generate.schism.base import SCHISMJSON
+from coupledmodeldriver.configure import Model
 from coupledmodeldriver.generate.schism.check import (
     check_schism_completion,
     CompletionStatus,
     is_schism_run_directory,
 )
 from coupledmodeldriver.utilities import ProcessPoolExecutorStackTraced
+from coupledmodeldriver._depend import optional_import
+
 
-MODELS = {model.name.lower(): model for model in ModelJSON.__subclasses__()}
+if (adcirc_check := optional_import('coupledmodeldriver.generate.adcirc.check')) is not None:
+    check_adcirc_completion = adcirc_check.check_adcirc_completion
+    CompletionStatus = adcirc_check.CompletionStatus
+    is_adcirc_run_directory = adcirc_check.is_adcirc_run_directory
 
 
 def parse_check_completion_arguments():
@@ -33,7 +31,9 @@ def parse_check_completion_arguments():
         default=Path.cwd(),
         help='directory containing model run configuration',
     )
-    argument_parser.add_argument('--model', help='model that is running, one of: `ADCIRC`')
+    argument_parser.add_argument(
+        '--model', help='model that is running, one of: `ADCIRC`, `SCHISM`'
+    )
     argument_parser.add_argument(
         '--verbose', action='store_true', help='list all errors and problems with runs'
     )
@@ -42,7 +42,7 @@ def parse_check_completion_arguments():
 
     model = arguments.model
     if model is not None:
-        model = MODELS[model.lower()]
+        model = Model[model.upper()]
 
     directory = convert_value(arguments.directory, [Path])
     if len(directory) == 1:
@@ -55,18 +55,18 @@ def parse_check_completion_arguments():
     }
 
 
-def is_model_directory(directory: PathLike, model: ModelJSON = None) -> bool:
+def is_model_directory(directory: PathLike, model: Model = None) -> bool:
     if directory is None:
         directory = Path.cwd()
     elif not isinstance(directory, Path):
         directory = Path(directory)
 
     if model is None:
-        model = ADCIRCJSON
+        model = Model.SCHISM
 
-    if model == ADCIRCJSON:
+    if model == Model.ADCIRC:
         is_model_directory = is_adcirc_run_directory(directory)
-    elif model == SCHISMJSON:
+    elif model == Model.SCHISM:
         is_model_directory = is_schism_run_directory(directory)
     else:
         raise NotImplementedError(f'model "{model}" not implemented')
@@ -75,33 +75,33 @@ def is_model_directory(directory: PathLike, model: ModelJSON = None) -> bool:
 
 
 def check_model_directory(
-    directory: PathLike, verbose: bool = False, model: ModelJSON = None
+    directory: PathLike, verbose: bool = False, model: Model = None
 ) -> Dict[str, Any]:
     if directory is None:
         directory = Path.cwd()
     elif not isinstance(directory, Path):
         directory = Path(directory)
 
     if model is None:
-        model = ADCIRCJSON
+        model = Model.SCHISM
 
-    if model == ADCIRCJSON:
+    if model == Model.ADCIRC:
         return check_adcirc_completion(directory, verbose=verbose)
-    elif model == SCHISMJSON:
+    elif model == Model.SCHISM:
         return check_schism_completion(directory, verbose=verbose)
     else:
         raise NotImplementedError(f'model "{model}" not implemented')
 
 
 def check_completion(
-    directory: PathLike = None, verbose: bool = False, model: ModelJSON = None
+    directory: PathLike = None, verbose: bool = False, model: Model = None
 ) -> Dict[str, Any]:
     """
     check the completion status of a running model
 
     :param directory: directory containing model run configuration
     :param verbose: list all errors and problems with runs
-    :param model: model that is running, one of: ``ADCIRC``
+    :param model: model that is running, one of: ``ADCIRC``, ``SCHISM``
     :return: JSON output of completion status
     """
 
@@ -123,8 +123,7 @@ def check_completion(
                 completion_status.update(subdirectory_completion_status)
     elif isinstance(directory, Path):
         if model is None:
-            # model = ADCIRCJSON
-            for model_type in MODELS.values():
+            for model_type in Model.values():
                 if not is_model_directory(directory, model=model_type):
                     continue
                 model = model_type

coupledmodeldriver/configure/base.py

@@ -1,3 +1,4 @@
+from __future__ import annotations
 from abc import ABC, abstractmethod
 from copy import copy
 from datetime import datetime, timedelta
@@ -7,7 +8,6 @@
 from pathlib import Path, PurePosixPath
 from typing import Any, Dict, List, Union
 
-from adcircpy.server import SlurmConfig as ADCIRCPySlurmConfig
 from nemspy import ModelingSystem
 from nemspy.model.base import ModelEntry
 from pyschism.server import SlurmConfig as PySCHISMSlurmConfig
@@ -16,6 +16,12 @@
 from coupledmodeldriver.platforms import Platform
 from coupledmodeldriver.script import SlurmEmailType
 from coupledmodeldriver.utilities import LOGGER
+from coupledmodeldriver._depend import requires_adcircpy, optional_import
+
+
+adcircpy = optional_import('adcircpy')
+if adcircpy is not None:
+    ADCIRCPySlurmConfig = adcircpy.server.SlurmConfig
 
 
 class NoRelPath(type(Path())):
@@ -308,6 +314,7 @@ def __init__(
             if self['email_address'] is not None:
                 self['email_type'] = SlurmEmailType.ALL
 
+    @requires_adcircpy
     def to_adcircpy(self) -> ADCIRCPySlurmConfig:
         return ADCIRCPySlurmConfig(
             account=self['account'],
@@ -328,6 +335,7 @@ def to_adcircpy(self) -> ADCIRCPySlurmConfig:
         )
 
     @classmethod
+    @requires_adcircpy
     def from_adcircpy(cls, slurm_config: ADCIRCPySlurmConfig):
         instance = cls(
             account=slurm_config._account,
@@ -361,7 +369,7 @@ def to_pyschism(self) -> PySCHISMSlurmConfig:
             mail_user=self['email_address'],
             log_filename=self['log_filename'],
             modules=self['modules'],
-            path_prefix=self['path_prefix'],
+            modulepath=self['path_prefix'],
             extra_commands=self['extra_commands'],
             launcher=self['launcher'],
             nodes=self['nodes'],

coupledmodeldriver/configure/configure.py

@@ -4,13 +4,20 @@
 from pathlib import Path
 from typing import Any, Collection, Dict, List, Mapping, Set, Union
 
+import coupledmodeldriver.configure.forcings.base
 from coupledmodeldriver.configure.base import ConfigurationJSON, ModelDriverJSON
 from coupledmodeldriver.configure.forcings.base import (
-    ADCIRCPY_FORCING_CLASSES,
     PYSCHISM_FORCING_CLASSES,
     ForcingJSON,
 )
 from coupledmodeldriver.utilities import LOGGER
+from coupledmodeldriver._depend import optional_import
+
+ADCIRCPY_FORCING_CLASSES = ()
+if optional_import('adcircpy'):
+    ADCIRCPY_FORCING_CLASSES = (
+        coupledmodeldriver.configure.forcings.base.ADCIRCPY_FORCING_CLASSES
+    )
 
 
 class RunConfiguration(ABC):