Merge pull request #480 from metno/v0.12.0

Pre-release of version 0.12.0

.coveragerc

@@ -1,18 +1,24 @@
 [run]
+source =
+  pyaerocom
 omit =
-  pyaerocom/conftest.py
-  pyaerocom/_conftest_helpers.py
-  pyaerocom/test/*
-  pyaerocom/io/test/*
-  pyaerocom/web/test/*
-  pyaerocom/plot/test/*
-  pyaerocom/scripts/test/*
-  pyaerocom/web/helpers_trends_iface.py
-  pyaerocom/web/trends_evaluation.py
-  pyaerocom/web/cli/main_trends_evaluation.py
+  # not used
+  pyaerocom/aeroval/utils.py
+  pyaerocom/io/read_aeronet_invv2.py
+  pyaerocom/io/read_aeronet_sdav2.py
+  pyaerocom/io/read_aeronet_sunv2.py
+  pyaerocom/io/read_gaw.py
+  pyaerocom/io/read_aeolus_l2a_data.py
+  pyaerocom/io/read_sentinel5p_data.py
+  pyaerocom/io/readsatellitel2base.py
+  pyaerocom/vert_coords.py
 
 [report]
+skip_covered = true
 exclude_lines =
+  pragma: no cover
+  NotImplementedError
+  abc.ABC
   if __name__=='__main__':
   if __name__=="__main__":
   if __name__ == "__main__":

.github/workflows/CI.yml

@@ -7,29 +7,58 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: [3.7, 3.8, 3.9]
+    steps:
+    - uses: actions/checkout@v2.3.4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Python info
+      run: |
+        which python
+        python --version
+    - name: Install system pacakges
+      run: sudo apt install libudunits2-dev libgeos-dev libproj-dev proj-data proj-bin
+    - name: Cache pip
+      uses: actions/cache@v2
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Install pyaerocom
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install wheel
+        python -m pip install .[test]
+    - name: Run pytest
+      run: pytest --cov=pyaerocom/
 
+  conda:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.8, 3.9]
     steps:
     - uses: actions/checkout@v2.3.4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: s-weigand/setup-conda@v1.0.5
       with:
         python-version: ${{ matrix.python-version }}
-    - uses: s-weigand/setup-conda@v1.0.5
     - name: Conda info
       run: |
         conda --version
+        conda info --envs
         which python
-    - name: Display current environments
-      run: conda info --envs
-    - name: Update environment
+        python --version
+    - name: Update test environment
       run: |
         conda env update -n base -f ./pyaerocom_env.yml
-    - name: Install pytest
-      run: |
-        conda install -c conda-forge --file ./test-requirements.txt
     - name: Install pyaerocom
-      run: python ./setup.py install
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install . --no-deps
     - name: Run pytest
       run: pytest --cov=pyaerocom/ --cov-report xml
     - name: Upload coverage to Codecov

.gitignore

@@ -85,6 +85,7 @@ celerybeat-schedule
 # virtualenv
 venv/
 ENV/
+.venv/
 
 # Spyder project settings
 .spyderproject

.readthedocs.yml

@@ -2,21 +2,27 @@
 # Read the Docs configuration file
 # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
 
-# Required
 version: 2
 
-# Build documentation in the docs/ directory with Sphinx
 sphinx:
    configuration: docs/conf.py
 
-# Optionally build your docs in additional formats such as PDF
 formats:
    - pdf
 
-conda:
-  environment: pyaerocom_env.yml
+build:
+  image: latest
+  apt_packages:
+    - libudunits2-dev
+    - libgeos-dev
+    - libproj-dev
+    - proj-data
+    - proj-bin
 
 python:
+  version: 3.8
   install:
-      - method: pip
-        path: .
+    - method: pip
+      path: .
+      extra_requirements:
+        - docs

README.rst

@@ -1,4 +1,4 @@
-|CI| |Docs|
+|CI| |Docs| |Coverage|
 
 =========
 pyaerocom
@@ -20,3 +20,7 @@ pyaerocom
 .. |Docs| image:: https://readthedocs.org/projects/pyaerocom/badge/?version=latest
 	 :target: https://pyaerocom.readthedocs.io/en/latest/?badge=latest
 	 :alt: Documentation Status
+
+.. |Coverage| image:: https://codecov.io/gh/metno/pyaerocom/branch/main-dev/graph/badge.svg?token=A0AdX8YciZ
+    :target: https://codecov.io/gh/metno/pyaerocom
+

VERSION.md

@@ -1 +1 @@
-0.11.1
+0.12.0rc1

changelog/v0120_release_summary.md

@@ -0,0 +1,52 @@
+# DRAFT pyaerocom version 0.12.0
+
+This release comes with many new features, major improvements and a more 
+stable API. Please see the individual points below for major changes.
+
+The notes below summarise the major updates since the release of version [0.
+10.0](https://github.com/metno/pyaerocom/releases/tag/v0.10.0). This 
+includes the changes that were done in the recent release of version 
+[0.11.0](https://github.com/metno/pyaerocom/releases/tag/v0.11.0)), which 
+is the last version that officially supports
+
+## Major updates
+
+### **NEW** AeroVal tools
+
+The web processing tools have seen major improvements and the backend has a 
+new API, which can be found in the **aeroval** sub-package. Compared to the 
+former AeroCom Evaluation tools (**web** sup-package in former releases), 
+the new tools are a lot more flexible and support, for instance:
+
+- Processing of time series of statistical parameters.
+- Option to calculate seasonal statistics.
+- Temporal and spatial correlation in heatmap display.
+- Combination of different observation datasets into "superobservations".
+- All this is packed into a new modular and intuitive API.
+
+**Follow AeroVal on twitter**: https://twitter.com/AeroVal_MetNo
+
+### **web** sub-package 
+
+- The code under **web** is deprecated and relevant code has been shipped 
+  and redesigned into the new **AeroVal** tools (**aeroval** subpackage).
+- Processing tools for old trends interface are deprecated, instead trends 
+  visualisation is included in the new **aeroval** interface and can be 
+  done via the new tools in the **aeroval** subpackage.
+
+### UNSORTED
+
+- EBAS evaluation of individual timestamps to resolve ts_type of NASA Ames 
+  file.
+- Processing of wet deposition and precipitation.
+
+## Unit conversions
+
+- Automatic detection and correction of implicit deposition units to 
+  explicit units, both for model and obs, centralised in 
+  [units_helpers.py](https://github.com/metno/pyaerocom/blob/master/pyaerocom/units_helpers.py) module. 
+  E.g. if precip is reported mm and if the associated sampling duration is 
+  known (e.g. daily), it is corrected to mm d-1. Same applies to wet 
+  deposition.
+
+

docs/api-aeroval.rst

@@ -0,0 +1,101 @@
+API of Aeroval tools
+====================
+
+Documentation of the pyaerocom AeroVal API, for high level web
+processing tools.
+
+
+Tools for AeroVal experiment setup
+----------------------------------
+
+High level analysis setup for AeroVal experiment
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. automodule:: pyaerocom.aeroval.setupclasses
+   :members:
+
+Specification of observation datasets
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. automodule:: pyaerocom.aeroval.obsentry
+   :members:
+
+Specification of model datasets
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. automodule:: pyaerocom.aeroval.modelentry
+   :members:
+
+Containers for model and observation setup
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Collection classes to specify a number of model entries and a
+number of observation entries for a given AeroVal experiment.
+
+.. automodule:: pyaerocom.aeroval.collections
+   :members:
+
+Processing tools
+----------------
+
+Experiment processing engine
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. automodule:: pyaerocom.aeroval.experiment_processor
+   :members:
+
+Model maps processing
+^^^^^^^^^^^^^^^^^^^^^
+
+.. automodule:: pyaerocom.aeroval.modelmaps_engine
+   :members:
+
+Model maps helper functions
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. automodule:: pyaerocom.aeroval.modelmaps_helpers
+   :members:
+
+Processing of super-observation entries
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Super observations refer to merged observation datasets to
+increase the number of stations.
+
+.. automodule:: pyaerocom.aeroval.superobs_engine
+   :members:
+
+Low-level base classes for processing engines
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. automodule:: pyaerocom.aeroval._processing_base
+   :members:
+
+Helpers for processing of auxiliary variables
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. automodule:: pyaerocom.aeroval.aux_io_helpers
+   :members:
+
+Conversion of co-located data to json output
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. automodule:: pyaerocom.aeroval.coldatatojson_engine
+   :members:
+
+Output management
+-----------------
+
+.. automodule:: pyaerocom.aeroval.experiment_output
+   :members:
+
+Global constants
+----------------
+
+.. automodule:: pyaerocom.aeroval.glob_defaults
+   :members:
+
+High-level utility functions
+----------------------------
+
+.. automodule:: pyaerocom.aeroval.utils
+   :members: