From 575d91c4ef1c503031623ae332f2081d83eb51af Mon Sep 17 00:00:00 2001 From: jgliss Date: Wed, 17 Feb 2021 10:49:14 +0100 Subject: [PATCH 1/5] Attempt to fix 319 --- VERSION.md | 2 +- pyaerocom/colocation_auto.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/VERSION.md b/VERSION.md index 78bc1abd1..571215736 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -0.10.0 +0.10.1 diff --git a/pyaerocom/colocation_auto.py b/pyaerocom/colocation_auto.py index a61a999a6..42ab7b049 100644 --- a/pyaerocom/colocation_auto.py +++ b/pyaerocom/colocation_auto.py @@ -661,7 +661,8 @@ def _eval_obs_filters(self): 'Colocator class, which is not a ' 'dictionary: {}'.format(obs_filters)) for key, val in obs_filters.items(): - if key in self: # can be handled + # keep ts_type filter in remaining (added on 17.2.21, 0.100 -> 0.10.1) + if key in self and not key == 'ts_type': # can be handled if isinstance(self[key], dict) and isinstance(val, dict): self[key].update(val) else: From a66ac7a5f785577801f449b26baca8bb80b7fa81 Mon Sep 17 00:00:00 2001 From: jgliss Date: Tue, 23 Feb 2021 09:02:56 +0100 Subject: [PATCH 2/5] Update pyaerocom_env.yml --- pyaerocom/io/iris_io.py | 14 -------------- pyaerocom/time_config.py | 3 --- pyaerocom_env.yml | 11 +++++------ 3 files changed, 5 insertions(+), 23 deletions(-) diff --git a/pyaerocom/io/iris_io.py b/pyaerocom/io/iris_io.py index 6d137b002..8e82ecca9 100644 --- a/pyaerocom/io/iris_io.py +++ b/pyaerocom/io/iris_io.py @@ -100,20 +100,6 @@ def load_cubes_custom(files, var_name=None, file_convention=None, - list, list of filenames that were successfully loaded """ from pyaerocom import const -# ============================================================================= -# if len(files) >= 1000: -# try: -# cubes = _load_cubes_custom_multiproc(files, var_name, -# file_convention, -# perform_fmt_checks, -# **kwargs) -# # if this worked, all input files were successfully loaded and the -# # function terminates, else, the retrieval is done file by file. -# # ... Maybe! -# return (cubes, files) -# except Exception: -# pass -# ============================================================================= cubes = [] loaded_files = [] print_where = False diff --git a/pyaerocom/time_config.py b/pyaerocom/time_config.py index 93b8f4823..9b039c338 100644 --- a/pyaerocom/time_config.py +++ b/pyaerocom/time_config.py @@ -11,9 +11,6 @@ # The following import was removed and the information about available unit # strings was copied from the netCDF4 module directly here -# from netCDF4 import (microsec_units, millisec_units, sec_units, min_units, -# hr_units, day_units) -# from netCDF4._netCDF4 import _dateparse microsec_units = ['microseconds', 'microsecond', 'microsec', 'microsecs'] millisec_units = ['milliseconds', 'millisecond', 'millisec', 'millisecs'] sec_units = ['second', 'seconds', 'sec', 'secs', 's'] diff --git a/pyaerocom_env.yml b/pyaerocom_env.yml index 1955e3d60..1a5e64616 100644 --- a/pyaerocom_env.yml +++ b/pyaerocom_env.yml @@ -1,17 +1,16 @@ -# This only installs the requirment of pyaerocom, not pyaerocom itself. -# Activate the envoirnment and run python setup.py. +# This only installs the requirments of pyaerocom, not pyaerocom itself. +# Activate the environment and run "python setup.py install". channels: - conda-forge dependencies: - - iris >= 2.1.0 - - xarray >= 0.10.8 + - iris >= 2.1.0, <3.0.0 + - xarray >= 0.16.0 - cartopy >= 0.16.0 - matplotlib >= 3.0.1 - scipy >= 1.1.0 - - pandas >= 0.23.0 + - pandas >= 0.23.0, <=1.2.2 - seaborn >= 0.8.0 - - netcdf4 - geonum - numpy - simplejson From 476c36bd3bdbfddda69b2e2d545046270a772efe Mon Sep 17 00:00:00 2001 From: jgliss Date: Tue, 23 Feb 2021 09:39:10 +0100 Subject: [PATCH 3/5] Update docs/install.rst --- docs/install.rst | 17 +++++++---------- pyaerocom_env.yml | 2 -- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index e3812285f..3ccc80d41 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -1,23 +1,20 @@ Installation ============ -You have several options to install pyaerocom, the first one is the easiest, but may not refer to the most recent (non-released) version of pyaerocom. So please check first, which version you are interested in. +You can install pyaerocom via conda, pip or from source. Via conda ^^^^^^^^^ -**NOTE:** This will install the latest release of pyaerocom. - -- It hence, may not include the most recent developments. -- Requirements are installed automatically. - -If you use *conda* as a package manager, the easiest way to install pyaerocom (and all requirements, see previous section) is to use the build provided in the *nordicesmhub* conda channel:: +The easiest way to install pyaerocom and all requirements using conda. You can install pyaerocom into a pre-existing conda environment via:: conda install -c conda-forge pyaerocom -This will install the latest release of pyaerocom including all requirements. Alternatively, you may install from source as described in the following. +Or into a new conda environment (recommended) named *pya* via:: + + conda create -c conda-forge --name pya pyaerocom -**NOTE**: installation support via conda as described above is quite recent, so please let us know if you run into problems with the installation (best way to do this is by raising an issue `here `__). +This will install the latest release of pyaerocom including all requirements. Alternatively, you may install via pip or from source as described in the following. Via PyPi ^^^^^^^^ @@ -42,7 +39,7 @@ Alternatively, if you plan to apply local changes to the pyaerocom source code, python setup.py develop -You may also download and extract (or clone) the `GitHub repo `__ to install the very latest (not yet released) version of pyaerocom. +You may also download and extract (or clone) the `GitHub repo `__ to install the very latest (not yet released) version of pyaerocom. Note, if you install in develop mode, make sure you do not have pyaerocom installed already in the site packages directory, check e.g. `conda list pyaerocom`. Requirements diff --git a/pyaerocom_env.yml b/pyaerocom_env.yml index 1a5e64616..09ee55335 100644 --- a/pyaerocom_env.yml +++ b/pyaerocom_env.yml @@ -1,6 +1,5 @@ # This only installs the requirments of pyaerocom, not pyaerocom itself. # Activate the environment and run "python setup.py install". - channels: - conda-forge dependencies: @@ -17,4 +16,3 @@ dependencies: - requests - reverse-geocode - tqdm - From 14de3c132b563630fd2cebcc81ebedc6ff98d030 Mon Sep 17 00:00:00 2001 From: jgliss Date: Tue, 23 Feb 2021 09:52:17 +0100 Subject: [PATCH 4/5] Checkout test_geodesy.py from dev-cams84 --- pyaerocom/test/test_geodesy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyaerocom/test/test_geodesy.py b/pyaerocom/test/test_geodesy.py index 32f32e84f..4d83a9f22 100644 --- a/pyaerocom/test/test_geodesy.py +++ b/pyaerocom/test/test_geodesy.py @@ -31,6 +31,7 @@ def test_is_within_radius_km(): assert geodesy.is_within_radius_km(0, 15, 0, 16, 1000, 111.2) @geonum_unavail +@pytest.mark.skip(reason='https://github.com/tkrajina/srtm.py/issues/51') def test_srtm_altitude(): npt.assert_almost_equal(geodesy.get_topo_altitude(TEST_LAT, TEST_LON), 207) From 064936d4d3daec29cd5970cce2d3ba346d94fb31 Mon Sep 17 00:00:00 2001 From: jgliss Date: Tue, 23 Feb 2021 10:51:47 +0100 Subject: [PATCH 5/5] Quickfix of #322 (affected sites will be skipped in colocation, needs further looking into for 0.11.0) --- pyaerocom/colocation.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/pyaerocom/colocation.py b/pyaerocom/colocation.py index 9ee5251d2..c9772e7a4 100644 --- a/pyaerocom/colocation.py +++ b/pyaerocom/colocation.py @@ -821,9 +821,23 @@ def colocate_gridded_ungridded(gridded_data, ungridded_data, ts_type=None, min_num_obs=min_num_obs, use_climatology_ref=use_climatology_ref) - # assign the unified timeseries data to the colocated data array - coldata[0, :, i] = _df['ref'].values - coldata[1, :, i] = _df['data'].values + + # this try/except block was introduced on 23/2/2021 as temporary fix from + # v0.10.0 -> v0.10.1 as a result of multi-weekly obsdata (EBAS) that + # can end up resulting in incorrect number of timestamps after resampling + # (the error was discovered using EBASMC, concpm10, 2019 and colocation + # frequency monthly) + try: + # assign the unified timeseries data to the colocated data array + coldata[0, :, i] = _df['ref'].values + coldata[1, :, i] = _df['data'].values + except ValueError as e: + const.print_log.warning( + f'Failed to colocate time for station {obs_stat.station_name}. ' + f'This station will be skipped (error: {e})' + ) + + lons.append(obs_stat.longitude) lats.append(obs_stat.latitude)