diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 79879f3..419c645 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -65,7 +65,7 @@ jobs: with: python-version: '3.x' - - name: Check that tag version matches climetlab version + - name: Check that tag version matches code version run: | tag=${GITHUB_REF#refs/tags/} version=$(python setup.py --version) diff --git a/.gitignore b/.gitignore index 4e366fd..02b3b82 100644 --- a/.gitignore +++ b/.gitignore @@ -137,3 +137,4 @@ data *.grib *.grib2 ? +*.download diff --git a/ecmwf/opendata/__init__.py b/ecmwf/opendata/__init__.py index 7cc3f4f..bb69af1 100644 --- a/ecmwf/opendata/__init__.py +++ b/ecmwf/opendata/__init__.py @@ -12,4 +12,4 @@ __version__ = "0.0.1" -__all__ = ["download", "Downloader"] +__all__ = ["Client"] diff --git a/ecmwf/opendata/client.py b/ecmwf/opendata/client.py index f99e8b9..7da8e9c 100644 --- a/ecmwf/opendata/client.py +++ b/ecmwf/opendata/client.py @@ -7,14 +7,15 @@ # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. # -import datetime + import itertools +import json import logging import os import re -import json + import requests -from multiurl import download +from multiurl import download, robust from .date import fulldate @@ -101,19 +102,19 @@ def get_parts(self, data_urls, for_index): base, _ = os.path.splitext(url) index_url = f"{base}.index" - r = requests.get(index_url) + r = robust(requests.get)(index_url) r.raise_for_status() - parts = [] for line in r.iter_lines(): line = json.loads(line) + # LOG.debug(line) matches = 0 for name, values in for_index.items(): - if line[name] in values: - matches +=1 + if line.get(name) in values: + matches += 1 if matches == count: - parts.append((line['_offset'], line['_length'])) + parts.append((line["_offset"], line["_length"])) if parts: result.append((url, parts)) diff --git a/setup.py b/setup.py index f4b47df..d5a50b0 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ def read(fname): url="https://github.com/ecmwf/ecmwf-opendata", packages=setuptools.find_namespace_packages(include=["ecmwf.*"]), include_package_data=True, - install_requires=["multiurl"], + install_requires=["multiurl>=0.0.6"], zip_safe=True, keywords="tool", classifiers=[