From 4ce4e78a299244ace2e19f24ca9c70c9ce35b69e Mon Sep 17 00:00:00 2001 From: Mattia Almansi Date: Tue, 5 Mar 2024 15:06:10 +0100 Subject: [PATCH 1/6] implement legacy retry options --- cads_api_client/legacy_api_client.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/cads_api_client/legacy_api_client.py b/cads_api_client/legacy_api_client.py index 67536fb..b812819 100644 --- a/cads_api_client/legacy_api_client.py +++ b/cads_api_client/legacy_api_client.py @@ -1,5 +1,6 @@ from __future__ import annotations +import functools import warnings from typing import Any, overload @@ -49,6 +50,11 @@ def __init__( url=self.url, key=self.key, session=self.session ) + self.retry_options = { + "maximum_tries": kwargs.pop("retry_max", 500), + "retry_after": kwargs.pop("sleep_max", 120), + } + if kwargs: warnings.warn( "This is a beta version." @@ -75,8 +81,18 @@ def retrieve( ) -> str | processing.Remote: if target is None: collection = self.client.collection(name) - return collection.submit(**request) - return self.client.retrieve(name, target, **request) + remote = collection.submit(**request) + remote.download = functools.partial( # type: ignore[method-assign] + remote.download, + retry_options=self.retry_options, + ) + return remote + return self.client.retrieve( + collection_id=name, + target=target, + retry_options=self.retry_options, + **request, + ) def service(self, name, *args, **kwargs): # type: ignore self.raise_not_implemented_error() From ec0f8552371aff8465680f02d9e6f37fd741b8f7 Mon Sep 17 00:00:00 2001 From: Mattia Almansi Date: Wed, 6 Mar 2024 13:51:23 +0100 Subject: [PATCH 2/6] implement sleep max --- cads_api_client/legacy_api_client.py | 38 +++++++++++++--------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/cads_api_client/legacy_api_client.py b/cads_api_client/legacy_api_client.py index b812819..02845c7 100644 --- a/cads_api_client/legacy_api_client.py +++ b/cads_api_client/legacy_api_client.py @@ -46,15 +46,12 @@ def __init__( self.url, self.key, _ = cdsapi.api.get_url_key_verify(url, key, None) self.session = kwargs.pop("session", requests.Session()) + self.retry_max = kwargs.pop("retry_max", 500) + self.sleep_max = kwargs.pop("sleep_max", 120) self.client = api_client.ApiClient( url=self.url, key=self.key, session=self.session ) - self.retry_options = { - "maximum_tries": kwargs.pop("retry_max", 500), - "retry_after": kwargs.pop("sleep_max", 120), - } - if kwargs: warnings.warn( "This is a beta version." @@ -74,25 +71,26 @@ def retrieve(self, name: str, request: dict[str, Any], target: str) -> str: ... @overload def retrieve( self, name: str, request: dict[str, Any], target: None = ... - ) -> processing.Remote: ... + ) -> processing.Results: ... def retrieve( self, name: str, request: dict[str, Any], target: str | None = None - ) -> str | processing.Remote: - if target is None: - collection = self.client.collection(name) - remote = collection.submit(**request) - remote.download = functools.partial( # type: ignore[method-assign] - remote.download, - retry_options=self.retry_options, - ) - return remote - return self.client.retrieve( - collection_id=name, - target=target, - retry_options=self.retry_options, - **request, + ) -> str | processing.Results: + retry_options = { + "maximum_tries": self.retry_max, + "retry_after": self.sleep_max, + } + result = self.client.submit_and_wait_on_result( + collection_id=name, retry_options=retry_options, **request + ) + if target is not None: + return result.download(target, retry_options=retry_options) + + result.download = functools.partial( # type: ignore[method-assign] + result.download, + retry_options=retry_options, ) + return result def service(self, name, *args, **kwargs): # type: ignore self.raise_not_implemented_error() From 948f40612cba0b72767905e3baf78e58ffc1b6a7 Mon Sep 17 00:00:00 2001 From: Mattia Almansi Date: Wed, 6 Mar 2024 14:17:04 +0100 Subject: [PATCH 3/6] add integration tests --- ci/environment-ci.yml | 2 ++ .../integration_test_40_legacy_api_client.py | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 tests/integration_test_40_legacy_api_client.py diff --git a/ci/environment-ci.yml b/ci/environment-ci.yml index c2ebae1..04dee41 100644 --- a/ci/environment-ci.yml +++ b/ci/environment-ci.yml @@ -17,3 +17,5 @@ dependencies: - types-requests - pip: - responses +# FIXME: install from B-Open fork until the PR gets merged + - git+https://github.com/bopen/cdsapi.git@COPDS-1531-beta-api diff --git a/tests/integration_test_40_legacy_api_client.py b/tests/integration_test_40_legacy_api_client.py new file mode 100644 index 0000000..ae81ddc --- /dev/null +++ b/tests/integration_test_40_legacy_api_client.py @@ -0,0 +1,21 @@ +import pathlib + +from cads_api_client import legacy_api_client + + +def test_retrieve(tmp_path: pathlib.Path, api_root_url: str, api_key: str) -> None: + client = legacy_api_client.LegacyApiClient(url=api_root_url, key=api_key) + + collection_id = "test-adaptor-dummy" + request = {"size": 1} + + target = tmp_path / "test-retrieve-with-target.grib" + actual_target = client.retrieve(collection_id, request, str(target)) + assert str(target) == actual_target + assert target.stat().st_size == 1 + + result = client.retrieve(collection_id, request) + target = tmp_path / "test-retrieve-no-target.grib" + actual_target = result.download(str(target)) + assert str(target) == actual_target + assert target.stat().st_size == 1 From ebe1007156b8d43f06174670b1e5726795c66191 Mon Sep 17 00:00:00 2001 From: Mattia Almansi Date: Wed, 6 Mar 2024 14:44:38 +0100 Subject: [PATCH 4/6] typing --- cads_api_client/legacy_api_client.py | 29 ++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/cads_api_client/legacy_api_client.py b/cads_api_client/legacy_api_client.py index 02845c7..2d51be8 100644 --- a/cads_api_client/legacy_api_client.py +++ b/cads_api_client/legacy_api_client.py @@ -2,7 +2,7 @@ import functools import warnings -from typing import Any, overload +from typing import Any, Callable, overload import cdsapi.api import requests @@ -46,12 +46,18 @@ def __init__( self.url, self.key, _ = cdsapi.api.get_url_key_verify(url, key, None) self.session = kwargs.pop("session", requests.Session()) + self.timeout = kwargs.pop("timeout", 60) self.retry_max = kwargs.pop("retry_max", 500) self.sleep_max = kwargs.pop("sleep_max", 120) self.client = api_client.ApiClient( url=self.url, key=self.key, session=self.session ) + self.retry_options = { + "maximum_tries": self.retry_max, + "retry_after": self.sleep_max, + } + if kwargs: warnings.warn( "This is a beta version." @@ -76,21 +82,20 @@ def retrieve( def retrieve( self, name: str, request: dict[str, Any], target: str | None = None ) -> str | processing.Results: - retry_options = { - "maximum_tries": self.retry_max, - "retry_after": self.sleep_max, - } result = self.client.submit_and_wait_on_result( - collection_id=name, retry_options=retry_options, **request + collection_id=name, + retry_options=self.retry_options, + **request, ) - if target is not None: - return result.download(target, retry_options=retry_options) - - result.download = functools.partial( # type: ignore[method-assign] + partial_download: Callable[..., str] = functools.partial( result.download, - retry_options=retry_options, + timeout=self.timeout, + retry_options=self.retry_options, ) - return result + result.download = partial_download # type: ignore[method-assign] + if target is None: + return result + return result.download(target) def service(self, name, *args, **kwargs): # type: ignore self.raise_not_implemented_error() From 3449b377b27997fd6a85e2c2c023dc6e4d3a6054 Mon Sep 17 00:00:00 2001 From: Mattia Almansi Date: Wed, 6 Mar 2024 14:49:34 +0100 Subject: [PATCH 5/6] add sleep_max --- cads_api_client/legacy_api_client.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cads_api_client/legacy_api_client.py b/cads_api_client/legacy_api_client.py index 2d51be8..320c7a9 100644 --- a/cads_api_client/legacy_api_client.py +++ b/cads_api_client/legacy_api_client.py @@ -46,13 +46,16 @@ def __init__( self.url, self.key, _ = cdsapi.api.get_url_key_verify(url, key, None) self.session = kwargs.pop("session", requests.Session()) - self.timeout = kwargs.pop("timeout", 60) - self.retry_max = kwargs.pop("retry_max", 500) self.sleep_max = kwargs.pop("sleep_max", 120) self.client = api_client.ApiClient( - url=self.url, key=self.key, session=self.session + url=self.url, + key=self.key, + session=self.session, + sleep_max=self.sleep_max, ) + self.timeout = kwargs.pop("timeout", 60) + self.retry_max = kwargs.pop("retry_max", 500) self.retry_options = { "maximum_tries": self.retry_max, "retry_after": self.sleep_max, From 4c7e3908c4526fb264dbebeabdfd5c926d28a89f Mon Sep 17 00:00:00 2001 From: Mattia Almansi Date: Wed, 6 Mar 2024 14:56:04 +0100 Subject: [PATCH 6/6] cleanup --- cads_api_client/legacy_api_client.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cads_api_client/legacy_api_client.py b/cads_api_client/legacy_api_client.py index 320c7a9..0a16064 100644 --- a/cads_api_client/legacy_api_client.py +++ b/cads_api_client/legacy_api_client.py @@ -96,9 +96,7 @@ def retrieve( retry_options=self.retry_options, ) result.download = partial_download # type: ignore[method-assign] - if target is None: - return result - return result.download(target) + return result if target is None else result.download(target) def service(self, name, *args, **kwargs): # type: ignore self.raise_not_implemented_error()