From bb42c455bb8f39b592ad8ea2e6dd339e7f9d094a Mon Sep 17 00:00:00 2001 From: Mattia Almansi Date: Wed, 25 Sep 2024 14:16:53 +0200 Subject: [PATCH] implement legacy remote (#87) --- cads_api_client/legacy_api_client.py | 39 +++++++++++++------ cads_api_client/processing.py | 10 ++--- .../integration_test_70_legacy_api_client.py | 21 ++++++++-- 3 files changed, 51 insertions(+), 19 deletions(-) diff --git a/cads_api_client/legacy_api_client.py b/cads_api_client/legacy_api_client.py index c25d7c5..1eb9283 100644 --- a/cads_api_client/legacy_api_client.py +++ b/cads_api_client/legacy_api_client.py @@ -9,10 +9,13 @@ from typing import Any, Callable, TypeVar, cast, overload import cdsapi.api +import multiurl import requests from . import __version__ as cads_api_client_version -from . import api_client, processing +from . import processing +from .api_client import ApiClient +from .processing import Remote, Results LEGACY_KWARGS = [ "full_stack", @@ -101,7 +104,7 @@ def __init__( UserWarning, ) - self.client = self.logging_decorator(api_client.ApiClient)( + self.client = self.logging_decorator(ApiClient)( url=self.url, key=self.key, verify=self.verify, @@ -139,7 +142,7 @@ def logging_decorator(self, func: F) -> F: @functools.wraps(func) def wrapper(*args: Any, **kwargs: Any) -> Any: with LoggingContext( - logger=processing.logger, quiet=self.quiet, debug=self._debug + logger=processing.LOGGER, quiet=self.quiet, debug=self._debug ): return func(*args, **kwargs) @@ -151,12 +154,12 @@ def retrieve(self, name: str, request: dict[str, Any], target: str) -> str: ... @overload def retrieve( self, name: str, request: dict[str, Any], target: None = ... - ) -> processing.Results: ... + ) -> Results: ... def retrieve( self, name: str, request: dict[str, Any], target: str | None = None - ) -> str | processing.Remote | processing.Results: - submitted: processing.Remote | processing.Results + ) -> str | Remote | Results: + submitted: Remote | Results if self.wait_until_complete: submitted = self.logging_decorator(self.client.submit_and_wait_on_results)( collection_id=name, @@ -206,7 +209,7 @@ def status(self, context: Any = None) -> dict[str, list[str]]: @typing.no_type_check def _download(self, results, targets=None): - if isinstance(results, (processing.Results, processing.Remote)): + if hasattr(results, "download"): if targets: path = targets.pop(0) else: @@ -221,8 +224,22 @@ def _download(self, results, targets=None): return results - def remote(self, url): # type: ignore - self.raise_not_implemented_error() + @typing.no_type_check + def download(self, results, targets=None): + if targets: + # Make a copy + targets = [t for t in targets] + return self._download(results, targets) + + def remote(self, url: str) -> cdsapi.api.Result: + r = requests.head(url) + reply = dict( + location=url, + content_length=r.headers["Content-Length"], + content_type=r.headers["Content-Type"], + ) + return cdsapi.api.Result(self, reply) - def robust(self, call): # type: ignore - self.raise_not_implemented_error() + def robust(self, call: F) -> F: + robust: F = multiurl.robust(call, **self.client._retry_options) + return robust diff --git a/cads_api_client/processing.py b/cads_api_client/processing.py index 997e9bd..88d4e8e 100644 --- a/cads_api_client/processing.py +++ b/cads_api_client/processing.py @@ -23,7 +23,7 @@ T_ApiResponse = TypeVar("T_ApiResponse", bound="ApiResponse") -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) LEVEL_NAMES_MAPPING = { "CRITICAL": 50, @@ -138,11 +138,11 @@ def from_request( robust_request = multiurl.robust(session.request, **retry_options) inputs = kwargs.get("json", {}).get("inputs", {}) - logger.debug(f"{method.upper()} {url} {inputs or ''}".strip()) + LOGGER.debug(f"{method.upper()} {url} {inputs or ''}".strip()) response = robust_request( method, url, headers=headers, **request_options, **kwargs ) - logger.debug(f"REPLY {response.text}") + LOGGER.debug(f"REPLY {response.text}") cads_raise_for_status(response) @@ -223,7 +223,7 @@ def _from_rel_href(self, rel: str) -> Self | None: return out def log(self, *args: Any, **kwargs: Any) -> None: - logger.log(*args, **kwargs) + LOGGER.log(*args, **kwargs) def info(self, *args: Any, **kwargs: Any) -> None: self.log(logging.INFO, *args, **kwargs) @@ -548,7 +548,7 @@ def reply(self) -> dict[str, Any]: return reply def log(self, *args: Any, **kwargs: Any) -> None: - logger.log(*args, **kwargs) + LOGGER.log(*args, **kwargs) def info(self, *args: Any, **kwargs: Any) -> None: self.log(logging.INFO, *args, **kwargs) diff --git a/tests/integration_test_70_legacy_api_client.py b/tests/integration_test_70_legacy_api_client.py index c41ebb7..099ecd6 100644 --- a/tests/integration_test_70_legacy_api_client.py +++ b/tests/integration_test_70_legacy_api_client.py @@ -18,7 +18,7 @@ @pytest.fixture def legacy_client(api_root_url: str, api_anon_key: str) -> LegacyApiClient: - return LegacyApiClient(url=api_root_url, key=api_anon_key, retry_max=0) + return LegacyApiClient(url=api_root_url, key=api_anon_key, retry_max=1) def legacy_update(remote: processing.Remote) -> None: @@ -218,9 +218,11 @@ def test_legacy_api_client_logging( def test_legacy_api_client_download( tmp_path: pathlib.Path, + monkeypatch: pytest.MonkeyPatch, api_root_url: str, api_anon_key: str, ) -> None: + monkeypatch.chdir(tmp_path) client = LegacyApiClient( url=api_root_url, key=api_anon_key, @@ -229,10 +231,12 @@ def test_legacy_api_client_download( ) remote = client.retrieve("test-adaptor-dummy", {"size": 1}) assert isinstance(remote, processing.Remote) + target = client.download(remote) + assert os.path.getsize(target) == 1 results = (remote, remote.make_results()) - target1 = str(tmp_path / "remote.grib") - target2 = str(tmp_path / "results.grib") + target1 = "remote.grib" + target2 = "results.grib" assert client.download(results, [target1, target2]) == [target1, target2] assert os.path.getsize(target1) == os.path.getsize(target2) == 1 @@ -254,3 +258,14 @@ def test_legacy_api_client_status(legacy_client: LegacyApiClient) -> None: for value in status.values() for string in value ) + + +def test_legacy_api_client_remote( + legacy_client: LegacyApiClient, tmp_path: pathlib.Path +) -> None: + results = legacy_client.retrieve("test-adaptor-dummy", {"size": 1}) + remote = legacy_client.remote(results.location) + target = str(tmp_path / "dummy.grib") + actual_target = remote.download(target) + assert target == actual_target + assert os.path.getsize(target) == 1