From 587ea86c517af48c92f51f94e4dc0fed14266e3e Mon Sep 17 00:00:00 2001 From: Goyo Date: Fri, 20 Dec 2024 16:28:22 +0100 Subject: [PATCH] Integration tests fix (#1561) * add support to new delete files endpoints * fix get parameter * fix tests * adapt files delete to the refactor * replace data with params * remove not used fixture * tests fixed * fix client integration * fix context manager in files when download * added a new test for provider end-points * fix black * migrated old tests * Update gateway/api/services/file_storage.py Co-authored-by: Goyo * remove additional line --------- Co-authored-by: David <9059044+Tansito@users.noreply.github.com> --- client/qiskit_serverless/core/files.py | 12 ++-- gateway/api/services/file_storage.py | 13 ++-- tests/docker/test_docker_experimental.py | 73 +++++++++++++++++---- tests/experimental/file_download.py | 4 +- tests/experimental/manage_data_directory.py | 18 ++--- 5 files changed, 84 insertions(+), 36 deletions(-) diff --git a/client/qiskit_serverless/core/files.py b/client/qiskit_serverless/core/files.py index 619c218f1..0a0d7a6a3 100644 --- a/client/qiskit_serverless/core/files.py +++ b/client/qiskit_serverless/core/files.py @@ -105,8 +105,8 @@ def download( file, download_location, function, - target_name, os.path.join(self._files_url, "download"), + target_name, ) def provider_download( @@ -126,8 +126,8 @@ def provider_download( file, download_location, function, - target_name, os.path.join(self._files_url, "provider", "download"), + target_name, ) def upload( @@ -138,7 +138,7 @@ def upload( with tracer.start_as_current_span("files.upload"): with open(file, "rb") as f: with requests.post( - os.path.join(self._files_url, "upload"), + os.path.join(self._files_url, "upload/"), files={"file": f}, params={"provider": provider, "function": function.title}, stream=True, @@ -158,7 +158,7 @@ def provider_upload( with tracer.start_as_current_span("files.upload"): with open(file, "rb") as f: with requests.post( - os.path.join(self._files_url, "upload"), + os.path.join(self._files_url, "upload/"), files={"file": f}, params={"provider": provider, "function": function.title}, stream=True, @@ -177,7 +177,7 @@ def list(self, function: QiskitFunction) -> List[str]: response_data = safe_json_request_as_dict( request=lambda: requests.get( self._files_url, - params={"title": function.title}, + params={"function": function.title}, headers={"Authorization": f"Bearer {self._token}"}, timeout=REQUESTS_TIMEOUT, ) @@ -194,7 +194,7 @@ def provider_list(self, function: QiskitFunction) -> List[str]: response_data = safe_json_request_as_dict( request=lambda: requests.get( os.path.join(self._files_url, "provider"), - params={"provider": function.provider, "title": function.title}, + params={"provider": function.provider, "function": function.title}, headers={"Authorization": f"Bearer {self._token}"}, timeout=REQUESTS_TIMEOUT, ) diff --git a/gateway/api/services/file_storage.py b/gateway/api/services/file_storage.py index 904eda48b..bdf6cdf73 100644 --- a/gateway/api/services/file_storage.py +++ b/gateway/api/services/file_storage.py @@ -151,13 +151,16 @@ def get_file(self, file_name: str) -> Optional[Tuple[FileWrapper, str, int]]: ) return None - with open(path_to_file, "rb") as file_object: - file_wrapper = FileWrapper(file_object) + # We can not use context manager here. Django close the file automatically: + # https://docs.djangoproject.com/en/5.1/ref/request-response/#fileresponse-objects + file_wrapper = FileWrapper( + open(path_to_file, "rb") # pylint: disable=consider-using-with + ) - file_type = mimetypes.guess_type(path_to_file)[0] - file_size = os.path.getsize(path_to_file) + file_type = mimetypes.guess_type(path_to_file)[0] + file_size = os.path.getsize(path_to_file) - return file_wrapper, file_type, file_size + return file_wrapper, file_type, file_size def upload_file(self, file: File) -> str: """ diff --git a/tests/docker/test_docker_experimental.py b/tests/docker/test_docker_experimental.py index 03f62f44c..60ba11d0f 100644 --- a/tests/docker/test_docker_experimental.py +++ b/tests/docker/test_docker_experimental.py @@ -23,14 +23,15 @@ class TestDockerExperimental: @mark.order(1) def test_file_producer(self, serverless_client: ServerlessClient): """Integration test for files.""" + functionTitle = "file-producer-for-consume" function = QiskitFunction( - title="file-producer-for-consume", + title=functionTitle, entrypoint="produce_files.py", working_dir=resources_path, ) serverless_client.upload(function) - file_producer_function = serverless_client.function("file-producer-for-consume") + file_producer_function = serverless_client.function(functionTitle) job = file_producer_function.run() @@ -39,7 +40,7 @@ def test_file_producer(self, serverless_client: ServerlessClient): assert job.status() == "DONE" assert isinstance(job.logs(), str) - assert len(serverless_client.files()) > 0 + assert len(serverless_client.files(functionTitle)) > 0 @mark.skip( reason="File producing and consuming is not working. Maybe write permissions for functions?" @@ -47,14 +48,15 @@ def test_file_producer(self, serverless_client: ServerlessClient): @mark.order(2) def test_file_consumer(self, serverless_client: ServerlessClient): """Integration test for files.""" + functionTitle = "file-consumer" function = QiskitFunction( - title="file-consumer", + title=functionTitle, entrypoint="consume_files.py", working_dir=resources_path, ) serverless_client.upload(function) - file_consumer_function = serverless_client.function("file-consumer") + file_consumer_function = serverless_client.function(functionTitle) job = file_consumer_function.run() assert job is not None @@ -62,7 +64,7 @@ def test_file_consumer(self, serverless_client: ServerlessClient): assert job.status() == "DONE" assert isinstance(job.logs(), str) - files = serverless_client.files() + files = serverless_client.files(functionTitle) assert files is not None @@ -70,18 +72,19 @@ def test_file_consumer(self, serverless_client: ServerlessClient): assert file_count > 0 - serverless_client.file_delete("uploaded_file.tar") + serverless_client.file_delete("uploaded_file.tar", functionTitle) - assert (file_count - len(serverless_client.files())) == 1 + assert (file_count - len(serverless_client.files(functionTitle))) == 1 @mark.order(1) def test_upload_download_delete(self, serverless_client: ServerlessClient): """Integration test for upload files.""" + function = serverless_client.function("hello-world") print("::: file_upload :::") - print(serverless_client.file_upload(filename_path)) + print(serverless_client.file_upload(filename_path, function)) - files = serverless_client.files() + files = serverless_client.files(function) print("::: files :::") print(files) @@ -92,19 +95,61 @@ def test_upload_download_delete(self, serverless_client: ServerlessClient): assert file_count == 1 print("::: file_download :::") - assert serverless_client.file_download(filename) is not None + assert serverless_client.file_download(filename, function) is not None - files = serverless_client.files() + files = serverless_client.files(function) print("::: files after download :::") print(files) assert file_count == len(files) print("::: file_delete :::") - print(serverless_client.file_delete(filename)) + print(serverless_client.file_delete(filename, function)) print("::: files after delete:::") - files = serverless_client.files() + files = serverless_client.files(function) + print(files) + + assert (file_count - len(files)) == 1 + + def test_provider_upload_download_delete(self, serverless_client: ServerlessClient): + """Integration test for upload files.""" + function = QiskitFunction( + title="provider-function", + provider="mockprovider", + image="test-local-provider-function:latest", + ) + serverless_client.upload(function) + + function = serverless_client.function("mockprovider/provider-function") + + print("::: file_upload :::") + print(serverless_client.file_upload(filename_path, function)) + + files = serverless_client.files(function) + print("::: files :::") + print(files) + + file_count = len(files) + print("::: file_count :::") + print(file_count) + + assert file_count == 1 + + print("::: file_download :::") + assert serverless_client.file_download(filename, function) is not None + + files = serverless_client.files(function) + print("::: files after download :::") + print(files) + + assert file_count == len(files) + + print("::: file_delete :::") + print(serverless_client.file_delete(filename, function)) + + print("::: files after delete:::") + files = serverless_client.files(function) print(files) assert (file_count - len(files)) == 1 diff --git a/tests/experimental/file_download.py b/tests/experimental/file_download.py index ee4da23f7..658740577 100644 --- a/tests/experimental/file_download.py +++ b/tests/experimental/file_download.py @@ -21,9 +21,9 @@ print(job.status()) print(job.logs()) -available_files = serverless.files() +available_files = serverless.files(function) print(available_files) if len(available_files) > 0: - serverless.file_download(available_files[0]) + serverless.file_download(available_files[0], function) print("Download complete") diff --git a/tests/experimental/manage_data_directory.py b/tests/experimental/manage_data_directory.py index 70455a923..9abbf563d 100644 --- a/tests/experimental/manage_data_directory.py +++ b/tests/experimental/manage_data_directory.py @@ -9,6 +9,11 @@ ) print(serverless) +function = QiskitFunction( + title="file-producer", entrypoint="produce_files.py", working_dir="./source_files/" +) +serverless.upload(function) + import tarfile filename = "uploaded_file.tar" @@ -16,12 +21,7 @@ file.add("manage_data_directory.py") file.close() -serverless.file_upload(filename) - -function = QiskitFunction( - title="file-producer", entrypoint="produce_files.py", working_dir="./source_files/" -) -serverless.upload(function) +serverless.file_upload(filename, function) functions = {f.title: f for f in serverless.list()} file_producer_function = functions.get("file-producer") @@ -33,7 +33,7 @@ print(job.logs()) -print(serverless.files()) +print(serverless.files(file_producer_function)) function = QiskitFunction( title="file-consumer", entrypoint="consume_files.py", working_dir="./source_files/" @@ -49,8 +49,8 @@ print(job.status()) print(job.logs()) -print(serverless.files()) +print(serverless.files(file_consumer_function)) -serverless.file_delete("uploaded_file.tar") +serverless.file_delete("uploaded_file.tar", file_consumer_function) print("Done deleting files")