Integration tests fix (#1561)

* add support to new delete files endpoints

* fix get parameter

* fix tests

* adapt files delete to the refactor

* replace data with params

* remove not used fixture

* tests fixed

* fix client integration

* fix context manager in files when download

* added a new test for provider end-points

* fix black

* migrated old tests

* Update gateway/api/services/file_storage.py

Co-authored-by: Goyo <korgan00@gmail.com>

* remove additional line

---------

Co-authored-by: David <9059044+Tansito@users.noreply.github.com>

client/qiskit_serverless/core/files.py

@@ -105,8 +105,8 @@ def download(
                 file,
                 download_location,
                 function,
-                target_name,
                 os.path.join(self._files_url, "download"),
+                target_name,
             )
 
     def provider_download(
@@ -126,8 +126,8 @@ def provider_download(
                 file,
                 download_location,
                 function,
-                target_name,
                 os.path.join(self._files_url, "provider", "download"),
+                target_name,
             )
 
     def upload(
@@ -138,7 +138,7 @@ def upload(
         with tracer.start_as_current_span("files.upload"):
             with open(file, "rb") as f:
                 with requests.post(
-                    os.path.join(self._files_url, "upload"),
+                    os.path.join(self._files_url, "upload/"),
                     files={"file": f},
                     params={"provider": provider, "function": function.title},
                     stream=True,
@@ -158,7 +158,7 @@ def provider_upload(
         with tracer.start_as_current_span("files.upload"):
             with open(file, "rb") as f:
                 with requests.post(
-                    os.path.join(self._files_url, "upload"),
+                    os.path.join(self._files_url, "upload/"),
                     files={"file": f},
                     params={"provider": provider, "function": function.title},
                     stream=True,
@@ -177,7 +177,7 @@ def list(self, function: QiskitFunction) -> List[str]:
             response_data = safe_json_request_as_dict(
                 request=lambda: requests.get(
                     self._files_url,
-                    params={"title": function.title},
+                    params={"function": function.title},
                     headers={"Authorization": f"Bearer {self._token}"},
                     timeout=REQUESTS_TIMEOUT,
                 )
@@ -194,7 +194,7 @@ def provider_list(self, function: QiskitFunction) -> List[str]:
             response_data = safe_json_request_as_dict(
                 request=lambda: requests.get(
                     os.path.join(self._files_url, "provider"),
-                    params={"provider": function.provider, "title": function.title},
+                    params={"provider": function.provider, "function": function.title},
                     headers={"Authorization": f"Bearer {self._token}"},
                     timeout=REQUESTS_TIMEOUT,
                 )

gateway/api/services/file_storage.py

@@ -151,13 +151,16 @@ def get_file(self, file_name: str) -> Optional[Tuple[FileWrapper, str, int]]:
             )
             return None
 
-        with open(path_to_file, "rb") as file_object:
-            file_wrapper = FileWrapper(file_object)
+        # We can not use context manager here. Django close the file automatically:
+        # https://docs.djangoproject.com/en/5.1/ref/request-response/#fileresponse-objects
+        file_wrapper = FileWrapper(
+            open(path_to_file, "rb")  # pylint: disable=consider-using-with
+        )
 
-            file_type = mimetypes.guess_type(path_to_file)[0]
-            file_size = os.path.getsize(path_to_file)
+        file_type = mimetypes.guess_type(path_to_file)[0]
+        file_size = os.path.getsize(path_to_file)
 
-            return file_wrapper, file_type, file_size
+        return file_wrapper, file_type, file_size
 
     def upload_file(self, file: File) -> str:
         """

tests/docker/test_docker_experimental.py

@@ -23,14 +23,15 @@ class TestDockerExperimental:
     @mark.order(1)
     def test_file_producer(self, serverless_client: ServerlessClient):
         """Integration test for files."""
+        functionTitle = "file-producer-for-consume"
         function = QiskitFunction(
-            title="file-producer-for-consume",
+            title=functionTitle,
             entrypoint="produce_files.py",
             working_dir=resources_path,
         )
         serverless_client.upload(function)
 
-        file_producer_function = serverless_client.function("file-producer-for-consume")
+        file_producer_function = serverless_client.function(functionTitle)
 
         job = file_producer_function.run()
 
@@ -39,49 +40,51 @@ def test_file_producer(self, serverless_client: ServerlessClient):
         assert job.status() == "DONE"
         assert isinstance(job.logs(), str)
 
-        assert len(serverless_client.files()) > 0
+        assert len(serverless_client.files(functionTitle)) > 0
 
     @mark.skip(
         reason="File producing and consuming is not working. Maybe write permissions for functions?"
     )
     @mark.order(2)
     def test_file_consumer(self, serverless_client: ServerlessClient):
         """Integration test for files."""
+        functionTitle = "file-consumer"
         function = QiskitFunction(
-            title="file-consumer",
+            title=functionTitle,
             entrypoint="consume_files.py",
             working_dir=resources_path,
         )
         serverless_client.upload(function)
 
-        file_consumer_function = serverless_client.function("file-consumer")
+        file_consumer_function = serverless_client.function(functionTitle)
 
         job = file_consumer_function.run()
         assert job is not None
         assert job.result() is not None
         assert job.status() == "DONE"
         assert isinstance(job.logs(), str)
 
-        files = serverless_client.files()
+        files = serverless_client.files(functionTitle)
 
         assert files is not None
 
         file_count = len(files)
 
         assert file_count > 0
 
-        serverless_client.file_delete("uploaded_file.tar")
+        serverless_client.file_delete("uploaded_file.tar", functionTitle)
 
-        assert (file_count - len(serverless_client.files())) == 1
+        assert (file_count - len(serverless_client.files(functionTitle))) == 1
 
     @mark.order(1)
     def test_upload_download_delete(self, serverless_client: ServerlessClient):
         """Integration test for upload files."""
+        function = serverless_client.function("hello-world")
 
         print("::: file_upload :::")
-        print(serverless_client.file_upload(filename_path))
+        print(serverless_client.file_upload(filename_path, function))
 
-        files = serverless_client.files()
+        files = serverless_client.files(function)
         print("::: files :::")
         print(files)
 
@@ -92,19 +95,61 @@ def test_upload_download_delete(self, serverless_client: ServerlessClient):
         assert file_count == 1
 
         print("::: file_download :::")
-        assert serverless_client.file_download(filename) is not None
+        assert serverless_client.file_download(filename, function) is not None
 
-        files = serverless_client.files()
+        files = serverless_client.files(function)
         print("::: files after download :::")
         print(files)
 
         assert file_count == len(files)
 
         print("::: file_delete :::")
-        print(serverless_client.file_delete(filename))
+        print(serverless_client.file_delete(filename, function))
 
         print("::: files after delete:::")
-        files = serverless_client.files()
+        files = serverless_client.files(function)
+        print(files)
+
+        assert (file_count - len(files)) == 1
+
+    def test_provider_upload_download_delete(self, serverless_client: ServerlessClient):
+        """Integration test for upload files."""
+        function = QiskitFunction(
+            title="provider-function",
+            provider="mockprovider",
+            image="test-local-provider-function:latest",
+        )
+        serverless_client.upload(function)
+
+        function = serverless_client.function("mockprovider/provider-function")
+
+        print("::: file_upload :::")
+        print(serverless_client.file_upload(filename_path, function))
+
+        files = serverless_client.files(function)
+        print("::: files :::")
+        print(files)
+
+        file_count = len(files)
+        print("::: file_count :::")
+        print(file_count)
+
+        assert file_count == 1
+
+        print("::: file_download :::")
+        assert serverless_client.file_download(filename, function) is not None
+
+        files = serverless_client.files(function)
+        print("::: files after download :::")
+        print(files)
+
+        assert file_count == len(files)
+
+        print("::: file_delete :::")
+        print(serverless_client.file_delete(filename, function))
+
+        print("::: files after delete:::")
+        files = serverless_client.files(function)
         print(files)
 
         assert (file_count - len(files)) == 1

tests/experimental/file_download.py

@@ -21,9 +21,9 @@
 print(job.status())
 print(job.logs())
 
-available_files = serverless.files()
+available_files = serverless.files(function)
 print(available_files)
 
 if len(available_files) > 0:
-    serverless.file_download(available_files[0])
+    serverless.file_download(available_files[0], function)
 print("Download complete")

tests/experimental/manage_data_directory.py

@@ -9,19 +9,19 @@
 )
 print(serverless)
 
+function = QiskitFunction(
+    title="file-producer", entrypoint="produce_files.py", working_dir="./source_files/"
+)
+serverless.upload(function)
+
 import tarfile
 
 filename = "uploaded_file.tar"
 file = tarfile.open(filename, "w")
 file.add("manage_data_directory.py")
 file.close()
 
-serverless.file_upload(filename)
-
-function = QiskitFunction(
-    title="file-producer", entrypoint="produce_files.py", working_dir="./source_files/"
-)
-serverless.upload(function)
+serverless.file_upload(filename, function)
 
 functions = {f.title: f for f in serverless.list()}
 file_producer_function = functions.get("file-producer")
@@ -33,7 +33,7 @@
 print(job.logs())
 
 
-print(serverless.files())
+print(serverless.files(file_producer_function))
 
 function = QiskitFunction(
     title="file-consumer", entrypoint="consume_files.py", working_dir="./source_files/"
@@ -49,8 +49,8 @@
 print(job.status())
 print(job.logs())
 
-print(serverless.files())
+print(serverless.files(file_consumer_function))
 
-serverless.file_delete("uploaded_file.tar")
+serverless.file_delete("uploaded_file.tar", file_consumer_function)
 
 print("Done deleting files")