Fix Delete more than 1000 items

MakisChristou · Oct 1, 2023 · aef23b6 · aef23b6
1 parent aa394af
commit aef23b6
Show file tree

Hide file tree

Showing 2 changed files with 33 additions and 23 deletions.
diff --git a/src/main.rs b/src/main.rs
@@ -308,6 +308,8 @@ async fn handle_download(
             config.key_bytes.clone(),
         );
 
+        println!("Assisiated Filenameas {}", associated_filenames.len());
+
         let mut total_size = 0;
 
         for assosiated_filename in associated_filenames {

diff --git a/src/storage/s3.rs b/src/storage/s3.rs
@@ -9,14 +9,14 @@ use bytes::Bytes;
 use futures_util::stream::StreamExt;
 use rusoto_core::Region;
 use rusoto_s3::{
-    Delete, DeleteObjectRequest, DeleteObjectsRequest, GetObjectRequest,
-    ObjectIdentifier, PutObjectRequest, S3Client, S3,
+    Delete, DeleteObjectRequest, DeleteObjectsRequest, GetObjectRequest, ObjectIdentifier,
+    PutObjectRequest, S3Client, S3,
 };
 extern crate dotenv;
-
+use super::Storage;
 use crate::config::Config;
 
-use super::Storage;
+const MAX_DELETE_COUNT: usize = 1000;
 
 #[derive(Clone)]
 pub struct S3Storage {
@@ -99,27 +99,35 @@ impl Storage for S3Storage {
     }
 
     async fn batch_delete(&self, filenames: HashSet<String>) -> Result<(), String> {
-        // Prepare the list of objects to delete
-        let objects: Vec<ObjectIdentifier> = filenames
-            .into_iter()
-            .map(|filename| ObjectIdentifier {
-                key: filename,
+        // Convert the filenames into a Vec
+        let filenames_vec: Vec<String> = filenames.into_iter().collect();
+
+        // Split the filenames into chunks of 1000 or fewer
+        for chunk in filenames_vec.chunks(MAX_DELETE_COUNT) {
+            // Prepare the list of objects to delete for this chunk
+            let objects: Vec<ObjectIdentifier> = chunk
+                .iter()
+                .map(|filename| ObjectIdentifier {
+                    key: filename.clone(),
+                    ..Default::default()
+                })
+                .collect();
+
+            let delete_req = DeleteObjectsRequest {
+                bucket: self.bucket_name.to_string(),
+                delete: Delete {
+                    objects,
+                    ..Default::default()
+                },
                 ..Default::default()
-            })
-            .collect();
+            };
 
-        let delete_req = DeleteObjectsRequest {
-            bucket: self.bucket_name.to_string(),
-            delete: Delete {
-                objects,
-                ..Default::default()
-            },
-            ..Default::default()
-        };
-
-        match self.s3_client.delete_objects(delete_req).await {
-            Ok(_) => Ok(()),
-            Err(e) => Err(format!("Could not batch delete files: {}", e)),
+            match self.s3_client.delete_objects(delete_req).await {
+                Ok(_) => continue,
+                Err(e) => return Err(format!("Could not batch delete files: {}", e)),
+            }
         }
+
+        Ok(())
     }
 }