improve removal on indexes to now include removing full directories

joshembling · Dec 17, 2023 · 1300c18 · 1300c18
1 parent 07ad185
commit 1300c18
Show file tree

Hide file tree

Showing 2 changed files with 61 additions and 53 deletions.
diff --git a/src/Commands/LaragenieCommand.php b/src/Commands/LaragenieCommand.php
@@ -241,84 +241,92 @@ public function removeIndexedFiles(OpenAI\Client $openai, Pinecone $pinecone)
             }
         }
 
-        $file = $this->ask('What file do you want to remove from your index? (You must provide the full namespace and file extension)');
+        $files = $this->ask('What file(s) do you want to remove from your index? (You can provide the full namespace and file extension or a directory with a wildcard)');
 
-        if (! $file) {
+        if (! $files) {
             $this->error('You must provide a filename.');
 
             $this->userAction($openai, $pinecone);
         }
 
-        $formatted_filename = str_replace('/', '-', $file);
-
         $this->question('Finding vectors...');
 
-        $this->findFilesToRemove($pinecone, $file, $formatted_filename);
+        $this->findFilesToRemove($pinecone, $files);
 
         $this->userAction($openai, $pinecone);
     }
 
-    public function flushFiles(OpenAI\Client $openai, Pinecone $pinecone)
+    public function findFilesToRemove(Pinecone $pinecone, string $user_files)
     {
-        $pinecone->index(env('PINECONE_INDEX'))->vectors()->delete(
-            deleteAll: true
-        );
-
-        $this->info('All files have been removed.');
+        $files = glob($user_files);
 
-        $this->userAction($openai, $pinecone);
-    }
+        foreach ($files as $file) {
+            $formatted_filename = str_replace('/', '-', $file);
 
-    public function findFilesToRemove(Pinecone $pinecone, string $file, string $formatted_filename)
-    {
-        for ($i = 1; $i < 100; $i++) {
-            try {
-                $pinecone_res = $pinecone->index(env('PINECONE_INDEX'))->vectors()->fetch([
-                    "{$formatted_filename}-{$i}",
-                ]);
-            } catch (\Throwable $th) {
-                $this->error('There has been an error.');
-                break;
-            }
+            $this->warn('Attempting to remove all "'.$file.'" indexes...');
 
-            if ($i === 1 && empty($pinecone_res->json()['vectors'])) {
-                $this->warn('No indexes were found for the file '.$file);
-                break;
-            } elseif ($i === 1) {
-                $choice = select(
-                    'Vectors have been found, are you sure you want to delete them? 🤔',
-                    [
-                        'y' => 'Yes',
-                        'n' => 'No',
-                    ],
-                );
+            for ($i = 0; $i < 100; $i++) {
+                try {
+                    $pinecone_res = $pinecone->index(env('PINECONE_INDEX'))->vectors()->fetch([
+                        "{$formatted_filename}-{$i}",
+                    ]);
+                } catch (\Throwable $th) {
+                    $this->error('There has been an error.');
+                    break;
+                }
 
-                if ($choice === 'y') {
-                    $this->question("Alright, let's bin those 🚽");
-                } else {
-                    $this->info('Nothing has been deleted 😅');
+                if ($i === 0 && empty($pinecone_res->json()['vectors'])) {
+                    $this->warn('No indexes were found for the file '.$file);
                     break;
                 }
-            }
 
-            try {
-                $response = $pinecone->index(env('PINECONE_INDEX'))->vectors()->delete(
-                    ids: ["{$formatted_filename}-{$i}"],
-                    deleteAll: false
-                );
-            } catch (\Throwable $th) {
-                $this->error($th);
-            }
+                if (config('laragenie.indexes.removal.strict') && $i === 0) {
+                    $choice = select(
+                        'Vectors have been found, are you sure you want to delete them? 🤔',
+                        [
+                            'y' => 'Yes',
+                            'n' => 'No',
+                        ],
+                    );
+
+                    if ($choice === 'y') {
+                        $this->question("Alright, let's bin those 🚽");
+                    } else {
+                        $this->info('Nothing has been deleted 😅');
+                        break;
+                    }
+                }
+
+                try {
+                    $response = $pinecone->index(env('PINECONE_INDEX'))->vectors()->delete(
+                        ids: ["{$formatted_filename}-{$i}"],
+                        deleteAll: false
+                    );
+                } catch (\Throwable $th) {
+                    $this->error($th);
+                }
 
-            if (empty($pinecone_res->json()['vectors'])) {
-                $this->newLine();
-                $this->info('Vectors have been deleted that were associated with '.$file);
+                if (empty($pinecone_res->json()['vectors'])) {
+                    $this->newLine();
+                    $this->info('Vectors have been deleted that were associated with '.$file);
 
-                break;
+                    break;
+                }
             }
         }
     }
 
+    public function flushFiles(OpenAI\Client $openai, Pinecone $pinecone)
+    {
+        $pinecone->index(env('PINECONE_INDEX'))->vectors()->delete(
+            deleteAll: true
+        );
+
+        $this->info('All files have been removed.');
+
+        $this->userAction($openai, $pinecone);
+    }
+
     public function somethingElse(OpenAI\Client $openai, Pinecone $pinecone)
     {
         $this->info('You can contact @joshembling on Github to suggest another feature.');

diff --git a/src/Helpers/Actions.php b/src/Helpers/Actions.php
@@ -55,7 +55,7 @@ public function removeAction()
         return select(
             'What do you want to do?',
             [
-                'one' => 'Remove data associated with one file',
+                'one' => 'Remove data associated with a directory or specific file',
                 'all' => 'Remove all chunked data',
             ],
         );