data_extraction/notebooks/rtf_extraction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RTF extraction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook shows examples of text extraction from RTF files with different packages"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Table of contents**<a id='toc0_'></a>    \n",
    "- 1. [Methods to load RTF files](#toc1_)    \n",
    "  - 1.1. [Load from unstructured local RTF loader](#toc1_1_)    \n",
    "  - 1.2. [Load from unstructured io API](#toc1_2_)    \n",
    "- 2. [Evaluate loded docs by embedding similarity](#toc2_)    \n",
    "  - 2.1. [Embedding & Storage](#toc2_1_)    \n",
    "  - 2.2. [Similarity search](#toc2_2_)    \n",
    "\n",
    "<!-- vscode-jupyter-toc-config\n",
    "\tnumbering=true\n",
    "\tanchor=true\n",
    "\tflat=false\n",
    "\tminLevel=2\n",
    "\tmaxLevel=4\n",
    "\t/vscode-jupyter-toc-config -->\n",
    "<!-- THIS CELL WILL BE REPLACED ON TOC UPDATE. DO NOT WRITE YOUR TEXT IN THIS CELL -->"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "\n",
    "current_dir = os.getcwd()\n",
    "kit_dir = os.path.abspath(os.path.join(current_dir, \"..\"))\n",
    "repo_dir = os.path.abspath(os.path.join(kit_dir, \"..\"))\n",
    "\n",
    "sys.path.append(kit_dir)\n",
    "sys.path.append(repo_dir)\n",
    "\n",
    "import glob\n",
    "import pandas as pd\n",
    "from dotenv import load_dotenv\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "from tqdm.autonotebook import trange\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. <a id='toc1_'></a>[Methods to load RTF files](#toc0_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "folder_loc = os.path.join(kit_dir,'data/sample_data/sample_files/')\n",
    "rtf_files = list(glob.glob(f'{folder_loc}/*.rtf'))\n",
    "file_path = rtf_files[0]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Load text splitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "        # Set a small chunk size, just to make splitting evident.\n",
    "        chunk_size = 200,\n",
    "        chunk_overlap  = 20,\n",
    "        length_function = len,\n",
    "        add_start_index = True,\n",
    "        separators = [\"\\n\\n\\n\",\"\\n\\n\", \"\\n\", \".\"]\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1. <a id='toc1_1_'></a>[Load from unstructured local RTF loader](#toc0_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "for using pypandoc is it required to install pandoc -> https://pandoc.org/installing.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "My First Heading\n",
      "---\n",
      "My first paragraph.\n",
      "---\n",
      "Table Example:\n",
      "---\n",
      "Column 1 Column 2 Row 1, Cell 1 Row 1, Cell 2 Row 2, Cell 1 Row 2, Cell 2\n",
      "---\n"
     ]
    }
   ],
   "source": [
    "from langchain.document_loaders import UnstructuredRTFLoader\n",
    "\n",
    "loader = UnstructuredRTFLoader(file_path, mode=\"elements\")\n",
    "docs_unstructured_local = loader.load_and_split(text_splitter = text_splitter)\n",
    "for doc in docs_unstructured_local:\n",
    "    print(f'{doc.page_content}\\n---')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.2. <a id='toc1_2_'></a>[Load from unstructured io API](#toc0_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "My First Heading\n",
      "---\n",
      "My first paragraph.\n",
      "---\n",
      "Table Example:\n",
      "---\n",
      "Column 1 Column 2 Row 1, Cell 1 Row 1, Cell 2 Row 2, Cell 1 Row 2, Cell 2\n",
      "---\n"
     ]
    }
   ],
   "source": [
    "from langchain.document_loaders import UnstructuredAPIFileLoader\n",
    "# register at Unstructured.io to get a free API Key\n",
    "load_dotenv(os.path.join(repo_dir,'.env'))\n",
    "\n",
    "loader = UnstructuredAPIFileLoader(file_path, \n",
    "                                   mode=\"elements\", \n",
    "                                   api_key=os.environ.get('UNSTRUCTURED_API_KEY'),\n",
    "                                   url=os.environ.get(\"UNSTRUCTURED_URL\"))\n",
    "docs_unstructured_api = loader.load_and_split(text_splitter = text_splitter)\n",
    "for doc in docs_unstructured_api:\n",
    "    print(f'{doc.page_content}\\n---')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. <a id='toc2_'></a>[Evaluate loded docs by embedding similarity](#toc0_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1. <a id='toc2_1_'></a>[Embedding & Storage](#toc0_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load INSTRUCTOR_Transformer\n",
      "max_seq_length  512\n"
     ]
    }
   ],
   "source": [
    "from langchain.embeddings import HuggingFaceInstructEmbeddings\n",
    "from langchain.vectorstores import FAISS\n",
    "\n",
    "encode_kwargs = {'normalize_embeddings': True}\n",
    "embd_model = HuggingFaceInstructEmbeddings( model_name='intfloat/e5-large-v2',\n",
    "                                            embed_instruction=\"\", # no instructions needed for candidate passages\n",
    "                                            query_instruction=\"Represent this sentence for searching relevant passages: \",\n",
    "                                            encode_kwargs=encode_kwargs)\n",
    "vectorstore_unstructured_local = FAISS.from_documents(documents=docs_unstructured_local, embedding=embd_model)\n",
    "vectorstore_unstructured_api = FAISS.from_documents(documents=docs_unstructured_api, embedding=embd_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2. <a id='toc2_2_'></a>[Similarity search](#toc0_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------Unstructured local Loader----------\n",
      "\n",
      "Column 1 Column 2 Row 1, Cell 1 Row 1, Cell 2 Row 2, Cell 1 Row 2, Cell 2\n",
      "--------Unstructured api loader------------\n",
      "\n",
      "Column 1 Column 2 Row 1, Cell 1 Row 1, Cell 2 Row 2, Cell 1 Row 2, Cell 2\n"
     ]
    }
   ],
   "source": [
    "query = \"how many columns are?\"\n",
    "\n",
    "ans = vectorstore_unstructured_local.similarity_search(query)\n",
    "print(\"-------Unstructured local Loader----------\\n\")\n",
    "print(ans[0].page_content)\n",
    "\n",
    "\n",
    "ans_2 = vectorstore_unstructured_api.similarity_search(query)\n",
    "print(\"--------Unstructured api loader------------\\n\")\n",
    "print(ans_2[0].page_content)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "c141c502b4874f6f7cd85d57d9f08ebe5bc7ea264c7dc3d59361b279d0075653"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}