Part 1.2: TF_IDF_and_Count_Vectorizer_sklearn.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TF-IDF_&&Count_Vectorizer_sklearn.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gulabpatel/NLP_Basics/blob/main/TF_IDF_and_Count_Vectorizer_sklearn.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3Tib-VoNf3SU"
      },
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
        "import numpy as np"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WShP9UhDdqqz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9c8c5227-9395-4af1-f5b6-e3186b201b42"
      },
      "source": [
        "sentences=['I have a credit card account','My account card, debit card is lost','My credit card stopped working']\r\n",
        "sentences"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['I have a credit card account',\n",
              " 'My account card, debit card is lost',\n",
              " 'My credit card stopped working']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bWwAqyZubjXg"
      },
      "source": [
        "vectorizer=CountVectorizer()\n",
        "countvec=vectorizer.fit_transform(sentences)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gidCFkK2bjUm",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "outputId": "b722a8a3-9b0e-40f5-bf89-5eb5a4f9e1e0"
      },
      "source": [
        "countvec.A #prints the matrix of count"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[1, 1, 1, 0, 1, 0, 0, 0, 0, 0],\n",
              "       [1, 2, 0, 1, 0, 1, 1, 1, 0, 0],\n",
              "       [0, 1, 1, 0, 0, 0, 0, 1, 1, 1]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IfQXzKR8bjRY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "319770ac-f974-431e-81f9-7bb20e7aafd4"
      },
      "source": [
        "vectorizer.get_feature_names()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['account',\n",
              " 'card',\n",
              " 'credit',\n",
              " 'debit',\n",
              " 'have',\n",
              " 'is',\n",
              " 'lost',\n",
              " 'my',\n",
              " 'stopped',\n",
              " 'working']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0HX9sTs-jGv-"
      },
      "source": [
        "vectorizer=CountVectorizer(max_features=4)\n",
        "countvec=vectorizer.fit_transform(sentences)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n0dnuMHijISd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "093b792f-d4d1-470a-9980-247216d51eca"
      },
      "source": [
        "print(countvec.A)\n",
        "print(vectorizer.get_feature_names())"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[1 1 1 0]\n",
            " [1 2 0 1]\n",
            " [0 1 1 1]]\n",
            "['account', 'card', 'credit', 'my']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qeGcIJgHc-rG"
      },
      "source": [
        "vectorizer=CountVectorizer(max_features=4, stop_words='english')\n",
        "countvec=vectorizer.fit_transform(sentences)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oMprZNb6c-oo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2fbe0f04-7ab5-40b4-fcee-40f64718302b"
      },
      "source": [
        "print(countvec.A)\n",
        "print(vectorizer.get_feature_names())"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[1 1 1 0]\n",
            " [1 2 0 1]\n",
            " [0 1 1 0]]\n",
            "['account', 'card', 'credit', 'debit']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vJOj7ucXbjIg"
      },
      "source": [
        "vectorizer=CountVectorizer(max_features=6, ngram_range=(1,2))\n",
        "countvec=vectorizer.fit_transform(sentences)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9VFcQsvKc-uy",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0c2622e0-9ef2-4552-871e-4b8cad3f0908"
      },
      "source": [
        "print(countvec.A)\n",
        "print(vectorizer.get_feature_names())"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[1 1 1 1 0 0]\n",
            " [1 2 0 0 1 0]\n",
            " [0 1 1 1 1 1]]\n",
            "['account', 'card', 'credit', 'credit card', 'my', 'stopped']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wKy15maGfXIJ"
      },
      "source": [
        "##TF = (# occurrences of term t in document) / (# of words in document)\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tqjK7ppXc-kp"
      },
      "source": [
        "vectorizer=TfidfVectorizer(use_idf=False, norm='l1') #use_idf=False means you are allowing only TF\n",
        "tfvec=vectorizer.fit_transform(sentences)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hpgLWhqodzOq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f14f4ec3-fb73-441c-843f-cfca80fb47d2"
      },
      "source": [
        "print(tfvec.A)\n",
        "print(vectorizer.get_feature_names())"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[0.25       0.25       0.25       0.         0.25       0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.14285714 0.28571429 0.         0.14285714 0.         0.14285714\n",
            "  0.14285714 0.14285714 0.         0.        ]\n",
            " [0.         0.2        0.2        0.         0.         0.\n",
            "  0.         0.2        0.2        0.2       ]]\n",
            "['account', 'card', 'credit', 'debit', 'have', 'is', 'lost', 'my', 'stopped', 'working']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aiBHUKVuWOSn",
        "outputId": "e2588dec-097c-4499-e314-d0f5591ec932"
      },
      "source": [
        "print(1/4), print(1/7), print(1/5)  #l1 norm calculation"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.25\n",
            "0.14285714285714285\n",
            "0.2\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(None, None, None)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6DGVPbtbh097"
      },
      "source": [
        "sentences=['I have a credit card account','My card, debit card account','My credit card is not working']"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wERaF8RodzLj"
      },
      "source": [
        "vectorizer=TfidfVectorizer(use_idf=False, norm='l2')\n",
        "tfvec=vectorizer.fit_transform(sentences)"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vfUA6DledzIr",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2d0e21b2-13a4-4d76-b209-32ff31941d47"
      },
      "source": [
        "print(tfvec.A)\n",
        "print(vectorizer.get_feature_names())"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[0.5        0.5        0.5        0.         0.5        0.\n",
            "  0.         0.         0.         0.        ]\n",
            " [0.33333333 0.66666667 0.         0.33333333 0.         0.33333333\n",
            "  0.33333333 0.33333333 0.         0.        ]\n",
            " [0.         0.4472136  0.4472136  0.         0.         0.\n",
            "  0.         0.4472136  0.4472136  0.4472136 ]]\n",
            "['account', 'card', 'credit', 'debit', 'have', 'is', 'lost', 'my', 'stopped', 'working']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yHviWpO4WMIs",
        "outputId": "dd036939-95fe-49a5-da80-0f3e1c5fa69c"
      },
      "source": [
        "print(1/np.sqrt(4)), print(1/np.sqrt(9)), print(2/np.sqrt(9))  #l2 norm calculation"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.5\n",
            "0.3333333333333333\n",
            "0.6666666666666666\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(None, None, None)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BKO1_ap3dzDP"
      },
      "source": [
        "vectorizer=TfidfVectorizer(use_idf=False, norm=None) #It will work as countvectorizer\n",
        "tfvec=vectorizer.fit_transform(sentences)"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aucBmRAOdzAo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "28f24b72-21f1-4e0f-e95f-35405721f7f7"
      },
      "source": [
        "print(tfvec.A)\n",
        "print(vectorizer.get_feature_names())"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[1. 1. 1. 0. 1. 0. 0. 0. 0. 0.]\n",
            " [1. 2. 0. 1. 0. 1. 1. 1. 0. 0.]\n",
            " [0. 1. 1. 0. 0. 0. 0. 1. 1. 1.]]\n",
            "['account', 'card', 'credit', 'debit', 'have', 'is', 'lost', 'my', 'stopped', 'working']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cuvVmX32AUup"
      },
      "source": [
        "IDF = log(# of documents / # documents with term t in it)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5UHJcaQMdy9h"
      },
      "source": [
        "vectorizer_idf=TfidfVectorizer(smooth_idf=False) #smooth_idf=False means it add 1 in log formula, e.g (log(#doc/#count)+1)\n",
        "tfidfvec=vectorizer_idf.fit_transform(sentences)"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TKFsfbK3AMLR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "aa253c87-facc-48f8-ef85-2c59bca0faa8"
      },
      "source": [
        "print(vectorizer_idf.idf_)\n",
        "print(vectorizer_idf.get_feature_names())"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1.40546511 1.         1.40546511 2.09861229 2.09861229 2.09861229\n",
            " 2.09861229 1.40546511 2.09861229 2.09861229]\n",
            "['account', 'card', 'credit', 'debit', 'have', 'is', 'lost', 'my', 'stopped', 'working']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MHhC4HZ_A53x"
      },
      "source": [
        "sentences=['I have a credit card account','My card, debit card account','My credit card is not working']"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x9Tsup0ZAMHe",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6c3f393f-6798-4d9b-beec-e6dfeff5d757"
      },
      "source": [
        "print(np.log(3/2)+1)"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1.4054651081081644\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "olxtjaNE2FYW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "457fee91-d855-470d-a1a4-05d9f9d01c12"
      },
      "source": [
        "tfidfvec.A"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.45951737, 0.3269504 , 0.45951737, 0.        , 0.68614212,\n",
              "        0.        , 0.        , 0.        , 0.        , 0.        ],\n",
              "       [0.3055129 , 0.43474989, 0.        , 0.45618573, 0.        ,\n",
              "        0.45618573, 0.45618573, 0.3055129 , 0.        , 0.        ],\n",
              "       [0.        , 0.26959162, 0.37890161, 0.        , 0.        ,\n",
              "        0.        , 0.        , 0.37890161, 0.56576828, 0.56576828]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nkHWUodzYHDu"
      },
      "source": [
        "-----"
      ]
    }
  ]
}