HW1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c67dc011",
   "metadata": {},
   "source": [
    "<h3 style='color:black'>Preparing the data</h3>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ec6cfda0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package words to\n",
      "[nltk_data]     C:\\Users\\naor2\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package words is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\naor2\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\naor2\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package names to\n",
      "[nltk_data]     C:\\Users\\naor2\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package names is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\naor2\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>twitter_screen_name</th>\n",
       "      <th>twitter_name</th>\n",
       "      <th>twitter_desc</th>\n",
       "      <th>twitter_location</th>\n",
       "      <th>wikidata_desc</th>\n",
       "      <th>type1</th>\n",
       "      <th>type2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BarackObama</td>\n",
       "      <td>Barack Obama</td>\n",
       "      <td>Dad, husband, President, citizen.</td>\n",
       "      <td>Washington, DC</td>\n",
       "      <td>44th president of the United States</td>\n",
       "      <td>Politician</td>\n",
       "      <td>Person</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>justinbieber</td>\n",
       "      <td>Justin Bieber</td>\n",
       "      <td>#Changes out now</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Canadian singer</td>\n",
       "      <td>MusicalArtist</td>\n",
       "      <td>Person</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>katyperry</td>\n",
       "      <td>KATY PERRY</td>\n",
       "      <td>Love. Light.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>American singer</td>\n",
       "      <td>MusicalArtist</td>\n",
       "      <td>Person</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>rihanna</td>\n",
       "      <td>Rihanna</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Barbadian singer, songwriter, and businesswoman</td>\n",
       "      <td>MusicalArtist</td>\n",
       "      <td>Person</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>taylorswift13</td>\n",
       "      <td>Taylor Swift</td>\n",
       "      <td>Lover out now</td>\n",
       "      <td>NaN</td>\n",
       "      <td>American singer-songwriter</td>\n",
       "      <td>MusicalArtist</td>\n",
       "      <td>Person</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29058</th>\n",
       "      <td>JessalynGilsig</td>\n",
       "      <td>Jessalyn Gilsig</td>\n",
       "      <td>Big Shot Vikings Glee Scandal Nip Tuck and any...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>actress</td>\n",
       "      <td>Person</td>\n",
       "      <td>Person</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29059</th>\n",
       "      <td>ReemaMajor</td>\n",
       "      <td>Reema Major</td>\n",
       "      <td>Dey Say Only Da Good Die Young So Fuck It ima ...</td>\n",
       "      <td>South Sudan - UAE</td>\n",
       "      <td>Canadian rapper</td>\n",
       "      <td>MusicalArtist</td>\n",
       "      <td>Person</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29060</th>\n",
       "      <td>EqualityTexas</td>\n",
       "      <td>Equality Texas</td>\n",
       "      <td>We work to secure full equality for LGBTQ Texa...</td>\n",
       "      <td>Austin, Texas</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Organisation</td>\n",
       "      <td>Organisation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29061</th>\n",
       "      <td>Margus_Hunt</td>\n",
       "      <td>Margus Hunt</td>\n",
       "      <td>Official Twitter page of Margus Hunt</td>\n",
       "      <td>Indianapolis, IN</td>\n",
       "      <td>Estonian track and field athlete, American foo...</td>\n",
       "      <td>AmericanFootballPlayer</td>\n",
       "      <td>Person</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29062</th>\n",
       "      <td>SandraBookman7</td>\n",
       "      <td>Sandra Bookman</td>\n",
       "      <td>Sandra is a reporter and weekend anchor for Ey...</td>\n",
       "      <td>New York City</td>\n",
       "      <td>American journalist and news anchor at WABC-TV</td>\n",
       "      <td>Person</td>\n",
       "      <td>Person</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>29063 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      twitter_screen_name     twitter_name  \\\n",
       "0             BarackObama     Barack Obama   \n",
       "1            justinbieber    Justin Bieber   \n",
       "2               katyperry       KATY PERRY   \n",
       "3                 rihanna          Rihanna   \n",
       "4           taylorswift13     Taylor Swift   \n",
       "...                   ...              ...   \n",
       "29058      JessalynGilsig  Jessalyn Gilsig   \n",
       "29059          ReemaMajor      Reema Major   \n",
       "29060       EqualityTexas   Equality Texas   \n",
       "29061         Margus_Hunt      Margus Hunt   \n",
       "29062      SandraBookman7   Sandra Bookman   \n",
       "\n",
       "                                            twitter_desc   twitter_location  \\\n",
       "0                      Dad, husband, President, citizen.     Washington, DC   \n",
       "1                                       #Changes out now                NaN   \n",
       "2                                           Love. Light.                NaN   \n",
       "3                                                    NaN                NaN   \n",
       "4                                          Lover out now                NaN   \n",
       "...                                                  ...                ...   \n",
       "29058  Big Shot Vikings Glee Scandal Nip Tuck and any...                NaN   \n",
       "29059  Dey Say Only Da Good Die Young So Fuck It ima ...  South Sudan - UAE   \n",
       "29060  We work to secure full equality for LGBTQ Texa...      Austin, Texas   \n",
       "29061               Official Twitter page of Margus Hunt   Indianapolis, IN   \n",
       "29062  Sandra is a reporter and weekend anchor for Ey...      New York City   \n",
       "\n",
       "                                           wikidata_desc  \\\n",
       "0                    44th president of the United States   \n",
       "1                                        Canadian singer   \n",
       "2                                        American singer   \n",
       "3        Barbadian singer, songwriter, and businesswoman   \n",
       "4                             American singer-songwriter   \n",
       "...                                                  ...   \n",
       "29058                                            actress   \n",
       "29059                                    Canadian rapper   \n",
       "29060                                                NaN   \n",
       "29061  Estonian track and field athlete, American foo...   \n",
       "29062     American journalist and news anchor at WABC-TV   \n",
       "\n",
       "                        type1         type2  \n",
       "0                  Politician        Person  \n",
       "1               MusicalArtist        Person  \n",
       "2               MusicalArtist        Person  \n",
       "3               MusicalArtist        Person  \n",
       "4               MusicalArtist        Person  \n",
       "...                       ...           ...  \n",
       "29058                  Person        Person  \n",
       "29059           MusicalArtist        Person  \n",
       "29060            Organisation  Organisation  \n",
       "29061  AmericanFootballPlayer        Person  \n",
       "29062                  Person        Person  \n",
       "\n",
       "[29063 rows x 7 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import nltk\n",
    "import math\n",
    "import re \n",
    "import collections\n",
    "nltk.download('words')\n",
    "nltk.download('stopwords')\n",
    "nltk.download('wordnet')\n",
    "nltk.download('names')\n",
    "nltk.download('punkt')\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.corpus import words\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.probability import FreqDist\n",
    "\n",
    "\n",
    "df = pd.read_csv('Twitter_data.csv')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1a2de761",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>twitter_screen_name</th>\n",
       "      <th>twitter_name</th>\n",
       "      <th>twitter_desc</th>\n",
       "      <th>twitter_location</th>\n",
       "      <th>wikidata_desc</th>\n",
       "      <th>type1</th>\n",
       "      <th>type2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BarackObama</td>\n",
       "      <td>Barack Obama</td>\n",
       "      <td>Dad, husband, President, citizen.</td>\n",
       "      <td>Washington, DC</td>\n",
       "      <td>44th president of the United States</td>\n",
       "      <td>Politician</td>\n",
       "      <td>Person</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>justinbieber</td>\n",
       "      <td>Justin Bieber</td>\n",
       "      <td>#Changes out now</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Canadian singer</td>\n",
       "      <td>MusicalArtist</td>\n",
       "      <td>Person</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>katyperry</td>\n",
       "      <td>KATY PERRY</td>\n",
       "      <td>Love. Light.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>American singer</td>\n",
       "      <td>MusicalArtist</td>\n",
       "      <td>Person</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>rihanna</td>\n",
       "      <td>Rihanna</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Barbadian singer, songwriter, and businesswoman</td>\n",
       "      <td>MusicalArtist</td>\n",
       "      <td>Person</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>taylorswift13</td>\n",
       "      <td>Taylor Swift</td>\n",
       "      <td>Lover out now</td>\n",
       "      <td>NaN</td>\n",
       "      <td>American singer-songwriter</td>\n",
       "      <td>MusicalArtist</td>\n",
       "      <td>Person</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29058</th>\n",
       "      <td>JessalynGilsig</td>\n",
       "      <td>Jessalyn Gilsig</td>\n",
       "      <td>Big Shot Vikings Glee Scandal Nip Tuck and any...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>actress</td>\n",
       "      <td>Person</td>\n",
       "      <td>Person</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29059</th>\n",
       "      <td>ReemaMajor</td>\n",
       "      <td>Reema Major</td>\n",
       "      <td>Dey Say Only Da Good Die Young So Fuck It ima ...</td>\n",
       "      <td>South Sudan - UAE</td>\n",
       "      <td>Canadian rapper</td>\n",
       "      <td>MusicalArtist</td>\n",
       "      <td>Person</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29060</th>\n",
       "      <td>EqualityTexas</td>\n",
       "      <td>Equality Texas</td>\n",
       "      <td>We work to secure full equality for LGBTQ Texa...</td>\n",
       "      <td>Austin, Texas</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Organisation</td>\n",
       "      <td>Organisation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29061</th>\n",
       "      <td>Margus_Hunt</td>\n",
       "      <td>Margus Hunt</td>\n",
       "      <td>Official Twitter page of Margus Hunt</td>\n",
       "      <td>Indianapolis, IN</td>\n",
       "      <td>Estonian track and field athlete, American foo...</td>\n",
       "      <td>AmericanFootballPlayer</td>\n",
       "      <td>Person</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29062</th>\n",
       "      <td>SandraBookman7</td>\n",
       "      <td>Sandra Bookman</td>\n",
       "      <td>Sandra is a reporter and weekend anchor for Ey...</td>\n",
       "      <td>New York City</td>\n",
       "      <td>American journalist and news anchor at WABC-TV</td>\n",
       "      <td>Person</td>\n",
       "      <td>Person</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>26585 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      twitter_screen_name     twitter_name  \\\n",
       "0             BarackObama     Barack Obama   \n",
       "1            justinbieber    Justin Bieber   \n",
       "2               katyperry       KATY PERRY   \n",
       "3                 rihanna          Rihanna   \n",
       "4           taylorswift13     Taylor Swift   \n",
       "...                   ...              ...   \n",
       "29058      JessalynGilsig  Jessalyn Gilsig   \n",
       "29059          ReemaMajor      Reema Major   \n",
       "29060       EqualityTexas   Equality Texas   \n",
       "29061         Margus_Hunt      Margus Hunt   \n",
       "29062      SandraBookman7   Sandra Bookman   \n",
       "\n",
       "                                            twitter_desc   twitter_location  \\\n",
       "0                      Dad, husband, President, citizen.     Washington, DC   \n",
       "1                                       #Changes out now                NaN   \n",
       "2                                           Love. Light.                NaN   \n",
       "3                                                    NaN                NaN   \n",
       "4                                          Lover out now                NaN   \n",
       "...                                                  ...                ...   \n",
       "29058  Big Shot Vikings Glee Scandal Nip Tuck and any...                NaN   \n",
       "29059  Dey Say Only Da Good Die Young So Fuck It ima ...  South Sudan - UAE   \n",
       "29060  We work to secure full equality for LGBTQ Texa...      Austin, Texas   \n",
       "29061               Official Twitter page of Margus Hunt   Indianapolis, IN   \n",
       "29062  Sandra is a reporter and weekend anchor for Ey...      New York City   \n",
       "\n",
       "                                           wikidata_desc  \\\n",
       "0                    44th president of the United States   \n",
       "1                                        Canadian singer   \n",
       "2                                        American singer   \n",
       "3        Barbadian singer, songwriter, and businesswoman   \n",
       "4                             American singer-songwriter   \n",
       "...                                                  ...   \n",
       "29058                                            actress   \n",
       "29059                                    Canadian rapper   \n",
       "29060                                                NaN   \n",
       "29061  Estonian track and field athlete, American foo...   \n",
       "29062     American journalist and news anchor at WABC-TV   \n",
       "\n",
       "                        type1         type2  \n",
       "0                  Politician        Person  \n",
       "1               MusicalArtist        Person  \n",
       "2               MusicalArtist        Person  \n",
       "3               MusicalArtist        Person  \n",
       "4               MusicalArtist        Person  \n",
       "...                       ...           ...  \n",
       "29058                  Person        Person  \n",
       "29059           MusicalArtist        Person  \n",
       "29060            Organisation  Organisation  \n",
       "29061  AmericanFootballPlayer        Person  \n",
       "29062                  Person        Person  \n",
       "\n",
       "[26585 rows x 7 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop duplicates (keeping only the first)\n",
    "df = df.drop_duplicates(subset='twitter_screen_name', keep='first')\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14c5cc44",
   "metadata": {},
   "source": [
    "<h3 style='color:black'>Helper Functions</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a6f85593",
   "metadata": {},
   "outputs": [],
   "source": [
    "# gets a list of sentences, splits to words and returns a list of them\n",
    "def list_splited_words(listWords):\n",
    "    english_vocab = sorted(set(w.lower() for w in nltk.corpus.words.words()))\n",
    "    stopwords = nltk.corpus.stopwords.words('english')\n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    splited =[]\n",
    "    for sen in listWords:\n",
    "        if(isinstance(sen, str)):  # only if not NaN\n",
    "            for word in nltk.word_tokenize(sen):\n",
    "                word=word.lower()\n",
    "                if word not in stopwords and word in english_vocab:\n",
    "                    splited.append(lemmatizer.lemmatize(word))  \n",
    "    return splited\n",
    "\n",
    "#-------------------------------------------------------------------------\n",
    "# calculates PMI\n",
    "def PMI(class_size, other_class_size, class_dataset, other_class_dataset):    \n",
    "    pmi_list=[]\n",
    "    pr_class= class_size / (class_size + other_class_size)\n",
    "    for word in set(class_dataset):\n",
    "        pr_w_class = class_dataset.count(word) / len(class_dataset)\n",
    "        pr_w = (other_class_dataset.count(word) + class_dataset.count(word)) / (len(class_dataset) + len(other_class_dataset))\n",
    "        pmi = math.log2(pr_w_class / (pr_w * pr_class))\n",
    "        pmi_list.append((word,pmi))\n",
    "    sorted_by_pmi = sorted(pmi_list, key=lambda tup: tup[1], reverse=True)   \n",
    "    return sorted_by_pmi\n",
    "\n",
    "#------------------------------------------------------------------------\n",
    "#check the sentence's sentiment (positive/negative)\n",
    "def check_sentiment(sen):\n",
    "    count=0\n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    for word in nltk.word_tokenize(sen):\n",
    "        word=lemmatizer.lemmatize(word)\n",
    "        if(word in positive_words):\n",
    "            count+=1\n",
    "        if(word in negative_words):\n",
    "            count-=1\n",
    "    return count # sentiment determined by the amount of positive/negative words"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37fe8ac9",
   "metadata": {},
   "source": [
    "<h1 style='color:red'>Question 1:</h1>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "54ba87a7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('official', 2741), ('twitter', 2344), ('news', 1784), ('new', 1423), ('world', 1384), ('account', 1343), ('author', 1143), ('u', 966), ('de', 826), ('follow', 731), ('former', 669), ('actor', 661), ('music', 649), ('time', 595), ('writer', 570), ('life', 554), ('host', 552), ('la', 530), ('love', 529), ('sport', 516), ('page', 511), ('people', 506), ('contact', 488), ('father', 488), ('husband', 486), ('since', 471), ('business', 464), ('album', 457), ('champion', 452), ('best', 448)]\n"
     ]
    }
   ],
   "source": [
    "english_vocab = sorted(set(w.lower() for w in nltk.corpus.words.words()))\n",
    "stopwords = nltk.corpus.stopwords.words('english')\n",
    "\n",
    "col_one_list = df['twitter_desc'].tolist() # putting all the twitter_desc into a list\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "splited =[]\n",
    "for sen in col_one_list:\n",
    "    if(isinstance(sen, str)): # only if not NaN\n",
    "        for word in nltk.word_tokenize(sen):\n",
    "            word=word.lower()\n",
    "            if word not in stopwords and word in english_vocab:\n",
    "                splited.append(lemmatizer.lemmatize(word))  \n",
    "fdist1=FreqDist(splited) # make a list of freq\n",
    "print(fdist1.most_common(30)) # print the 30 most frequent "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5bb4d96",
   "metadata": {},
   "source": [
    "<h1 style='color:red'>Question 2:</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2e22907b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Amount of female users: 3558\n",
      "Amount of male users: 6370\n"
     ]
    }
   ],
   "source": [
    "names = nltk.corpus.names\n",
    "names.fileids()\n",
    "male_names = names.words('male.txt')\n",
    "female_names = names.words('female.txt')\n",
    "\n",
    "names_and_desc = []\n",
    "\n",
    "# create a tuple of (name, twitter_desc)\n",
    "for row in df.itertuples():\n",
    "    if(isinstance(row[2], str)):\n",
    "        name=re.sub('\\W+','-', row[2]) # (row[2] - full name)\n",
    "        names_and_desc.append((name.split('-')[0].capitalize(),row[3]))  # split the first and last name \n",
    "        \n",
    "male_list_name=[]\n",
    "female_list_name=[]\n",
    "\n",
    "male_words=[]\n",
    "female_words=[]\n",
    "# for each name add the name and desc for the correct list(male/female)\n",
    "for tup in names_and_desc:\n",
    "    if tup[0] in male_names and tup[0] not in female_names:\n",
    "        male_list_name.append(name)\n",
    "        male_words.append(tup[1])\n",
    "    if tup[0] in female_names and tup[0] not in male_names:\n",
    "        female_list_name.append(name)\n",
    "        female_words.append(tup[1])\n",
    "\n",
    "print(\"Amount of female users: \" + str(len(female_list_name)))\n",
    "print(\"Amount of male users: \" + str(len(male_list_name)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b7dfcb87",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Female most common:\n",
      "[('author', 328), ('actress', 249), ('new', 197), ('official', 191), ('twitter', 160), ('writer', 159), ('wife', 143), ('world', 136), ('host', 126), ('former', 126), ('lover', 121), ('actor', 116), ('de', 115), ('time', 114), ('mother', 112), ('love', 107), ('la', 100), ('book', 93), ('life', 91), ('news', 89), ('founder', 84), ('singer', 79), ('director', 79), ('girl', 78), ('account', 78), ('producer', 73), ('champion', 72), ('speaker', 66), ('contact', 64), ('u', 62)]\n"
     ]
    }
   ],
   "source": [
    "print(\"Female most common:\")\n",
    "splited_female=list_splited_words(female_words)\n",
    "fdist1=FreqDist(splited_female)\n",
    "print(fdist1.most_common(30))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b886f663",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Male most common:\n",
      "[('official', 512), ('author', 453), ('twitter', 440), ('new', 363), ('husband', 298), ('father', 289), ('former', 281), ('actor', 280), ('world', 235), ('host', 213), ('account', 212), ('de', 203), ('writer', 200), ('time', 187), ('news', 172), ('champion', 163), ('player', 160), ('director', 154), ('dad', 147), ('founder', 146), ('life', 142), ('music', 140), ('love', 128), ('book', 127), ('page', 123), ('professional', 117), ('producer', 117), ('u', 115), ('la', 113), ('university', 113)]\n"
     ]
    }
   ],
   "source": [
    "print(\"Male most common:\")\n",
    "splited_male=list_splited_words(male_words)\n",
    "fdist1=FreqDist(splited_male)\n",
    "print(fdist1.most_common(30))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "55a16f67",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('silicon', 1.2824596202239196),\n",
       " ('roger', 1.2824596202239196),\n",
       " ('budget', 1.2824596202239196),\n",
       " ('course', 1.2824596202239196),\n",
       " ('important', 1.2824596202239196),\n",
       " ('castle', 1.2824596202239196),\n",
       " ('quarterback', 1.2824596202239196),\n",
       " ('related', 1.2824596202239196),\n",
       " ('chuck', 1.2824596202239196),\n",
       " ('jim', 1.2824596202239196),\n",
       " ('offering', 1.2824596202239196),\n",
       " ('pundit', 1.2824596202239196),\n",
       " ('austin', 1.2824596202239196),\n",
       " ('eagle', 1.2824596202239196),\n",
       " ('raconteur', 1.2824596202239196),\n",
       " ('peter', 1.2824596202239196),\n",
       " ('river', 1.2824596202239196),\n",
       " ('racer', 1.2824596202239196),\n",
       " ('session', 1.2824596202239196),\n",
       " ('madrid', 1.2824596202239196),\n",
       " ('solve', 1.2824596202239196),\n",
       " ('palmer', 1.2824596202239196),\n",
       " ('effective', 1.2824596202239196),\n",
       " ('venezolano', 1.2824596202239196),\n",
       " ('memory', 1.2824596202239196),\n",
       " ('cast', 1.2824596202239196),\n",
       " ('tight', 1.2824596202239196),\n",
       " ('viking', 1.2824596202239196),\n",
       " ('padre', 1.2824596202239196),\n",
       " ('cash', 1.2824596202239196)]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#--------------PMI Male----------------\n",
    "male_pmi=PMI(len(male_list_name), len(female_list_name), splited_male, splited_female)\n",
    "highest_pmi_male = male_pmi[0:30]\n",
    "highest_pmi_male"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "89840d70",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('mad', 2.957227883424215),\n",
       " ('lesbian', 2.957227883424215),\n",
       " ('bittersweet', 2.957227883424215),\n",
       " ('fancy', 2.957227883424215),\n",
       " ('contract', 2.957227883424215),\n",
       " ('lynn', 2.957227883424215),\n",
       " ('vera', 2.957227883424215),\n",
       " ('katy', 2.957227883424215),\n",
       " ('marie', 2.957227883424215),\n",
       " ('mum', 2.957227883424215),\n",
       " ('sit', 2.957227883424215),\n",
       " ('disabled', 2.957227883424215),\n",
       " ('bailey', 2.957227883424215),\n",
       " ('kristen', 2.957227883424215),\n",
       " ('sept', 2.957227883424215),\n",
       " ('bonnie', 2.957227883424215),\n",
       " ('lena', 2.957227883424215),\n",
       " ('glamour', 2.957227883424215),\n",
       " ('soprano', 2.957227883424215),\n",
       " ('edit', 2.957227883424215),\n",
       " ('dental', 2.957227883424215),\n",
       " ('ballet', 2.957227883424215),\n",
       " ('congresswoman', 2.957227883424215),\n",
       " ('bloom', 2.957227883424215),\n",
       " ('feminism', 2.957227883424215),\n",
       " ('wander', 2.957227883424215),\n",
       " ('ordinary', 2.957227883424215),\n",
       " ('normal', 2.957227883424215),\n",
       " ('katie', 2.957227883424215),\n",
       " ('bitch', 2.957227883424215)]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#--------------PMI Female----------------\n",
    "female_pmi=PMI(len(female_list_name), len(male_list_name), splited_female, splited_male)\n",
    "highest_pmi_female\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f390f1c",
   "metadata": {},
   "source": [
    "<h1 style='color:red'>Question 3:</h1>\n",
    "<h3 style='color:red'>Section A:</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "84201a1e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('siamese', 6.415122790145467),\n",
       " ('brick', 6.415122790145467),\n",
       " ('rooted', 6.415122790145467),\n",
       " ('douche', 6.415122790145467),\n",
       " ('benelux', 6.415122790145467),\n",
       " ('dixie', 6.415122790145467),\n",
       " ('caustic', 6.415122790145467),\n",
       " ('spelunker', 6.415122790145467),\n",
       " ('asylum', 6.415122790145467),\n",
       " ('flytrap', 6.415122790145467),\n",
       " ('heartbreaker', 6.415122790145467),\n",
       " ('banger', 6.415122790145467),\n",
       " ('mana', 6.415122790145467),\n",
       " ('shortly', 6.415122790145467),\n",
       " ('nee', 6.415122790145467),\n",
       " ('wakefulness', 6.415122790145467),\n",
       " ('pothead', 6.415122790145467),\n",
       " ('cantor', 6.415122790145467),\n",
       " ('artemis', 6.415122790145467),\n",
       " ('mantra', 6.415122790145467),\n",
       " ('thyself', 6.415122790145467),\n",
       " ('miner', 6.415122790145467),\n",
       " ('gastronome', 6.415122790145467),\n",
       " ('staunch', 6.415122790145467),\n",
       " ('branded', 6.415122790145467),\n",
       " ('jubilee', 6.415122790145467),\n",
       " ('hedonistic', 6.415122790145467),\n",
       " ('grip', 6.415122790145467),\n",
       " ('tarzan', 6.415122790145467),\n",
       " ('tong', 6.415122790145467)]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "musical_artist_words=[] # will contain a list of words of musicians \n",
    "not_musical_artist_words=[] # will contain a list of words of non musicians \n",
    "count_musicians=0 # amount of musical artists on the dataset\n",
    "count_non_musicians=0 # amount of the rest\n",
    "\n",
    "for row in df.itertuples():\n",
    "    if(isinstance(row[6], str) and row[6]=='MusicalArtist'): # row[6] - type1\n",
    "        musical_artist_words.append(row[3]) # add the twitter_desc to the list\n",
    "        count_musicians+=1\n",
    "    else:\n",
    "        not_musical_artist_words.append(row[3]) # add the twitter_desc to the list\n",
    "        count_non_musicians+=1\n",
    "        \n",
    "# split the twitter_desc to a list of words        \n",
    "musical_artist_words=list_splited_words(musical_artist_words) \n",
    "not_musical_artist_words=list_splited_words(not_musical_artist_words)\n",
    "\n",
    "#---------PMI MUSICIANS--------\n",
    "musical_pmi=PMI(count_musicians, count_non_musicians, musical_artist_words, not_musical_artist_words)\n",
    "highest_pmi_musicians = musical_pmi[0:30]\n",
    "highest_pmi_musicians\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9cb75660",
   "metadata": {},
   "source": [
    "<h3 style='color:red'>Section B:</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "bed6208f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('extravagantly', 8.70364562347373),\n",
       " ('mem', 8.70364562347373),\n",
       " ('undaunted', 8.70364562347373),\n",
       " ('qualifier', 8.70364562347373),\n",
       " ('coloradan', 8.70364562347373),\n",
       " ('elijah', 8.70364562347373),\n",
       " ('granite', 8.70364562347373),\n",
       " ('ohioan', 8.70364562347373),\n",
       " ('governorship', 8.70364562347373),\n",
       " ('casework', 8.70364562347373),\n",
       " ('guelph', 8.70364562347373),\n",
       " ('ringmaster', 8.70364562347373),\n",
       " ('nomination', 8.70364562347373),\n",
       " ('unsettled', 8.70364562347373),\n",
       " ('coopery', 8.70364562347373),\n",
       " ('vota', 8.70364562347373),\n",
       " ('rodeo', 8.70364562347373),\n",
       " ('projection', 8.70364562347373),\n",
       " ('greenwood', 8.70364562347373),\n",
       " ('treasury', 8.70364562347373),\n",
       " ('serfdom', 8.70364562347373),\n",
       " ('lolo', 8.70364562347373),\n",
       " ('granddad', 8.70364562347373),\n",
       " ('councilor', 8.70364562347373),\n",
       " ('ontarian', 8.70364562347373),\n",
       " ('succubus', 8.70364562347373),\n",
       " ('detroiter', 8.70364562347373),\n",
       " ('triumph', 8.70364562347373),\n",
       " ('outrage', 8.70364562347373),\n",
       " ('lieutenant', 8.70364562347373)]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "political_words=[]  # will contain a list of words of politicans \n",
    "not_political_words=[] # will contain a list of words of non politicans\n",
    "count_political=0 # amount of politicans on the dataset\n",
    "count_non_political=0 # amount of the rest\n",
    "\n",
    "for row in df.itertuples():\n",
    "    if(isinstance(row[6], str) and row[6]=='Politician'): # row[6] - type1\n",
    "        political_words.append(row[3]) # add the twitter_desc to the list\n",
    "        count_political+=1\n",
    "    else:\n",
    "        not_political_words.append(row[3]) # add the twitter_desc to the list\n",
    "        count_non_political+=1\n",
    "\n",
    "# split the twitter_desc to a list of words\n",
    "political_words=list_splited_words(political_words)\n",
    "not_political_words=list_splited_words(not_political_words)\n",
    "\n",
    "#---------PMI POLITICANS--------\n",
    "political_pmi=PMI(count_political, count_non_political, political_words, not_political_words)\n",
    "highest_pmi_political = political_pmi[0:30]\n",
    "highest_pmi_political"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74046141",
   "metadata": {},
   "source": [
    "<h1 style='color:red'>Question 4:</h1>\n",
    "    *the lists of positive and negative words were taken from: http://www.cs.uic.edu/~liub/FBS/opinion-lexicon-English.rar\n",
    "    (we took it from Subjectivity and Sentiment Analysis chapter)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d2d12fa2",
   "metadata": {},
   "outputs": [],
   "source": [
    "positive_file = open(\"positive-words.txt\", \"r\") # read the file of positive words \n",
    "  \n",
    "# reading the file\n",
    "data = positive_file.read()\n",
    "  \n",
    "# replacing end splitting the text \n",
    "# when newline ('\\n') is seen.\n",
    "positive_words = data.split(\"\\n\")\n",
    "positive_file.close()\n",
    "\n",
    "negative_file = open(\"negative-words.txt\", \"r\") # read the file of negative words \n",
    "  \n",
    "# reading the file\n",
    "data = negative_file.read()\n",
    "  \n",
    "# replacing end splitting the text \n",
    "# when newline ('\\n') is seen.\n",
    "negative_words = data.split(\"\\n\")\n",
    "negative_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3a274c5a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "most positive sen:\n",
      "\n",
      "('With an intelligent workspace, suddenly work doesn’t feel like work. Citrix Workspace empowers your work style, and gives you more time to do your best work.', 7)\n",
      "('Upwork is the leading online talent solution. We empower businesses with flexible access to quality talent, on demand. Tweet @UpworkHelp for support.', 6)\n",
      "('Robb Report is the manual of modern luxury, synonymous around the world with affluence, luxury, and the best of the best.', 6)\n",
      "(\"We believe that reliable, clean and affordable energy is essential to a brighter, more sustainable future. That's performance that drives progress.\", 6)\n",
      "('Live well. Take care. Do good. \\n\\nWe make stuff that makes life better. ✨ Shop our aisles: plant based wellness, clean beauty, sustainable travel, and more.', 6)\n",
      "(\"Every summer, we present some of the world's most exciting singers in stunning opera productions, offering a mix of beloved classics and thrilling new works.\", 6)\n",
      "('A husband, father, actor, director, and a climate change advocate with an eye out for on a better, brighter, cleaner, more hopeful future for all of us.', 5)\n",
      "('Actively compassionate advocate for freedom, equity and joy, culture of beauty expander...and CEO/founder of @PatternBeauty! 💛', 5)\n",
      "('Being a father is my pride & joy, & playing basketball is my dream coming true...\\nHungry, humble & grateful.\\nGod is good.\\n907 AK', 5)\n",
      "('I love Jesus. I love Scripture. I like people. I like diversity. My interactions and follows do not equal endorsements or reflections of doctrinal alignment.', 5)\n",
      "/nmost negative sen:\n",
      "\n",
      "('The rumors of my demise have been greatly exaggerated.\\n\\nI was once that fat youtuber guy that broke shit.', -5)\n",
      "('International Criminal Court (ICC): fighting impunity for war crimes, crimes against humanity, genocide and aggression Follow/RT≠Endorsement FR: @CourPenaleInt', -5)\n",
      "('Fear, doubt, and worry destroys your dreams. I have no fear,doubt, or worries and im living my dream.', -5)\n",
      "(\"Economics professor, quietly writing obscure economic texts for years, until thrust onto the public scene by Europe's inane handling of an inevitable crisis\", -4)\n",
      "('To report a non-urgent crime visit our website (below). For anything urgent or if life is in danger, call 999.  You cannot report crimes here.', -4)\n",
      "('Marxist pig. Liberal fascist. Queer scum. He/Him. YouTube profits off of hate speech. IG: gaywonk https://t.co/VKhcUD6WGQ', -4)\n",
      "('Fleet Street hack and author. Often sarcastic, occasionally right. Fox (n) carnivore of genus vulpes; crafty person; scavenger; (vb) to confuse; to be drunk', -4)\n",
      "('He who fights with monsters might take care lest he thereby become a monster. And if you gaze for long into an abyss, the abyss gazes also into you...', -4)\n",
      "(\"Entrepreneur. Shark tank. Investor. Mishegas. Author. https://t.co/fzdaBg6JpD , Dad, world's worst joke teller\", -3)\n",
      "('my book has been number one for I can’t even count how many weeks straight because of u guys 😍😍 I talk about abuse, molestation, self hatred,etc', -3)\n"
     ]
    }
   ],
   "source": [
    "col_one_list = df['twitter_desc'].tolist() # create a list of twitter_desc \n",
    "rank_list = [] # tuple of (sentence, rank(positive/negative))\n",
    "\n",
    "for sen in col_one_list:\n",
    "    if(isinstance(sen, str)):\n",
    "        rank_list.append((sen, check_sentiment(sen)))\n",
    "        \n",
    "sorted_rank_list = sorted(rank_list, key=lambda tup: tup[1], reverse=True) # sort - most positive first\n",
    "positive_sorted_rank_list = sorted_rank_list[0:10]\n",
    "print(\"most positive sen:\\n\")\n",
    "print(*positive_sorted_rank_list,sep='\\n')\n",
    "\n",
    "sorted_rank_list = sorted(rank_list, key=lambda tup: tup[1], reverse=False)  # sort - most negative first\n",
    "negative_sorted_rank_list = sorted_rank_list[0:10]\n",
    "print(\"/nmost negative sen:\\n\")\n",
    "print(*negative_sorted_rank_list,sep='\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89277b2f",
   "metadata": {},
   "source": [
    "<h1 style='color:red'>Question 5:</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "20c5befd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hebrew: 10   0.0003761519653940192%\n",
      "arabic: 54   0.002031220613127704%\n"
     ]
    }
   ],
   "source": [
    "col_one_list = df['twitter_desc'].tolist() # create a list of twitter_desc\n",
    "count_hebrew=0 # amount of sentences that contain at least one char in hebrew\n",
    "count_arabic=0 # amount of sentences that contain at least one char in arabic\n",
    "\n",
    "for sen in col_one_list:\n",
    "    if(isinstance(sen,str) and re.search(r'[\\u0600-\\u06FF\\u0750-\\u077F]+', sen) is not None): # if contains an arabic char\n",
    "        count_arabic+=1\n",
    "    if(isinstance(sen,str) and re.search(r\"[א-ת]+\", sen) is not None): # if contains a hebrew char\n",
    "        count_hebrew+=1\n",
    "        \n",
    "hebrew_percentage = count_hebrew / len(col_one_list)\n",
    "print(\"hebrew: \"+ str(count_hebrew) + \"   \" + str(hebrew_percentage)+\"%\")\n",
    "\n",
    "arabic_percentage = count_arabic / len(col_one_list)\n",
    "print(\"arabic: \"+str(count_arabic)+ \"   \"+ str(arabic_percentage)+\"%\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5aadd6c6",
   "metadata": {},
   "source": [
    "<h1 style='color:red'>Question 6:</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "7029fb28",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('🙏', 66),\n",
       " ('🙌', 29),\n",
       " ('😈', 18),\n",
       " ('😎', 17),\n",
       " ('😘', 12),\n",
       " ('🙋', 12),\n",
       " ('😍', 11),\n",
       " ('😊', 11),\n",
       " ('😁', 11),\n",
       " ('🙂', 10)]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a string that contains all the emoticons in the twitter_desc \n",
    "desc_string = \"\" \n",
    "col_one_list = df['twitter_desc'].tolist() # create a list of twitter_desc \n",
    "for sen in col_one_list:\n",
    "    if(isinstance(sen,str)): # only if not NaN\n",
    "        for char in re.findall(r'[\\U0001f600-\\U0001f650]+',sen): # return only emoticons \n",
    "            desc_string=desc_string + str(char)\n",
    "\n",
    "emoji_counter = collections.Counter(desc_string) # count the occurrence of each emoticon \n",
    "emoji_counter.most_common(10)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}