Chapter 1 - Must I Exist.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 1 – Introduction: Must I exist?\n",
    "\n",
    "In the introduction to *Contingent Selves*, I describe the concept of the 'contingent self', and indicate what makes this concept important for the study of Romantic literature.\n",
    "\n",
    "As part of this introduction, I analyse a corpus of academic documents from JSTOR Data for Research, and consider how the word 'self' is used in them. This notebook shows the upper-level code required to reproduce the analysis in the text. More experienced users can investigate the rest of this repository to see the details of the implementation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle as p\n",
    "import os\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "import pandas as pd\n",
    "import nltk\n",
    "\n",
    "from utils import JSTORCorpus, TargetedTrigramAssocFinder, RobustBigramAssocMeasures, RobustTrigramAssocMeasures\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus loaded from data/last-15-years-corpus.p\n"
     ]
    }
   ],
   "source": [
    "DATA_DIR = 'data/'\n",
    "STUB = 'associations-2020-07-07-wn'\n",
    "\n",
    "corpus = JSTORCorpus.load(DATA_DIR + 'last-15-years-corpus.p')\n",
    "\n",
    "analysis = {}\n",
    "\n",
    "for wn in [15,25,35]:\n",
    "    analysis[wn] = {}\n",
    "    for file_name in os.listdir(DATA_DIR + STUB + str(wn)):\n",
    "        with open(DATA_DIR + STUB + str(wn) + '/' + file_name, mode='rb') as file:\n",
    "            analysis[wn][file_name] = p.load(file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The shape of the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unigrams in the corpus: 112,023,237\n",
      "\n",
      "Trigrams with \"self\" and \"romantic\":\n",
      "    types: 4,594\n",
      "    tokens: 12,342\n",
      "\n",
      "Bigrams with \"self\" or \"romantic\":\n",
      "    types: 104,704\n",
      "    tokens: 2,578,060\n",
      "Bigrams with \"self\":\n",
      "    types: 54,157\n",
      "    tokens: 666,241\n",
      "Bigrams with \"romantic\":\n",
      "    types: 43,892\n",
      "    tokens: 548,300\n"
     ]
    }
   ],
   "source": [
    "print(f'Unigrams in the corpus: {analysis[25][\"romantic-self-trigrams.p\"].N:,}\\n')\n",
    "\n",
    "print(f'Trigrams with \"self\" and \"romantic\":')\n",
    "print(f'    types: {analysis[25][\"romantic-self-trigrams.p\"].ngram_fd.B():,}')\n",
    "print(f'    tokens: {analysis[25][\"romantic-self-trigrams.p\"].ngram_fd.N():,}\\n')\n",
    "\n",
    "print(f'Bigrams with \"self\" or \"romantic\":')\n",
    "print(f'    types: {analysis[25][\"romantic-self-trigrams.p\"].bigram_fd.B():,}')\n",
    "print(f'    tokens: {analysis[25][\"romantic-self-trigrams.p\"].bigram_fd.N():,}')\n",
    "print(f'Bigrams with \"self\":')\n",
    "print(f'    types: {analysis[25][\"self-bigrams.p\"].ngram_fd.B():,}')\n",
    "print(f'    tokens: {analysis[25][\"self-bigrams.p\"].ngram_fd.N():,}')\n",
    "print(f'Bigrams with \"romantic\":')\n",
    "print(f'    types: {analysis[25][\"romantic-bigrams.p\"].ngram_fd.B():,}')\n",
    "print(f'    tokens: {analysis[25][\"romantic-bigrams.p\"].ngram_fd.N():,}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus_df = pd.DataFrame.from_dict(corpus.corpus_meta, orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "research-article    6422\n",
       "book-review         3014\n",
       "misc                1732\n",
       "book-chapter         179\n",
       "other                 59\n",
       "review-article        42\n",
       "books-received         3\n",
       "news                   3\n",
       "announcement           2\n",
       "front-matter           2\n",
       "abstract               1\n",
       "obituary               1\n",
       "discussion             1\n",
       "editorial              1\n",
       "Name: type, dtype: int64"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus_df['type'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Studies in Romanticism                      553\n",
       "The Wordsworth Circle                       551\n",
       "The Modern Language Review                  295\n",
       "Keats-Shelley Journal                       263\n",
       "PMLA                                        234\n",
       "Nineteenth-Century Literature               220\n",
       "Victorian Studies                           169\n",
       "The Review of English Studies               166\n",
       "ELH                                         153\n",
       "The Slavic and East European Journal        151\n",
       "The German Quarterly                        145\n",
       "Studies in English Literature, 1500-1900    144\n",
       "Modern Philology                            143\n",
       "German Studies Review                       140\n",
       "Modern Language Review                      135\n",
       "Name: journal, dtype: int64"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus_df['journal'].value_counts()[:15]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2001    881\n",
       "2009    850\n",
       "2002    847\n",
       "2008    827\n",
       "2005    818\n",
       "2013    817\n",
       "2003    808\n",
       "2011    806\n",
       "2007    803\n",
       "2004    780\n",
       "2006    769\n",
       "2012    764\n",
       "2010    750\n",
       "2014    487\n",
       "2015    455\n",
       "Name: year, dtype: int64"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus_df['year'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11462"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus_df.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>title</th>\n",
       "      <th>year</th>\n",
       "      <th>journal</th>\n",
       "      <th>part-of</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>data/ocr/journal-article-10.2307_1261338.txt</td>\n",
       "      <td>misc</td>\n",
       "      <td>Divisions and Discussion Groups</td>\n",
       "      <td>2003</td>\n",
       "      <td>PMLA</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>data/ocr/journal-article-10.2307_41468101.txt</td>\n",
       "      <td>misc</td>\n",
       "      <td>Notes on Contributors</td>\n",
       "      <td>2010</td>\n",
       "      <td>The Eighteenth Century</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>data/ocr/journal-article-10.2307_41319897.txt</td>\n",
       "      <td>misc</td>\n",
       "      <td>Back Matter</td>\n",
       "      <td>2011</td>\n",
       "      <td>Studies in the Novel</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>data/ocr/journal-article-10.2307_25601599.txt</td>\n",
       "      <td>misc</td>\n",
       "      <td>Front Matter</td>\n",
       "      <td>2003</td>\n",
       "      <td>Studies in Romanticism</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>data/ocr/journal-article-10.2307_3251788.txt</td>\n",
       "      <td>misc</td>\n",
       "      <td>Volume Information</td>\n",
       "      <td>2001</td>\n",
       "      <td>MLN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>data/ocr/journal-article-10.2307_41058287.txt</td>\n",
       "      <td>misc</td>\n",
       "      <td>Front Matter</td>\n",
       "      <td>2010</td>\n",
       "      <td>PMLA</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>data/ocr/journal-article-10.2307_3593540.txt</td>\n",
       "      <td>misc</td>\n",
       "      <td>Back Matter</td>\n",
       "      <td>2004</td>\n",
       "      <td>Contemporary Literature</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>data/ocr/journal-article-10.2307_25833909.txt</td>\n",
       "      <td>misc</td>\n",
       "      <td>Back Matter</td>\n",
       "      <td>2004</td>\n",
       "      <td>The Year's Work in Modern Language Studies</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>data/ocr/journal-article-10.2307_23348595.txt</td>\n",
       "      <td>misc</td>\n",
       "      <td>Living in Poetry</td>\n",
       "      <td>2011</td>\n",
       "      <td>Indian Literature</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>data/ocr/journal-article-10.2307_23474940.txt</td>\n",
       "      <td>misc</td>\n",
       "      <td>Back Matter</td>\n",
       "      <td>2012</td>\n",
       "      <td>Italica</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>data/ocr/journal-article-10.2307_40040002.txt</td>\n",
       "      <td>misc</td>\n",
       "      <td>Volume Information</td>\n",
       "      <td>2007</td>\n",
       "      <td>South Central Review</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>data/ocr/journal-article-10.2307_3070601.txt</td>\n",
       "      <td>misc</td>\n",
       "      <td>Volume Information</td>\n",
       "      <td>2001</td>\n",
       "      <td>The Review of English Studies</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>data/ocr/journal-article-10.2307_43973906.txt</td>\n",
       "      <td>misc</td>\n",
       "      <td>Front Matter</td>\n",
       "      <td>2015</td>\n",
       "      <td>Studies in Romanticism</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>data/ocr/journal-article-10.2307_41679744.txt</td>\n",
       "      <td>misc</td>\n",
       "      <td>Front Matter</td>\n",
       "      <td>2012</td>\n",
       "      <td>Shakespeare Quarterly</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>data/ocr/journal-article-10.2307_25602187.txt</td>\n",
       "      <td>misc</td>\n",
       "      <td>Back Matter</td>\n",
       "      <td>2009</td>\n",
       "      <td>Studies in Romanticism</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>data/ocr/journal-article-10.2307_23402965.txt</td>\n",
       "      <td>misc</td>\n",
       "      <td>Notes from Concord</td>\n",
       "      <td>2009</td>\n",
       "      <td>The Thoreau Society Bulletin</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>data/ocr/journal-article-10.2307_41616789.txt</td>\n",
       "      <td>misc</td>\n",
       "      <td>Back Matter</td>\n",
       "      <td>2011</td>\n",
       "      <td>PMLA</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>data/ocr/journal-article-10.2307_24247245.txt</td>\n",
       "      <td>misc</td>\n",
       "      <td>Back Matter</td>\n",
       "      <td>2012</td>\n",
       "      <td>Studies in Romanticism</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>data/ocr/journal-article-10.2307_25602163.txt</td>\n",
       "      <td>misc</td>\n",
       "      <td>Front Matter</td>\n",
       "      <td>2008</td>\n",
       "      <td>Studies in Romanticism</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>data/ocr/journal-article-10.2307_27784790.txt</td>\n",
       "      <td>misc</td>\n",
       "      <td>Back Matter</td>\n",
       "      <td>2008</td>\n",
       "      <td>South Atlantic Review</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               type  \\\n",
       "data/ocr/journal-article-10.2307_1261338.txt   misc   \n",
       "data/ocr/journal-article-10.2307_41468101.txt  misc   \n",
       "data/ocr/journal-article-10.2307_41319897.txt  misc   \n",
       "data/ocr/journal-article-10.2307_25601599.txt  misc   \n",
       "data/ocr/journal-article-10.2307_3251788.txt   misc   \n",
       "data/ocr/journal-article-10.2307_41058287.txt  misc   \n",
       "data/ocr/journal-article-10.2307_3593540.txt   misc   \n",
       "data/ocr/journal-article-10.2307_25833909.txt  misc   \n",
       "data/ocr/journal-article-10.2307_23348595.txt  misc   \n",
       "data/ocr/journal-article-10.2307_23474940.txt  misc   \n",
       "data/ocr/journal-article-10.2307_40040002.txt  misc   \n",
       "data/ocr/journal-article-10.2307_3070601.txt   misc   \n",
       "data/ocr/journal-article-10.2307_43973906.txt  misc   \n",
       "data/ocr/journal-article-10.2307_41679744.txt  misc   \n",
       "data/ocr/journal-article-10.2307_25602187.txt  misc   \n",
       "data/ocr/journal-article-10.2307_23402965.txt  misc   \n",
       "data/ocr/journal-article-10.2307_41616789.txt  misc   \n",
       "data/ocr/journal-article-10.2307_24247245.txt  misc   \n",
       "data/ocr/journal-article-10.2307_25602163.txt  misc   \n",
       "data/ocr/journal-article-10.2307_27784790.txt  misc   \n",
       "\n",
       "                                                                         title  \\\n",
       "data/ocr/journal-article-10.2307_1261338.txt   Divisions and Discussion Groups   \n",
       "data/ocr/journal-article-10.2307_41468101.txt            Notes on Contributors   \n",
       "data/ocr/journal-article-10.2307_41319897.txt                      Back Matter   \n",
       "data/ocr/journal-article-10.2307_25601599.txt                     Front Matter   \n",
       "data/ocr/journal-article-10.2307_3251788.txt                Volume Information   \n",
       "data/ocr/journal-article-10.2307_41058287.txt                     Front Matter   \n",
       "data/ocr/journal-article-10.2307_3593540.txt                       Back Matter   \n",
       "data/ocr/journal-article-10.2307_25833909.txt                      Back Matter   \n",
       "data/ocr/journal-article-10.2307_23348595.txt                 Living in Poetry   \n",
       "data/ocr/journal-article-10.2307_23474940.txt                      Back Matter   \n",
       "data/ocr/journal-article-10.2307_40040002.txt               Volume Information   \n",
       "data/ocr/journal-article-10.2307_3070601.txt                Volume Information   \n",
       "data/ocr/journal-article-10.2307_43973906.txt                     Front Matter   \n",
       "data/ocr/journal-article-10.2307_41679744.txt                     Front Matter   \n",
       "data/ocr/journal-article-10.2307_25602187.txt                      Back Matter   \n",
       "data/ocr/journal-article-10.2307_23402965.txt               Notes from Concord   \n",
       "data/ocr/journal-article-10.2307_41616789.txt                      Back Matter   \n",
       "data/ocr/journal-article-10.2307_24247245.txt                      Back Matter   \n",
       "data/ocr/journal-article-10.2307_25602163.txt                     Front Matter   \n",
       "data/ocr/journal-article-10.2307_27784790.txt                      Back Matter   \n",
       "\n",
       "                                               year  \\\n",
       "data/ocr/journal-article-10.2307_1261338.txt   2003   \n",
       "data/ocr/journal-article-10.2307_41468101.txt  2010   \n",
       "data/ocr/journal-article-10.2307_41319897.txt  2011   \n",
       "data/ocr/journal-article-10.2307_25601599.txt  2003   \n",
       "data/ocr/journal-article-10.2307_3251788.txt   2001   \n",
       "data/ocr/journal-article-10.2307_41058287.txt  2010   \n",
       "data/ocr/journal-article-10.2307_3593540.txt   2004   \n",
       "data/ocr/journal-article-10.2307_25833909.txt  2004   \n",
       "data/ocr/journal-article-10.2307_23348595.txt  2011   \n",
       "data/ocr/journal-article-10.2307_23474940.txt  2012   \n",
       "data/ocr/journal-article-10.2307_40040002.txt  2007   \n",
       "data/ocr/journal-article-10.2307_3070601.txt   2001   \n",
       "data/ocr/journal-article-10.2307_43973906.txt  2015   \n",
       "data/ocr/journal-article-10.2307_41679744.txt  2012   \n",
       "data/ocr/journal-article-10.2307_25602187.txt  2009   \n",
       "data/ocr/journal-article-10.2307_23402965.txt  2009   \n",
       "data/ocr/journal-article-10.2307_41616789.txt  2011   \n",
       "data/ocr/journal-article-10.2307_24247245.txt  2012   \n",
       "data/ocr/journal-article-10.2307_25602163.txt  2008   \n",
       "data/ocr/journal-article-10.2307_27784790.txt  2008   \n",
       "\n",
       "                                                                                  journal  \\\n",
       "data/ocr/journal-article-10.2307_1261338.txt                                         PMLA   \n",
       "data/ocr/journal-article-10.2307_41468101.txt                      The Eighteenth Century   \n",
       "data/ocr/journal-article-10.2307_41319897.txt                        Studies in the Novel   \n",
       "data/ocr/journal-article-10.2307_25601599.txt                      Studies in Romanticism   \n",
       "data/ocr/journal-article-10.2307_3251788.txt                                          MLN   \n",
       "data/ocr/journal-article-10.2307_41058287.txt                                        PMLA   \n",
       "data/ocr/journal-article-10.2307_3593540.txt                      Contemporary Literature   \n",
       "data/ocr/journal-article-10.2307_25833909.txt  The Year's Work in Modern Language Studies   \n",
       "data/ocr/journal-article-10.2307_23348595.txt                           Indian Literature   \n",
       "data/ocr/journal-article-10.2307_23474940.txt                                     Italica   \n",
       "data/ocr/journal-article-10.2307_40040002.txt                        South Central Review   \n",
       "data/ocr/journal-article-10.2307_3070601.txt                The Review of English Studies   \n",
       "data/ocr/journal-article-10.2307_43973906.txt                      Studies in Romanticism   \n",
       "data/ocr/journal-article-10.2307_41679744.txt                       Shakespeare Quarterly   \n",
       "data/ocr/journal-article-10.2307_25602187.txt                      Studies in Romanticism   \n",
       "data/ocr/journal-article-10.2307_23402965.txt                The Thoreau Society Bulletin   \n",
       "data/ocr/journal-article-10.2307_41616789.txt                                        PMLA   \n",
       "data/ocr/journal-article-10.2307_24247245.txt                      Studies in Romanticism   \n",
       "data/ocr/journal-article-10.2307_25602163.txt                      Studies in Romanticism   \n",
       "data/ocr/journal-article-10.2307_27784790.txt                       South Atlantic Review   \n",
       "\n",
       "                                              part-of  \n",
       "data/ocr/journal-article-10.2307_1261338.txt      NaN  \n",
       "data/ocr/journal-article-10.2307_41468101.txt     NaN  \n",
       "data/ocr/journal-article-10.2307_41319897.txt     NaN  \n",
       "data/ocr/journal-article-10.2307_25601599.txt     NaN  \n",
       "data/ocr/journal-article-10.2307_3251788.txt      NaN  \n",
       "data/ocr/journal-article-10.2307_41058287.txt     NaN  \n",
       "data/ocr/journal-article-10.2307_3593540.txt      NaN  \n",
       "data/ocr/journal-article-10.2307_25833909.txt     NaN  \n",
       "data/ocr/journal-article-10.2307_23348595.txt     NaN  \n",
       "data/ocr/journal-article-10.2307_23474940.txt     NaN  \n",
       "data/ocr/journal-article-10.2307_40040002.txt     NaN  \n",
       "data/ocr/journal-article-10.2307_3070601.txt      NaN  \n",
       "data/ocr/journal-article-10.2307_43973906.txt     NaN  \n",
       "data/ocr/journal-article-10.2307_41679744.txt     NaN  \n",
       "data/ocr/journal-article-10.2307_25602187.txt     NaN  \n",
       "data/ocr/journal-article-10.2307_23402965.txt     NaN  \n",
       "data/ocr/journal-article-10.2307_41616789.txt     NaN  \n",
       "data/ocr/journal-article-10.2307_24247245.txt     NaN  \n",
       "data/ocr/journal-article-10.2307_25602163.txt     NaN  \n",
       "data/ocr/journal-article-10.2307_27784790.txt     NaN  "
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus_df[corpus_df['type'] == 'misc'].sample(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stopwords?\n",
    "\n",
    "What happens if we filter for stopwords?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "english = stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stopword_filter(*ngram):\n",
    "    \"\"\"Returns true if the ngram contains junk or a stopword\"\"\"\n",
    "\n",
    "    sw = set(stopwords.words('english'))\n",
    "\n",
    "    if not sw.isdisjoint(ngram):\n",
    "        return True\n",
    "    elif any([re.match(r'[\\W\\d]+', wd) for wd in ngram]):\n",
    "        return True\n",
    "    else:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "for wn, folder in analysis.items():\n",
    "    for finder in folder.values():\n",
    "        finder.apply_ngram_filter(stopword_filter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "rom_self_trigrams.apply_ngram_filter(stopword_filter)\n",
    "romantic_bigrams.apply_ngram_filter(stopword_filter)\n",
    "self_bigrams.apply_ngram_filter(stopword_filter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "rom_scores = analysis[25]['romantic-bigrams.p'].score_ngrams(RobustBigramAssocMeasures.likelihood_ratio)\n",
    "self_scores = analysis[25]['self-bigrams.p'].score_ngrams(RobustBigramAssocMeasures.likelihood_ratio)\n",
    "rom_self_scores = analysis[25]['romantic-self-trigrams.p'].score_ngrams(RobustTrigramAssocMeasures.likelihood_ratio)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rnk: ? + romantic            : ? + self                  : ? + romantic & self\n",
      "1  : romantic        829,835 : self            1,049,517 : self           1,601,037\n",
      "2  : period           27,522 : conscious          19,889 : romantic       1,262,582\n",
      "3  : poetry           22,248 : consciousness      18,869 : period            65,069\n",
      "4  : era              18,052 : one                16,113 : poetry            61,877\n",
      "5  : romanticism      15,009 : romantic           14,979 : one               57,504\n",
      "6  : self             14,979 : identity           10,942 : conscious         52,548\n",
      "7  : literature       14,803 : consciously        10,924 : consciousness     52,490\n",
      "8  : poets            13,701 : sense              10,074 : era               50,331\n",
      "9  : british          12,709 : also                9,845 : literature        48,807\n",
      "10 : literary         12,026 : world               9,711 : romanticism       47,934\n",
      "11 : writers          11,453 : reflexive           9,175 : literary          47,506\n",
      "12 : culture           9,425 : subject             7,630 : also              47,038\n",
      "13 : english           9,136 : writing             7,552 : poets             44,775\n",
      "14 : tradition         9,034 : nature              7,311 : nature            43,910\n",
      "15 : wordsworth        8,367 : fashioning          7,181 : writing           43,630\n",
      "16 : imagination       7,894 : life                6,640 : british           42,584\n",
      "17 : century           7,721 : knowledge           6,561 : culture           41,884\n",
      "18 : cambridge         7,687 : interest            6,494 : writers           41,771\n",
      "19 : new               7,655 : individual          6,371 : sense             41,518\n",
      "20 : university        7,172 : reliance            6,268 : new               41,509\n",
      "21 : one               7,170 : representation      6,217 : world             41,208\n",
      "22 : nature            7,123 : form                6,049 : identity          40,859\n",
      "23 : love              6,971 : rather              6,021 : love              39,641\n",
      "24 : rev               6,809 : social              5,854 : poet              39,513\n",
      "25 : poet              6,799 : even                5,814 : consciously       39,032\n",
      "26 : novel             6,615 : expression          5,712 : century           38,755\n",
      "27 : history           6,601 : thus                5,589 : wordsworth        38,681\n",
      "28 : also              6,517 : image               5,563 : life              38,601\n",
      "29 : writing           6,493 : awareness           5,546 : work              38,574\n",
      "30 : studies           6,294 : narrative           5,460 : tradition         38,487\n"
     ]
    }
   ],
   "source": [
    "out = \"rank,romantic_bg,score,self_bg,score,rom_self_tg,score\\n\"\n",
    "print(\"rnk: ? + romantic            : ? + self                  : ? + romantic & self\")\n",
    "for idx, (rom, self, rom_self) in enumerate(zip(rom_scores[:30], self_scores[:30], rom_self_scores[:30])):\n",
    "    \n",
    "    rank = idx + 1\n",
    "    \n",
    "    rom_bg, r_sc = rom\n",
    "    self_bg, s_sc = self\n",
    "    trigram, t_sc = rom_self\n",
    "    \n",
    "    r_wd = rom_bg[0]\n",
    "    s_wd = self_bg[0]\n",
    "    t_wd = trigram[0]\n",
    "    \n",
    "    out += f\"{rank},{r_wd},{r_sc},{s_wd},{s_sc},{t_wd},{t_sc}\\n\"\n",
    "    \n",
    "    print(f\"{rank:<2} : {r_wd:<11} {r_sc:>11,.0f} : {s_wd:<14} {s_sc:>10,.0f} : {t_wd:<13} {t_sc:>10,.0f}\")\n",
    "\n",
    "with open('data/association_analysis_dump.csv', 'wt') as file:\n",
    "    file.write(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rank_word(scores, word):\n",
    "    for rank, (ngram, score) in enumerate(scores):\n",
    "        if word in ngram:\n",
    "            return rank, score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1563, 23265.226372689973)"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rank_word(rom_self_scores, 'necessity')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# What words are excluded in the context of 'romantic' and 'self'\n",
    "\n",
    "It appears that several words, including 'abnegation' and 'contingent', never occur when both 'romantic' and 'self' are within 14 words either side. Are there other such words?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_rom_scores = []\n",
    "for ngram, score in rom_scores:\n",
    "    if (ngram[0],'self','romantic') not in rom_self_trigrams.ngram_fd:\n",
    "        filtered_rom_scores.append((ngram, score))\n",
    "\n",
    "filtered_self_scores = []\n",
    "for ngram, score in self_scores:\n",
    "    if (ngram[0],'self','romantic') not in rom_self_trigrams.ngram_fd:\n",
    "        filtered_self_scores.append((ngram, score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0  : abrams               3,086.69 : cannot               2,650.72\n",
      "1  : circles              2,878.80 : preservation         2,531.96\n",
      "2  : edited               1,718.78 : denial               2,085.75\n",
      "3  : michael              1,657.13 : make                 2,004.17\n",
      "4  : ecology              1,484.96 : interested           1,819.75\n",
      "5  : scholars             1,469.87 : discipline           1,817.74\n",
      "6  : melancholy           1,442.19 : effacing             1,784.51\n",
      "7  : praxis               1,426.95 : esteem               1,690.77\n",
      "8  : writings             1,354.74 : respect              1,621.89\n",
      "9  : trumpener            1,244.66 : styled               1,531.31\n",
      "10 : companion            1,234.64 : envelope             1,342.85\n",
      "11 : err                  1,212.37 : things               1,297.58\n",
      "12 : anthology            1,195.80 : boundaries           1,283.58\n",
      "13 : scholarship          1,185.10 : addressed            1,233.29\n",
      "14 : bate                 1,185.04 : abnegation           1,223.02\n",
      "15 : baltimore            1,171.96 : psychological        1,192.15\n",
      "16 : make                 1,166.87 : hatred               1,183.38\n",
      "17 : cornell              1,110.91 : display              1,129.27\n",
      "18 : ithaca               1,105.77 : government           1,122.05\n",
      "19 : manifesto            1,088.71 : narcissism           1,108.46\n",
      "20 : rc                   1,040.76 : requires             1,057.42\n",
      "21 : comedy               1,026.17 : regulating           1,038.24\n",
      "22 : katie                1,021.49 : quite                1,031.65\n",
      "23 : elfenbein            1,010.86 : ability              1,022.40\n",
      "24 : tim                    982.03 : congratulatory       1,019.02\n",
      "25 : mclane                 962.75 : face                 1,006.42\n",
      "26 : leask                  948.41 : appointed              990.55\n",
      "27 : journal                944.68 : mere                   963.49\n",
      "28 : morton                 928.63 : stable                 962.17\n",
      "29 : esterhammer            926.61 : demands                960.80\n",
      "30 : edinburgh              925.04 : identical              932.60\n",
      "31 : sir                    922.35 : false                  929.30\n",
      "32 : sociability            920.46 : audience               927.63\n",
      "33 : supernaturalism        914.28 : censorship             901.52\n",
      "34 : godwin                 909.03 : identities             870.67\n",
      "35 : williams               896.55 : portraiture            867.03\n",
      "36 : spanish                871.15 : unconscious            865.44\n",
      "37 : exotic                 870.13 : satisfied              849.77\n",
      "38 : rajan                  853.16 : capable                848.47\n",
      "39 : prophecy               851.97 : righteous              846.35\n",
      "40 : italy                  851.79 : sustaining             844.48\n",
      "41 : throughout             846.38 : show                   834.34\n",
      "42 : sobriety               836.13 : line                   833.91\n",
      "43 : lamp                   823.45 : increasingly           833.42\n",
      "44 : ferris                 817.76 : entity                 825.95\n",
      "45 : things                 805.61 : psychology             811.40\n",
      "46 : jonathan               793.57 : inflicted              806.78\n",
      "47 : artists                789.64 : deprecation            806.47\n",
      "48 : alan                   784.39 : auto                   804.66\n",
      "49 : posterity              781.53 : writings               790.86\n"
     ]
    }
   ],
   "source": [
    "for idx, (rom, self) in enumerate(zip(filtered_rom_scores[:50], filtered_self_scores[:50])):\n",
    "    rom_bg, r_sc = rom\n",
    "    self_bg, s_sc = self\n",
    "    \n",
    "    r_wd = rom_bg[0]\n",
    "    s_wd = self_bg[0]\n",
    "    \n",
    "    print(f\"{idx:<2} : {r_wd:<15} {r_sc:>13,.2f} : {s_wd:<15} {s_sc:>13,.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "foo = [n for n in self_bigrams.score_ngram(w1='romantic',w2='self', score_fn=RobustBigramAssocMeasures._contingency)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2378.0, 64569.0, 48458.0, 120826232.0]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "foo"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}