Topic Modelling.ipynb

{
  "cells": [
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "!pip install pyLDAvis\n",
        "#Import the necessary packages\n",
        "import pickle\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from datetime import datetime\n",
        "import nltk\n",
        "nltk.download('wordnet')\n",
        "nltk.download('punkt')\n",
        "nltk.download('stopwords')\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.tokenize import sent_tokenize\n",
        "from nltk.stem import WordNetLemmatizer, SnowballStemmer\n",
        "from nltk.corpus import stopwords\n",
        "import gensim\n",
        "from gensim.utils import simple_preprocess\n",
        "from gensim.corpora.dictionary import Dictionary\n",
        "import gensim.corpora as corpora\n",
        "from gensim.models import CoherenceModel\n",
        "from gensim.models.ldamodel import LdaModel\n",
        "#import spacy\n",
        "import string\n",
        "import re\n",
        "import pyLDAvis\n",
        "import pyLDAvis.gensim\n",
        "import matplotlib.pyplot as plt\n",
        "import matplotlib.font_manager\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Load the krebs dataset (saved as a pickle file) into a dataframe df\n",
        "df = pickle.load(open('krebs_dataset.pickle', 'rb'))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Inspect the dataframe\n",
        "df.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Set the index to the date column\n",
        "df = df.set_index('Date')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "df.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "**Text Preprocessing**"
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Convert each article from a list into strings\n",
        "df['Body'] = df['Body'].apply(', '.join)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Define a function that removes urls from text using regex\n",
        "def remove_urls(text):\n",
        "    URLess_string = re.sub(r'\\w+:\\/{2}[\\d\\w-]+(\\.[\\d\\w-]+)*(?:(?:\\/[^\\s/]*))*', '', text)\n",
        "    return URLess_string"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Define a function that removes punctuation from the text which may impact the model's ability to extract topics\n",
        "def remove_punctuation(text):\n",
        "    no_punct = re.sub(r'[^\\w\\s]', '', text)\n",
        "    return no_punct"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "#I treate the Body and Title series separately because their text produce different results when the LDA model is applied later."
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Apply the remove url function to the body series\n",
        "df['Body'] = df['Body'].apply(lambda x: remove_urls(x))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Apply the remove punctuation function to the body series\n",
        "df['Body'] = df['Body'].apply(lambda x: remove_punctuation(x))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Apply the remove url function to the Title series\n",
        "df['Title'] = df['Title'].apply(lambda x: remove_urls(x))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Apply the remove punctuation function to the Title series\n",
        "df['Title'] = df['Title'].apply(lambda x: remove_punctuation(x))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "df.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Define a function that takes input string, converts it all to lower case then tokenizes it using nltk.tokenize\n",
        "def word_tokenizer(text):\n",
        "    tokens = nltk.word_tokenize(text.lower())\n",
        "    return tokens"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Apply the tokenizer function to the entire Body series\n",
        "df.Body = df.Body.apply(lambda x: word_tokenizer(x))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Apply the tokenizer function to the Title series\n",
        "df['Title'] = df['Title'].apply(lambda x: word_tokenizer(x))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "df.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Create a function which removes stopwords using nltk's stop word feature\n",
        "all_stopwords = stopwords.words('english')\n",
        "new_stopwords = [\"krebsonsecurity\", \"krebs\",\"security\", \"cyber\", \"\\'\", \"\"\" \" \"\"\", \"-\", \"one\", \"get\", \"like\", \"may\", \"would\", \"said\", \"`\", \"\\\"\", \"...\", 'krebsonsecuritycom']\n",
        "all_stopwords.extend(new_stopwords)\n",
        "def remove_stopwords(text):\n",
        "    words = [w for w in text if w not in all_stopwords]\n",
        "    return words"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Apply the stopwords function to the entire Body series\n",
        "df.Body = df.Body.apply(lambda x: remove_stopwords(x))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Apply the tokenizer function to the Title  series\n",
        "df['Title'] = df['Title'].apply(lambda x: remove_stopwords(x))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Create a function which removes lemmas using nltk's stop word feature\n",
        "lemmatizer = WordNetLemmatizer()\n",
        "def word_lemmatizer(text):\n",
        "    lem_text = [lemmatizer.lemmatize(i) for i in text]\n",
        "    return lem_text"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Apply the lemmas function to the entire Body series\n",
        "df.Body = df.Body.apply(lambda x: word_lemmatizer(x))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Apply the lemmas function to the Title  series\n",
        "df['Title'] = df['Title'].apply(lambda x: word_lemmatizer(x))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Save the preprocessed data to a pickle file: 'df_preprocessed.pickle'\n",
        "with open('df_preprocessed.pickle', 'wb') as f:\n",
        "    pickle.dump(df, f)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "df.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "# Create a dictionary of all unique tokens by passing df.Body to the Dictionary method\n",
        "dictionary = corpora.Dictionary(df.Body)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Each unique token is mapped to an id number e.g. 'chip': 140\n",
        "#dictionary.token2id"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Turn the dictionary into a corpus (a Bag of Words) that contains the word id and its frequency in each document (article in our case)\n",
        "texts = df.Body\n",
        "corpus = [dictionary.doc2bow(text) for text in texts]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "# Create a dictionary of all unique tokens by passing df.Title to the Dictionary method\n",
        "dictionary_from_titles = corpora.Dictionary(df.Title)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "#Turn the dictionary into a corpus (a Bag of Words) that contains the word id and its frequency in each document (article in our case)\n",
        "texts_from_titles = df.Title\n",
        "corpus_from_titles = [dictionary_from_titles.doc2bow(text) for text in texts_from_titles]"
      ]
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#We can now see the first 10 word ids with their frequency counts from the fifth document\n",
        "#corpus[4][:10]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#You can produce the corpus in human readable format for the first article\n",
        "#[[(dictionary[id], freq) for id, freq in cp] for cp in corpus[:1]]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Create an LDA model from the Body corpus which extracts 10 topics\n",
        "lda_model = gensim.models.ldamodel.LdaModel(corpus=corpus, id2word=dictionary, num_topics=10, random_state=42, update_every=1, chunksize=100, passes=10, alpha='auto', per_word_topics=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "lda_model.print_topics()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Create an LDA model from the Title corpus which extracts 10 topics\n",
        "lda_model_from_titles = gensim.models.ldamodel.LdaModel(corpus=corpus_from_titles, id2word=dictionary_from_titles, num_topics=30, random_state=42, update_every=1, chunksize=100, passes=10, alpha='auto', per_word_topics=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#print each topic\n",
        "for i, topic in enumerate(lda_model_from_titles.print_topics(14)):\n",
        "    print('{} --- {}'.format(i, topic))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "print('\\nPerplexity for Corpus from Body: ', lda_model.log_perplexity(corpus) )\n",
        "print('\\nPerplexity for Corpus from Titles: ', lda_model_from_titles.log_perplexity(corpus_from_titles) )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "coherence_model_lda_body = CoherenceModel(model=lda_model, texts=df.Body, dictionary=dictionary, coherence='c_v')\n",
        "coherence_lda_body = coherence_model_lda_body.get_coherence()\n",
        "print('\\nCoherence score from Body: ', coherence_lda_body)\n",
        "\n",
        "coherence_model_lda_titles = CoherenceModel(model=lda_model_from_titles, texts=df.Title, dictionary=dictionary_from_titles, coherence='c_v')\n",
        "coherence_lda_titles = coherence_model_lda_titles.get_coherence()\n",
        "print('\\nCoherence score from Titles: ', coherence_lda_titles)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "def compute_coherence_values(dictionary, corpus, texts, limit, start=2, step=3):\n",
        "    coherence_values = []\n",
        "    model_list = []\n",
        "    for num_topics in range(start, limit, step):\n",
        "        model = gensim.models.ldamodel.LdaModel(corpus=corpus, id2word=dictionary, num_topics=num_topics, random_state=42, update_every=1, chunksize=100, passes=10, alpha='auto', per_word_topics=True)\n",
        "        model_list.append(model)\n",
        "        coherence_model = CoherenceModel(model=model, texts=texts, dictionary=dictionary, coherence='c_v')\n",
        "        coherence_values.append(coherence_model.get_coherence())\n",
        "        \n",
        "    return model_list, coherence_values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "model_list_titles, coherence_values_titles = compute_coherence_values(dictionary=dictionary_from_titles, corpus=corpus_from_titles, texts=df.Title, limit=100, start=2, step=6)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "limit=100; start=2; step=6;\n",
        "x = range(start, limit, step)\n",
        "plt.plot(x, coherence_values_titles, 'bo--', linewidth=2, markersize=6, alpha=0.5)\n",
        "plt.xlabel(\"Num Topics\")\n",
        "plt.ylabel(\"Coherence Value\")\n",
        "plt.legend((\"coherence_values\"), loc='best')\n",
        "plt.title('Coherence Value as a function of topic numbers\\n from Krebsonsecurity Title Corpus')\n",
        "plt.show()\n",
        "plt.savefig('Coherence_values_titles')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "for m, cv in zip(x, coherence_values_titles):\n",
        "    print('Num Topics =', m, 'has Coherence ValueErrorlue of', round(cv,4))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "pyLDAvis.enable_notebook()\n",
        "vis = pyLDAvis.gensim.prepare(lda_model_from_titles, corpus_from_titles, dictionary_from_titles)\n",
        "vis"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "pyLDAvis.enable_notebook()\n",
        "vis = pyLDAvis.gensim.prepare(lda_model, corpus, dictionary)\n",
        "vis"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Save the lda model from body to a pickle file\n",
        "with open('lda_body.pickle', 'wb') as f:\n",
        "    pickle.dump(lda_model, f)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true
      },
      "cell_type": "code",
      "source": [
        "#Save the lda model from titles to a pickle file\n",
        "with open('lda_titles.pickle', 'wb') as f:\n",
        "    pickle.dump(lda_model_from_titles, f)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "name": "python36",
      "display_name": "Python 3.6",
      "language": "python"
    },
    "language_info": {
      "mimetype": "text/x-python",
      "nbconvert_exporter": "python",
      "name": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.6",
      "file_extension": ".py",
      "codemirror_mode": {
        "version": 3,
        "name": "ipython"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}