1.AnalyzeData_Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, sys, glob, csv\n",
    "# os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"\n",
    "# The GPU id to use, usually either \"0\" or \"1\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"2\" \n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import shutil\n",
    "import pickle\n",
    "import tqdm\n",
    "from tqdm import tqdm\n",
    "import xlsxwriter\n",
    "import random\n",
    "from random import shuffle\n",
    "from math import log, floor\n",
    "import re\n",
    "import collections\n",
    "from collections import Counter\n",
    "import string\n",
    "import unicodedata as udata\n",
    "# import pause, datetime\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from pathlib import Path\n",
    "from distutils.dir_util import copy_tree\n",
    "import sklearn\n",
    "from sklearn.metrics import *\n",
    "import itertools as it\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import wordpunct_tokenize\n",
    "from nltk import word_tokenize\n",
    "from nltk.stem.porter import *\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from collections import Counter\n",
    "import functools\n",
    "import spacy\n",
    "\n",
    "import tensorflow.keras.preprocessing.text as T\n",
    "import torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Prepare preprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2196009"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sp_tokenizer = spacy.load(\"en\")\n",
    "dil= r\"[!\\\"#$%&()*+,\\-\\./:;<\\=>?\\@[\\\\]\\^_`{|}~\\º]+\\ *\" \n",
    "dil_filter = 'ºÑ!\\\"#$%&()*+,-./:;<=>?@[\\\\]^_\\`{|}~\\t\\n'\n",
    "glove = pd.read_table('./data/preprocess/glove.840B.300d.txt', sep=\" \", index_col=0, header=None, quoting=csv.QUOTE_NONE)\n",
    "glove_words = set(glove.index.tolist())\n",
    "len(glove_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def text_preprcess2(text):\n",
    "    '''\n",
    "    Goal: filter out punctuations、words not in GloVe\n",
    "    * cased\n",
    "    * spacy\n",
    "    * for RoBERTa + GloVe embedding\n",
    "    Input: text string\n",
    "    Return: preprcessed text string\n",
    "    '''\n",
    "    tokens = [i for i in text if i not in dil_filter] #濾掉符號\n",
    "    tokens = ''.join(tokens)\n",
    "    tokens = sp_tokenizer(tokens)\n",
    "    token_li = []\n",
    "    for token in tokens:\n",
    "        token_li.append(token.text)\n",
    "    tokens_li = [i for i in token_li if i in glove_words] #只留有在glove的字\n",
    "    tokens = list(filter(None, tokens_li))\n",
    "    tokens_str = ' '.join(tokens)\n",
    "    tokens_str = re.sub(dil,\" \",tokens_str) #在確認濾一次符號\n",
    "    return tokens_str\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/leoqaz12/.cache/torch/hub/pytorch_fairseq_master\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading archive file http://dl.fbaipublicfiles.com/fairseq/models/roberta.large.tar.gz from cache at /home/leoqaz12/.cache/torch/pytorch_fairseq/83e3a689e28e5e4696ecb0bbb05a77355444a5c8a3437e0f736d8a564e80035e.c687083d14776c1979f3f71654febb42f2bb3d9a94ff7ebdfe1ac6748dba89d2\n",
      "| dictionary: 50264 types\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RobertaHubInterface(\n",
       "  (model): RobertaModel(\n",
       "    (decoder): RobertaEncoder(\n",
       "      (sentence_encoder): TransformerSentenceEncoder(\n",
       "        (embed_tokens): Embedding(50265, 1024, padding_idx=1)\n",
       "        (embed_positions): LearnedPositionalEmbedding(514, 1024, padding_idx=1)\n",
       "        (layers): ModuleList(\n",
       "          (0): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (1): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (2): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (3): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (4): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (5): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (6): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (7): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (8): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (9): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (10): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (11): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (12): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (13): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (14): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (15): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (16): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (17): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (18): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (19): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (20): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (21): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (22): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (23): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "        )\n",
       "        (emb_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (lm_head): RobertaLMHead(\n",
       "        (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "        (layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "    )\n",
       "    (classification_heads): ModuleDict()\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roberta = torch.hub.load('pytorch/fairseq', 'roberta.large')\n",
    "roberta.cuda() #gpu\n",
    "# roberta.cpu()\n",
    "roberta.eval() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/20 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 18857  =====\n",
      "==ERR== 19309  =====\n",
      "==ERR== 19148  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  5%|▌         | 1/20 [00:33<10:37, 33.56s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 21139  =====\n",
      "==ERR== 21056  =====\n",
      "==ERR== 21194  =====\n",
      "==ERR== 21159  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 10%|█         | 2/20 [01:02<09:38, 32.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 21844  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 15%|█▌        | 3/20 [01:34<09:08, 32.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 22811  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 20%|██        | 4/20 [02:06<08:32, 32.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 23437  =====\n",
      "==ERR== 23490  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 25%|██▌       | 5/20 [02:48<08:47, 35.18s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 19698  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▌      | 7/20 [04:02<07:53, 36.39s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 27445  =====\n",
      "==ERR== 27125  =====\n",
      "==ERR== 27502  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 40%|████      | 8/20 [04:31<06:50, 34.17s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 25988  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 45%|████▌     | 9/20 [04:59<05:54, 32.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 28469  =====\n",
      "==ERR== 28384  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 50%|█████     | 10/20 [05:28<05:13, 31.34s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 25096  =====\n",
      "==ERR== 25469  =====\n",
      "==ERR== 25237  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 55%|█████▌    | 11/20 [05:55<04:28, 29.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 17597  =====\n",
      "==ERR== 17601  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 60%|██████    | 12/20 [06:22<03:51, 29.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 25604  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 65%|██████▌   | 13/20 [06:44<03:08, 26.97s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 25564  =====\n",
      "==ERR== 24144  =====\n",
      "==ERR== 23980  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 70%|███████   | 14/20 [07:19<02:57, 29.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 26597  =====\n",
      "==ERR== 26784  =====\n",
      "==ERR== 26706  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 75%|███████▌  | 15/20 [07:46<02:23, 28.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 18716  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 80%|████████  | 16/20 [08:28<02:10, 32.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 20301  =====\n",
      "==ERR== 20416  =====\n",
      "==ERR== 20438  =====\n",
      "==ERR== 20199  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 85%|████████▌ | 17/20 [09:04<01:40, 33.54s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 24750  =====\n",
      "==ERR== 24546  =====\n",
      "==ERR== 24744  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 90%|█████████ | 18/20 [09:33<01:04, 32.36s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 28128  =====\n",
      "==ERR== 28005  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [10:29<00:00, 31.45s/it]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>category</th>\n",
       "      <th>preprocess_text</th>\n",
       "      <th>tokens_num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>19305</td>\n",
       "      <td>rec.sport.hockey</td>\n",
       "      <td>My votes FWIW Team MVP Pat Verbeek He fans on ...</td>\n",
       "      <td>514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>19313</td>\n",
       "      <td>rec.sport.hockey</td>\n",
       "      <td>Kovalev is too talented a player to play for R...</td>\n",
       "      <td>165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>19122</td>\n",
       "      <td>rec.sport.hockey</td>\n",
       "      <td>New Jersey 1 0 23 Pittsburgh 2 3 16 First peri...</td>\n",
       "      <td>1116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>19312</td>\n",
       "      <td>rec.sport.hockey</td>\n",
       "      <td>First of all the Penguins WILL win the cup aga...</td>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>19115</td>\n",
       "      <td>rec.sport.hockey</td>\n",
       "      <td>You ca n't But good luck trying</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11309</td>\n",
       "      <td>20797</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>When the object of their belief is said to be ...</td>\n",
       "      <td>273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11310</td>\n",
       "      <td>20537</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>Koff You mean that as long as I put you to sle...</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11311</td>\n",
       "      <td>20757</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>Mr Connor 's assertion that more complex later...</td>\n",
       "      <td>131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11312</td>\n",
       "      <td>20677</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>excess stuff deleted I know of a similar incid...</td>\n",
       "      <td>142</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11313</td>\n",
       "      <td>20719</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>The System refered to a moral system You shown...</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11314 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id          category  \\\n",
       "0      19305  rec.sport.hockey   \n",
       "1      19313  rec.sport.hockey   \n",
       "2      19122  rec.sport.hockey   \n",
       "3      19312  rec.sport.hockey   \n",
       "4      19115  rec.sport.hockey   \n",
       "...      ...               ...   \n",
       "11309  20797       alt.atheism   \n",
       "11310  20537       alt.atheism   \n",
       "11311  20757       alt.atheism   \n",
       "11312  20677       alt.atheism   \n",
       "11313  20719       alt.atheism   \n",
       "\n",
       "                                         preprocess_text tokens_num  \n",
       "0      My votes FWIW Team MVP Pat Verbeek He fans on ...        514  \n",
       "1      Kovalev is too talented a player to play for R...        165  \n",
       "2      New Jersey 1 0 23 Pittsburgh 2 3 16 First peri...       1116  \n",
       "3      First of all the Penguins WILL win the cup aga...         68  \n",
       "4                        You ca n't But good luck trying          9  \n",
       "...                                                  ...        ...  \n",
       "11309  When the object of their belief is said to be ...        273  \n",
       "11310  Koff You mean that as long as I put you to sle...         25  \n",
       "11311  Mr Connor 's assertion that more complex later...        131  \n",
       "11312  excess stuff deleted I know of a similar incid...        142  \n",
       "11313  The System refered to a moral system You shown...         38  \n",
       "\n",
       "[11314 rows x 4 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.DataFrame(columns=['id','category','preprocess_text','tokens_num'])#,'preprocess_text','preprocss_tokenize_num'])\n",
    "train_dir = './data/20news-bydate-v3/20news-bydate-train/'\n",
    "cat_dir = next(os.walk(train_dir))[1]\n",
    "count = 0\n",
    "for cat in tqdm(cat_dir):\n",
    "    in_dir = train_dir + cat + '/'\n",
    "    news_list = next(os.walk(in_dir))[2]\n",
    "    news_list = [os.path.join(in_dir, f) for f in news_list]\n",
    "    news_list = list(filter(lambda f: f.endswith(\".txt\"), news_list))\n",
    "    for news_path in news_list:\n",
    "        id_num = news_path.split('/')[-1].split('.')[0]\n",
    "        with open(news_path,'r',encoding='latin1') as f:\n",
    "            news_text = f.read()\n",
    "            news_text = re.sub('\\n',' ',news_text)\n",
    "            news_text = re.sub('\\t',' ',news_text)\n",
    "            news_text_pre = text_preprcess2(news_text)\n",
    "        try:\n",
    "            doc = roberta.extract_features_aligned_to_words(news_text_pre)\n",
    "            tokens_num = len(doc)\n",
    "        except AssertionError:\n",
    "            tokens_num = np.nan\n",
    "        except Exception as e:\n",
    "            tokens_num_ = str(e)\n",
    "            try:\n",
    "                tokens_num = int(tokens_num_.split(':')[-1].split()[0])\n",
    "            except IndexError:\n",
    "                tokens_num = 0\n",
    "                print('==ERR==',id_num,tokens_num_,'=====')\n",
    "        train_df.loc[count] = [int(id_num),cat,news_text_pre,tokens_num]\n",
    "        count+=1        \n",
    "# GPU: 10:06\n",
    "# CPU: 75\n",
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>category</th>\n",
       "      <th>preprocess_text</th>\n",
       "      <th>tokens_num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>8440</td>\n",
       "      <td>26596</td>\n",
       "      <td>comp.sys.mac.hardware</td>\n",
       "      <td>Send follow ups to Sorry about the header but ...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id               category  \\\n",
       "8440  26596  comp.sys.mac.hardware   \n",
       "\n",
       "                                        preprocess_text tokens_num  \n",
       "8440  Send follow ups to Sorry about the header but ...        NaN  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df[train_df['tokens_num'].isnull()] #assertion err\n",
    "# train_df[train_df['id'] == 26339]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>category</th>\n",
       "      <th>preprocess_text</th>\n",
       "      <th>tokens_num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>150</td>\n",
       "      <td>18857</td>\n",
       "      <td>rec.sport.hockey</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>252</td>\n",
       "      <td>19309</td>\n",
       "      <td>rec.sport.hockey</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>318</td>\n",
       "      <td>19148</td>\n",
       "      <td>rec.sport.hockey</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>734</td>\n",
       "      <td>21139</td>\n",
       "      <td>comp.graphics</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>757</td>\n",
       "      <td>21056</td>\n",
       "      <td>comp.graphics</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>853</td>\n",
       "      <td>21194</td>\n",
       "      <td>comp.graphics</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>881</td>\n",
       "      <td>21159</td>\n",
       "      <td>comp.graphics</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1222</td>\n",
       "      <td>21844</td>\n",
       "      <td>talk.politics.guns</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1761</td>\n",
       "      <td>22811</td>\n",
       "      <td>talk.politics.misc</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2433</td>\n",
       "      <td>23437</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>23490</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3225</td>\n",
       "      <td>19698</td>\n",
       "      <td>sci.space</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3959</td>\n",
       "      <td>27445</td>\n",
       "      <td>comp.sys.ibm.pc.hardware</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4215</td>\n",
       "      <td>27125</td>\n",
       "      <td>comp.sys.ibm.pc.hardware</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4366</td>\n",
       "      <td>27502</td>\n",
       "      <td>comp.sys.ibm.pc.hardware</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5022</td>\n",
       "      <td>25988</td>\n",
       "      <td>rec.sport.baseball</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5276</td>\n",
       "      <td>28469</td>\n",
       "      <td>sci.electronics</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5583</td>\n",
       "      <td>28384</td>\n",
       "      <td>sci.electronics</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5758</td>\n",
       "      <td>25096</td>\n",
       "      <td>misc.forsale</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5991</td>\n",
       "      <td>25469</td>\n",
       "      <td>misc.forsale</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6134</td>\n",
       "      <td>25237</td>\n",
       "      <td>misc.forsale</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6700</td>\n",
       "      <td>17597</td>\n",
       "      <td>rec.motorcycles</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6849</td>\n",
       "      <td>17601</td>\n",
       "      <td>rec.motorcycles</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7122</td>\n",
       "      <td>25604</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7283</td>\n",
       "      <td>25564</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7318</td>\n",
       "      <td>24144</td>\n",
       "      <td>sci.med</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7681</td>\n",
       "      <td>23980</td>\n",
       "      <td>sci.med</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7879</td>\n",
       "      <td>26597</td>\n",
       "      <td>comp.sys.mac.hardware</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8067</td>\n",
       "      <td>26784</td>\n",
       "      <td>comp.sys.mac.hardware</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8070</td>\n",
       "      <td>26706</td>\n",
       "      <td>comp.sys.mac.hardware</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8756</td>\n",
       "      <td>18716</td>\n",
       "      <td>soc.religion.christian</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9127</td>\n",
       "      <td>20301</td>\n",
       "      <td>comp.windows.x</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9448</td>\n",
       "      <td>20416</td>\n",
       "      <td>comp.windows.x</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9454</td>\n",
       "      <td>20438</td>\n",
       "      <td>comp.windows.x</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9470</td>\n",
       "      <td>20199</td>\n",
       "      <td>comp.windows.x</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9774</td>\n",
       "      <td>24750</td>\n",
       "      <td>comp.os.ms-windows.misc</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9791</td>\n",
       "      <td>24546</td>\n",
       "      <td>comp.os.ms-windows.misc</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9902</td>\n",
       "      <td>24744</td>\n",
       "      <td>comp.os.ms-windows.misc</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10331</td>\n",
       "      <td>28128</td>\n",
       "      <td>rec.autos</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10735</td>\n",
       "      <td>28005</td>\n",
       "      <td>rec.autos</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                  category preprocess_text tokens_num\n",
       "150    18857          rec.sport.hockey                          0\n",
       "252    19309          rec.sport.hockey                          0\n",
       "318    19148          rec.sport.hockey                          0\n",
       "734    21139             comp.graphics                          0\n",
       "757    21056             comp.graphics                          0\n",
       "853    21194             comp.graphics                          0\n",
       "881    21159             comp.graphics                          0\n",
       "1222   21844        talk.politics.guns                          0\n",
       "1761   22811        talk.politics.misc                          0\n",
       "2433   23437     talk.politics.mideast                          0\n",
       "2500   23490     talk.politics.mideast                          0\n",
       "3225   19698                 sci.space                          0\n",
       "3959   27445  comp.sys.ibm.pc.hardware                          0\n",
       "4215   27125  comp.sys.ibm.pc.hardware                          0\n",
       "4366   27502  comp.sys.ibm.pc.hardware                          0\n",
       "5022   25988        rec.sport.baseball                          0\n",
       "5276   28469           sci.electronics                          0\n",
       "5583   28384           sci.electronics                          0\n",
       "5758   25096              misc.forsale                          0\n",
       "5991   25469              misc.forsale                          0\n",
       "6134   25237              misc.forsale                          0\n",
       "6700   17597           rec.motorcycles                          0\n",
       "6849   17601           rec.motorcycles                          0\n",
       "7122   25604        talk.religion.misc                          0\n",
       "7283   25564        talk.religion.misc                          0\n",
       "7318   24144                   sci.med                          0\n",
       "7681   23980                   sci.med                          0\n",
       "7879   26597     comp.sys.mac.hardware                          0\n",
       "8067   26784     comp.sys.mac.hardware                          0\n",
       "8070   26706     comp.sys.mac.hardware                          0\n",
       "8756   18716    soc.religion.christian                          0\n",
       "9127   20301            comp.windows.x                          0\n",
       "9448   20416            comp.windows.x                          0\n",
       "9454   20438            comp.windows.x                          0\n",
       "9470   20199            comp.windows.x                          0\n",
       "9774   24750   comp.os.ms-windows.misc                          0\n",
       "9791   24546   comp.os.ms-windows.misc                          0\n",
       "9902   24744   comp.os.ms-windows.misc                          0\n",
       "10331  28128                 rec.autos                          0\n",
       "10735  28005                 rec.autos                          0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df[train_df['tokens_num'] == 0] #no text tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def basic_statistics(all_length):\n",
    "    '''\n",
    "    input: length list of elements e.g.[1,1,1,3,5,9,4,2,1,3,54,78,5...]\n",
    "    output1: mean、std、mode、min、q1、median(q2)、q3、max、iqr、outlier、far out\n",
    "    output2: statistics graph、10%~90% form\n",
    "    '''\n",
    "    stat_dict = {}\n",
    "    stat_dict['mean'] = np.mean(all_length)\n",
    "    stat_dict['std'] = np.std(all_length)\n",
    "    stat_dict['mode'] = np.argmax(np.bincount(all_length))\n",
    "    stat_dict['min'] = np.min(all_length)\n",
    "    stat_dict['q1'] = np.quantile(all_length,0.25)\n",
    "    stat_dict['median'] = np.quantile(all_length,0.5)\n",
    "    stat_dict['q3'] = np.quantile(all_length,0.75)\n",
    "    stat_dict['max'] = np.max(all_length)\n",
    "    stat_dict['iqr'] = stat_dict['q3'] - stat_dict['q1']\n",
    "    stat_dict['outlier'] = stat_dict['q3'] + 1.5*stat_dict['iqr']\n",
    "    stat_dict['far_out'] = stat_dict['q3'] + 3*stat_dict['iqr']\n",
    "    for i in [10,20,30,40,50,60,70,80,90,100]:\n",
    "        stat_dict[str(i)+'%'] = np.percentile(all_length,i)\n",
    "    return pd.DataFrame.from_dict(stat_dict,orient='index',columns=['length'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>mean</td>\n",
       "      <td>190.796959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>std</td>\n",
       "      <td>543.958521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mode</td>\n",
       "      <td>41.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>min</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>q1</td>\n",
       "      <td>43.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>median</td>\n",
       "      <td>84.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>q3</td>\n",
       "      <td>166.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>max</td>\n",
       "      <td>12700.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>iqr</td>\n",
       "      <td>123.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>outlier</td>\n",
       "      <td>350.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>far_out</td>\n",
       "      <td>535.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10%</td>\n",
       "      <td>23.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20%</td>\n",
       "      <td>36.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30%</td>\n",
       "      <td>50.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40%</td>\n",
       "      <td>65.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50%</td>\n",
       "      <td>84.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>60%</td>\n",
       "      <td>109.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>70%</td>\n",
       "      <td>143.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>80%</td>\n",
       "      <td>199.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>90%</td>\n",
       "      <td>330.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100%</td>\n",
       "      <td>12700.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               length\n",
       "mean       190.796959\n",
       "std        543.958521\n",
       "mode        41.000000\n",
       "min          0.000000\n",
       "q1          43.000000\n",
       "median      84.000000\n",
       "q3         166.000000\n",
       "max      12700.000000\n",
       "iqr        123.000000\n",
       "outlier    350.500000\n",
       "far_out    535.000000\n",
       "10%         23.000000\n",
       "20%         36.000000\n",
       "30%         50.000000\n",
       "40%         65.000000\n",
       "50%         84.000000\n",
       "60%        109.000000\n",
       "70%        143.000000\n",
       "80%        199.000000\n",
       "90%        330.000000\n",
       "100%     12700.000000"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kk = train_df.tokens_num.tolist()\n",
    "kk = [x for x in kk if str(x) != 'nan']\n",
    "basic_statistics(kk)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>mean</td>\n",
       "      <td>107.045188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>std</td>\n",
       "      <td>91.447652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mode</td>\n",
       "      <td>41.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>min</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>q1</td>\n",
       "      <td>41.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>median</td>\n",
       "      <td>78.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>q3</td>\n",
       "      <td>144.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>max</td>\n",
       "      <td>498.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>iqr</td>\n",
       "      <td>103.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>outlier</td>\n",
       "      <td>298.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>far_out</td>\n",
       "      <td>453.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10%</td>\n",
       "      <td>22.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20%</td>\n",
       "      <td>35.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30%</td>\n",
       "      <td>47.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40%</td>\n",
       "      <td>61.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50%</td>\n",
       "      <td>78.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>60%</td>\n",
       "      <td>99.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>70%</td>\n",
       "      <td>126.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>80%</td>\n",
       "      <td>166.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>90%</td>\n",
       "      <td>239.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100%</td>\n",
       "      <td>498.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             length\n",
       "mean     107.045188\n",
       "std       91.447652\n",
       "mode      41.000000\n",
       "min        3.000000\n",
       "q1        41.000000\n",
       "median    78.000000\n",
       "q3       144.000000\n",
       "max      498.000000\n",
       "iqr      103.000000\n",
       "outlier  298.500000\n",
       "far_out  453.000000\n",
       "10%       22.000000\n",
       "20%       35.000000\n",
       "30%       47.000000\n",
       "40%       61.000000\n",
       "50%       78.000000\n",
       "60%       99.000000\n",
       "70%      126.000000\n",
       "80%      166.000000\n",
       "90%      239.000000\n",
       "100%     498.000000"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#只留下>=3 tokens & <=512 tokens的文章\n",
    "train_df_select = train_df[(train_df['tokens_num'] >=3) & (train_df['tokens_num']<=512)]\n",
    "basic_statistics(train_df_select.tokens_num.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>category</th>\n",
       "      <th>preprocess_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>19313</td>\n",
       "      <td>rec.sport.hockey</td>\n",
       "      <td>Kovalev is too talented a player to play for R...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>19312</td>\n",
       "      <td>rec.sport.hockey</td>\n",
       "      <td>First of all the Penguins WILL win the cup aga...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>19115</td>\n",
       "      <td>rec.sport.hockey</td>\n",
       "      <td>You ca n't But good luck trying</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>19224</td>\n",
       "      <td>rec.sport.hockey</td>\n",
       "      <td>As the subject suggests the Flames were not im...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>19299</td>\n",
       "      <td>rec.sport.hockey</td>\n",
       "      <td>Well I got ta tell ya last night 's Leafs game...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11309</td>\n",
       "      <td>20797</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>When the object of their belief is said to be ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11310</td>\n",
       "      <td>20537</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>Koff You mean that as long as I put you to sle...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11311</td>\n",
       "      <td>20757</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>Mr Connor 's assertion that more complex later...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11312</td>\n",
       "      <td>20677</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>excess stuff deleted I know of a similar incid...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11313</td>\n",
       "      <td>20719</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>The System refered to a moral system You shown...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10556 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id          category  \\\n",
       "1      19313  rec.sport.hockey   \n",
       "3      19312  rec.sport.hockey   \n",
       "4      19115  rec.sport.hockey   \n",
       "5      19224  rec.sport.hockey   \n",
       "6      19299  rec.sport.hockey   \n",
       "...      ...               ...   \n",
       "11309  20797       alt.atheism   \n",
       "11310  20537       alt.atheism   \n",
       "11311  20757       alt.atheism   \n",
       "11312  20677       alt.atheism   \n",
       "11313  20719       alt.atheism   \n",
       "\n",
       "                                         preprocess_text  \n",
       "1      Kovalev is too talented a player to play for R...  \n",
       "3      First of all the Penguins WILL win the cup aga...  \n",
       "4                        You ca n't But good luck trying  \n",
       "5      As the subject suggests the Flames were not im...  \n",
       "6      Well I got ta tell ya last night 's Leafs game...  \n",
       "...                                                  ...  \n",
       "11309  When the object of their belief is said to be ...  \n",
       "11310  Koff You mean that as long as I put you to sle...  \n",
       "11311  Mr Connor 's assertion that more complex later...  \n",
       "11312  excess stuff deleted I know of a similar incid...  \n",
       "11313  The System refered to a moral system You shown...  \n",
       "\n",
       "[10556 rows x 3 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df_select_ = train_df_select.filter(['id','category','preprocess_text'])\n",
    "train_df_select_.to_csv('data/20news-bydate-v3/train_df_1.csv')\n",
    "train_df_select_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Valid/Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/20 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 11033  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|█         | 2/20 [00:44<06:28, 21.60s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 12989  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 15%|█▌        | 3/20 [01:03<05:52, 20.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 13486  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 20%|██        | 4/20 [01:23<05:31, 20.74s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 13990  =====\n",
      "==ERR== 14021  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 25%|██▌       | 5/20 [01:53<05:50, 23.40s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 11230  =====\n",
      "==ERR== 11539  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 30%|███       | 6/20 [02:15<05:21, 22.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 13263  =====\n",
      "==ERR== 13166  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 35%|███▌      | 7/20 [02:36<04:49, 22.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 16471  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 40%|████      | 8/20 [02:54<04:13, 21.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 15649  =====\n",
      "==ERR== 15593  =====\n",
      "==ERR== 15680  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 45%|████▌     | 9/20 [03:14<03:48, 20.75s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 17456  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 50%|█████     | 10/20 [03:32<03:19, 19.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 15017  =====\n",
      "==ERR== 15020  =====\n",
      "==ERR== 15167  =====\n",
      "==ERR== 15174  =====\n",
      "==ERR== 15094  =====\n",
      "==ERR== 15147  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 12/20 [04:06<02:27, 18.39s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 15420  =====\n",
      "==ERR== 15388  =====\n",
      "==ERR== 15357  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 65%|██████▌   | 13/20 [04:21<02:01, 17.35s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 14310  =====\n",
      "==ERR== 14211  =====\n",
      "==ERR== 14507  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 70%|███████   | 14/20 [04:45<01:55, 19.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 15986  =====\n",
      "==ERR== 16115  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 16/20 [05:31<01:27, 21.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 11790  =====\n",
      "==ERR== 11766  =====\n",
      "==ERR== 11831  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 85%|████████▌ | 17/20 [05:53<01:05, 21.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 14705  =====\n",
      "==ERR== 14648  =====\n",
      "==ERR== 14551  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 90%|█████████ | 18/20 [06:12<00:41, 20.96s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 16832  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 95%|█████████▌| 19/20 [06:30<00:20, 20.17s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==ERR== 12269  =====\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [06:49<00:00, 20.45s/it]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>category</th>\n",
       "      <th>preprocess_text</th>\n",
       "      <th>tokens_num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>10812</td>\n",
       "      <td>rec.sport.hockey</td>\n",
       "      <td>FULL 1993 CALDER CUP PLAYOFF SCHEDULE AND RESU...</td>\n",
       "      <td>537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>11191</td>\n",
       "      <td>rec.sport.hockey</td>\n",
       "      <td>I can understand your anger about the German a...</td>\n",
       "      <td>427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>10983</td>\n",
       "      <td>rec.sport.hockey</td>\n",
       "      <td>No and in fact that was Ferreira 's original s...</td>\n",
       "      <td>313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>10991</td>\n",
       "      <td>rec.sport.hockey</td>\n",
       "      <td>Come on Boston where the hell are you Seven wi...</td>\n",
       "      <td>76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>11174</td>\n",
       "      <td>rec.sport.hockey</td>\n",
       "      <td>I would appreciate it if someone could volunte...</td>\n",
       "      <td>1739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7527</td>\n",
       "      <td>12126</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>Another one rescued from the bit bucket Over t...</td>\n",
       "      <td>308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7528</td>\n",
       "      <td>12044</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>I asked around in one of the areas you suggest...</td>\n",
       "      <td>205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7529</td>\n",
       "      <td>12098</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>In the part of the posting you have so helpful...</td>\n",
       "      <td>73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7530</td>\n",
       "      <td>12271</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>The 24 children were of course killed by a lon...</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7531</td>\n",
       "      <td>12169</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>I replied to this query via email but I think ...</td>\n",
       "      <td>382</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7532 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         id          category  \\\n",
       "0     10812  rec.sport.hockey   \n",
       "1     11191  rec.sport.hockey   \n",
       "2     10983  rec.sport.hockey   \n",
       "3     10991  rec.sport.hockey   \n",
       "4     11174  rec.sport.hockey   \n",
       "...     ...               ...   \n",
       "7527  12126       alt.atheism   \n",
       "7528  12044       alt.atheism   \n",
       "7529  12098       alt.atheism   \n",
       "7530  12271       alt.atheism   \n",
       "7531  12169       alt.atheism   \n",
       "\n",
       "                                        preprocess_text tokens_num  \n",
       "0     FULL 1993 CALDER CUP PLAYOFF SCHEDULE AND RESU...        537  \n",
       "1     I can understand your anger about the German a...        427  \n",
       "2     No and in fact that was Ferreira 's original s...        313  \n",
       "3     Come on Boston where the hell are you Seven wi...         76  \n",
       "4     I would appreciate it if someone could volunte...       1739  \n",
       "...                                                 ...        ...  \n",
       "7527  Another one rescued from the bit bucket Over t...        308  \n",
       "7528  I asked around in one of the areas you suggest...        205  \n",
       "7529  In the part of the posting you have so helpful...         73  \n",
       "7530  The 24 children were of course killed by a lon...         28  \n",
       "7531  I replied to this query via email but I think ...        382  \n",
       "\n",
       "[7532 rows x 4 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_df = pd.DataFrame(columns=['id','category','preprocess_text','tokens_num'])#,'preprocess_text','preprocss_tokenize_num'])\n",
    "train_dir = './data/20news-bydate-v3/20news-bydate-test/'\n",
    "cat_dir = next(os.walk(train_dir))[1]\n",
    "count = 0\n",
    "for cat in tqdm(cat_dir):\n",
    "    in_dir = train_dir + cat + '/'\n",
    "    news_list = next(os.walk(in_dir))[2]\n",
    "    news_list = [os.path.join(in_dir, f) for f in news_list]\n",
    "    news_list = list(filter(lambda f: f.endswith(\".txt\"), news_list))\n",
    "    for news_path in news_list:\n",
    "        id_num = news_path.split('/')[-1].split('.')[0]\n",
    "        with open(news_path,'r',encoding='latin1') as f:\n",
    "            news_text = f.read()\n",
    "            news_text = re.sub('\\n',' ',news_text)\n",
    "            news_text = re.sub('\\t',' ',news_text)\n",
    "            news_text_pre = text_preprcess2(news_text)\n",
    "        try:\n",
    "            doc = roberta.extract_features_aligned_to_words(news_text_pre)\n",
    "            tokens_num = len(doc)\n",
    "        except AssertionError:\n",
    "            tokens_num = np.nan\n",
    "        except Exception as e:\n",
    "            tokens_num_ = str(e)\n",
    "            try:\n",
    "                tokens_num = int(tokens_num_.split(':')[-1].split()[0])\n",
    "            except IndexError:\n",
    "                tokens_num = 0\n",
    "                print('==ERR==',id_num,tokens_num_,'=====')\n",
    "        dev_df.loc[count] = [int(id_num),cat,news_text_pre,tokens_num]\n",
    "        count+=1        \n",
    "# GPU: 7:06\n",
    "# CPU: 75\n",
    "dev_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>category</th>\n",
       "      <th>preprocess_text</th>\n",
       "      <th>tokens_num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [id, category, preprocess_text, tokens_num]\n",
       "Index: []"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_df[dev_df['tokens_num'].isnull()] #assertion err"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>category</th>\n",
       "      <th>preprocess_text</th>\n",
       "      <th>tokens_num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>257</td>\n",
       "      <td>11033</td>\n",
       "      <td>rec.sport.hockey</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>908</td>\n",
       "      <td>12989</td>\n",
       "      <td>talk.politics.guns</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1188</td>\n",
       "      <td>13486</td>\n",
       "      <td>talk.politics.misc</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1480</td>\n",
       "      <td>13990</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1729</td>\n",
       "      <td>14021</td>\n",
       "      <td>talk.politics.mideast</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1918</td>\n",
       "      <td>11230</td>\n",
       "      <td>sci.space</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2134</td>\n",
       "      <td>11539</td>\n",
       "      <td>sci.space</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2294</td>\n",
       "      <td>13263</td>\n",
       "      <td>sci.crypt</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2350</td>\n",
       "      <td>13166</td>\n",
       "      <td>sci.crypt</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3008</td>\n",
       "      <td>16471</td>\n",
       "      <td>comp.sys.ibm.pc.hardware</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3023</td>\n",
       "      <td>15649</td>\n",
       "      <td>rec.sport.baseball</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3294</td>\n",
       "      <td>15593</td>\n",
       "      <td>rec.sport.baseball</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3347</td>\n",
       "      <td>15680</td>\n",
       "      <td>rec.sport.baseball</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3417</td>\n",
       "      <td>17456</td>\n",
       "      <td>sci.electronics</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3966</td>\n",
       "      <td>15017</td>\n",
       "      <td>misc.forsale</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3974</td>\n",
       "      <td>15020</td>\n",
       "      <td>misc.forsale</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4023</td>\n",
       "      <td>15167</td>\n",
       "      <td>misc.forsale</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4034</td>\n",
       "      <td>15174</td>\n",
       "      <td>misc.forsale</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4054</td>\n",
       "      <td>15094</td>\n",
       "      <td>misc.forsale</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4099</td>\n",
       "      <td>15147</td>\n",
       "      <td>misc.forsale</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4674</td>\n",
       "      <td>15420</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4744</td>\n",
       "      <td>15388</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4838</td>\n",
       "      <td>15357</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5121</td>\n",
       "      <td>14310</td>\n",
       "      <td>sci.med</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5152</td>\n",
       "      <td>14211</td>\n",
       "      <td>sci.med</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5240</td>\n",
       "      <td>14507</td>\n",
       "      <td>sci.med</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5246</td>\n",
       "      <td>15986</td>\n",
       "      <td>comp.sys.mac.hardware</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5547</td>\n",
       "      <td>16115</td>\n",
       "      <td>comp.sys.mac.hardware</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6180</td>\n",
       "      <td>11790</td>\n",
       "      <td>comp.windows.x</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6182</td>\n",
       "      <td>11766</td>\n",
       "      <td>comp.windows.x</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6316</td>\n",
       "      <td>11831</td>\n",
       "      <td>comp.windows.x</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6460</td>\n",
       "      <td>14705</td>\n",
       "      <td>comp.os.ms-windows.misc</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6593</td>\n",
       "      <td>14648</td>\n",
       "      <td>comp.os.ms-windows.misc</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6596</td>\n",
       "      <td>14551</td>\n",
       "      <td>comp.os.ms-windows.misc</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6942</td>\n",
       "      <td>16832</td>\n",
       "      <td>rec.autos</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7456</td>\n",
       "      <td>12269</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id                  category preprocess_text tokens_num\n",
       "257   11033          rec.sport.hockey                          0\n",
       "908   12989        talk.politics.guns                          0\n",
       "1188  13486        talk.politics.misc                          0\n",
       "1480  13990     talk.politics.mideast                          0\n",
       "1729  14021     talk.politics.mideast                          0\n",
       "1918  11230                 sci.space                          0\n",
       "2134  11539                 sci.space                          0\n",
       "2294  13263                 sci.crypt                          0\n",
       "2350  13166                 sci.crypt                          0\n",
       "3008  16471  comp.sys.ibm.pc.hardware                          0\n",
       "3023  15649        rec.sport.baseball                          0\n",
       "3294  15593        rec.sport.baseball                          0\n",
       "3347  15680        rec.sport.baseball                          0\n",
       "3417  17456           sci.electronics                          0\n",
       "3966  15017              misc.forsale                          0\n",
       "3974  15020              misc.forsale                          0\n",
       "4023  15167              misc.forsale                          0\n",
       "4034  15174              misc.forsale                          0\n",
       "4054  15094              misc.forsale                          0\n",
       "4099  15147              misc.forsale                          0\n",
       "4674  15420        talk.religion.misc                          0\n",
       "4744  15388        talk.religion.misc                          0\n",
       "4838  15357        talk.religion.misc                          0\n",
       "5121  14310                   sci.med                          0\n",
       "5152  14211                   sci.med                          0\n",
       "5240  14507                   sci.med                          0\n",
       "5246  15986     comp.sys.mac.hardware                          0\n",
       "5547  16115     comp.sys.mac.hardware                          0\n",
       "6180  11790            comp.windows.x                          0\n",
       "6182  11766            comp.windows.x                          0\n",
       "6316  11831            comp.windows.x                          0\n",
       "6460  14705   comp.os.ms-windows.misc                          0\n",
       "6593  14648   comp.os.ms-windows.misc                          0\n",
       "6596  14551   comp.os.ms-windows.misc                          0\n",
       "6942  16832                 rec.autos                          0\n",
       "7456  12269               alt.atheism                          0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_df[dev_df['tokens_num'] == 0] #no text tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>mean</td>\n",
       "      <td>182.255842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>std</td>\n",
       "      <td>497.224604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mode</td>\n",
       "      <td>29.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>min</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>q1</td>\n",
       "      <td>42.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>median</td>\n",
       "      <td>83.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>q3</td>\n",
       "      <td>163.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>max</td>\n",
       "      <td>10428.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>iqr</td>\n",
       "      <td>121.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>outlier</td>\n",
       "      <td>344.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>far_out</td>\n",
       "      <td>526.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10%</td>\n",
       "      <td>21.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20%</td>\n",
       "      <td>35.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30%</td>\n",
       "      <td>50.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40%</td>\n",
       "      <td>65.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50%</td>\n",
       "      <td>83.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>60%</td>\n",
       "      <td>107.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>70%</td>\n",
       "      <td>139.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>80%</td>\n",
       "      <td>197.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>90%</td>\n",
       "      <td>319.800000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100%</td>\n",
       "      <td>10428.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               length\n",
       "mean       182.255842\n",
       "std        497.224604\n",
       "mode        29.000000\n",
       "min          0.000000\n",
       "q1          42.000000\n",
       "median      83.000000\n",
       "q3         163.000000\n",
       "max      10428.000000\n",
       "iqr        121.000000\n",
       "outlier    344.500000\n",
       "far_out    526.000000\n",
       "10%         21.000000\n",
       "20%         35.000000\n",
       "30%         50.000000\n",
       "40%         65.000000\n",
       "50%         83.000000\n",
       "60%        107.000000\n",
       "70%        139.000000\n",
       "80%        197.000000\n",
       "90%        319.800000\n",
       "100%     10428.000000"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kk = dev_df.tokens_num.tolist()\n",
    "kk = [x for x in kk if str(x) != 'nan']\n",
    "basic_statistics(kk)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>mean</td>\n",
       "      <td>106.575792</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>std</td>\n",
       "      <td>92.066336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mode</td>\n",
       "      <td>29.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>min</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>q1</td>\n",
       "      <td>41.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>median</td>\n",
       "      <td>78.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>q3</td>\n",
       "      <td>143.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>max</td>\n",
       "      <td>481.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>iqr</td>\n",
       "      <td>102.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>outlier</td>\n",
       "      <td>296.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>far_out</td>\n",
       "      <td>449.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10%</td>\n",
       "      <td>21.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20%</td>\n",
       "      <td>34.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30%</td>\n",
       "      <td>47.400000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40%</td>\n",
       "      <td>62.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50%</td>\n",
       "      <td>78.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>60%</td>\n",
       "      <td>98.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>70%</td>\n",
       "      <td>124.600000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>80%</td>\n",
       "      <td>164.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>90%</td>\n",
       "      <td>237.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100%</td>\n",
       "      <td>481.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             length\n",
       "mean     106.575792\n",
       "std       92.066336\n",
       "mode      29.000000\n",
       "min        3.000000\n",
       "q1        41.000000\n",
       "median    78.000000\n",
       "q3       143.000000\n",
       "max      481.000000\n",
       "iqr      102.000000\n",
       "outlier  296.000000\n",
       "far_out  449.000000\n",
       "10%       21.000000\n",
       "20%       34.000000\n",
       "30%       47.400000\n",
       "40%       62.000000\n",
       "50%       78.000000\n",
       "60%       98.000000\n",
       "70%      124.600000\n",
       "80%      164.000000\n",
       "90%      237.000000\n",
       "100%     481.000000"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#只留下>=3 tokens & <=512 tokens的文章\n",
    "dev_df_select = dev_df[(dev_df['tokens_num'] >=3) & (dev_df['tokens_num']<=512)]\n",
    "basic_statistics(dev_df_select.tokens_num.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>category</th>\n",
       "      <th>preprocess_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>11191</td>\n",
       "      <td>rec.sport.hockey</td>\n",
       "      <td>I can understand your anger about the German a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>10983</td>\n",
       "      <td>rec.sport.hockey</td>\n",
       "      <td>No and in fact that was Ferreira 's original s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>10991</td>\n",
       "      <td>rec.sport.hockey</td>\n",
       "      <td>Come on Boston where the hell are you Seven wi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>11088</td>\n",
       "      <td>rec.sport.hockey</td>\n",
       "      <td>Quoting The Logistician to All TL I am in need...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>11035</td>\n",
       "      <td>rec.sport.hockey</td>\n",
       "      <td>Quoting Greg Rogers to All GR Hi all GR Due to...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7527</td>\n",
       "      <td>12126</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>Another one rescued from the bit bucket Over t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7528</td>\n",
       "      <td>12044</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>I asked around in one of the areas you suggest...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7529</td>\n",
       "      <td>12098</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>In the part of the posting you have so helpful...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7530</td>\n",
       "      <td>12271</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>The 24 children were of course killed by a lon...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7531</td>\n",
       "      <td>12169</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>I replied to this query via email but I think ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7039 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         id          category  \\\n",
       "1     11191  rec.sport.hockey   \n",
       "2     10983  rec.sport.hockey   \n",
       "3     10991  rec.sport.hockey   \n",
       "5     11088  rec.sport.hockey   \n",
       "6     11035  rec.sport.hockey   \n",
       "...     ...               ...   \n",
       "7527  12126       alt.atheism   \n",
       "7528  12044       alt.atheism   \n",
       "7529  12098       alt.atheism   \n",
       "7530  12271       alt.atheism   \n",
       "7531  12169       alt.atheism   \n",
       "\n",
       "                                        preprocess_text  \n",
       "1     I can understand your anger about the German a...  \n",
       "2     No and in fact that was Ferreira 's original s...  \n",
       "3     Come on Boston where the hell are you Seven wi...  \n",
       "5     Quoting The Logistician to All TL I am in need...  \n",
       "6     Quoting Greg Rogers to All GR Hi all GR Due to...  \n",
       "...                                                 ...  \n",
       "7527  Another one rescued from the bit bucket Over t...  \n",
       "7528  I asked around in one of the areas you suggest...  \n",
       "7529  In the part of the posting you have so helpful...  \n",
       "7530  The 24 children were of course killed by a lon...  \n",
       "7531  I replied to this query via email but I think ...  \n",
       "\n",
       "[7039 rows x 3 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_df_select_ =  dev_df_select.filter(['id','category','preprocess_text'])\n",
    "dev_df_select_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'alt.atheism': [7215,\n",
       "   7218,\n",
       "   7219,\n",
       "   7220,\n",
       "   7222,\n",
       "   7224,\n",
       "   7226,\n",
       "   7227,\n",
       "   7230,\n",
       "   7233,\n",
       "   7237,\n",
       "   7239,\n",
       "   7240,\n",
       "   7243,\n",
       "   7245,\n",
       "   7252,\n",
       "   7256,\n",
       "   7258,\n",
       "   7259,\n",
       "   7262,\n",
       "   7265,\n",
       "   7269,\n",
       "   7270,\n",
       "   7271,\n",
       "   7275,\n",
       "   7276,\n",
       "   7277,\n",
       "   7278,\n",
       "   7279,\n",
       "   7280,\n",
       "   7281,\n",
       "   7284,\n",
       "   7285,\n",
       "   7289,\n",
       "   7290,\n",
       "   7292,\n",
       "   7298,\n",
       "   7299,\n",
       "   7300,\n",
       "   7302,\n",
       "   7303,\n",
       "   7305,\n",
       "   7306,\n",
       "   7307,\n",
       "   7309,\n",
       "   7312,\n",
       "   7316,\n",
       "   7319,\n",
       "   7320,\n",
       "   7322,\n",
       "   7325,\n",
       "   7326,\n",
       "   7331,\n",
       "   7333,\n",
       "   7335,\n",
       "   7340,\n",
       "   7346,\n",
       "   7347,\n",
       "   7348,\n",
       "   7349,\n",
       "   7350,\n",
       "   7352,\n",
       "   7354,\n",
       "   7356,\n",
       "   7357,\n",
       "   7359,\n",
       "   7360,\n",
       "   7361,\n",
       "   7363,\n",
       "   7369,\n",
       "   7371,\n",
       "   7372,\n",
       "   7375,\n",
       "   7377,\n",
       "   7379,\n",
       "   7380,\n",
       "   7383,\n",
       "   7384,\n",
       "   7385,\n",
       "   7391,\n",
       "   7393,\n",
       "   7396,\n",
       "   7398,\n",
       "   7400,\n",
       "   7402,\n",
       "   7404,\n",
       "   7405,\n",
       "   7406,\n",
       "   7407,\n",
       "   7409,\n",
       "   7410,\n",
       "   7415,\n",
       "   7416,\n",
       "   7419,\n",
       "   7428,\n",
       "   7430,\n",
       "   7431,\n",
       "   7432,\n",
       "   7433,\n",
       "   7434,\n",
       "   7435,\n",
       "   7440,\n",
       "   7444,\n",
       "   7449,\n",
       "   7450,\n",
       "   7452,\n",
       "   7454,\n",
       "   7455,\n",
       "   7457,\n",
       "   7459,\n",
       "   7460,\n",
       "   7461,\n",
       "   7462,\n",
       "   7463,\n",
       "   7467,\n",
       "   7469,\n",
       "   7472,\n",
       "   7474,\n",
       "   7477,\n",
       "   7480,\n",
       "   7482,\n",
       "   7486,\n",
       "   7491,\n",
       "   7492,\n",
       "   7493,\n",
       "   7494,\n",
       "   7498,\n",
       "   7499,\n",
       "   7500,\n",
       "   7502,\n",
       "   7503,\n",
       "   7506,\n",
       "   7507,\n",
       "   7509,\n",
       "   7511,\n",
       "   7512,\n",
       "   7513,\n",
       "   7516,\n",
       "   7518,\n",
       "   7521,\n",
       "   7524,\n",
       "   7527,\n",
       "   7528,\n",
       "   7530,\n",
       "   7531],\n",
       "  'comp.graphics': [512,\n",
       "   513,\n",
       "   517,\n",
       "   522,\n",
       "   527,\n",
       "   528,\n",
       "   529,\n",
       "   531,\n",
       "   533,\n",
       "   534,\n",
       "   535,\n",
       "   536,\n",
       "   537,\n",
       "   538,\n",
       "   539,\n",
       "   542,\n",
       "   544,\n",
       "   546,\n",
       "   548,\n",
       "   552,\n",
       "   554,\n",
       "   555,\n",
       "   559,\n",
       "   563,\n",
       "   565,\n",
       "   567,\n",
       "   568,\n",
       "   573,\n",
       "   575,\n",
       "   579,\n",
       "   580,\n",
       "   583,\n",
       "   585,\n",
       "   588,\n",
       "   589,\n",
       "   591,\n",
       "   595,\n",
       "   596,\n",
       "   597,\n",
       "   598,\n",
       "   600,\n",
       "   604,\n",
       "   605,\n",
       "   606,\n",
       "   609,\n",
       "   611,\n",
       "   613,\n",
       "   614,\n",
       "   618,\n",
       "   619,\n",
       "   620,\n",
       "   621,\n",
       "   622,\n",
       "   627,\n",
       "   628,\n",
       "   629,\n",
       "   630,\n",
       "   632,\n",
       "   633,\n",
       "   636,\n",
       "   637,\n",
       "   639,\n",
       "   645,\n",
       "   648,\n",
       "   649,\n",
       "   653,\n",
       "   654,\n",
       "   655,\n",
       "   656,\n",
       "   660,\n",
       "   663,\n",
       "   665,\n",
       "   666,\n",
       "   667,\n",
       "   668,\n",
       "   670,\n",
       "   671,\n",
       "   673,\n",
       "   674,\n",
       "   675,\n",
       "   676,\n",
       "   679,\n",
       "   680,\n",
       "   681,\n",
       "   683,\n",
       "   684,\n",
       "   685,\n",
       "   694,\n",
       "   696,\n",
       "   697,\n",
       "   699,\n",
       "   700,\n",
       "   701,\n",
       "   704,\n",
       "   706,\n",
       "   707,\n",
       "   709,\n",
       "   713,\n",
       "   714,\n",
       "   715,\n",
       "   718,\n",
       "   720,\n",
       "   722,\n",
       "   724,\n",
       "   726,\n",
       "   727,\n",
       "   729,\n",
       "   730,\n",
       "   732,\n",
       "   734,\n",
       "   735,\n",
       "   736,\n",
       "   737,\n",
       "   738,\n",
       "   739,\n",
       "   740,\n",
       "   745,\n",
       "   746,\n",
       "   748,\n",
       "   750,\n",
       "   756,\n",
       "   762,\n",
       "   763,\n",
       "   764,\n",
       "   768,\n",
       "   771,\n",
       "   773,\n",
       "   776,\n",
       "   778,\n",
       "   779,\n",
       "   782,\n",
       "   783,\n",
       "   784,\n",
       "   785,\n",
       "   787,\n",
       "   401,\n",
       "   404,\n",
       "   409,\n",
       "   413,\n",
       "   414,\n",
       "   415,\n",
       "   416,\n",
       "   418,\n",
       "   420,\n",
       "   425,\n",
       "   430,\n",
       "   435,\n",
       "   436,\n",
       "   437,\n",
       "   438,\n",
       "   440,\n",
       "   442,\n",
       "   448,\n",
       "   449,\n",
       "   453,\n",
       "   454,\n",
       "   455,\n",
       "   456,\n",
       "   457,\n",
       "   459,\n",
       "   467,\n",
       "   469,\n",
       "   471,\n",
       "   473,\n",
       "   475,\n",
       "   477,\n",
       "   479,\n",
       "   480,\n",
       "   484,\n",
       "   486,\n",
       "   488,\n",
       "   492,\n",
       "   493,\n",
       "   494,\n",
       "   502,\n",
       "   504,\n",
       "   508,\n",
       "   509,\n",
       "   510,\n",
       "   511],\n",
       "  'comp.os.ms-windows.misc': [6657,\n",
       "   6658,\n",
       "   6659,\n",
       "   6666,\n",
       "   6668,\n",
       "   6669,\n",
       "   6670,\n",
       "   6672,\n",
       "   6673,\n",
       "   6674,\n",
       "   6677,\n",
       "   6679,\n",
       "   6680,\n",
       "   6681,\n",
       "   6682,\n",
       "   6683,\n",
       "   6685,\n",
       "   6687,\n",
       "   6688,\n",
       "   6689,\n",
       "   6690,\n",
       "   6691,\n",
       "   6692,\n",
       "   6694,\n",
       "   6698,\n",
       "   6703,\n",
       "   6705,\n",
       "   6706,\n",
       "   6707,\n",
       "   6708,\n",
       "   6710,\n",
       "   6713,\n",
       "   6714,\n",
       "   6716,\n",
       "   6718,\n",
       "   6721,\n",
       "   6724,\n",
       "   6726,\n",
       "   6728,\n",
       "   6729,\n",
       "   6731,\n",
       "   6733,\n",
       "   6734,\n",
       "   6736,\n",
       "   6737,\n",
       "   6740,\n",
       "   6743,\n",
       "   6747,\n",
       "   6750,\n",
       "   6751,\n",
       "   6752,\n",
       "   6757,\n",
       "   6758,\n",
       "   6759,\n",
       "   6765,\n",
       "   6766,\n",
       "   6767,\n",
       "   6771,\n",
       "   6772,\n",
       "   6773,\n",
       "   6774,\n",
       "   6775,\n",
       "   6777,\n",
       "   6779,\n",
       "   6782,\n",
       "   6784,\n",
       "   6790,\n",
       "   6792,\n",
       "   6794,\n",
       "   6797,\n",
       "   6799,\n",
       "   6800,\n",
       "   6803,\n",
       "   6806,\n",
       "   6807,\n",
       "   6809,\n",
       "   6810,\n",
       "   6811,\n",
       "   6813,\n",
       "   6815,\n",
       "   6423,\n",
       "   6425,\n",
       "   6426,\n",
       "   6429,\n",
       "   6432,\n",
       "   6433,\n",
       "   6436,\n",
       "   6438,\n",
       "   6444,\n",
       "   6445,\n",
       "   6446,\n",
       "   6447,\n",
       "   6448,\n",
       "   6450,\n",
       "   6453,\n",
       "   6454,\n",
       "   6455,\n",
       "   6456,\n",
       "   6457,\n",
       "   6459,\n",
       "   6461,\n",
       "   6464,\n",
       "   6468,\n",
       "   6469,\n",
       "   6470,\n",
       "   6471,\n",
       "   6472,\n",
       "   6475,\n",
       "   6476,\n",
       "   6481,\n",
       "   6483,\n",
       "   6484,\n",
       "   6485,\n",
       "   6487,\n",
       "   6488,\n",
       "   6489,\n",
       "   6490,\n",
       "   6492,\n",
       "   6496,\n",
       "   6497,\n",
       "   6502,\n",
       "   6503,\n",
       "   6504,\n",
       "   6507,\n",
       "   6508,\n",
       "   6511,\n",
       "   6512,\n",
       "   6513,\n",
       "   6515,\n",
       "   6517,\n",
       "   6519,\n",
       "   6521,\n",
       "   6523,\n",
       "   6528,\n",
       "   6530,\n",
       "   6532,\n",
       "   6534,\n",
       "   6536,\n",
       "   6537,\n",
       "   6539,\n",
       "   6540,\n",
       "   6545,\n",
       "   6546,\n",
       "   6554,\n",
       "   6555,\n",
       "   6557,\n",
       "   6558,\n",
       "   6559,\n",
       "   6562,\n",
       "   6564,\n",
       "   6566,\n",
       "   6568,\n",
       "   6569,\n",
       "   6571,\n",
       "   6577,\n",
       "   6578,\n",
       "   6579,\n",
       "   6581,\n",
       "   6582,\n",
       "   6583,\n",
       "   6587,\n",
       "   6589,\n",
       "   6595,\n",
       "   6598,\n",
       "   6600,\n",
       "   6601,\n",
       "   6603,\n",
       "   6605,\n",
       "   6609,\n",
       "   6613,\n",
       "   6615,\n",
       "   6617,\n",
       "   6619,\n",
       "   6620,\n",
       "   6623,\n",
       "   6624,\n",
       "   6625,\n",
       "   6626,\n",
       "   6627,\n",
       "   6635,\n",
       "   6636,\n",
       "   6637,\n",
       "   6641,\n",
       "   6643,\n",
       "   6644,\n",
       "   6647,\n",
       "   6650,\n",
       "   6652,\n",
       "   6653,\n",
       "   6655],\n",
       "  'comp.sys.ibm.pc.hardware': [2628,\n",
       "   2630,\n",
       "   2637,\n",
       "   2639,\n",
       "   2645,\n",
       "   2648,\n",
       "   2651,\n",
       "   2652,\n",
       "   2653,\n",
       "   2654,\n",
       "   2655,\n",
       "   2656,\n",
       "   2657,\n",
       "   2659,\n",
       "   2661,\n",
       "   2664,\n",
       "   2665,\n",
       "   2666,\n",
       "   2668,\n",
       "   2669,\n",
       "   2670,\n",
       "   2671,\n",
       "   2672,\n",
       "   2673,\n",
       "   2674,\n",
       "   2675,\n",
       "   2676,\n",
       "   2677,\n",
       "   2678,\n",
       "   2680,\n",
       "   2684,\n",
       "   2686,\n",
       "   2687,\n",
       "   2689,\n",
       "   2692,\n",
       "   2693,\n",
       "   2694,\n",
       "   2695,\n",
       "   2702,\n",
       "   2704,\n",
       "   2705,\n",
       "   2709,\n",
       "   2711,\n",
       "   2712,\n",
       "   2716,\n",
       "   2717,\n",
       "   2720,\n",
       "   2721,\n",
       "   2723,\n",
       "   2725,\n",
       "   2726,\n",
       "   2730,\n",
       "   2731,\n",
       "   2733,\n",
       "   2734,\n",
       "   2735,\n",
       "   2737,\n",
       "   2741,\n",
       "   2747,\n",
       "   2748,\n",
       "   2750,\n",
       "   2753,\n",
       "   2755,\n",
       "   2757,\n",
       "   2764,\n",
       "   2765,\n",
       "   2767,\n",
       "   2768,\n",
       "   2769,\n",
       "   2771,\n",
       "   2773,\n",
       "   2775,\n",
       "   2777,\n",
       "   2778,\n",
       "   2779,\n",
       "   2780,\n",
       "   2781,\n",
       "   2782,\n",
       "   2783,\n",
       "   2785,\n",
       "   2787,\n",
       "   2791,\n",
       "   2792,\n",
       "   2793,\n",
       "   2798,\n",
       "   2800,\n",
       "   2802,\n",
       "   2807,\n",
       "   2808,\n",
       "   2814,\n",
       "   2819,\n",
       "   2820,\n",
       "   2821,\n",
       "   2822,\n",
       "   2825,\n",
       "   2834,\n",
       "   2835,\n",
       "   2836,\n",
       "   2837,\n",
       "   2838,\n",
       "   2839,\n",
       "   2840,\n",
       "   2841,\n",
       "   2842,\n",
       "   2846,\n",
       "   2849,\n",
       "   2851,\n",
       "   2855,\n",
       "   2857,\n",
       "   2858,\n",
       "   2861,\n",
       "   2865,\n",
       "   2867,\n",
       "   2868,\n",
       "   2869,\n",
       "   2871,\n",
       "   2873,\n",
       "   2874,\n",
       "   2875,\n",
       "   2877,\n",
       "   2878,\n",
       "   2879,\n",
       "   2880,\n",
       "   2881,\n",
       "   2882,\n",
       "   2883,\n",
       "   2884,\n",
       "   2885,\n",
       "   2886,\n",
       "   2888,\n",
       "   2891,\n",
       "   2896,\n",
       "   2900,\n",
       "   2901,\n",
       "   2903,\n",
       "   2904,\n",
       "   2911,\n",
       "   2917,\n",
       "   2919,\n",
       "   2923,\n",
       "   2924,\n",
       "   2925,\n",
       "   2927,\n",
       "   2928,\n",
       "   2929,\n",
       "   2931,\n",
       "   2932,\n",
       "   2933,\n",
       "   2938,\n",
       "   2942,\n",
       "   2944,\n",
       "   2945,\n",
       "   2950,\n",
       "   2952,\n",
       "   2953,\n",
       "   2954,\n",
       "   2955,\n",
       "   2956,\n",
       "   2957,\n",
       "   2958,\n",
       "   2959,\n",
       "   2961,\n",
       "   2962,\n",
       "   2964,\n",
       "   2965,\n",
       "   2966,\n",
       "   2974,\n",
       "   2975,\n",
       "   2976,\n",
       "   2977,\n",
       "   2978,\n",
       "   2980,\n",
       "   2984,\n",
       "   2986,\n",
       "   2988,\n",
       "   2989,\n",
       "   2990,\n",
       "   2991,\n",
       "   2992,\n",
       "   2993,\n",
       "   2994,\n",
       "   2995,\n",
       "   2997,\n",
       "   3000,\n",
       "   3005,\n",
       "   3007,\n",
       "   3010,\n",
       "   3011,\n",
       "   3012,\n",
       "   3015,\n",
       "   3017],\n",
       "  'comp.sys.mac.hardware': [5245,\n",
       "   5247,\n",
       "   5250,\n",
       "   5251,\n",
       "   5253,\n",
       "   5256,\n",
       "   5260,\n",
       "   5262,\n",
       "   5264,\n",
       "   5265,\n",
       "   5270,\n",
       "   5271,\n",
       "   5275,\n",
       "   5276,\n",
       "   5277,\n",
       "   5279,\n",
       "   5280,\n",
       "   5282,\n",
       "   5283,\n",
       "   5284,\n",
       "   5285,\n",
       "   5286,\n",
       "   5290,\n",
       "   5291,\n",
       "   5292,\n",
       "   5293,\n",
       "   5294,\n",
       "   5295,\n",
       "   5296,\n",
       "   5297,\n",
       "   5299,\n",
       "   5301,\n",
       "   5305,\n",
       "   5307,\n",
       "   5309,\n",
       "   5311,\n",
       "   5314,\n",
       "   5316,\n",
       "   5322,\n",
       "   5323,\n",
       "   5324,\n",
       "   5327,\n",
       "   5331,\n",
       "   5333,\n",
       "   5336,\n",
       "   5339,\n",
       "   5340,\n",
       "   5343,\n",
       "   5345,\n",
       "   5348,\n",
       "   5349,\n",
       "   5352,\n",
       "   5353,\n",
       "   5354,\n",
       "   5356,\n",
       "   5358,\n",
       "   5359,\n",
       "   5360,\n",
       "   5364,\n",
       "   5370,\n",
       "   5371,\n",
       "   5373,\n",
       "   5376,\n",
       "   5378,\n",
       "   5379,\n",
       "   5380,\n",
       "   5384,\n",
       "   5386,\n",
       "   5387,\n",
       "   5390,\n",
       "   5392,\n",
       "   5394,\n",
       "   5399,\n",
       "   5400,\n",
       "   5401,\n",
       "   5402,\n",
       "   5404,\n",
       "   5409,\n",
       "   5410,\n",
       "   5411,\n",
       "   5412,\n",
       "   5413,\n",
       "   5417,\n",
       "   5419,\n",
       "   5421,\n",
       "   5422,\n",
       "   5426,\n",
       "   5428,\n",
       "   5431,\n",
       "   5433,\n",
       "   5434,\n",
       "   5438,\n",
       "   5439,\n",
       "   5445,\n",
       "   5447,\n",
       "   5448,\n",
       "   5450,\n",
       "   5452,\n",
       "   5455,\n",
       "   5457,\n",
       "   5458,\n",
       "   5459,\n",
       "   5463,\n",
       "   5464,\n",
       "   5465,\n",
       "   5468,\n",
       "   5471,\n",
       "   5473,\n",
       "   5477,\n",
       "   5478,\n",
       "   5480,\n",
       "   5482,\n",
       "   5485,\n",
       "   5486,\n",
       "   5488,\n",
       "   5489,\n",
       "   5490,\n",
       "   5491,\n",
       "   5495,\n",
       "   5496,\n",
       "   5499,\n",
       "   5500,\n",
       "   5502,\n",
       "   5503,\n",
       "   5509,\n",
       "   5512,\n",
       "   5515,\n",
       "   5517,\n",
       "   5518,\n",
       "   5519,\n",
       "   5520,\n",
       "   5521,\n",
       "   5523,\n",
       "   5525,\n",
       "   5527,\n",
       "   5528,\n",
       "   5529,\n",
       "   5530,\n",
       "   5533,\n",
       "   5534,\n",
       "   5536,\n",
       "   5537,\n",
       "   5538,\n",
       "   5539,\n",
       "   5540,\n",
       "   5541,\n",
       "   5542,\n",
       "   5546,\n",
       "   5549,\n",
       "   5550,\n",
       "   5551,\n",
       "   5558,\n",
       "   5559,\n",
       "   5561,\n",
       "   5562,\n",
       "   5563,\n",
       "   5567,\n",
       "   5572,\n",
       "   5575,\n",
       "   5576,\n",
       "   5577,\n",
       "   5580,\n",
       "   5584,\n",
       "   5588,\n",
       "   5589,\n",
       "   5590,\n",
       "   5593,\n",
       "   5595,\n",
       "   5597,\n",
       "   5598,\n",
       "   5599,\n",
       "   5600,\n",
       "   5602,\n",
       "   5605,\n",
       "   5606,\n",
       "   5608,\n",
       "   5609,\n",
       "   5610,\n",
       "   5611,\n",
       "   5612,\n",
       "   5613,\n",
       "   5615,\n",
       "   5617,\n",
       "   5619,\n",
       "   5621,\n",
       "   5624,\n",
       "   5626,\n",
       "   5627,\n",
       "   5628],\n",
       "  'comp.windows.x': [6144,\n",
       "   6145,\n",
       "   6150,\n",
       "   6151,\n",
       "   6154,\n",
       "   6157,\n",
       "   6158,\n",
       "   6160,\n",
       "   6163,\n",
       "   6164,\n",
       "   6165,\n",
       "   6167,\n",
       "   6168,\n",
       "   6170,\n",
       "   6172,\n",
       "   6173,\n",
       "   6174,\n",
       "   6181,\n",
       "   6184,\n",
       "   6188,\n",
       "   6190,\n",
       "   6192,\n",
       "   6199,\n",
       "   6201,\n",
       "   6203,\n",
       "   6207,\n",
       "   6209,\n",
       "   6210,\n",
       "   6211,\n",
       "   6213,\n",
       "   6214,\n",
       "   6215,\n",
       "   6217,\n",
       "   6221,\n",
       "   6224,\n",
       "   6228,\n",
       "   6231,\n",
       "   6236,\n",
       "   6237,\n",
       "   6238,\n",
       "   6239,\n",
       "   6240,\n",
       "   6241,\n",
       "   6242,\n",
       "   6243,\n",
       "   6246,\n",
       "   6248,\n",
       "   6249,\n",
       "   6250,\n",
       "   6251,\n",
       "   6252,\n",
       "   6255,\n",
       "   6257,\n",
       "   6261,\n",
       "   6263,\n",
       "   6264,\n",
       "   6266,\n",
       "   6270,\n",
       "   6271,\n",
       "   6272,\n",
       "   6273,\n",
       "   6276,\n",
       "   6277,\n",
       "   6280,\n",
       "   6281,\n",
       "   6282,\n",
       "   6285,\n",
       "   6286,\n",
       "   6288,\n",
       "   6289,\n",
       "   6290,\n",
       "   6294,\n",
       "   6296,\n",
       "   6298,\n",
       "   6300,\n",
       "   6307,\n",
       "   6308,\n",
       "   6311,\n",
       "   6312,\n",
       "   6313,\n",
       "   6317,\n",
       "   6322,\n",
       "   6324,\n",
       "   6326,\n",
       "   6330,\n",
       "   6332,\n",
       "   6333,\n",
       "   6335,\n",
       "   6336,\n",
       "   6337,\n",
       "   6339,\n",
       "   6342,\n",
       "   6346,\n",
       "   6347,\n",
       "   6349,\n",
       "   6353,\n",
       "   6354,\n",
       "   6355,\n",
       "   6358,\n",
       "   6360,\n",
       "   6361,\n",
       "   6362,\n",
       "   6363,\n",
       "   6364,\n",
       "   6368,\n",
       "   6369,\n",
       "   6373,\n",
       "   6378,\n",
       "   6381,\n",
       "   6382,\n",
       "   6383,\n",
       "   6384,\n",
       "   6388,\n",
       "   6391,\n",
       "   6395,\n",
       "   6396,\n",
       "   6397,\n",
       "   6398,\n",
       "   6400,\n",
       "   6401,\n",
       "   6402,\n",
       "   6403,\n",
       "   6404,\n",
       "   6405,\n",
       "   6406,\n",
       "   6407,\n",
       "   6408,\n",
       "   6412,\n",
       "   6413,\n",
       "   6415,\n",
       "   6416,\n",
       "   6419,\n",
       "   6421,\n",
       "   6029,\n",
       "   6031,\n",
       "   6032,\n",
       "   6033,\n",
       "   6035,\n",
       "   6036,\n",
       "   6037,\n",
       "   6039,\n",
       "   6040,\n",
       "   6041,\n",
       "   6043,\n",
       "   6046,\n",
       "   6048,\n",
       "   6049,\n",
       "   6051,\n",
       "   6054,\n",
       "   6056,\n",
       "   6059,\n",
       "   6061,\n",
       "   6063,\n",
       "   6064,\n",
       "   6066,\n",
       "   6067,\n",
       "   6070,\n",
       "   6072,\n",
       "   6073,\n",
       "   6078,\n",
       "   6083,\n",
       "   6084,\n",
       "   6085,\n",
       "   6087,\n",
       "   6088,\n",
       "   6089,\n",
       "   6093,\n",
       "   6097,\n",
       "   6098,\n",
       "   6101,\n",
       "   6103,\n",
       "   6105,\n",
       "   6107,\n",
       "   6109,\n",
       "   6110,\n",
       "   6111,\n",
       "   6115,\n",
       "   6116,\n",
       "   6118,\n",
       "   6119,\n",
       "   6120,\n",
       "   6123,\n",
       "   6124,\n",
       "   6125,\n",
       "   6128,\n",
       "   6133,\n",
       "   6134,\n",
       "   6141],\n",
       "  'misc.forsale': [4098,\n",
       "   4100,\n",
       "   4102,\n",
       "   4104,\n",
       "   4106,\n",
       "   4107,\n",
       "   4108,\n",
       "   4109,\n",
       "   4110,\n",
       "   4115,\n",
       "   4116,\n",
       "   4120,\n",
       "   4125,\n",
       "   4126,\n",
       "   4127,\n",
       "   4129,\n",
       "   4132,\n",
       "   4134,\n",
       "   4136,\n",
       "   4137,\n",
       "   4141,\n",
       "   4142,\n",
       "   4148,\n",
       "   4149,\n",
       "   4150,\n",
       "   4152,\n",
       "   4155,\n",
       "   4156,\n",
       "   4157,\n",
       "   4159,\n",
       "   4161,\n",
       "   4163,\n",
       "   4164,\n",
       "   4165,\n",
       "   4166,\n",
       "   4172,\n",
       "   4178,\n",
       "   4180,\n",
       "   4181,\n",
       "   4184,\n",
       "   4186,\n",
       "   4187,\n",
       "   4189,\n",
       "   4192,\n",
       "   4194,\n",
       "   4195,\n",
       "   3811,\n",
       "   3816,\n",
       "   3817,\n",
       "   3819,\n",
       "   3821,\n",
       "   3822,\n",
       "   3823,\n",
       "   3826,\n",
       "   3828,\n",
       "   3829,\n",
       "   3830,\n",
       "   3832,\n",
       "   3834,\n",
       "   3837,\n",
       "   3840,\n",
       "   3844,\n",
       "   3845,\n",
       "   3846,\n",
       "   3847,\n",
       "   3848,\n",
       "   3849,\n",
       "   3851,\n",
       "   3859,\n",
       "   3860,\n",
       "   3867,\n",
       "   3872,\n",
       "   3873,\n",
       "   3876,\n",
       "   3880,\n",
       "   3882,\n",
       "   3885,\n",
       "   3886,\n",
       "   3887,\n",
       "   3890,\n",
       "   3892,\n",
       "   3894,\n",
       "   3895,\n",
       "   3896,\n",
       "   3897,\n",
       "   3898,\n",
       "   3899,\n",
       "   3900,\n",
       "   3902,\n",
       "   3903,\n",
       "   3904,\n",
       "   3906,\n",
       "   3907,\n",
       "   3908,\n",
       "   3909,\n",
       "   3910,\n",
       "   3913,\n",
       "   3915,\n",
       "   3916,\n",
       "   3918,\n",
       "   3921,\n",
       "   3925,\n",
       "   3926,\n",
       "   3927,\n",
       "   3928,\n",
       "   3929,\n",
       "   3930,\n",
       "   3936,\n",
       "   3940,\n",
       "   3941,\n",
       "   3942,\n",
       "   3946,\n",
       "   3947,\n",
       "   3952,\n",
       "   3954,\n",
       "   3955,\n",
       "   3958,\n",
       "   3959,\n",
       "   3960,\n",
       "   3964,\n",
       "   3968,\n",
       "   3976,\n",
       "   3977,\n",
       "   3984,\n",
       "   3986,\n",
       "   3987,\n",
       "   3988,\n",
       "   3989,\n",
       "   3990,\n",
       "   3991,\n",
       "   3992,\n",
       "   3993,\n",
       "   3994,\n",
       "   3996,\n",
       "   3998,\n",
       "   3999,\n",
       "   4002,\n",
       "   4004,\n",
       "   4005,\n",
       "   4008,\n",
       "   4009,\n",
       "   4011,\n",
       "   4012,\n",
       "   4014,\n",
       "   4015,\n",
       "   4016,\n",
       "   4021,\n",
       "   4022,\n",
       "   4025,\n",
       "   4027,\n",
       "   4029,\n",
       "   4031,\n",
       "   4035,\n",
       "   4038,\n",
       "   4039,\n",
       "   4041,\n",
       "   4042,\n",
       "   4043,\n",
       "   4044,\n",
       "   4045,\n",
       "   4047,\n",
       "   4048,\n",
       "   4050,\n",
       "   4056,\n",
       "   4058,\n",
       "   4059,\n",
       "   4060,\n",
       "   4063,\n",
       "   4067,\n",
       "   4068,\n",
       "   4069,\n",
       "   4070,\n",
       "   4071,\n",
       "   4073,\n",
       "   4074,\n",
       "   4076,\n",
       "   4078,\n",
       "   4085,\n",
       "   4086,\n",
       "   4087,\n",
       "   4088,\n",
       "   4089,\n",
       "   4095],\n",
       "  'rec.autos': [7168,\n",
       "   7169,\n",
       "   7170,\n",
       "   7172,\n",
       "   7173,\n",
       "   7174,\n",
       "   7176,\n",
       "   7178,\n",
       "   7181,\n",
       "   7185,\n",
       "   7186,\n",
       "   7188,\n",
       "   7190,\n",
       "   7191,\n",
       "   7192,\n",
       "   7194,\n",
       "   7196,\n",
       "   7197,\n",
       "   7198,\n",
       "   7200,\n",
       "   7201,\n",
       "   7202,\n",
       "   7203,\n",
       "   7204,\n",
       "   7206,\n",
       "   7207,\n",
       "   6818,\n",
       "   6819,\n",
       "   6820,\n",
       "   6822,\n",
       "   6827,\n",
       "   6828,\n",
       "   6829,\n",
       "   6832,\n",
       "   6835,\n",
       "   6837,\n",
       "   6840,\n",
       "   6841,\n",
       "   6842,\n",
       "   6843,\n",
       "   6846,\n",
       "   6848,\n",
       "   6849,\n",
       "   6851,\n",
       "   6852,\n",
       "   6855,\n",
       "   6856,\n",
       "   6859,\n",
       "   6861,\n",
       "   6863,\n",
       "   6864,\n",
       "   6865,\n",
       "   6870,\n",
       "   6871,\n",
       "   6872,\n",
       "   6873,\n",
       "   6875,\n",
       "   6877,\n",
       "   6881,\n",
       "   6883,\n",
       "   6889,\n",
       "   6891,\n",
       "   6892,\n",
       "   6894,\n",
       "   6899,\n",
       "   6902,\n",
       "   6908,\n",
       "   6909,\n",
       "   6911,\n",
       "   6912,\n",
       "   6915,\n",
       "   6917,\n",
       "   6918,\n",
       "   6919,\n",
       "   6920,\n",
       "   6925,\n",
       "   6928,\n",
       "   6930,\n",
       "   6932,\n",
       "   6934,\n",
       "   6935,\n",
       "   6936,\n",
       "   6938,\n",
       "   6941,\n",
       "   6945,\n",
       "   6946,\n",
       "   6949,\n",
       "   6955,\n",
       "   6958,\n",
       "   6959,\n",
       "   6962,\n",
       "   6964,\n",
       "   6966,\n",
       "   6968,\n",
       "   6969,\n",
       "   6970,\n",
       "   6972,\n",
       "   6974,\n",
       "   6976,\n",
       "   6979,\n",
       "   6980,\n",
       "   6984,\n",
       "   6987,\n",
       "   6989,\n",
       "   6991,\n",
       "   6994,\n",
       "   6995,\n",
       "   6997,\n",
       "   6999,\n",
       "   7001,\n",
       "   7003,\n",
       "   7006,\n",
       "   7007,\n",
       "   7008,\n",
       "   7009,\n",
       "   7012,\n",
       "   7013,\n",
       "   7015,\n",
       "   7017,\n",
       "   7018,\n",
       "   7020,\n",
       "   7021,\n",
       "   7022,\n",
       "   7023,\n",
       "   7025,\n",
       "   7028,\n",
       "   7029,\n",
       "   7031,\n",
       "   7033,\n",
       "   7034,\n",
       "   7035,\n",
       "   7037,\n",
       "   7039,\n",
       "   7041,\n",
       "   7043,\n",
       "   7047,\n",
       "   7048,\n",
       "   7049,\n",
       "   7050,\n",
       "   7052,\n",
       "   7054,\n",
       "   7056,\n",
       "   7058,\n",
       "   7059,\n",
       "   7063,\n",
       "   7064,\n",
       "   7068,\n",
       "   7070,\n",
       "   7071,\n",
       "   7074,\n",
       "   7077,\n",
       "   7078,\n",
       "   7080,\n",
       "   7081,\n",
       "   7082,\n",
       "   7083,\n",
       "   7085,\n",
       "   7091,\n",
       "   7092,\n",
       "   7094,\n",
       "   7096,\n",
       "   7097,\n",
       "   7098,\n",
       "   7101,\n",
       "   7102,\n",
       "   7103,\n",
       "   7104,\n",
       "   7105,\n",
       "   7106,\n",
       "   7107,\n",
       "   7110,\n",
       "   7112,\n",
       "   7115,\n",
       "   7116,\n",
       "   7117,\n",
       "   7121,\n",
       "   7122,\n",
       "   7123,\n",
       "   7125,\n",
       "   7130,\n",
       "   7131,\n",
       "   7134,\n",
       "   7135,\n",
       "   7136,\n",
       "   7139,\n",
       "   7141,\n",
       "   7145,\n",
       "   7146,\n",
       "   7149,\n",
       "   7150,\n",
       "   7157,\n",
       "   7160,\n",
       "   7161,\n",
       "   7163,\n",
       "   7164],\n",
       "  'rec.motorcycles': [4200,\n",
       "   4201,\n",
       "   4202,\n",
       "   4203,\n",
       "   4204,\n",
       "   4205,\n",
       "   4206,\n",
       "   4208,\n",
       "   4209,\n",
       "   4211,\n",
       "   4213,\n",
       "   4216,\n",
       "   4217,\n",
       "   4220,\n",
       "   4221,\n",
       "   4222,\n",
       "   4223,\n",
       "   4225,\n",
       "   4227,\n",
       "   4230,\n",
       "   4231,\n",
       "   4232,\n",
       "   4235,\n",
       "   4236,\n",
       "   4237,\n",
       "   4239,\n",
       "   4240,\n",
       "   4241,\n",
       "   4242,\n",
       "   4243,\n",
       "   4245,\n",
       "   4247,\n",
       "   4249,\n",
       "   4250,\n",
       "   4253,\n",
       "   4261,\n",
       "   4265,\n",
       "   4266,\n",
       "   4267,\n",
       "   4268,\n",
       "   4273,\n",
       "   4274,\n",
       "   4275,\n",
       "   4276,\n",
       "   4279,\n",
       "   4280,\n",
       "   4281,\n",
       "   4285,\n",
       "   4286,\n",
       "   4288,\n",
       "   4290,\n",
       "   4291,\n",
       "   4292,\n",
       "   4294,\n",
       "   4300,\n",
       "   4301,\n",
       "   4303,\n",
       "   4306,\n",
       "   4307,\n",
       "   4308,\n",
       "   4309,\n",
       "   4310,\n",
       "   4311,\n",
       "   4315,\n",
       "   4317,\n",
       "   4323,\n",
       "   4328,\n",
       "   4329,\n",
       "   4333,\n",
       "   4334,\n",
       "   4335,\n",
       "   4336,\n",
       "   4337,\n",
       "   4338,\n",
       "   4341,\n",
       "   4344,\n",
       "   4346,\n",
       "   4347,\n",
       "   4349,\n",
       "   4356,\n",
       "   4359,\n",
       "   4363,\n",
       "   4364,\n",
       "   4365,\n",
       "   4367,\n",
       "   4368,\n",
       "   4372,\n",
       "   4376,\n",
       "   4380,\n",
       "   4381,\n",
       "   4382,\n",
       "   4383,\n",
       "   4384,\n",
       "   4386,\n",
       "   4390,\n",
       "   4391,\n",
       "   4392,\n",
       "   4395,\n",
       "   4396,\n",
       "   4398,\n",
       "   4401,\n",
       "   4402,\n",
       "   4406,\n",
       "   4407,\n",
       "   4414,\n",
       "   4420,\n",
       "   4425,\n",
       "   4426,\n",
       "   4427,\n",
       "   4429,\n",
       "   4430,\n",
       "   4431,\n",
       "   4433,\n",
       "   4434,\n",
       "   4435,\n",
       "   4436,\n",
       "   4437,\n",
       "   4439,\n",
       "   4442,\n",
       "   4443,\n",
       "   4444,\n",
       "   4446,\n",
       "   4447,\n",
       "   4448,\n",
       "   4450,\n",
       "   4451,\n",
       "   4453,\n",
       "   4457,\n",
       "   4459,\n",
       "   4463,\n",
       "   4464,\n",
       "   4465,\n",
       "   4469,\n",
       "   4471,\n",
       "   4472,\n",
       "   4474,\n",
       "   4475,\n",
       "   4476,\n",
       "   4478,\n",
       "   4479,\n",
       "   4483,\n",
       "   4484,\n",
       "   4488,\n",
       "   4489,\n",
       "   4492,\n",
       "   4493,\n",
       "   4494,\n",
       "   4496,\n",
       "   4498,\n",
       "   4506,\n",
       "   4509,\n",
       "   4516,\n",
       "   4517,\n",
       "   4523,\n",
       "   4524,\n",
       "   4525,\n",
       "   4526,\n",
       "   4527,\n",
       "   4528,\n",
       "   4534,\n",
       "   4536,\n",
       "   4537,\n",
       "   4538,\n",
       "   4539,\n",
       "   4540,\n",
       "   4542,\n",
       "   4545,\n",
       "   4547,\n",
       "   4548,\n",
       "   4549,\n",
       "   4550,\n",
       "   4553,\n",
       "   4554,\n",
       "   4556,\n",
       "   4557,\n",
       "   4560,\n",
       "   4562,\n",
       "   4567,\n",
       "   4568,\n",
       "   4569,\n",
       "   4571,\n",
       "   4573,\n",
       "   4576,\n",
       "   4577,\n",
       "   4579,\n",
       "   4580,\n",
       "   4581,\n",
       "   4583,\n",
       "   4585,\n",
       "   4586,\n",
       "   4587,\n",
       "   4588,\n",
       "   4589,\n",
       "   4592,\n",
       "   4593,\n",
       "   4595],\n",
       "  'rec.sport.baseball': [3072,\n",
       "   3074,\n",
       "   3075,\n",
       "   3076,\n",
       "   3079,\n",
       "   3080,\n",
       "   3083,\n",
       "   3086,\n",
       "   3089,\n",
       "   3091,\n",
       "   3092,\n",
       "   3096,\n",
       "   3097,\n",
       "   3099,\n",
       "   3101,\n",
       "   3102,\n",
       "   3103,\n",
       "   3106,\n",
       "   3107,\n",
       "   3110,\n",
       "   3112,\n",
       "   3117,\n",
       "   3118,\n",
       "   3119,\n",
       "   3120,\n",
       "   3121,\n",
       "   3125,\n",
       "   3127,\n",
       "   3129,\n",
       "   3131,\n",
       "   3132,\n",
       "   3133,\n",
       "   3134,\n",
       "   3136,\n",
       "   3138,\n",
       "   3139,\n",
       "   3140,\n",
       "   3141,\n",
       "   3143,\n",
       "   3146,\n",
       "   3150,\n",
       "   3152,\n",
       "   3157,\n",
       "   3158,\n",
       "   3162,\n",
       "   3163,\n",
       "   3164,\n",
       "   3166,\n",
       "   3167,\n",
       "   3168,\n",
       "   3169,\n",
       "   3170,\n",
       "   3174,\n",
       "   3177,\n",
       "   3179,\n",
       "   3181,\n",
       "   3182,\n",
       "   3184,\n",
       "   3186,\n",
       "   3188,\n",
       "   3189,\n",
       "   3191,\n",
       "   3195,\n",
       "   3196,\n",
       "   3197,\n",
       "   3198,\n",
       "   3199,\n",
       "   3200,\n",
       "   3204,\n",
       "   3205,\n",
       "   3206,\n",
       "   3207,\n",
       "   3209,\n",
       "   3211,\n",
       "   3214,\n",
       "   3220,\n",
       "   3221,\n",
       "   3228,\n",
       "   3231,\n",
       "   3232,\n",
       "   3235,\n",
       "   3236,\n",
       "   3237,\n",
       "   3240,\n",
       "   3241,\n",
       "   3242,\n",
       "   3246,\n",
       "   3247,\n",
       "   3250,\n",
       "   3251,\n",
       "   3256,\n",
       "   3257,\n",
       "   3258,\n",
       "   3259,\n",
       "   3262,\n",
       "   3263,\n",
       "   3265,\n",
       "   3269,\n",
       "   3270,\n",
       "   3271,\n",
       "   3274,\n",
       "   3276,\n",
       "   3278,\n",
       "   3279,\n",
       "   3282,\n",
       "   3286,\n",
       "   3287,\n",
       "   3288,\n",
       "   3289,\n",
       "   3290,\n",
       "   3291,\n",
       "   3292,\n",
       "   3293,\n",
       "   3299,\n",
       "   3303,\n",
       "   3304,\n",
       "   3305,\n",
       "   3306,\n",
       "   3307,\n",
       "   3308,\n",
       "   3310,\n",
       "   3311,\n",
       "   3312,\n",
       "   3314,\n",
       "   3318,\n",
       "   3320,\n",
       "   3324,\n",
       "   3325,\n",
       "   3327,\n",
       "   3328,\n",
       "   3329,\n",
       "   3332,\n",
       "   3338,\n",
       "   3340,\n",
       "   3341,\n",
       "   3342,\n",
       "   3345,\n",
       "   3346,\n",
       "   3348,\n",
       "   3349,\n",
       "   3351,\n",
       "   3353,\n",
       "   3354,\n",
       "   3355,\n",
       "   3356,\n",
       "   3357,\n",
       "   3363,\n",
       "   3366,\n",
       "   3367,\n",
       "   3373,\n",
       "   3374,\n",
       "   3379,\n",
       "   3380,\n",
       "   3381,\n",
       "   3383,\n",
       "   3385,\n",
       "   3387,\n",
       "   3388,\n",
       "   3392,\n",
       "   3397,\n",
       "   3399,\n",
       "   3404,\n",
       "   3405,\n",
       "   3411,\n",
       "   3413,\n",
       "   3415,\n",
       "   3020,\n",
       "   3021,\n",
       "   3024,\n",
       "   3025,\n",
       "   3026,\n",
       "   3027,\n",
       "   3029,\n",
       "   3031,\n",
       "   3034,\n",
       "   3038,\n",
       "   3039,\n",
       "   3045,\n",
       "   3047,\n",
       "   3048,\n",
       "   3053,\n",
       "   3057,\n",
       "   3060,\n",
       "   3061,\n",
       "   3064,\n",
       "   3067,\n",
       "   3069,\n",
       "   3071],\n",
       "  'rec.sport.hockey': [5,\n",
       "   7,\n",
       "   8,\n",
       "   12,\n",
       "   15,\n",
       "   16,\n",
       "   17,\n",
       "   19,\n",
       "   22,\n",
       "   23,\n",
       "   28,\n",
       "   29,\n",
       "   31,\n",
       "   32,\n",
       "   33,\n",
       "   35,\n",
       "   37,\n",
       "   39,\n",
       "   41,\n",
       "   44,\n",
       "   46,\n",
       "   52,\n",
       "   57,\n",
       "   58,\n",
       "   60,\n",
       "   61,\n",
       "   63,\n",
       "   65,\n",
       "   66,\n",
       "   67,\n",
       "   68,\n",
       "   69,\n",
       "   70,\n",
       "   74,\n",
       "   75,\n",
       "   76,\n",
       "   77,\n",
       "   79,\n",
       "   83,\n",
       "   86,\n",
       "   87,\n",
       "   89,\n",
       "   93,\n",
       "   95,\n",
       "   96,\n",
       "   97,\n",
       "   99,\n",
       "   100,\n",
       "   104,\n",
       "   106,\n",
       "   108,\n",
       "   109,\n",
       "   110,\n",
       "   112,\n",
       "   115,\n",
       "   116,\n",
       "   118,\n",
       "   119,\n",
       "   123,\n",
       "   124,\n",
       "   125,\n",
       "   127,\n",
       "   128,\n",
       "   129,\n",
       "   130,\n",
       "   131,\n",
       "   132,\n",
       "   137,\n",
       "   138,\n",
       "   142,\n",
       "   145,\n",
       "   146,\n",
       "   152,\n",
       "   157,\n",
       "   158,\n",
       "   165,\n",
       "   166,\n",
       "   168,\n",
       "   176,\n",
       "   177,\n",
       "   179,\n",
       "   180,\n",
       "   185,\n",
       "   188,\n",
       "   190,\n",
       "   193,\n",
       "   194,\n",
       "   195,\n",
       "   205,\n",
       "   209,\n",
       "   213,\n",
       "   215,\n",
       "   216,\n",
       "   217,\n",
       "   219,\n",
       "   220,\n",
       "   221,\n",
       "   223,\n",
       "   224,\n",
       "   225,\n",
       "   226,\n",
       "   228,\n",
       "   232,\n",
       "   233,\n",
       "   234,\n",
       "   243,\n",
       "   245,\n",
       "   246,\n",
       "   247,\n",
       "   250,\n",
       "   251,\n",
       "   256,\n",
       "   258,\n",
       "   259,\n",
       "   261,\n",
       "   264,\n",
       "   266,\n",
       "   268,\n",
       "   270,\n",
       "   271,\n",
       "   273,\n",
       "   274,\n",
       "   275,\n",
       "   277,\n",
       "   278,\n",
       "   279,\n",
       "   280,\n",
       "   283,\n",
       "   285,\n",
       "   286,\n",
       "   289,\n",
       "   291,\n",
       "   292,\n",
       "   293,\n",
       "   294,\n",
       "   295,\n",
       "   296,\n",
       "   297,\n",
       "   299,\n",
       "   307,\n",
       "   309,\n",
       "   311,\n",
       "   315,\n",
       "   316,\n",
       "   318,\n",
       "   319,\n",
       "   320,\n",
       "   321,\n",
       "   322,\n",
       "   323,\n",
       "   325,\n",
       "   326,\n",
       "   329,\n",
       "   333,\n",
       "   335,\n",
       "   336,\n",
       "   339,\n",
       "   341,\n",
       "   343,\n",
       "   344,\n",
       "   346,\n",
       "   348,\n",
       "   349,\n",
       "   355,\n",
       "   356,\n",
       "   359,\n",
       "   360,\n",
       "   361,\n",
       "   363,\n",
       "   364,\n",
       "   365,\n",
       "   368,\n",
       "   369,\n",
       "   374,\n",
       "   375,\n",
       "   377,\n",
       "   378,\n",
       "   380,\n",
       "   382,\n",
       "   386,\n",
       "   389,\n",
       "   390,\n",
       "   391,\n",
       "   392,\n",
       "   395,\n",
       "   396],\n",
       "  'sci.crypt': [2561,\n",
       "   2565,\n",
       "   2566,\n",
       "   2568,\n",
       "   2569,\n",
       "   2571,\n",
       "   2572,\n",
       "   2574,\n",
       "   2577,\n",
       "   2579,\n",
       "   2584,\n",
       "   2587,\n",
       "   2588,\n",
       "   2589,\n",
       "   2591,\n",
       "   2592,\n",
       "   2593,\n",
       "   2595,\n",
       "   2597,\n",
       "   2598,\n",
       "   2599,\n",
       "   2602,\n",
       "   2604,\n",
       "   2608,\n",
       "   2610,\n",
       "   2612,\n",
       "   2614,\n",
       "   2615,\n",
       "   2617,\n",
       "   2619,\n",
       "   2622,\n",
       "   2625,\n",
       "   2232,\n",
       "   2238,\n",
       "   2241,\n",
       "   2242,\n",
       "   2244,\n",
       "   2245,\n",
       "   2246,\n",
       "   2248,\n",
       "   2251,\n",
       "   2252,\n",
       "   2255,\n",
       "   2256,\n",
       "   2257,\n",
       "   2258,\n",
       "   2259,\n",
       "   2263,\n",
       "   2264,\n",
       "   2265,\n",
       "   2266,\n",
       "   2268,\n",
       "   2269,\n",
       "   2270,\n",
       "   2274,\n",
       "   2276,\n",
       "   2279,\n",
       "   2282,\n",
       "   2283,\n",
       "   2284,\n",
       "   2285,\n",
       "   2287,\n",
       "   2296,\n",
       "   2297,\n",
       "   2298,\n",
       "   2299,\n",
       "   2303,\n",
       "   2306,\n",
       "   2309,\n",
       "   2311,\n",
       "   2313,\n",
       "   2314,\n",
       "   2316,\n",
       "   2317,\n",
       "   2319,\n",
       "   2320,\n",
       "   2328,\n",
       "   2330,\n",
       "   2332,\n",
       "   2334,\n",
       "   2335,\n",
       "   2336,\n",
       "   2339,\n",
       "   2340,\n",
       "   2341,\n",
       "   2342,\n",
       "   2343,\n",
       "   2346,\n",
       "   2348,\n",
       "   2352,\n",
       "   2353,\n",
       "   2355,\n",
       "   2361,\n",
       "   2362,\n",
       "   2365,\n",
       "   2366,\n",
       "   2369,\n",
       "   2372,\n",
       "   2376,\n",
       "   2379,\n",
       "   2380,\n",
       "   2381,\n",
       "   2382,\n",
       "   2388,\n",
       "   2389,\n",
       "   2390,\n",
       "   2392,\n",
       "   2396,\n",
       "   2397,\n",
       "   2398,\n",
       "   2400,\n",
       "   2402,\n",
       "   2403,\n",
       "   2405,\n",
       "   2411,\n",
       "   2414,\n",
       "   2416,\n",
       "   2417,\n",
       "   2418,\n",
       "   2419,\n",
       "   2422,\n",
       "   2423,\n",
       "   2424,\n",
       "   2425,\n",
       "   2426,\n",
       "   2428,\n",
       "   2432,\n",
       "   2434,\n",
       "   2436,\n",
       "   2441,\n",
       "   2442,\n",
       "   2443,\n",
       "   2446,\n",
       "   2447,\n",
       "   2449,\n",
       "   2451,\n",
       "   2455,\n",
       "   2459,\n",
       "   2466,\n",
       "   2468,\n",
       "   2469,\n",
       "   2470,\n",
       "   2472,\n",
       "   2473,\n",
       "   2474,\n",
       "   2475,\n",
       "   2477,\n",
       "   2479,\n",
       "   2480,\n",
       "   2483,\n",
       "   2485,\n",
       "   2486,\n",
       "   2489,\n",
       "   2491,\n",
       "   2494,\n",
       "   2495,\n",
       "   2496,\n",
       "   2499,\n",
       "   2500,\n",
       "   2501,\n",
       "   2504,\n",
       "   2508,\n",
       "   2511,\n",
       "   2514,\n",
       "   2515,\n",
       "   2516,\n",
       "   2517,\n",
       "   2518,\n",
       "   2519,\n",
       "   2522,\n",
       "   2525,\n",
       "   2527,\n",
       "   2528,\n",
       "   2529,\n",
       "   2530,\n",
       "   2531,\n",
       "   2534,\n",
       "   2537,\n",
       "   2538,\n",
       "   2540,\n",
       "   2541,\n",
       "   2544,\n",
       "   2545,\n",
       "   2547,\n",
       "   2550,\n",
       "   2552,\n",
       "   2554,\n",
       "   2558],\n",
       "  'sci.electronics': [3585,\n",
       "   3587,\n",
       "   3588,\n",
       "   3589,\n",
       "   3591,\n",
       "   3592,\n",
       "   3594,\n",
       "   3596,\n",
       "   3598,\n",
       "   3600,\n",
       "   3602,\n",
       "   3604,\n",
       "   3605,\n",
       "   3606,\n",
       "   3607,\n",
       "   3608,\n",
       "   3610,\n",
       "   3611,\n",
       "   3612,\n",
       "   3613,\n",
       "   3615,\n",
       "   3617,\n",
       "   3620,\n",
       "   3625,\n",
       "   3626,\n",
       "   3629,\n",
       "   3631,\n",
       "   3633,\n",
       "   3634,\n",
       "   3637,\n",
       "   3638,\n",
       "   3640,\n",
       "   3642,\n",
       "   3643,\n",
       "   3644,\n",
       "   3650,\n",
       "   3651,\n",
       "   3653,\n",
       "   3654,\n",
       "   3656,\n",
       "   3662,\n",
       "   3663,\n",
       "   3666,\n",
       "   3670,\n",
       "   3671,\n",
       "   3672,\n",
       "   3674,\n",
       "   3676,\n",
       "   3678,\n",
       "   3682,\n",
       "   3683,\n",
       "   3685,\n",
       "   3686,\n",
       "   3687,\n",
       "   3688,\n",
       "   3689,\n",
       "   3692,\n",
       "   3694,\n",
       "   3697,\n",
       "   3698,\n",
       "   3700,\n",
       "   3701,\n",
       "   3703,\n",
       "   3704,\n",
       "   3706,\n",
       "   3708,\n",
       "   3715,\n",
       "   3717,\n",
       "   3718,\n",
       "   3720,\n",
       "   3722,\n",
       "   3723,\n",
       "   3731,\n",
       "   3734,\n",
       "   3740,\n",
       "   3743,\n",
       "   3746,\n",
       "   3748,\n",
       "   3749,\n",
       "   3753,\n",
       "   3755,\n",
       "   3756,\n",
       "   3757,\n",
       "   3760,\n",
       "   3761,\n",
       "   3762,\n",
       "   3763,\n",
       "   3764,\n",
       "   3765,\n",
       "   3767,\n",
       "   3768,\n",
       "   3769,\n",
       "   3771,\n",
       "   3772,\n",
       "   3774,\n",
       "   3778,\n",
       "   3784,\n",
       "   3786,\n",
       "   3788,\n",
       "   3790,\n",
       "   3792,\n",
       "   3796,\n",
       "   3801,\n",
       "   3802,\n",
       "   3803,\n",
       "   3807,\n",
       "   3420,\n",
       "   3421,\n",
       "   3425,\n",
       "   3427,\n",
       "   3430,\n",
       "   3433,\n",
       "   3434,\n",
       "   3435,\n",
       "   3437,\n",
       "   3438,\n",
       "   3439,\n",
       "   3446,\n",
       "   3452,\n",
       "   3454,\n",
       "   3456,\n",
       "   3457,\n",
       "   3459,\n",
       "   3460,\n",
       "   3462,\n",
       "   3463,\n",
       "   3464,\n",
       "   3465,\n",
       "   3466,\n",
       "   3469,\n",
       "   3470,\n",
       "   3471,\n",
       "   3475,\n",
       "   3477,\n",
       "   3478,\n",
       "   3482,\n",
       "   3484,\n",
       "   3486,\n",
       "   3487,\n",
       "   3489,\n",
       "   3490,\n",
       "   3491,\n",
       "   3494,\n",
       "   3496,\n",
       "   3498,\n",
       "   3499,\n",
       "   3500,\n",
       "   3502,\n",
       "   3503,\n",
       "   3505,\n",
       "   3506,\n",
       "   3511,\n",
       "   3513,\n",
       "   3514,\n",
       "   3516,\n",
       "   3517,\n",
       "   3518,\n",
       "   3519,\n",
       "   3524,\n",
       "   3525,\n",
       "   3526,\n",
       "   3527,\n",
       "   3528,\n",
       "   3529,\n",
       "   3530,\n",
       "   3532,\n",
       "   3533,\n",
       "   3535,\n",
       "   3536,\n",
       "   3537,\n",
       "   3544,\n",
       "   3545,\n",
       "   3546,\n",
       "   3547,\n",
       "   3550,\n",
       "   3551,\n",
       "   3552,\n",
       "   3554,\n",
       "   3555,\n",
       "   3556,\n",
       "   3560,\n",
       "   3563,\n",
       "   3567,\n",
       "   3568,\n",
       "   3569,\n",
       "   3571,\n",
       "   3573,\n",
       "   3574,\n",
       "   3575,\n",
       "   3577,\n",
       "   3578,\n",
       "   3582,\n",
       "   3583],\n",
       "  'sci.med': [5123,\n",
       "   5124,\n",
       "   5125,\n",
       "   5127,\n",
       "   5129,\n",
       "   5130,\n",
       "   5133,\n",
       "   5135,\n",
       "   5138,\n",
       "   5139,\n",
       "   5140,\n",
       "   5142,\n",
       "   5147,\n",
       "   5149,\n",
       "   5150,\n",
       "   5151,\n",
       "   5153,\n",
       "   5155,\n",
       "   5157,\n",
       "   5160,\n",
       "   5162,\n",
       "   5163,\n",
       "   5166,\n",
       "   5167,\n",
       "   5172,\n",
       "   5173,\n",
       "   5174,\n",
       "   5176,\n",
       "   5178,\n",
       "   5180,\n",
       "   5181,\n",
       "   5182,\n",
       "   5183,\n",
       "   5187,\n",
       "   5188,\n",
       "   5191,\n",
       "   5192,\n",
       "   5198,\n",
       "   5199,\n",
       "   5200,\n",
       "   5202,\n",
       "   5205,\n",
       "   5206,\n",
       "   5207,\n",
       "   5208,\n",
       "   5212,\n",
       "   5219,\n",
       "   5223,\n",
       "   5224,\n",
       "   5225,\n",
       "   5226,\n",
       "   5227,\n",
       "   5230,\n",
       "   5235,\n",
       "   5237,\n",
       "   5238,\n",
       "   5241,\n",
       "   4849,\n",
       "   4850,\n",
       "   4854,\n",
       "   4855,\n",
       "   4859,\n",
       "   4861,\n",
       "   4866,\n",
       "   4867,\n",
       "   4868,\n",
       "   4870,\n",
       "   4876,\n",
       "   4878,\n",
       "   4880,\n",
       "   4882,\n",
       "   4884,\n",
       "   4885,\n",
       "   4888,\n",
       "   4893,\n",
       "   4894,\n",
       "   4899,\n",
       "   4901,\n",
       "   4903,\n",
       "   4904,\n",
       "   4906,\n",
       "   4907,\n",
       "   4909,\n",
       "   4914,\n",
       "   4919,\n",
       "   4921,\n",
       "   4923,\n",
       "   4924,\n",
       "   4927,\n",
       "   4930,\n",
       "   4932,\n",
       "   4933,\n",
       "   4934,\n",
       "   4940,\n",
       "   4943,\n",
       "   4947,\n",
       "   4950,\n",
       "   4951,\n",
       "   4952,\n",
       "   4953,\n",
       "   4954,\n",
       "   4955,\n",
       "   4958,\n",
       "   4962,\n",
       "   4963,\n",
       "   4964,\n",
       "   4965,\n",
       "   4968,\n",
       "   4969,\n",
       "   4972,\n",
       "   4976,\n",
       "   4980,\n",
       "   4983,\n",
       "   4985,\n",
       "   4986,\n",
       "   4989,\n",
       "   4990,\n",
       "   4991,\n",
       "   4995,\n",
       "   4996,\n",
       "   4997,\n",
       "   4998,\n",
       "   4999,\n",
       "   5000,\n",
       "   5004,\n",
       "   5005,\n",
       "   5008,\n",
       "   5009,\n",
       "   5011,\n",
       "   5013,\n",
       "   5015,\n",
       "   5016,\n",
       "   5018,\n",
       "   5019,\n",
       "   5022,\n",
       "   5028,\n",
       "   5032,\n",
       "   5035,\n",
       "   5037,\n",
       "   5038,\n",
       "   5039,\n",
       "   5041,\n",
       "   5045,\n",
       "   5047,\n",
       "   5050,\n",
       "   5051,\n",
       "   5053,\n",
       "   5055,\n",
       "   5060,\n",
       "   5063,\n",
       "   5066,\n",
       "   5067,\n",
       "   5068,\n",
       "   5070,\n",
       "   5071,\n",
       "   5072,\n",
       "   5073,\n",
       "   5074,\n",
       "   5076,\n",
       "   5077,\n",
       "   5078,\n",
       "   5082,\n",
       "   5084,\n",
       "   5088,\n",
       "   5093,\n",
       "   5094,\n",
       "   5095,\n",
       "   5096,\n",
       "   5097,\n",
       "   5098,\n",
       "   5100,\n",
       "   5101,\n",
       "   5102,\n",
       "   5105,\n",
       "   5106,\n",
       "   5109,\n",
       "   5110,\n",
       "   5112,\n",
       "   5114,\n",
       "   5115,\n",
       "   5116,\n",
       "   5118,\n",
       "   5119],\n",
       "  'sci.space': [2048,\n",
       "   2050,\n",
       "   2051,\n",
       "   2055,\n",
       "   2056,\n",
       "   2058,\n",
       "   2059,\n",
       "   2061,\n",
       "   2062,\n",
       "   2063,\n",
       "   2067,\n",
       "   2074,\n",
       "   2077,\n",
       "   2078,\n",
       "   2079,\n",
       "   2080,\n",
       "   2082,\n",
       "   2084,\n",
       "   2085,\n",
       "   2087,\n",
       "   2088,\n",
       "   2091,\n",
       "   2093,\n",
       "   2094,\n",
       "   2095,\n",
       "   2100,\n",
       "   2102,\n",
       "   2103,\n",
       "   2104,\n",
       "   2106,\n",
       "   2107,\n",
       "   2109,\n",
       "   2116,\n",
       "   2118,\n",
       "   2119,\n",
       "   2120,\n",
       "   2121,\n",
       "   2123,\n",
       "   2124,\n",
       "   2130,\n",
       "   2131,\n",
       "   2135,\n",
       "   2136,\n",
       "   2137,\n",
       "   2138,\n",
       "   2144,\n",
       "   2145,\n",
       "   2150,\n",
       "   2152,\n",
       "   2153,\n",
       "   2156,\n",
       "   2157,\n",
       "   2158,\n",
       "   2160,\n",
       "   2161,\n",
       "   2162,\n",
       "   2164,\n",
       "   2166,\n",
       "   2167,\n",
       "   2168,\n",
       "   2171,\n",
       "   2174,\n",
       "   2175,\n",
       "   2176,\n",
       "   2178,\n",
       "   2179,\n",
       "   2180,\n",
       "   2181,\n",
       "   2182,\n",
       "   2183,\n",
       "   2184,\n",
       "   2185,\n",
       "   2186,\n",
       "   2191,\n",
       "   2192,\n",
       "   2199,\n",
       "   2200,\n",
       "   2205,\n",
       "   2208,\n",
       "   2210,\n",
       "   2212,\n",
       "   2216,\n",
       "   2217,\n",
       "   2218,\n",
       "   2219,\n",
       "   2222,\n",
       "   2224,\n",
       "   2225,\n",
       "   2227,\n",
       "   2228,\n",
       "   1838,\n",
       "   1839,\n",
       "   1840,\n",
       "   1841,\n",
       "   1842,\n",
       "   1843,\n",
       "   1844,\n",
       "   1845,\n",
       "   1847,\n",
       "   1848,\n",
       "   1849,\n",
       "   1852,\n",
       "   1857,\n",
       "   1858,\n",
       "   1860,\n",
       "   1861,\n",
       "   1862,\n",
       "   1866,\n",
       "   1867,\n",
       "   1868,\n",
       "   1869,\n",
       "   1870,\n",
       "   1871,\n",
       "   1872,\n",
       "   1876,\n",
       "   1877,\n",
       "   1883,\n",
       "   1884,\n",
       "   1885,\n",
       "   1886,\n",
       "   1889,\n",
       "   1890,\n",
       "   1891,\n",
       "   1892,\n",
       "   1894,\n",
       "   1900,\n",
       "   1901,\n",
       "   1906,\n",
       "   1907,\n",
       "   1915,\n",
       "   1916,\n",
       "   1917,\n",
       "   1921,\n",
       "   1923,\n",
       "   1926,\n",
       "   1929,\n",
       "   1933,\n",
       "   1936,\n",
       "   1937,\n",
       "   1939,\n",
       "   1942,\n",
       "   1944,\n",
       "   1945,\n",
       "   1946,\n",
       "   1947,\n",
       "   1950,\n",
       "   1953,\n",
       "   1954,\n",
       "   1957,\n",
       "   1959,\n",
       "   1962,\n",
       "   1963,\n",
       "   1969,\n",
       "   1970,\n",
       "   1971,\n",
       "   1972,\n",
       "   1975,\n",
       "   1976,\n",
       "   1977,\n",
       "   1980,\n",
       "   1983,\n",
       "   1984,\n",
       "   1985,\n",
       "   1988,\n",
       "   1989,\n",
       "   1990,\n",
       "   1996,\n",
       "   1998,\n",
       "   1999,\n",
       "   2000,\n",
       "   2003,\n",
       "   2004,\n",
       "   2006,\n",
       "   2011,\n",
       "   2013,\n",
       "   2014,\n",
       "   2018,\n",
       "   2021,\n",
       "   2022,\n",
       "   2027,\n",
       "   2032,\n",
       "   2033,\n",
       "   2035,\n",
       "   2036,\n",
       "   2039,\n",
       "   2044,\n",
       "   2045],\n",
       "  'soc.religion.christian': [5633,\n",
       "   5634,\n",
       "   5635,\n",
       "   5638,\n",
       "   5642,\n",
       "   5644,\n",
       "   5646,\n",
       "   5648,\n",
       "   5649,\n",
       "   5654,\n",
       "   5655,\n",
       "   5662,\n",
       "   5666,\n",
       "   5675,\n",
       "   5680,\n",
       "   5682,\n",
       "   5683,\n",
       "   5684,\n",
       "   5688,\n",
       "   5689,\n",
       "   5690,\n",
       "   5691,\n",
       "   5696,\n",
       "   5699,\n",
       "   5703,\n",
       "   5704,\n",
       "   5711,\n",
       "   5715,\n",
       "   5716,\n",
       "   5718,\n",
       "   5719,\n",
       "   5721,\n",
       "   5722,\n",
       "   5724,\n",
       "   5729,\n",
       "   5731,\n",
       "   5736,\n",
       "   5737,\n",
       "   5744,\n",
       "   5751,\n",
       "   5753,\n",
       "   5754,\n",
       "   5755,\n",
       "   5756,\n",
       "   5758,\n",
       "   5760,\n",
       "   5763,\n",
       "   5768,\n",
       "   5770,\n",
       "   5771,\n",
       "   5772,\n",
       "   5774,\n",
       "   5775,\n",
       "   5777,\n",
       "   5779,\n",
       "   5783,\n",
       "   5784,\n",
       "   5786,\n",
       "   5788,\n",
       "   5791,\n",
       "   5795,\n",
       "   5797,\n",
       "   5798,\n",
       "   5800,\n",
       "   5801,\n",
       "   5805,\n",
       "   5807,\n",
       "   5808,\n",
       "   5809,\n",
       "   5813,\n",
       "   5817,\n",
       "   5819,\n",
       "   5820,\n",
       "   5821,\n",
       "   5822,\n",
       "   5823,\n",
       "   5825,\n",
       "   5832,\n",
       "   5834,\n",
       "   5836,\n",
       "   5839,\n",
       "   5840,\n",
       "   5842,\n",
       "   5845,\n",
       "   5846,\n",
       "   5847,\n",
       "   5848,\n",
       "   5850,\n",
       "   5854,\n",
       "   5855,\n",
       "   5860,\n",
       "   5865,\n",
       "   5866,\n",
       "   5867,\n",
       "   5868,\n",
       "   5871,\n",
       "   5875,\n",
       "   5877,\n",
       "   5879,\n",
       "   5881,\n",
       "   5884,\n",
       "   5885,\n",
       "   5888,\n",
       "   5890,\n",
       "   5893,\n",
       "   5894,\n",
       "   5895,\n",
       "   5899,\n",
       "   5904,\n",
       "   5905,\n",
       "   5906,\n",
       "   5907,\n",
       "   5909,\n",
       "   5910,\n",
       "   5911,\n",
       "   5913,\n",
       "   5914,\n",
       "   5915,\n",
       "   5919,\n",
       "   5921,\n",
       "   5927,\n",
       "   5928,\n",
       "   5933,\n",
       "   5936,\n",
       "   5937,\n",
       "   5938,\n",
       "   5940,\n",
       "   5941,\n",
       "   5945,\n",
       "   5946,\n",
       "   5954,\n",
       "   5955,\n",
       "   5956,\n",
       "   5958,\n",
       "   5960,\n",
       "   5963,\n",
       "   5965,\n",
       "   5966,\n",
       "   5969,\n",
       "   5971,\n",
       "   5974,\n",
       "   5977,\n",
       "   5979,\n",
       "   5981,\n",
       "   5983,\n",
       "   5986,\n",
       "   5988,\n",
       "   5990,\n",
       "   5993,\n",
       "   5997,\n",
       "   5998,\n",
       "   5999,\n",
       "   6001,\n",
       "   6002,\n",
       "   6006,\n",
       "   6008,\n",
       "   6009,\n",
       "   6010,\n",
       "   6011,\n",
       "   6012,\n",
       "   6013,\n",
       "   6014,\n",
       "   6016,\n",
       "   6018,\n",
       "   6020,\n",
       "   6023,\n",
       "   6024],\n",
       "  'talk.politics.guns': [1025,\n",
       "   1028,\n",
       "   1029,\n",
       "   1030,\n",
       "   1031,\n",
       "   1033,\n",
       "   1034,\n",
       "   1039,\n",
       "   1044,\n",
       "   1045,\n",
       "   1047,\n",
       "   1048,\n",
       "   1049,\n",
       "   1050,\n",
       "   1053,\n",
       "   1054,\n",
       "   1055,\n",
       "   1056,\n",
       "   1057,\n",
       "   1059,\n",
       "   1061,\n",
       "   1062,\n",
       "   1067,\n",
       "   1068,\n",
       "   1070,\n",
       "   1074,\n",
       "   1075,\n",
       "   1077,\n",
       "   1078,\n",
       "   1079,\n",
       "   1080,\n",
       "   1083,\n",
       "   1085,\n",
       "   1087,\n",
       "   1089,\n",
       "   1090,\n",
       "   1092,\n",
       "   1094,\n",
       "   1095,\n",
       "   1099,\n",
       "   1101,\n",
       "   1102,\n",
       "   1104,\n",
       "   1105,\n",
       "   1109,\n",
       "   1112,\n",
       "   1113,\n",
       "   1115,\n",
       "   1120,\n",
       "   1121,\n",
       "   1123,\n",
       "   1126,\n",
       "   1128,\n",
       "   1129,\n",
       "   1131,\n",
       "   1132,\n",
       "   1133,\n",
       "   1134,\n",
       "   1136,\n",
       "   1137,\n",
       "   1138,\n",
       "   1141,\n",
       "   1144,\n",
       "   1148,\n",
       "   1149,\n",
       "   1151,\n",
       "   791,\n",
       "   792,\n",
       "   795,\n",
       "   801,\n",
       "   803,\n",
       "   804,\n",
       "   806,\n",
       "   807,\n",
       "   808,\n",
       "   811,\n",
       "   815,\n",
       "   819,\n",
       "   820,\n",
       "   823,\n",
       "   826,\n",
       "   828,\n",
       "   830,\n",
       "   831,\n",
       "   836,\n",
       "   837,\n",
       "   838,\n",
       "   841,\n",
       "   842,\n",
       "   845,\n",
       "   846,\n",
       "   847,\n",
       "   848,\n",
       "   850,\n",
       "   854,\n",
       "   855,\n",
       "   858,\n",
       "   860,\n",
       "   861,\n",
       "   862,\n",
       "   863,\n",
       "   865,\n",
       "   869,\n",
       "   872,\n",
       "   881,\n",
       "   885,\n",
       "   888,\n",
       "   889,\n",
       "   893,\n",
       "   894,\n",
       "   895,\n",
       "   897,\n",
       "   898,\n",
       "   904,\n",
       "   906,\n",
       "   907,\n",
       "   911,\n",
       "   915,\n",
       "   917,\n",
       "   918,\n",
       "   919,\n",
       "   922,\n",
       "   927,\n",
       "   928,\n",
       "   929,\n",
       "   930,\n",
       "   933,\n",
       "   934,\n",
       "   936,\n",
       "   938,\n",
       "   939,\n",
       "   943,\n",
       "   944,\n",
       "   948,\n",
       "   950,\n",
       "   951,\n",
       "   952,\n",
       "   954,\n",
       "   958,\n",
       "   960,\n",
       "   961,\n",
       "   964,\n",
       "   967,\n",
       "   972,\n",
       "   974,\n",
       "   976,\n",
       "   979,\n",
       "   980,\n",
       "   981,\n",
       "   982,\n",
       "   987,\n",
       "   988,\n",
       "   991,\n",
       "   994,\n",
       "   995,\n",
       "   997,\n",
       "   1000,\n",
       "   1002,\n",
       "   1003,\n",
       "   1004,\n",
       "   1009,\n",
       "   1013,\n",
       "   1014,\n",
       "   1015,\n",
       "   1016,\n",
       "   1017,\n",
       "   1018,\n",
       "   1020,\n",
       "   1021,\n",
       "   1023],\n",
       "  'talk.politics.mideast': [1536,\n",
       "   1540,\n",
       "   1543,\n",
       "   1548,\n",
       "   1550,\n",
       "   1556,\n",
       "   1557,\n",
       "   1558,\n",
       "   1561,\n",
       "   1562,\n",
       "   1569,\n",
       "   1570,\n",
       "   1572,\n",
       "   1574,\n",
       "   1576,\n",
       "   1577,\n",
       "   1578,\n",
       "   1579,\n",
       "   1580,\n",
       "   1581,\n",
       "   1584,\n",
       "   1591,\n",
       "   1592,\n",
       "   1593,\n",
       "   1597,\n",
       "   1599,\n",
       "   1600,\n",
       "   1601,\n",
       "   1603,\n",
       "   1605,\n",
       "   1607,\n",
       "   1611,\n",
       "   1612,\n",
       "   1617,\n",
       "   1620,\n",
       "   1624,\n",
       "   1626,\n",
       "   1627,\n",
       "   1628,\n",
       "   1630,\n",
       "   1631,\n",
       "   1632,\n",
       "   1634,\n",
       "   1637,\n",
       "   1639,\n",
       "   1641,\n",
       "   1642,\n",
       "   1643,\n",
       "   1646,\n",
       "   1651,\n",
       "   1652,\n",
       "   1653,\n",
       "   1662,\n",
       "   1664,\n",
       "   1667,\n",
       "   1669,\n",
       "   1677,\n",
       "   1678,\n",
       "   1681,\n",
       "   1685,\n",
       "   1686,\n",
       "   1688,\n",
       "   1689,\n",
       "   1690,\n",
       "   1691,\n",
       "   1692,\n",
       "   1695,\n",
       "   1697,\n",
       "   1698,\n",
       "   1699,\n",
       "   1702,\n",
       "   1711,\n",
       "   1713,\n",
       "   1717,\n",
       "   1720,\n",
       "   1723,\n",
       "   1724,\n",
       "   1725,\n",
       "   1727,\n",
       "   1730,\n",
       "   1732,\n",
       "   1734,\n",
       "   1735,\n",
       "   1737,\n",
       "   1738,\n",
       "   1740,\n",
       "   1741,\n",
       "   1743,\n",
       "   1750,\n",
       "   1751,\n",
       "   1755,\n",
       "   1758,\n",
       "   1761,\n",
       "   1767,\n",
       "   1768,\n",
       "   1769,\n",
       "   1776,\n",
       "   1777,\n",
       "   1780,\n",
       "   1782,\n",
       "   1783,\n",
       "   1784,\n",
       "   1788,\n",
       "   1789,\n",
       "   1791,\n",
       "   1795,\n",
       "   1796,\n",
       "   1802,\n",
       "   1806,\n",
       "   1808,\n",
       "   1812,\n",
       "   1813,\n",
       "   1816,\n",
       "   1821,\n",
       "   1823,\n",
       "   1825,\n",
       "   1828,\n",
       "   1830,\n",
       "   1831,\n",
       "   1837,\n",
       "   1464,\n",
       "   1465,\n",
       "   1466,\n",
       "   1469,\n",
       "   1470,\n",
       "   1471,\n",
       "   1474,\n",
       "   1475,\n",
       "   1477,\n",
       "   1479,\n",
       "   1481,\n",
       "   1483,\n",
       "   1484,\n",
       "   1486,\n",
       "   1487,\n",
       "   1489,\n",
       "   1493,\n",
       "   1495,\n",
       "   1496,\n",
       "   1500,\n",
       "   1501,\n",
       "   1505,\n",
       "   1507,\n",
       "   1511,\n",
       "   1517,\n",
       "   1518,\n",
       "   1519,\n",
       "   1522,\n",
       "   1523,\n",
       "   1527,\n",
       "   1528,\n",
       "   1529,\n",
       "   1531,\n",
       "   1532],\n",
       "  'talk.politics.misc': [1153,\n",
       "   1154,\n",
       "   1155,\n",
       "   1157,\n",
       "   1160,\n",
       "   1161,\n",
       "   1163,\n",
       "   1164,\n",
       "   1165,\n",
       "   1166,\n",
       "   1167,\n",
       "   1168,\n",
       "   1170,\n",
       "   1172,\n",
       "   1173,\n",
       "   1174,\n",
       "   1177,\n",
       "   1179,\n",
       "   1180,\n",
       "   1181,\n",
       "   1184,\n",
       "   1185,\n",
       "   1189,\n",
       "   1190,\n",
       "   1194,\n",
       "   1197,\n",
       "   1198,\n",
       "   1201,\n",
       "   1202,\n",
       "   1206,\n",
       "   1209,\n",
       "   1211,\n",
       "   1214,\n",
       "   1217,\n",
       "   1218,\n",
       "   1219,\n",
       "   1223,\n",
       "   1224,\n",
       "   1226,\n",
       "   1227,\n",
       "   1228,\n",
       "   1234,\n",
       "   1237,\n",
       "   1238,\n",
       "   1239,\n",
       "   1242,\n",
       "   1247,\n",
       "   1255,\n",
       "   1258,\n",
       "   1259,\n",
       "   1260,\n",
       "   1263,\n",
       "   1264,\n",
       "   1269,\n",
       "   1272,\n",
       "   1273,\n",
       "   1275,\n",
       "   1277,\n",
       "   1281,\n",
       "   1284,\n",
       "   1287,\n",
       "   1289,\n",
       "   1291,\n",
       "   1292,\n",
       "   1294,\n",
       "   1295,\n",
       "   1296,\n",
       "   1297,\n",
       "   1298,\n",
       "   1303,\n",
       "   1305,\n",
       "   1309,\n",
       "   1310,\n",
       "   1311,\n",
       "   1312,\n",
       "   1314,\n",
       "   1315,\n",
       "   1316,\n",
       "   1321,\n",
       "   1322,\n",
       "   1323,\n",
       "   1324,\n",
       "   1325,\n",
       "   1331,\n",
       "   1334,\n",
       "   1336,\n",
       "   1337,\n",
       "   1339,\n",
       "   1340,\n",
       "   1344,\n",
       "   1346,\n",
       "   1347,\n",
       "   1349,\n",
       "   1353,\n",
       "   1355,\n",
       "   1357,\n",
       "   1358,\n",
       "   1361,\n",
       "   1365,\n",
       "   1368,\n",
       "   1369,\n",
       "   1370,\n",
       "   1373,\n",
       "   1380,\n",
       "   1383,\n",
       "   1385,\n",
       "   1387,\n",
       "   1389,\n",
       "   1393,\n",
       "   1394,\n",
       "   1396,\n",
       "   1398,\n",
       "   1401,\n",
       "   1403,\n",
       "   1405,\n",
       "   1406,\n",
       "   1413,\n",
       "   1414,\n",
       "   1415,\n",
       "   1416,\n",
       "   1417,\n",
       "   1420,\n",
       "   1421,\n",
       "   1424,\n",
       "   1426,\n",
       "   1427,\n",
       "   1431,\n",
       "   1433,\n",
       "   1434,\n",
       "   1435,\n",
       "   1436,\n",
       "   1437,\n",
       "   1441,\n",
       "   1442,\n",
       "   1445,\n",
       "   1446,\n",
       "   1454,\n",
       "   1456,\n",
       "   1457],\n",
       "  'talk.religion.misc': [4608,\n",
       "   4609,\n",
       "   4610,\n",
       "   4613,\n",
       "   4614,\n",
       "   4615,\n",
       "   4617,\n",
       "   4620,\n",
       "   4621,\n",
       "   4625,\n",
       "   4626,\n",
       "   4629,\n",
       "   4630,\n",
       "   4631,\n",
       "   4632,\n",
       "   4634,\n",
       "   4639,\n",
       "   4641,\n",
       "   4644,\n",
       "   4646,\n",
       "   4647,\n",
       "   4653,\n",
       "   4655,\n",
       "   4657,\n",
       "   4660,\n",
       "   4661,\n",
       "   4665,\n",
       "   4672,\n",
       "   4677,\n",
       "   4678,\n",
       "   4679,\n",
       "   4686,\n",
       "   4691,\n",
       "   4695,\n",
       "   4697,\n",
       "   4701,\n",
       "   4704,\n",
       "   4705,\n",
       "   4711,\n",
       "   4713,\n",
       "   4714,\n",
       "   4715,\n",
       "   4716,\n",
       "   4721,\n",
       "   4723,\n",
       "   4727,\n",
       "   4728,\n",
       "   4729,\n",
       "   4731,\n",
       "   4732,\n",
       "   4733,\n",
       "   4734,\n",
       "   4738,\n",
       "   4739,\n",
       "   4741,\n",
       "   4742,\n",
       "   4747,\n",
       "   4748,\n",
       "   4749,\n",
       "   4755,\n",
       "   4758,\n",
       "   4759,\n",
       "   4760,\n",
       "   4764,\n",
       "   4765,\n",
       "   4766,\n",
       "   4771,\n",
       "   4774,\n",
       "   4776,\n",
       "   4779,\n",
       "   4780,\n",
       "   4781,\n",
       "   4785,\n",
       "   4786,\n",
       "   4790,\n",
       "   4796,\n",
       "   4797,\n",
       "   4798,\n",
       "   4799,\n",
       "   4800,\n",
       "   4801,\n",
       "   4802,\n",
       "   4804,\n",
       "   4806,\n",
       "   4808,\n",
       "   4809,\n",
       "   4810,\n",
       "   4811,\n",
       "   4814,\n",
       "   4816,\n",
       "   4817,\n",
       "   4818,\n",
       "   4819,\n",
       "   4821,\n",
       "   4824,\n",
       "   4825,\n",
       "   4828,\n",
       "   4829,\n",
       "   4830,\n",
       "   4831,\n",
       "   4832,\n",
       "   4834,\n",
       "   4836,\n",
       "   4837,\n",
       "   4841,\n",
       "   4843,\n",
       "   4845,\n",
       "   4846,\n",
       "   4602,\n",
       "   4603,\n",
       "   4604,\n",
       "   4605]},\n",
       " {'alt.atheism': [7213,\n",
       "   7451,\n",
       "   7420,\n",
       "   7328,\n",
       "   7273,\n",
       "   7342,\n",
       "   7425,\n",
       "   7381,\n",
       "   7523,\n",
       "   7251,\n",
       "   7386,\n",
       "   7293,\n",
       "   7529,\n",
       "   7370,\n",
       "   7228,\n",
       "   7382,\n",
       "   7508,\n",
       "   7315,\n",
       "   7310,\n",
       "   7221,\n",
       "   7517,\n",
       "   7244,\n",
       "   7423,\n",
       "   7287,\n",
       "   7345,\n",
       "   7329,\n",
       "   7272,\n",
       "   7355,\n",
       "   7438,\n",
       "   7394,\n",
       "   7318,\n",
       "   7437,\n",
       "   7351,\n",
       "   7288,\n",
       "   7286,\n",
       "   7411,\n",
       "   7464,\n",
       "   7483,\n",
       "   7496,\n",
       "   7414,\n",
       "   7475,\n",
       "   7337,\n",
       "   7308,\n",
       "   7466,\n",
       "   7282,\n",
       "   7426,\n",
       "   7250,\n",
       "   7214,\n",
       "   7231,\n",
       "   7294,\n",
       "   7408,\n",
       "   7389,\n",
       "   7395,\n",
       "   7353,\n",
       "   7479,\n",
       "   7296,\n",
       "   7510,\n",
       "   7338,\n",
       "   7334,\n",
       "   7266,\n",
       "   7525,\n",
       "   7429,\n",
       "   7314,\n",
       "   7413,\n",
       "   7362,\n",
       "   7263,\n",
       "   7399,\n",
       "   7343,\n",
       "   7427,\n",
       "   7439,\n",
       "   7246,\n",
       "   7373,\n",
       "   7490,\n",
       "   7336,\n",
       "   7473,\n",
       "   7403,\n",
       "   7217,\n",
       "   7223,\n",
       "   7458,\n",
       "   7249,\n",
       "   7468,\n",
       "   7234,\n",
       "   7297,\n",
       "   7504,\n",
       "   7453,\n",
       "   7229,\n",
       "   7387,\n",
       "   7489,\n",
       "   7216,\n",
       "   7397,\n",
       "   7442,\n",
       "   7520,\n",
       "   7248,\n",
       "   7366,\n",
       "   7225,\n",
       "   7418,\n",
       "   7358,\n",
       "   7257,\n",
       "   7324,\n",
       "   7392,\n",
       "   7235,\n",
       "   7519,\n",
       "   7364,\n",
       "   7241,\n",
       "   7476,\n",
       "   7436,\n",
       "   7341,\n",
       "   7505,\n",
       "   7344,\n",
       "   7522,\n",
       "   7441,\n",
       "   7501,\n",
       "   7236,\n",
       "   7478,\n",
       "   7485,\n",
       "   7514,\n",
       "   7330,\n",
       "   7526,\n",
       "   7378,\n",
       "   7304,\n",
       "   7323,\n",
       "   7374,\n",
       "   7497,\n",
       "   7448,\n",
       "   7488,\n",
       "   7446,\n",
       "   7313,\n",
       "   7267,\n",
       "   7515,\n",
       "   7255,\n",
       "   7365,\n",
       "   7253,\n",
       "   7260,\n",
       "   7254,\n",
       "   7471,\n",
       "   7295,\n",
       "   7367,\n",
       "   7495,\n",
       "   7412,\n",
       "   7421,\n",
       "   7242,\n",
       "   7261,\n",
       "   7424,\n",
       "   7445],\n",
       "  'comp.graphics': [721,\n",
       "   615,\n",
       "   743,\n",
       "   431,\n",
       "   638,\n",
       "   662,\n",
       "   481,\n",
       "   602,\n",
       "   657,\n",
       "   472,\n",
       "   570,\n",
       "   406,\n",
       "   781,\n",
       "   710,\n",
       "   433,\n",
       "   582,\n",
       "   503,\n",
       "   489,\n",
       "   549,\n",
       "   577,\n",
       "   476,\n",
       "   432,\n",
       "   429,\n",
       "   407,\n",
       "   497,\n",
       "   586,\n",
       "   523,\n",
       "   461,\n",
       "   631,\n",
       "   518,\n",
       "   460,\n",
       "   712,\n",
       "   643,\n",
       "   754,\n",
       "   587,\n",
       "   692,\n",
       "   519,\n",
       "   572,\n",
       "   526,\n",
       "   752,\n",
       "   408,\n",
       "   774,\n",
       "   677,\n",
       "   466,\n",
       "   553,\n",
       "   626,\n",
       "   616,\n",
       "   607,\n",
       "   524,\n",
       "   698,\n",
       "   521,\n",
       "   647,\n",
       "   495,\n",
       "   777,\n",
       "   451,\n",
       "   547,\n",
       "   686,\n",
       "   551,\n",
       "   728,\n",
       "   723,\n",
       "   564,\n",
       "   651,\n",
       "   478,\n",
       "   424,\n",
       "   576,\n",
       "   703,\n",
       "   541,\n",
       "   682,\n",
       "   695,\n",
       "   569,\n",
       "   708,\n",
       "   767,\n",
       "   687,\n",
       "   468,\n",
       "   757,\n",
       "   705,\n",
       "   617,\n",
       "   664,\n",
       "   659,\n",
       "   624,\n",
       "   761,\n",
       "   716,\n",
       "   525,\n",
       "   458,\n",
       "   672,\n",
       "   540,\n",
       "   550,\n",
       "   443,\n",
       "   485,\n",
       "   487,\n",
       "   464,\n",
       "   470,\n",
       "   426,\n",
       "   558,\n",
       "   594,\n",
       "   691,\n",
       "   678,\n",
       "   766,\n",
       "   482,\n",
       "   690,\n",
       "   446,\n",
       "   499,\n",
       "   543,\n",
       "   410,\n",
       "   753,\n",
       "   562,\n",
       "   669,\n",
       "   593,\n",
       "   447,\n",
       "   412,\n",
       "   507,\n",
       "   501,\n",
       "   411,\n",
       "   462,\n",
       "   561,\n",
       "   530,\n",
       "   421,\n",
       "   642,\n",
       "   693,\n",
       "   399,\n",
       "   578,\n",
       "   417,\n",
       "   610,\n",
       "   744,\n",
       "   402,\n",
       "   505,\n",
       "   422,\n",
       "   532,\n",
       "   770,\n",
       "   749,\n",
       "   725,\n",
       "   644,\n",
       "   741,\n",
       "   514,\n",
       "   786,\n",
       "   445,\n",
       "   780,\n",
       "   608,\n",
       "   717,\n",
       "   755,\n",
       "   515,\n",
       "   601,\n",
       "   574,\n",
       "   625,\n",
       "   590,\n",
       "   571,\n",
       "   434,\n",
       "   759,\n",
       "   441,\n",
       "   775,\n",
       "   719,\n",
       "   742,\n",
       "   731,\n",
       "   520,\n",
       "   652,\n",
       "   423,\n",
       "   483,\n",
       "   490,\n",
       "   702,\n",
       "   641,\n",
       "   650,\n",
       "   581,\n",
       "   661,\n",
       "   733,\n",
       "   599,\n",
       "   603,\n",
       "   751,\n",
       "   403,\n",
       "   439,\n",
       "   765,\n",
       "   584,\n",
       "   545,\n",
       "   516,\n",
       "   557,\n",
       "   506,\n",
       "   769,\n",
       "   758,\n",
       "   772,\n",
       "   711,\n",
       "   400],\n",
       "  'comp.os.ms-windows.misc': [6463,\n",
       "   6452,\n",
       "   6493,\n",
       "   6700,\n",
       "   6514,\n",
       "   6745,\n",
       "   6525,\n",
       "   6798,\n",
       "   6671,\n",
       "   6749,\n",
       "   6732,\n",
       "   6788,\n",
       "   6634,\n",
       "   6656,\n",
       "   6611,\n",
       "   6742,\n",
       "   6739,\n",
       "   6574,\n",
       "   6631,\n",
       "   6533,\n",
       "   6664,\n",
       "   6816,\n",
       "   6667,\n",
       "   6725,\n",
       "   6529,\n",
       "   6791,\n",
       "   6498,\n",
       "   6783,\n",
       "   6594,\n",
       "   6538,\n",
       "   6755,\n",
       "   6720,\n",
       "   6805,\n",
       "   6565,\n",
       "   6660,\n",
       "   6695,\n",
       "   6719,\n",
       "   6789,\n",
       "   6570,\n",
       "   6573,\n",
       "   6709,\n",
       "   6580,\n",
       "   6606,\n",
       "   6520,\n",
       "   6697,\n",
       "   6479,\n",
       "   6451,\n",
       "   6586,\n",
       "   6572,\n",
       "   6549,\n",
       "   6435,\n",
       "   6640,\n",
       "   6588,\n",
       "   6486,\n",
       "   6785,\n",
       "   6730,\n",
       "   6506,\n",
       "   6544,\n",
       "   6531,\n",
       "   6441,\n",
       "   6712,\n",
       "   6678,\n",
       "   6576,\n",
       "   6510,\n",
       "   6563,\n",
       "   6630,\n",
       "   6754,\n",
       "   6808,\n",
       "   6526,\n",
       "   6505,\n",
       "   6522,\n",
       "   6778,\n",
       "   6735,\n",
       "   6542,\n",
       "   6561,\n",
       "   6491,\n",
       "   6753,\n",
       "   6717,\n",
       "   6628,\n",
       "   6480,\n",
       "   6723,\n",
       "   6795,\n",
       "   6428,\n",
       "   6802,\n",
       "   6684,\n",
       "   6704,\n",
       "   6632,\n",
       "   6780,\n",
       "   6722,\n",
       "   6761,\n",
       "   6612,\n",
       "   6478,\n",
       "   6801,\n",
       "   6814,\n",
       "   6439,\n",
       "   6781,\n",
       "   6604,\n",
       "   6699,\n",
       "   6527,\n",
       "   6430,\n",
       "   6762,\n",
       "   6494,\n",
       "   6804,\n",
       "   6638,\n",
       "   6662,\n",
       "   6675,\n",
       "   6535,\n",
       "   6654,\n",
       "   6812,\n",
       "   6651,\n",
       "   6622,\n",
       "   6676,\n",
       "   6618,\n",
       "   6686,\n",
       "   6440,\n",
       "   6746,\n",
       "   6616,\n",
       "   6663,\n",
       "   6646,\n",
       "   6614,\n",
       "   6701,\n",
       "   6633,\n",
       "   6585,\n",
       "   6786,\n",
       "   6621,\n",
       "   6741,\n",
       "   6649,\n",
       "   6639,\n",
       "   6509,\n",
       "   6599,\n",
       "   6702,\n",
       "   6431,\n",
       "   6760,\n",
       "   6524,\n",
       "   6769,\n",
       "   6727,\n",
       "   6500,\n",
       "   6748,\n",
       "   6474,\n",
       "   6584,\n",
       "   6473,\n",
       "   6756,\n",
       "   6443,\n",
       "   6427,\n",
       "   6693,\n",
       "   6552,\n",
       "   6547,\n",
       "   6560,\n",
       "   6449,\n",
       "   6465,\n",
       "   6744,\n",
       "   6467,\n",
       "   6477,\n",
       "   6629,\n",
       "   6608,\n",
       "   6466,\n",
       "   6556,\n",
       "   6597,\n",
       "   6738,\n",
       "   6550,\n",
       "   6424,\n",
       "   6696,\n",
       "   6592,\n",
       "   6551,\n",
       "   6642,\n",
       "   6458,\n",
       "   6567,\n",
       "   6575,\n",
       "   6645,\n",
       "   6715,\n",
       "   6518,\n",
       "   6543,\n",
       "   6661,\n",
       "   6648,\n",
       "   6499,\n",
       "   6590,\n",
       "   6462,\n",
       "   6434,\n",
       "   6553,\n",
       "   6610,\n",
       "   6796,\n",
       "   6770,\n",
       "   6763,\n",
       "   6516,\n",
       "   6495,\n",
       "   6787,\n",
       "   6793,\n",
       "   6548,\n",
       "   6591,\n",
       "   6607],\n",
       "  'comp.sys.ibm.pc.hardware': [2647,\n",
       "   2894,\n",
       "   2968,\n",
       "   2895,\n",
       "   2918,\n",
       "   2679,\n",
       "   3013,\n",
       "   3001,\n",
       "   2831,\n",
       "   2908,\n",
       "   2914,\n",
       "   2844,\n",
       "   2947,\n",
       "   2905,\n",
       "   2646,\n",
       "   2685,\n",
       "   2641,\n",
       "   2920,\n",
       "   2815,\n",
       "   2650,\n",
       "   2912,\n",
       "   2754,\n",
       "   2967,\n",
       "   2703,\n",
       "   2856,\n",
       "   2960,\n",
       "   2872,\n",
       "   2860,\n",
       "   2796,\n",
       "   2971,\n",
       "   2718,\n",
       "   2642,\n",
       "   2963,\n",
       "   2887,\n",
       "   2854,\n",
       "   3014,\n",
       "   2936,\n",
       "   2809,\n",
       "   2635,\n",
       "   2921,\n",
       "   2698,\n",
       "   2681,\n",
       "   2890,\n",
       "   2948,\n",
       "   2999,\n",
       "   2743,\n",
       "   2828,\n",
       "   2943,\n",
       "   2715,\n",
       "   3004,\n",
       "   2631,\n",
       "   2758,\n",
       "   2749,\n",
       "   2691,\n",
       "   2697,\n",
       "   3003,\n",
       "   2981,\n",
       "   2892,\n",
       "   2797,\n",
       "   2784,\n",
       "   2852,\n",
       "   2632,\n",
       "   3018,\n",
       "   2864,\n",
       "   2799,\n",
       "   2811,\n",
       "   2662,\n",
       "   2663,\n",
       "   2985,\n",
       "   2727,\n",
       "   2813,\n",
       "   2804,\n",
       "   2752,\n",
       "   3006,\n",
       "   2939,\n",
       "   2644,\n",
       "   2746,\n",
       "   2761,\n",
       "   2766,\n",
       "   2634,\n",
       "   2649,\n",
       "   2847,\n",
       "   2913,\n",
       "   2763,\n",
       "   2739,\n",
       "   2922,\n",
       "   2915,\n",
       "   2906,\n",
       "   3019,\n",
       "   2788,\n",
       "   2996,\n",
       "   2937,\n",
       "   2935,\n",
       "   2870,\n",
       "   3016,\n",
       "   2760,\n",
       "   2742,\n",
       "   2770,\n",
       "   2772,\n",
       "   2970,\n",
       "   2707,\n",
       "   2745,\n",
       "   2728,\n",
       "   2774,\n",
       "   2987,\n",
       "   2714,\n",
       "   2973,\n",
       "   2713,\n",
       "   2863,\n",
       "   2786,\n",
       "   2729,\n",
       "   2658,\n",
       "   2790,\n",
       "   2910,\n",
       "   2972,\n",
       "   2776,\n",
       "   3009,\n",
       "   2940,\n",
       "   2710,\n",
       "   2806,\n",
       "   2949,\n",
       "   2983,\n",
       "   3002,\n",
       "   2829,\n",
       "   2683,\n",
       "   2667,\n",
       "   2876,\n",
       "   2862,\n",
       "   2832,\n",
       "   2889,\n",
       "   2898,\n",
       "   2762,\n",
       "   2907,\n",
       "   2827,\n",
       "   2660,\n",
       "   2732,\n",
       "   2751,\n",
       "   2701,\n",
       "   2902,\n",
       "   2700,\n",
       "   2916,\n",
       "   2638,\n",
       "   2817,\n",
       "   2845,\n",
       "   2979,\n",
       "   2848,\n",
       "   2899,\n",
       "   2803,\n",
       "   2740,\n",
       "   2934,\n",
       "   2636,\n",
       "   2736,\n",
       "   2629,\n",
       "   2859,\n",
       "   2823,\n",
       "   2810,\n",
       "   2759,\n",
       "   2643,\n",
       "   2816,\n",
       "   2833,\n",
       "   2706,\n",
       "   2893,\n",
       "   2930,\n",
       "   2853,\n",
       "   2794,\n",
       "   2998,\n",
       "   2830,\n",
       "   2866,\n",
       "   2738,\n",
       "   2682,\n",
       "   2690,\n",
       "   2843,\n",
       "   2818,\n",
       "   2850,\n",
       "   2724,\n",
       "   2699,\n",
       "   2897,\n",
       "   2640,\n",
       "   2826,\n",
       "   2969,\n",
       "   2756,\n",
       "   2744,\n",
       "   2812,\n",
       "   2722,\n",
       "   2719,\n",
       "   2909,\n",
       "   2951,\n",
       "   2696,\n",
       "   2824,\n",
       "   2633,\n",
       "   2982],\n",
       "  'comp.sys.mac.hardware': [5267,\n",
       "   5514,\n",
       "   5268,\n",
       "   5470,\n",
       "   5449,\n",
       "   5585,\n",
       "   5596,\n",
       "   5313,\n",
       "   5259,\n",
       "   5306,\n",
       "   5435,\n",
       "   5607,\n",
       "   5388,\n",
       "   5254,\n",
       "   5484,\n",
       "   5369,\n",
       "   5492,\n",
       "   5344,\n",
       "   5304,\n",
       "   5397,\n",
       "   5351,\n",
       "   5320,\n",
       "   5263,\n",
       "   5357,\n",
       "   5365,\n",
       "   5372,\n",
       "   5398,\n",
       "   5441,\n",
       "   5341,\n",
       "   5406,\n",
       "   5312,\n",
       "   5266,\n",
       "   5494,\n",
       "   5425,\n",
       "   5511,\n",
       "   5560,\n",
       "   5405,\n",
       "   5474,\n",
       "   5326,\n",
       "   5443,\n",
       "   5273,\n",
       "   5361,\n",
       "   5591,\n",
       "   5427,\n",
       "   5432,\n",
       "   5423,\n",
       "   5319,\n",
       "   5505,\n",
       "   5481,\n",
       "   5334,\n",
       "   5416,\n",
       "   5248,\n",
       "   5396,\n",
       "   5460,\n",
       "   5601,\n",
       "   5557,\n",
       "   5289,\n",
       "   5330,\n",
       "   5350,\n",
       "   5453,\n",
       "   5466,\n",
       "   5614,\n",
       "   5389,\n",
       "   5553,\n",
       "   5368,\n",
       "   5385,\n",
       "   5382,\n",
       "   5347,\n",
       "   5329,\n",
       "   5487,\n",
       "   5581,\n",
       "   5278,\n",
       "   5543,\n",
       "   5298,\n",
       "   5261,\n",
       "   5462,\n",
       "   5269,\n",
       "   5363,\n",
       "   5552,\n",
       "   5272,\n",
       "   5274,\n",
       "   5569,\n",
       "   5346,\n",
       "   5524,\n",
       "   5393,\n",
       "   5618,\n",
       "   5420,\n",
       "   5472,\n",
       "   5446,\n",
       "   5317,\n",
       "   5548,\n",
       "   5281,\n",
       "   5362,\n",
       "   5497,\n",
       "   5574,\n",
       "   5583,\n",
       "   5440,\n",
       "   5381,\n",
       "   5366,\n",
       "   5555,\n",
       "   5252,\n",
       "   5451,\n",
       "   5586,\n",
       "   5328,\n",
       "   5407,\n",
       "   5355,\n",
       "   5403,\n",
       "   5257,\n",
       "   5620,\n",
       "   5629,\n",
       "   5338,\n",
       "   5444,\n",
       "   5414,\n",
       "   5287,\n",
       "   5479,\n",
       "   5578,\n",
       "   5415,\n",
       "   5258,\n",
       "   5603,\n",
       "   5315,\n",
       "   5342,\n",
       "   5532,\n",
       "   5508,\n",
       "   5335,\n",
       "   5467,\n",
       "   5408,\n",
       "   5310,\n",
       "   5321,\n",
       "   5476,\n",
       "   5249,\n",
       "   5300,\n",
       "   5573,\n",
       "   5374,\n",
       "   5592,\n",
       "   5504,\n",
       "   5568,\n",
       "   5625,\n",
       "   5565,\n",
       "   5493,\n",
       "   5383,\n",
       "   5579,\n",
       "   5616,\n",
       "   5288,\n",
       "   5429,\n",
       "   5506,\n",
       "   5582,\n",
       "   5604,\n",
       "   5510,\n",
       "   5395,\n",
       "   5622,\n",
       "   5255,\n",
       "   5498,\n",
       "   5325,\n",
       "   5375,\n",
       "   5507,\n",
       "   5526,\n",
       "   5391,\n",
       "   5377,\n",
       "   5566,\n",
       "   5531,\n",
       "   5424,\n",
       "   5623,\n",
       "   5308,\n",
       "   5367,\n",
       "   5483,\n",
       "   5594,\n",
       "   5337,\n",
       "   5535,\n",
       "   5571,\n",
       "   5522,\n",
       "   5516,\n",
       "   5430,\n",
       "   5544,\n",
       "   5302,\n",
       "   5436,\n",
       "   5456,\n",
       "   5303,\n",
       "   5556,\n",
       "   5554,\n",
       "   5442,\n",
       "   5461,\n",
       "   5475,\n",
       "   5437,\n",
       "   5332,\n",
       "   5418,\n",
       "   5545,\n",
       "   5469,\n",
       "   5513],\n",
       "  'comp.windows.x': [6112,\n",
       "   6102,\n",
       "   6379,\n",
       "   6042,\n",
       "   6082,\n",
       "   6159,\n",
       "   6393,\n",
       "   6343,\n",
       "   6260,\n",
       "   6302,\n",
       "   6206,\n",
       "   6038,\n",
       "   6269,\n",
       "   6152,\n",
       "   6414,\n",
       "   6060,\n",
       "   6131,\n",
       "   6301,\n",
       "   6295,\n",
       "   6279,\n",
       "   6235,\n",
       "   6262,\n",
       "   6365,\n",
       "   6191,\n",
       "   6034,\n",
       "   6371,\n",
       "   6113,\n",
       "   6331,\n",
       "   6153,\n",
       "   6229,\n",
       "   6161,\n",
       "   6392,\n",
       "   6253,\n",
       "   6366,\n",
       "   6351,\n",
       "   6278,\n",
       "   6305,\n",
       "   6314,\n",
       "   6299,\n",
       "   6127,\n",
       "   6420,\n",
       "   6143,\n",
       "   6418,\n",
       "   6216,\n",
       "   6071,\n",
       "   6284,\n",
       "   6176,\n",
       "   6315,\n",
       "   6057,\n",
       "   6137,\n",
       "   6390,\n",
       "   6259,\n",
       "   6196,\n",
       "   6275,\n",
       "   6334,\n",
       "   6162,\n",
       "   6121,\n",
       "   6356,\n",
       "   6254,\n",
       "   6077,\n",
       "   6062,\n",
       "   6222,\n",
       "   6156,\n",
       "   6265,\n",
       "   6374,\n",
       "   6359,\n",
       "   6100,\n",
       "   6178,\n",
       "   6219,\n",
       "   6200,\n",
       "   6135,\n",
       "   6256,\n",
       "   6065,\n",
       "   6387,\n",
       "   6376,\n",
       "   6140,\n",
       "   6304,\n",
       "   6345,\n",
       "   6108,\n",
       "   6198,\n",
       "   6053,\n",
       "   6195,\n",
       "   6309,\n",
       "   6179,\n",
       "   6399,\n",
       "   6283,\n",
       "   6352,\n",
       "   6114,\n",
       "   6325,\n",
       "   6327,\n",
       "   6139,\n",
       "   6096,\n",
       "   6287,\n",
       "   6303,\n",
       "   6323,\n",
       "   6245,\n",
       "   6340,\n",
       "   6075,\n",
       "   6090,\n",
       "   6422,\n",
       "   6028,\n",
       "   6293,\n",
       "   6086,\n",
       "   6348,\n",
       "   6234,\n",
       "   6208,\n",
       "   6030,\n",
       "   6220,\n",
       "   6106,\n",
       "   6306,\n",
       "   6258,\n",
       "   6212,\n",
       "   6268,\n",
       "   6166,\n",
       "   6117,\n",
       "   6104,\n",
       "   6230,\n",
       "   6320,\n",
       "   6193,\n",
       "   6202,\n",
       "   6095,\n",
       "   6394,\n",
       "   6227,\n",
       "   6225,\n",
       "   6292,\n",
       "   6344,\n",
       "   6410,\n",
       "   6318,\n",
       "   6183,\n",
       "   6186,\n",
       "   6055,\n",
       "   6321,\n",
       "   6132,\n",
       "   6081,\n",
       "   6058,\n",
       "   6122,\n",
       "   6226,\n",
       "   6187,\n",
       "   6148,\n",
       "   6409,\n",
       "   6074,\n",
       "   6380,\n",
       "   6197,\n",
       "   6185,\n",
       "   6138,\n",
       "   6050,\n",
       "   6204,\n",
       "   6357,\n",
       "   6223,\n",
       "   6329,\n",
       "   6297,\n",
       "   6080,\n",
       "   6244,\n",
       "   6149,\n",
       "   6175,\n",
       "   6044,\n",
       "   6126,\n",
       "   6069,\n",
       "   6372,\n",
       "   6367,\n",
       "   6370,\n",
       "   6411,\n",
       "   6094,\n",
       "   6341,\n",
       "   6385,\n",
       "   6129,\n",
       "   6047,\n",
       "   6136,\n",
       "   6171,\n",
       "   6386,\n",
       "   6076,\n",
       "   6169,\n",
       "   6350,\n",
       "   6092,\n",
       "   6375,\n",
       "   6130,\n",
       "   6247,\n",
       "   6218,\n",
       "   6068,\n",
       "   6310,\n",
       "   6291,\n",
       "   6091,\n",
       "   6417,\n",
       "   6338,\n",
       "   6232,\n",
       "   6274,\n",
       "   6205,\n",
       "   6142],\n",
       "  'misc.forsale': [3843,\n",
       "   4055,\n",
       "   4083,\n",
       "   4191,\n",
       "   3820,\n",
       "   3972,\n",
       "   3891,\n",
       "   3983,\n",
       "   4033,\n",
       "   3852,\n",
       "   3934,\n",
       "   3858,\n",
       "   3842,\n",
       "   4105,\n",
       "   3857,\n",
       "   3914,\n",
       "   3854,\n",
       "   3869,\n",
       "   3827,\n",
       "   3975,\n",
       "   4057,\n",
       "   4066,\n",
       "   3939,\n",
       "   3905,\n",
       "   3971,\n",
       "   3957,\n",
       "   3815,\n",
       "   4046,\n",
       "   3871,\n",
       "   3962,\n",
       "   3861,\n",
       "   4010,\n",
       "   3945,\n",
       "   3853,\n",
       "   3835,\n",
       "   3881,\n",
       "   4018,\n",
       "   4091,\n",
       "   4090,\n",
       "   3874,\n",
       "   4075,\n",
       "   3973,\n",
       "   4190,\n",
       "   4092,\n",
       "   4003,\n",
       "   4162,\n",
       "   4077,\n",
       "   3995,\n",
       "   3855,\n",
       "   3961,\n",
       "   4175,\n",
       "   3943,\n",
       "   3893,\n",
       "   4112,\n",
       "   4053,\n",
       "   4182,\n",
       "   4171,\n",
       "   4113,\n",
       "   3911,\n",
       "   4198,\n",
       "   4177,\n",
       "   3866,\n",
       "   3856,\n",
       "   4072,\n",
       "   3825,\n",
       "   3919,\n",
       "   4160,\n",
       "   3938,\n",
       "   3889,\n",
       "   3863,\n",
       "   4135,\n",
       "   3978,\n",
       "   3831,\n",
       "   3922,\n",
       "   4094,\n",
       "   4013,\n",
       "   3879,\n",
       "   3814,\n",
       "   4158,\n",
       "   4122,\n",
       "   4020,\n",
       "   4037,\n",
       "   3833,\n",
       "   4118,\n",
       "   4111,\n",
       "   3862,\n",
       "   4121,\n",
       "   4052,\n",
       "   3953,\n",
       "   3865,\n",
       "   3997,\n",
       "   4006,\n",
       "   4032,\n",
       "   3877,\n",
       "   4093,\n",
       "   4151,\n",
       "   3813,\n",
       "   3838,\n",
       "   3810,\n",
       "   3870,\n",
       "   4197,\n",
       "   3836,\n",
       "   4146,\n",
       "   4176,\n",
       "   4173,\n",
       "   3868,\n",
       "   4000,\n",
       "   4019,\n",
       "   3923,\n",
       "   4114,\n",
       "   3884,\n",
       "   3920,\n",
       "   3901,\n",
       "   3985,\n",
       "   4024,\n",
       "   3875,\n",
       "   3937,\n",
       "   3967,\n",
       "   4061,\n",
       "   3963,\n",
       "   4124,\n",
       "   4051,\n",
       "   4082,\n",
       "   4007,\n",
       "   4143,\n",
       "   3951,\n",
       "   4170,\n",
       "   4017,\n",
       "   3979,\n",
       "   4153,\n",
       "   4154,\n",
       "   4103,\n",
       "   4001,\n",
       "   4128,\n",
       "   3950,\n",
       "   4123,\n",
       "   4080,\n",
       "   3965,\n",
       "   4097,\n",
       "   3948,\n",
       "   3912,\n",
       "   4199,\n",
       "   3888,\n",
       "   4062,\n",
       "   3864,\n",
       "   4030,\n",
       "   4138,\n",
       "   4174,\n",
       "   3841,\n",
       "   4026,\n",
       "   3839,\n",
       "   4196,\n",
       "   4028,\n",
       "   3981,\n",
       "   3931,\n",
       "   4084,\n",
       "   3980,\n",
       "   3818,\n",
       "   4144,\n",
       "   4145,\n",
       "   4079,\n",
       "   3917,\n",
       "   3824,\n",
       "   4169,\n",
       "   3933,\n",
       "   3924,\n",
       "   4036,\n",
       "   4193,\n",
       "   4119,\n",
       "   4140,\n",
       "   4117,\n",
       "   4168,\n",
       "   4183,\n",
       "   3878,\n",
       "   4049,\n",
       "   4101,\n",
       "   4131,\n",
       "   4040,\n",
       "   4185,\n",
       "   4188,\n",
       "   3850,\n",
       "   3883],\n",
       "  'rec.autos': [7124,\n",
       "   7069,\n",
       "   7062,\n",
       "   6853,\n",
       "   6893,\n",
       "   7177,\n",
       "   6940,\n",
       "   7120,\n",
       "   6924,\n",
       "   6914,\n",
       "   7093,\n",
       "   7144,\n",
       "   7162,\n",
       "   6953,\n",
       "   7199,\n",
       "   7189,\n",
       "   7095,\n",
       "   6886,\n",
       "   6900,\n",
       "   7184,\n",
       "   7148,\n",
       "   6981,\n",
       "   7147,\n",
       "   6975,\n",
       "   7140,\n",
       "   6857,\n",
       "   7111,\n",
       "   6830,\n",
       "   6898,\n",
       "   6831,\n",
       "   6996,\n",
       "   7114,\n",
       "   6978,\n",
       "   6922,\n",
       "   6944,\n",
       "   6874,\n",
       "   6844,\n",
       "   7153,\n",
       "   7155,\n",
       "   7046,\n",
       "   6824,\n",
       "   6960,\n",
       "   6838,\n",
       "   7002,\n",
       "   6858,\n",
       "   7109,\n",
       "   7010,\n",
       "   6992,\n",
       "   7053,\n",
       "   7087,\n",
       "   7209,\n",
       "   6926,\n",
       "   7143,\n",
       "   7205,\n",
       "   6845,\n",
       "   7100,\n",
       "   6910,\n",
       "   6906,\n",
       "   7086,\n",
       "   7166,\n",
       "   6947,\n",
       "   7040,\n",
       "   7142,\n",
       "   7180,\n",
       "   6897,\n",
       "   6988,\n",
       "   6867,\n",
       "   6847,\n",
       "   6921,\n",
       "   6885,\n",
       "   7182,\n",
       "   6950,\n",
       "   6882,\n",
       "   7187,\n",
       "   6951,\n",
       "   7133,\n",
       "   6817,\n",
       "   7075,\n",
       "   6904,\n",
       "   6913,\n",
       "   6967,\n",
       "   7151,\n",
       "   6931,\n",
       "   7088,\n",
       "   6821,\n",
       "   7154,\n",
       "   6839,\n",
       "   6895,\n",
       "   6869,\n",
       "   7208,\n",
       "   7137,\n",
       "   7118,\n",
       "   7051,\n",
       "   7042,\n",
       "   7156,\n",
       "   6868,\n",
       "   6834,\n",
       "   7066,\n",
       "   6952,\n",
       "   7055,\n",
       "   7057,\n",
       "   6866,\n",
       "   7072,\n",
       "   7079,\n",
       "   6961,\n",
       "   6939,\n",
       "   6905,\n",
       "   6907,\n",
       "   7119,\n",
       "   7073,\n",
       "   7067,\n",
       "   7138,\n",
       "   6876,\n",
       "   7026,\n",
       "   7060,\n",
       "   6879,\n",
       "   6982,\n",
       "   6948,\n",
       "   6826,\n",
       "   7171,\n",
       "   6937,\n",
       "   7113,\n",
       "   6903,\n",
       "   7127,\n",
       "   6927,\n",
       "   7016,\n",
       "   7038,\n",
       "   7027,\n",
       "   7014,\n",
       "   7019,\n",
       "   6860,\n",
       "   7036,\n",
       "   7126,\n",
       "   7089,\n",
       "   6977,\n",
       "   7175,\n",
       "   7193,\n",
       "   6923,\n",
       "   6957,\n",
       "   7167,\n",
       "   6836,\n",
       "   6963,\n",
       "   6884,\n",
       "   7005,\n",
       "   7030,\n",
       "   7090,\n",
       "   7000,\n",
       "   7084,\n",
       "   7210,\n",
       "   6854,\n",
       "   7179,\n",
       "   7158,\n",
       "   7212,\n",
       "   7024,\n",
       "   7159,\n",
       "   6971,\n",
       "   7195,\n",
       "   6833,\n",
       "   7132,\n",
       "   7004,\n",
       "   6933,\n",
       "   6823,\n",
       "   6929,\n",
       "   7032,\n",
       "   7128,\n",
       "   7044,\n",
       "   7211,\n",
       "   7061,\n",
       "   6862,\n",
       "   7011,\n",
       "   7099,\n",
       "   6916,\n",
       "   6983,\n",
       "   6973,\n",
       "   6896,\n",
       "   6880,\n",
       "   6943,\n",
       "   7152,\n",
       "   7076,\n",
       "   7045,\n",
       "   7165,\n",
       "   6993,\n",
       "   7065,\n",
       "   6850,\n",
       "   6888,\n",
       "   6998,\n",
       "   6825,\n",
       "   7183,\n",
       "   7129,\n",
       "   6954,\n",
       "   7108,\n",
       "   6956,\n",
       "   6985,\n",
       "   6986],\n",
       "  'rec.motorcycles': [4389,\n",
       "   4218,\n",
       "   4501,\n",
       "   4481,\n",
       "   4345,\n",
       "   4321,\n",
       "   4452,\n",
       "   4348,\n",
       "   4440,\n",
       "   4330,\n",
       "   4533,\n",
       "   4409,\n",
       "   4370,\n",
       "   4233,\n",
       "   4340,\n",
       "   4371,\n",
       "   4461,\n",
       "   4299,\n",
       "   4332,\n",
       "   4212,\n",
       "   4564,\n",
       "   4360,\n",
       "   4272,\n",
       "   4558,\n",
       "   4432,\n",
       "   4441,\n",
       "   4470,\n",
       "   4590,\n",
       "   4530,\n",
       "   4499,\n",
       "   4594,\n",
       "   4495,\n",
       "   4454,\n",
       "   4417,\n",
       "   4515,\n",
       "   4497,\n",
       "   4214,\n",
       "   4415,\n",
       "   4477,\n",
       "   4503,\n",
       "   4252,\n",
       "   4278,\n",
       "   4486,\n",
       "   4350,\n",
       "   4270,\n",
       "   4379,\n",
       "   4246,\n",
       "   4289,\n",
       "   4466,\n",
       "   4385,\n",
       "   4399,\n",
       "   4224,\n",
       "   4490,\n",
       "   4428,\n",
       "   4319,\n",
       "   4375,\n",
       "   4377,\n",
       "   4388,\n",
       "   4284,\n",
       "   4312,\n",
       "   4419,\n",
       "   4529,\n",
       "   4373,\n",
       "   4393,\n",
       "   4416,\n",
       "   4438,\n",
       "   4505,\n",
       "   4343,\n",
       "   4473,\n",
       "   4566,\n",
       "   4535,\n",
       "   4353,\n",
       "   4403,\n",
       "   4322,\n",
       "   4512,\n",
       "   4244,\n",
       "   4342,\n",
       "   4445,\n",
       "   4570,\n",
       "   4326,\n",
       "   4460,\n",
       "   4318,\n",
       "   4510,\n",
       "   4324,\n",
       "   4500,\n",
       "   4374,\n",
       "   4491,\n",
       "   4544,\n",
       "   4304,\n",
       "   4541,\n",
       "   4277,\n",
       "   4361,\n",
       "   4295,\n",
       "   4305,\n",
       "   4555,\n",
       "   4387,\n",
       "   4458,\n",
       "   4551,\n",
       "   4219,\n",
       "   4485,\n",
       "   4314,\n",
       "   4565,\n",
       "   4400,\n",
       "   4411,\n",
       "   4320,\n",
       "   4260,\n",
       "   4584,\n",
       "   4357,\n",
       "   4546,\n",
       "   4552,\n",
       "   4339,\n",
       "   4455,\n",
       "   4238,\n",
       "   4511,\n",
       "   4258,\n",
       "   4487,\n",
       "   4257,\n",
       "   4591,\n",
       "   4397,\n",
       "   4462,\n",
       "   4504,\n",
       "   4228,\n",
       "   4482,\n",
       "   4264,\n",
       "   4282,\n",
       "   4413,\n",
       "   4518,\n",
       "   4519,\n",
       "   4327,\n",
       "   4251,\n",
       "   4502,\n",
       "   4226,\n",
       "   4262,\n",
       "   4582,\n",
       "   4302,\n",
       "   4293,\n",
       "   4234,\n",
       "   4423,\n",
       "   4422,\n",
       "   4532,\n",
       "   4559,\n",
       "   4297,\n",
       "   4597,\n",
       "   4283,\n",
       "   4207,\n",
       "   4263,\n",
       "   4563,\n",
       "   4316,\n",
       "   4543,\n",
       "   4229,\n",
       "   4351,\n",
       "   4210,\n",
       "   4296,\n",
       "   4412,\n",
       "   4575,\n",
       "   4507,\n",
       "   4513,\n",
       "   4574,\n",
       "   4421,\n",
       "   4256,\n",
       "   4596,\n",
       "   4378,\n",
       "   4369,\n",
       "   4468,\n",
       "   4215,\n",
       "   4362,\n",
       "   4254,\n",
       "   4331,\n",
       "   4325,\n",
       "   4355,\n",
       "   4418,\n",
       "   4298,\n",
       "   4521,\n",
       "   4572,\n",
       "   4408,\n",
       "   4424,\n",
       "   4352,\n",
       "   4248,\n",
       "   4508,\n",
       "   4456,\n",
       "   4522,\n",
       "   4404,\n",
       "   4255,\n",
       "   4366,\n",
       "   4410,\n",
       "   4561,\n",
       "   4449,\n",
       "   4467,\n",
       "   4354,\n",
       "   4520,\n",
       "   4578,\n",
       "   4514,\n",
       "   4358,\n",
       "   4405,\n",
       "   4394],\n",
       "  'rec.sport.baseball': [3202,\n",
       "   3402,\n",
       "   3368,\n",
       "   3407,\n",
       "   3176,\n",
       "   3272,\n",
       "   3203,\n",
       "   3230,\n",
       "   3386,\n",
       "   3316,\n",
       "   3055,\n",
       "   3350,\n",
       "   3193,\n",
       "   3260,\n",
       "   3098,\n",
       "   3065,\n",
       "   3142,\n",
       "   3161,\n",
       "   3213,\n",
       "   3022,\n",
       "   3227,\n",
       "   3267,\n",
       "   3149,\n",
       "   3147,\n",
       "   3372,\n",
       "   3344,\n",
       "   3226,\n",
       "   3326,\n",
       "   3116,\n",
       "   3248,\n",
       "   3105,\n",
       "   3365,\n",
       "   3364,\n",
       "   3056,\n",
       "   3238,\n",
       "   3412,\n",
       "   3313,\n",
       "   3335,\n",
       "   3222,\n",
       "   3062,\n",
       "   3233,\n",
       "   3095,\n",
       "   3054,\n",
       "   3040,\n",
       "   3336,\n",
       "   3414,\n",
       "   3087,\n",
       "   3190,\n",
       "   3331,\n",
       "   3043,\n",
       "   3063,\n",
       "   3078,\n",
       "   3180,\n",
       "   3393,\n",
       "   3360,\n",
       "   3264,\n",
       "   3130,\n",
       "   3295,\n",
       "   3284,\n",
       "   3128,\n",
       "   3400,\n",
       "   3352,\n",
       "   3042,\n",
       "   3032,\n",
       "   3155,\n",
       "   3378,\n",
       "   3334,\n",
       "   3390,\n",
       "   3100,\n",
       "   3145,\n",
       "   3172,\n",
       "   3234,\n",
       "   3408,\n",
       "   3394,\n",
       "   3239,\n",
       "   3403,\n",
       "   3280,\n",
       "   3319,\n",
       "   3058,\n",
       "   3252,\n",
       "   3398,\n",
       "   3088,\n",
       "   3094,\n",
       "   3160,\n",
       "   3275,\n",
       "   3036,\n",
       "   3124,\n",
       "   3330,\n",
       "   3077,\n",
       "   3377,\n",
       "   3041,\n",
       "   3030,\n",
       "   3215,\n",
       "   3361,\n",
       "   3051,\n",
       "   3037,\n",
       "   3216,\n",
       "   3123,\n",
       "   3073,\n",
       "   3208,\n",
       "   3033,\n",
       "   3369,\n",
       "   3194,\n",
       "   3212,\n",
       "   3035,\n",
       "   3173,\n",
       "   3396,\n",
       "   3253,\n",
       "   3358,\n",
       "   3059,\n",
       "   3178,\n",
       "   3244,\n",
       "   3315,\n",
       "   3218,\n",
       "   3028,\n",
       "   3255,\n",
       "   3154,\n",
       "   3192,\n",
       "   3111,\n",
       "   3219,\n",
       "   3113,\n",
       "   3109,\n",
       "   3217,\n",
       "   3254,\n",
       "   3285,\n",
       "   3375,\n",
       "   3249,\n",
       "   3229,\n",
       "   3302,\n",
       "   3410,\n",
       "   3201,\n",
       "   3370,\n",
       "   3409,\n",
       "   3401,\n",
       "   3406,\n",
       "   3185,\n",
       "   3389,\n",
       "   3277,\n",
       "   3223,\n",
       "   3323,\n",
       "   3362,\n",
       "   3243,\n",
       "   3070,\n",
       "   3159,\n",
       "   3391,\n",
       "   3245,\n",
       "   3108,\n",
       "   3183,\n",
       "   3090,\n",
       "   3395,\n",
       "   3115,\n",
       "   3126,\n",
       "   3093,\n",
       "   3273,\n",
       "   3268,\n",
       "   3082,\n",
       "   3261,\n",
       "   3122,\n",
       "   3084,\n",
       "   3296,\n",
       "   3317,\n",
       "   3144,\n",
       "   3297,\n",
       "   3151,\n",
       "   3321,\n",
       "   3175,\n",
       "   3068,\n",
       "   3044,\n",
       "   3225,\n",
       "   3135,\n",
       "   3298,\n",
       "   3343,\n",
       "   3309,\n",
       "   3085,\n",
       "   3283,\n",
       "   3156,\n",
       "   3339,\n",
       "   3337,\n",
       "   3052,\n",
       "   3066,\n",
       "   3376,\n",
       "   3384,\n",
       "   3382,\n",
       "   3210,\n",
       "   3333,\n",
       "   3137,\n",
       "   3266,\n",
       "   3322],\n",
       "  'rec.sport.hockey': [122,\n",
       "   211,\n",
       "   198,\n",
       "   282,\n",
       "   255,\n",
       "   372,\n",
       "   170,\n",
       "   62,\n",
       "   227,\n",
       "   175,\n",
       "   331,\n",
       "   281,\n",
       "   155,\n",
       "   133,\n",
       "   384,\n",
       "   6,\n",
       "   370,\n",
       "   161,\n",
       "   114,\n",
       "   203,\n",
       "   252,\n",
       "   276,\n",
       "   27,\n",
       "   78,\n",
       "   249,\n",
       "   64,\n",
       "   25,\n",
       "   398,\n",
       "   47,\n",
       "   53,\n",
       "   3,\n",
       "   383,\n",
       "   290,\n",
       "   214,\n",
       "   347,\n",
       "   353,\n",
       "   167,\n",
       "   164,\n",
       "   367,\n",
       "   242,\n",
       "   98,\n",
       "   30,\n",
       "   314,\n",
       "   373,\n",
       "   159,\n",
       "   310,\n",
       "   134,\n",
       "   337,\n",
       "   354,\n",
       "   352,\n",
       "   72,\n",
       "   121,\n",
       "   191,\n",
       "   136,\n",
       "   174,\n",
       "   241,\n",
       "   334,\n",
       "   162,\n",
       "   197,\n",
       "   192,\n",
       "   82,\n",
       "   11,\n",
       "   385,\n",
       "   204,\n",
       "   305,\n",
       "   253,\n",
       "   218,\n",
       "   154,\n",
       "   379,\n",
       "   2,\n",
       "   202,\n",
       "   20,\n",
       "   178,\n",
       "   147,\n",
       "   73,\n",
       "   105,\n",
       "   362,\n",
       "   14,\n",
       "   49,\n",
       "   393,\n",
       "   148,\n",
       "   199,\n",
       "   140,\n",
       "   288,\n",
       "   317,\n",
       "   332,\n",
       "   117,\n",
       "   40,\n",
       "   272,\n",
       "   254,\n",
       "   81,\n",
       "   103,\n",
       "   21,\n",
       "   340,\n",
       "   267,\n",
       "   156,\n",
       "   120,\n",
       "   201,\n",
       "   328,\n",
       "   184,\n",
       "   196,\n",
       "   151,\n",
       "   244,\n",
       "   24,\n",
       "   397,\n",
       "   80,\n",
       "   237,\n",
       "   298,\n",
       "   71,\n",
       "   187,\n",
       "   301,\n",
       "   235,\n",
       "   172,\n",
       "   92,\n",
       "   10,\n",
       "   248,\n",
       "   48,\n",
       "   45,\n",
       "   126,\n",
       "   59,\n",
       "   303,\n",
       "   149,\n",
       "   312,\n",
       "   230,\n",
       "   38,\n",
       "   357,\n",
       "   18,\n",
       "   85,\n",
       "   153,\n",
       "   43,\n",
       "   36,\n",
       "   101,\n",
       "   173,\n",
       "   1,\n",
       "   212,\n",
       "   150,\n",
       "   238,\n",
       "   139,\n",
       "   183,\n",
       "   56,\n",
       "   313,\n",
       "   351,\n",
       "   171,\n",
       "   240,\n",
       "   330,\n",
       "   350,\n",
       "   342,\n",
       "   113,\n",
       "   189,\n",
       "   50,\n",
       "   263,\n",
       "   284,\n",
       "   287,\n",
       "   169,\n",
       "   358,\n",
       "   143,\n",
       "   94,\n",
       "   13,\n",
       "   102,\n",
       "   376,\n",
       "   236,\n",
       "   51,\n",
       "   181,\n",
       "   182,\n",
       "   54,\n",
       "   26,\n",
       "   229,\n",
       "   210,\n",
       "   90,\n",
       "   231,\n",
       "   371,\n",
       "   345,\n",
       "   163,\n",
       "   207,\n",
       "   141,\n",
       "   206,\n",
       "   84,\n",
       "   135,\n",
       "   302,\n",
       "   88,\n",
       "   222,\n",
       "   200,\n",
       "   9,\n",
       "   260,\n",
       "   300,\n",
       "   308],\n",
       "  'sci.crypt': [2502,\n",
       "   2430,\n",
       "   2367,\n",
       "   2627,\n",
       "   2323,\n",
       "   2233,\n",
       "   2624,\n",
       "   2293,\n",
       "   2513,\n",
       "   2580,\n",
       "   2318,\n",
       "   2272,\n",
       "   2567,\n",
       "   2438,\n",
       "   2308,\n",
       "   2377,\n",
       "   2542,\n",
       "   2262,\n",
       "   2289,\n",
       "   2578,\n",
       "   2260,\n",
       "   2410,\n",
       "   2412,\n",
       "   2239,\n",
       "   2329,\n",
       "   2607,\n",
       "   2546,\n",
       "   2345,\n",
       "   2458,\n",
       "   2421,\n",
       "   2292,\n",
       "   2249,\n",
       "   2267,\n",
       "   2261,\n",
       "   2611,\n",
       "   2304,\n",
       "   2623,\n",
       "   2384,\n",
       "   2457,\n",
       "   2484,\n",
       "   2585,\n",
       "   2243,\n",
       "   2413,\n",
       "   2273,\n",
       "   2305,\n",
       "   2533,\n",
       "   2520,\n",
       "   2620,\n",
       "   2290,\n",
       "   2281,\n",
       "   2548,\n",
       "   2582,\n",
       "   2331,\n",
       "   2404,\n",
       "   2613,\n",
       "   2488,\n",
       "   2399,\n",
       "   2600,\n",
       "   2393,\n",
       "   2481,\n",
       "   2490,\n",
       "   2503,\n",
       "   2363,\n",
       "   2452,\n",
       "   2526,\n",
       "   2461,\n",
       "   2236,\n",
       "   2453,\n",
       "   2581,\n",
       "   2394,\n",
       "   2444,\n",
       "   2476,\n",
       "   2507,\n",
       "   2347,\n",
       "   2310,\n",
       "   2356,\n",
       "   2462,\n",
       "   2536,\n",
       "   2375,\n",
       "   2235,\n",
       "   2482,\n",
       "   2326,\n",
       "   2357,\n",
       "   2535,\n",
       "   2621,\n",
       "   2506,\n",
       "   2560,\n",
       "   2463,\n",
       "   2456,\n",
       "   2368,\n",
       "   2583,\n",
       "   2557,\n",
       "   2321,\n",
       "   2601,\n",
       "   2570,\n",
       "   2510,\n",
       "   2467,\n",
       "   2471,\n",
       "   2437,\n",
       "   2307,\n",
       "   2358,\n",
       "   2370,\n",
       "   2391,\n",
       "   2487,\n",
       "   2549,\n",
       "   2586,\n",
       "   2415,\n",
       "   2618,\n",
       "   2478,\n",
       "   2439,\n",
       "   2427,\n",
       "   2280,\n",
       "   2559,\n",
       "   2497,\n",
       "   2575,\n",
       "   2333,\n",
       "   2450,\n",
       "   2237,\n",
       "   2378,\n",
       "   2551,\n",
       "   2344,\n",
       "   2395,\n",
       "   2383,\n",
       "   2498,\n",
       "   2301,\n",
       "   2445,\n",
       "   2302,\n",
       "   2505,\n",
       "   2539,\n",
       "   2364,\n",
       "   2626,\n",
       "   2371,\n",
       "   2406,\n",
       "   2609,\n",
       "   2324,\n",
       "   2325,\n",
       "   2555,\n",
       "   2590,\n",
       "   2373,\n",
       "   2408,\n",
       "   2465,\n",
       "   2440,\n",
       "   2401,\n",
       "   2576,\n",
       "   2596,\n",
       "   2616,\n",
       "   2247,\n",
       "   2322,\n",
       "   2351,\n",
       "   2286,\n",
       "   2433,\n",
       "   2573,\n",
       "   2563,\n",
       "   2275,\n",
       "   2543,\n",
       "   2291,\n",
       "   2492,\n",
       "   2387,\n",
       "   2524,\n",
       "   2594,\n",
       "   2409,\n",
       "   2564,\n",
       "   2300,\n",
       "   2240,\n",
       "   2349,\n",
       "   2354,\n",
       "   2562,\n",
       "   2234,\n",
       "   2295,\n",
       "   2386,\n",
       "   2420,\n",
       "   2337,\n",
       "   2315,\n",
       "   2448,\n",
       "   2429,\n",
       "   2407,\n",
       "   2521,\n",
       "   2360,\n",
       "   2493,\n",
       "   2385,\n",
       "   2553,\n",
       "   2523,\n",
       "   2271,\n",
       "   2254,\n",
       "   2606,\n",
       "   2460,\n",
       "   2312,\n",
       "   2250],\n",
       "  'sci.electronics': [3752,\n",
       "   3564,\n",
       "   3472,\n",
       "   3628,\n",
       "   3515,\n",
       "   3473,\n",
       "   3468,\n",
       "   3432,\n",
       "   3559,\n",
       "   3476,\n",
       "   3800,\n",
       "   3657,\n",
       "   3422,\n",
       "   3483,\n",
       "   3539,\n",
       "   3758,\n",
       "   3599,\n",
       "   3609,\n",
       "   3444,\n",
       "   3735,\n",
       "   3691,\n",
       "   3627,\n",
       "   3440,\n",
       "   3744,\n",
       "   3447,\n",
       "   3512,\n",
       "   3729,\n",
       "   3453,\n",
       "   3541,\n",
       "   3586,\n",
       "   3488,\n",
       "   3581,\n",
       "   3495,\n",
       "   3619,\n",
       "   3521,\n",
       "   3423,\n",
       "   3561,\n",
       "   3493,\n",
       "   3675,\n",
       "   3603,\n",
       "   3782,\n",
       "   3510,\n",
       "   3779,\n",
       "   3655,\n",
       "   3451,\n",
       "   3646,\n",
       "   3780,\n",
       "   3652,\n",
       "   3736,\n",
       "   3431,\n",
       "   3485,\n",
       "   3614,\n",
       "   3520,\n",
       "   3726,\n",
       "   3508,\n",
       "   3419,\n",
       "   3523,\n",
       "   3724,\n",
       "   3711,\n",
       "   3647,\n",
       "   3630,\n",
       "   3542,\n",
       "   3716,\n",
       "   3681,\n",
       "   3725,\n",
       "   3590,\n",
       "   3445,\n",
       "   3659,\n",
       "   3441,\n",
       "   3809,\n",
       "   3635,\n",
       "   3426,\n",
       "   3719,\n",
       "   3424,\n",
       "   3696,\n",
       "   3794,\n",
       "   3543,\n",
       "   3737,\n",
       "   3641,\n",
       "   3713,\n",
       "   3636,\n",
       "   3616,\n",
       "   3727,\n",
       "   3601,\n",
       "   3645,\n",
       "   3442,\n",
       "   3754,\n",
       "   3721,\n",
       "   3664,\n",
       "   3562,\n",
       "   3738,\n",
       "   3428,\n",
       "   3540,\n",
       "   3497,\n",
       "   3558,\n",
       "   3507,\n",
       "   3443,\n",
       "   3714,\n",
       "   3593,\n",
       "   3667,\n",
       "   3795,\n",
       "   3789,\n",
       "   3741,\n",
       "   3481,\n",
       "   3693,\n",
       "   3522,\n",
       "   3808,\n",
       "   3750,\n",
       "   3770,\n",
       "   3467,\n",
       "   3668,\n",
       "   3739,\n",
       "   3632,\n",
       "   3649,\n",
       "   3504,\n",
       "   3695,\n",
       "   3669,\n",
       "   3450,\n",
       "   3787,\n",
       "   3673,\n",
       "   3745,\n",
       "   3548,\n",
       "   3709,\n",
       "   3429,\n",
       "   3565,\n",
       "   3690,\n",
       "   3766,\n",
       "   3730,\n",
       "   3804,\n",
       "   3661,\n",
       "   3742,\n",
       "   3557,\n",
       "   3679,\n",
       "   3597,\n",
       "   3805,\n",
       "   3595,\n",
       "   3712,\n",
       "   3797,\n",
       "   3793,\n",
       "   3710,\n",
       "   3707,\n",
       "   3579,\n",
       "   3449,\n",
       "   3566,\n",
       "   3531,\n",
       "   3684,\n",
       "   3509,\n",
       "   3501,\n",
       "   3791,\n",
       "   3660,\n",
       "   3448,\n",
       "   3570,\n",
       "   3648,\n",
       "   3584,\n",
       "   3665,\n",
       "   3751,\n",
       "   3658,\n",
       "   3572,\n",
       "   3534,\n",
       "   3622,\n",
       "   3799,\n",
       "   3699,\n",
       "   3549,\n",
       "   3479,\n",
       "   3705,\n",
       "   3418,\n",
       "   3618,\n",
       "   3798,\n",
       "   3777,\n",
       "   3783,\n",
       "   3732,\n",
       "   3775,\n",
       "   3580,\n",
       "   3680,\n",
       "   3785,\n",
       "   3781,\n",
       "   3677,\n",
       "   3639,\n",
       "   3728,\n",
       "   3461,\n",
       "   3458,\n",
       "   3480,\n",
       "   3806,\n",
       "   3492,\n",
       "   3623,\n",
       "   3759,\n",
       "   3474,\n",
       "   3624,\n",
       "   3702,\n",
       "   3576,\n",
       "   3776,\n",
       "   3621],\n",
       "  'sci.med': [5003,\n",
       "   5228,\n",
       "   5220,\n",
       "   5027,\n",
       "   5042,\n",
       "   4856,\n",
       "   5034,\n",
       "   5154,\n",
       "   4993,\n",
       "   5033,\n",
       "   5215,\n",
       "   5169,\n",
       "   5218,\n",
       "   5111,\n",
       "   5236,\n",
       "   5113,\n",
       "   4896,\n",
       "   5158,\n",
       "   4948,\n",
       "   5143,\n",
       "   5010,\n",
       "   5144,\n",
       "   4877,\n",
       "   5020,\n",
       "   5092,\n",
       "   5104,\n",
       "   4873,\n",
       "   4982,\n",
       "   5204,\n",
       "   5107,\n",
       "   5159,\n",
       "   5029,\n",
       "   5146,\n",
       "   4891,\n",
       "   5221,\n",
       "   4872,\n",
       "   4863,\n",
       "   4974,\n",
       "   4966,\n",
       "   4902,\n",
       "   4879,\n",
       "   5132,\n",
       "   5171,\n",
       "   4881,\n",
       "   5006,\n",
       "   5043,\n",
       "   5007,\n",
       "   5145,\n",
       "   5233,\n",
       "   4931,\n",
       "   4984,\n",
       "   5195,\n",
       "   5134,\n",
       "   5128,\n",
       "   5069,\n",
       "   5048,\n",
       "   4895,\n",
       "   4941,\n",
       "   5065,\n",
       "   5193,\n",
       "   5079,\n",
       "   5017,\n",
       "   5052,\n",
       "   5081,\n",
       "   4942,\n",
       "   5156,\n",
       "   4916,\n",
       "   5086,\n",
       "   5058,\n",
       "   5137,\n",
       "   5056,\n",
       "   5201,\n",
       "   5196,\n",
       "   5024,\n",
       "   4889,\n",
       "   4935,\n",
       "   4915,\n",
       "   5131,\n",
       "   5091,\n",
       "   5203,\n",
       "   4926,\n",
       "   5064,\n",
       "   4992,\n",
       "   5026,\n",
       "   4883,\n",
       "   4874,\n",
       "   4945,\n",
       "   4937,\n",
       "   4908,\n",
       "   4875,\n",
       "   5170,\n",
       "   5209,\n",
       "   5211,\n",
       "   4910,\n",
       "   5023,\n",
       "   4869,\n",
       "   5040,\n",
       "   4887,\n",
       "   5117,\n",
       "   4979,\n",
       "   4865,\n",
       "   4905,\n",
       "   4971,\n",
       "   5001,\n",
       "   4922,\n",
       "   4913,\n",
       "   5031,\n",
       "   4957,\n",
       "   4973,\n",
       "   5036,\n",
       "   5059,\n",
       "   4851,\n",
       "   5217,\n",
       "   4912,\n",
       "   5210,\n",
       "   4975,\n",
       "   4925,\n",
       "   4960,\n",
       "   5046,\n",
       "   4994,\n",
       "   5120,\n",
       "   4892,\n",
       "   4967,\n",
       "   5197,\n",
       "   5085,\n",
       "   4970,\n",
       "   4853,\n",
       "   5012,\n",
       "   5075,\n",
       "   5168,\n",
       "   4911,\n",
       "   4920,\n",
       "   5141,\n",
       "   4956,\n",
       "   5165,\n",
       "   5222,\n",
       "   4857,\n",
       "   5231,\n",
       "   5213,\n",
       "   5103,\n",
       "   4860,\n",
       "   5242,\n",
       "   5239,\n",
       "   5108,\n",
       "   4978,\n",
       "   4917,\n",
       "   5214,\n",
       "   5089,\n",
       "   5161,\n",
       "   4929,\n",
       "   5243,\n",
       "   4981,\n",
       "   5216,\n",
       "   4961,\n",
       "   4898,\n",
       "   4897,\n",
       "   4946,\n",
       "   5122,\n",
       "   4890,\n",
       "   5175,\n",
       "   5179,\n",
       "   4944,\n",
       "   5002,\n",
       "   5030,\n",
       "   5044,\n",
       "   5229,\n",
       "   4928,\n",
       "   4862,\n",
       "   4959,\n",
       "   4918,\n",
       "   4939,\n",
       "   5049,\n",
       "   4936,\n",
       "   5136,\n",
       "   5054,\n",
       "   5057,\n",
       "   4987,\n",
       "   4988,\n",
       "   5184,\n",
       "   5185,\n",
       "   5234,\n",
       "   5148],\n",
       "  'sci.space': [2140,\n",
       "   1855,\n",
       "   2194,\n",
       "   1967,\n",
       "   1932,\n",
       "   2019,\n",
       "   1903,\n",
       "   2195,\n",
       "   1873,\n",
       "   1854,\n",
       "   2139,\n",
       "   1968,\n",
       "   2029,\n",
       "   1940,\n",
       "   1995,\n",
       "   2068,\n",
       "   1992,\n",
       "   2005,\n",
       "   2009,\n",
       "   1856,\n",
       "   2037,\n",
       "   2188,\n",
       "   1993,\n",
       "   1910,\n",
       "   2071,\n",
       "   2024,\n",
       "   1899,\n",
       "   2007,\n",
       "   1925,\n",
       "   1958,\n",
       "   2190,\n",
       "   1956,\n",
       "   2169,\n",
       "   1982,\n",
       "   2128,\n",
       "   2143,\n",
       "   1966,\n",
       "   1943,\n",
       "   2081,\n",
       "   2053,\n",
       "   2213,\n",
       "   1931,\n",
       "   1874,\n",
       "   1927,\n",
       "   2202,\n",
       "   2096,\n",
       "   2165,\n",
       "   2002,\n",
       "   1846,\n",
       "   2010,\n",
       "   2177,\n",
       "   2017,\n",
       "   2075,\n",
       "   1978,\n",
       "   1997,\n",
       "   1991,\n",
       "   1897,\n",
       "   2049,\n",
       "   2147,\n",
       "   2101,\n",
       "   1935,\n",
       "   2054,\n",
       "   2072,\n",
       "   2193,\n",
       "   2115,\n",
       "   2209,\n",
       "   1896,\n",
       "   2092,\n",
       "   1979,\n",
       "   2064,\n",
       "   2043,\n",
       "   2196,\n",
       "   2016,\n",
       "   2133,\n",
       "   2149,\n",
       "   2142,\n",
       "   2172,\n",
       "   2159,\n",
       "   2226,\n",
       "   2038,\n",
       "   2214,\n",
       "   2122,\n",
       "   2025,\n",
       "   1928,\n",
       "   1850,\n",
       "   2148,\n",
       "   2229,\n",
       "   2221,\n",
       "   2083,\n",
       "   1912,\n",
       "   2170,\n",
       "   1865,\n",
       "   1859,\n",
       "   1913,\n",
       "   2198,\n",
       "   1987,\n",
       "   2086,\n",
       "   2211,\n",
       "   2126,\n",
       "   2207,\n",
       "   2189,\n",
       "   1974,\n",
       "   2057,\n",
       "   1994,\n",
       "   1964,\n",
       "   2098,\n",
       "   2026,\n",
       "   2197,\n",
       "   2060,\n",
       "   1864,\n",
       "   2220,\n",
       "   2206,\n",
       "   1898,\n",
       "   2155,\n",
       "   2215,\n",
       "   1893,\n",
       "   2028,\n",
       "   2047,\n",
       "   1981,\n",
       "   2108,\n",
       "   2129,\n",
       "   2231,\n",
       "   1965,\n",
       "   2127,\n",
       "   2031,\n",
       "   1938,\n",
       "   1853,\n",
       "   2042,\n",
       "   2110,\n",
       "   2117,\n",
       "   2151,\n",
       "   1955,\n",
       "   1851,\n",
       "   1882,\n",
       "   2001,\n",
       "   2066,\n",
       "   2112,\n",
       "   2052,\n",
       "   2020,\n",
       "   2163,\n",
       "   1905,\n",
       "   1952,\n",
       "   2041,\n",
       "   2089,\n",
       "   1878,\n",
       "   1961,\n",
       "   1875,\n",
       "   1881,\n",
       "   1902,\n",
       "   1863,\n",
       "   1934,\n",
       "   1914,\n",
       "   2040,\n",
       "   1880,\n",
       "   2008,\n",
       "   1951,\n",
       "   1888,\n",
       "   2111,\n",
       "   2201,\n",
       "   2113,\n",
       "   2034,\n",
       "   2114,\n",
       "   2076,\n",
       "   2023,\n",
       "   2230,\n",
       "   2187,\n",
       "   2012,\n",
       "   2099,\n",
       "   1924,\n",
       "   1930,\n",
       "   2125,\n",
       "   1973,\n",
       "   2069,\n",
       "   1949,\n",
       "   2065,\n",
       "   2105,\n",
       "   1941,\n",
       "   1887,\n",
       "   2223,\n",
       "   2070,\n",
       "   2141,\n",
       "   1909,\n",
       "   1879,\n",
       "   2132,\n",
       "   2046,\n",
       "   1960,\n",
       "   1908],\n",
       "  'soc.religion.christian': [5922,\n",
       "   5826,\n",
       "   5698,\n",
       "   5804,\n",
       "   5887,\n",
       "   5851,\n",
       "   5735,\n",
       "   5640,\n",
       "   5929,\n",
       "   5728,\n",
       "   5692,\n",
       "   6027,\n",
       "   5864,\n",
       "   5705,\n",
       "   5872,\n",
       "   5732,\n",
       "   5651,\n",
       "   5967,\n",
       "   5898,\n",
       "   5794,\n",
       "   5920,\n",
       "   5752,\n",
       "   5765,\n",
       "   5647,\n",
       "   5897,\n",
       "   6005,\n",
       "   5652,\n",
       "   5769,\n",
       "   5891,\n",
       "   5658,\n",
       "   5874,\n",
       "   5780,\n",
       "   5659,\n",
       "   5792,\n",
       "   5901,\n",
       "   5976,\n",
       "   6019,\n",
       "   6025,\n",
       "   5939,\n",
       "   5811,\n",
       "   5924,\n",
       "   5681,\n",
       "   5668,\n",
       "   5980,\n",
       "   5806,\n",
       "   5665,\n",
       "   5660,\n",
       "   5876,\n",
       "   5748,\n",
       "   5713,\n",
       "   5931,\n",
       "   5685,\n",
       "   6015,\n",
       "   5947,\n",
       "   5657,\n",
       "   5679,\n",
       "   5726,\n",
       "   5870,\n",
       "   5725,\n",
       "   5950,\n",
       "   5930,\n",
       "   5828,\n",
       "   5686,\n",
       "   5738,\n",
       "   6003,\n",
       "   5745,\n",
       "   5835,\n",
       "   5934,\n",
       "   5695,\n",
       "   5948,\n",
       "   5707,\n",
       "   5733,\n",
       "   5661,\n",
       "   5837,\n",
       "   5742,\n",
       "   6021,\n",
       "   5984,\n",
       "   5968,\n",
       "   5717,\n",
       "   5673,\n",
       "   5720,\n",
       "   5796,\n",
       "   5903,\n",
       "   5740,\n",
       "   5645,\n",
       "   6000,\n",
       "   5743,\n",
       "   5730,\n",
       "   5889,\n",
       "   5878,\n",
       "   5793,\n",
       "   5991,\n",
       "   5844,\n",
       "   5829,\n",
       "   5710,\n",
       "   5746,\n",
       "   5767,\n",
       "   5818,\n",
       "   5678,\n",
       "   5869,\n",
       "   5856,\n",
       "   5859,\n",
       "   5833,\n",
       "   5687,\n",
       "   5693,\n",
       "   5787,\n",
       "   5883,\n",
       "   5892,\n",
       "   5853,\n",
       "   5827,\n",
       "   5802,\n",
       "   5749,\n",
       "   5663,\n",
       "   5987,\n",
       "   5757,\n",
       "   5712,\n",
       "   5830,\n",
       "   5838,\n",
       "   5727,\n",
       "   5949,\n",
       "   5862,\n",
       "   6007,\n",
       "   5708,\n",
       "   5701,\n",
       "   5762,\n",
       "   5943,\n",
       "   5789,\n",
       "   5664,\n",
       "   5843,\n",
       "   5902,\n",
       "   5849,\n",
       "   5631,\n",
       "   5816,\n",
       "   6017,\n",
       "   5639,\n",
       "   5896,\n",
       "   5873,\n",
       "   5761,\n",
       "   5912,\n",
       "   5925,\n",
       "   5900,\n",
       "   5697,\n",
       "   5824,\n",
       "   5810,\n",
       "   5978,\n",
       "   5778,\n",
       "   5803,\n",
       "   5812,\n",
       "   5636,\n",
       "   5776,\n",
       "   5959,\n",
       "   5766,\n",
       "   5773,\n",
       "   5961,\n",
       "   5632,\n",
       "   5799,\n",
       "   5782,\n",
       "   6026,\n",
       "   5995,\n",
       "   5653,\n",
       "   5916,\n",
       "   5982,\n",
       "   5814,\n",
       "   5985,\n",
       "   5952,\n",
       "   5741,\n",
       "   5709],\n",
       "  'talk.politics.guns': [1052,\n",
       "   1117,\n",
       "   956,\n",
       "   1040,\n",
       "   937,\n",
       "   839,\n",
       "   996,\n",
       "   859,\n",
       "   793,\n",
       "   1098,\n",
       "   870,\n",
       "   912,\n",
       "   1145,\n",
       "   809,\n",
       "   805,\n",
       "   851,\n",
       "   901,\n",
       "   1043,\n",
       "   966,\n",
       "   920,\n",
       "   999,\n",
       "   796,\n",
       "   849,\n",
       "   876,\n",
       "   1086,\n",
       "   931,\n",
       "   963,\n",
       "   813,\n",
       "   993,\n",
       "   892,\n",
       "   878,\n",
       "   945,\n",
       "   962,\n",
       "   1073,\n",
       "   883,\n",
       "   1130,\n",
       "   1036,\n",
       "   1076,\n",
       "   1066,\n",
       "   973,\n",
       "   1037,\n",
       "   800,\n",
       "   1005,\n",
       "   1032,\n",
       "   942,\n",
       "   1035,\n",
       "   802,\n",
       "   814,\n",
       "   990,\n",
       "   1125,\n",
       "   874,\n",
       "   968,\n",
       "   1122,\n",
       "   799,\n",
       "   984,\n",
       "   788,\n",
       "   1041,\n",
       "   925,\n",
       "   1072,\n",
       "   864,\n",
       "   965,\n",
       "   857,\n",
       "   1111,\n",
       "   882,\n",
       "   1064,\n",
       "   1097,\n",
       "   957,\n",
       "   1140,\n",
       "   959,\n",
       "   989,\n",
       "   868,\n",
       "   1096,\n",
       "   941,\n",
       "   1135,\n",
       "   790,\n",
       "   1008,\n",
       "   1001,\n",
       "   985,\n",
       "   1019,\n",
       "   1011,\n",
       "   884,\n",
       "   986,\n",
       "   1071,\n",
       "   1124,\n",
       "   875,\n",
       "   873,\n",
       "   903,\n",
       "   935,\n",
       "   1127,\n",
       "   1051,\n",
       "   992,\n",
       "   1139,\n",
       "   1106,\n",
       "   971,\n",
       "   1091,\n",
       "   1010,\n",
       "   970,\n",
       "   1006,\n",
       "   789,\n",
       "   1038,\n",
       "   1118,\n",
       "   1116,\n",
       "   1114,\n",
       "   840,\n",
       "   825,\n",
       "   1069,\n",
       "   871,\n",
       "   1146,\n",
       "   1110,\n",
       "   900,\n",
       "   916,\n",
       "   902,\n",
       "   829,\n",
       "   921,\n",
       "   822,\n",
       "   946,\n",
       "   1007,\n",
       "   818,\n",
       "   955,\n",
       "   844,\n",
       "   1143,\n",
       "   1012,\n",
       "   975,\n",
       "   798,\n",
       "   913,\n",
       "   905,\n",
       "   1024,\n",
       "   812,\n",
       "   890,\n",
       "   833,\n",
       "   1108,\n",
       "   1093,\n",
       "   923,\n",
       "   914,\n",
       "   910,\n",
       "   821,\n",
       "   832,\n",
       "   853,\n",
       "   983,\n",
       "   969,\n",
       "   953,\n",
       "   891,\n",
       "   816,\n",
       "   1082,\n",
       "   1103,\n",
       "   852,\n",
       "   1063,\n",
       "   1058,\n",
       "   998,\n",
       "   867,\n",
       "   834,\n",
       "   827,\n",
       "   1081,\n",
       "   1119,\n",
       "   810,\n",
       "   1046,\n",
       "   896,\n",
       "   797,\n",
       "   886,\n",
       "   1100,\n",
       "   932,\n",
       "   794,\n",
       "   824,\n",
       "   947,\n",
       "   877,\n",
       "   899,\n",
       "   1022,\n",
       "   940,\n",
       "   1147],\n",
       "  'talk.politics.mideast': [1781,\n",
       "   1693,\n",
       "   1565,\n",
       "   1674,\n",
       "   1805,\n",
       "   1709,\n",
       "   1595,\n",
       "   1752,\n",
       "   1619,\n",
       "   1508,\n",
       "   1586,\n",
       "   1659,\n",
       "   1815,\n",
       "   1834,\n",
       "   1623,\n",
       "   1658,\n",
       "   1763,\n",
       "   1739,\n",
       "   1587,\n",
       "   1736,\n",
       "   1833,\n",
       "   1655,\n",
       "   1757,\n",
       "   1647,\n",
       "   1672,\n",
       "   1701,\n",
       "   1803,\n",
       "   1772,\n",
       "   1707,\n",
       "   1824,\n",
       "   1779,\n",
       "   1827,\n",
       "   1606,\n",
       "   1585,\n",
       "   1715,\n",
       "   1549,\n",
       "   1829,\n",
       "   1811,\n",
       "   1665,\n",
       "   1774,\n",
       "   1596,\n",
       "   1506,\n",
       "   1786,\n",
       "   1622,\n",
       "   1650,\n",
       "   1488,\n",
       "   1571,\n",
       "   1564,\n",
       "   1706,\n",
       "   1513,\n",
       "   1625,\n",
       "   1759,\n",
       "   1680,\n",
       "   1684,\n",
       "   1539,\n",
       "   1644,\n",
       "   1534,\n",
       "   1756,\n",
       "   1636,\n",
       "   1608,\n",
       "   1648,\n",
       "   1820,\n",
       "   1745,\n",
       "   1525,\n",
       "   1504,\n",
       "   1542,\n",
       "   1491,\n",
       "   1744,\n",
       "   1822,\n",
       "   1515,\n",
       "   1799,\n",
       "   1765,\n",
       "   1482,\n",
       "   1554,\n",
       "   1817,\n",
       "   1615,\n",
       "   1544,\n",
       "   1555,\n",
       "   1712,\n",
       "   1705,\n",
       "   1485,\n",
       "   1640,\n",
       "   1537,\n",
       "   1468,\n",
       "   1656,\n",
       "   1613,\n",
       "   1696,\n",
       "   1785,\n",
       "   1826,\n",
       "   1787,\n",
       "   1835,\n",
       "   1731,\n",
       "   1520,\n",
       "   1621,\n",
       "   1716,\n",
       "   1462,\n",
       "   1514,\n",
       "   1633,\n",
       "   1819,\n",
       "   1708,\n",
       "   1748,\n",
       "   1798,\n",
       "   1575,\n",
       "   1545,\n",
       "   1568,\n",
       "   1490,\n",
       "   1770,\n",
       "   1801,\n",
       "   1797,\n",
       "   1538,\n",
       "   1683,\n",
       "   1809,\n",
       "   1610,\n",
       "   1560,\n",
       "   1547,\n",
       "   1668,\n",
       "   1747,\n",
       "   1551,\n",
       "   1766,\n",
       "   1478,\n",
       "   1778,\n",
       "   1494,\n",
       "   1671,\n",
       "   1516,\n",
       "   1760,\n",
       "   1726,\n",
       "   1589,\n",
       "   1661,\n",
       "   1722,\n",
       "   1524,\n",
       "   1657,\n",
       "   1749,\n",
       "   1682,\n",
       "   1794,\n",
       "   1771,\n",
       "   1675,\n",
       "   1552,\n",
       "   1793,\n",
       "   1567,\n",
       "   1594,\n",
       "   1718,\n",
       "   1790,\n",
       "   1583,\n",
       "   1526,\n",
       "   1541,\n",
       "   1509,\n",
       "   1649,\n",
       "   1503,\n",
       "   1679,\n",
       "   1728,\n",
       "   1616,\n",
       "   1753,\n",
       "   1719,\n",
       "   1762],\n",
       "  'talk.politics.misc': [1350,\n",
       "   1290,\n",
       "   1444,\n",
       "   1450,\n",
       "   1386,\n",
       "   1319,\n",
       "   1203,\n",
       "   1205,\n",
       "   1248,\n",
       "   1169,\n",
       "   1366,\n",
       "   1246,\n",
       "   1261,\n",
       "   1306,\n",
       "   1245,\n",
       "   1382,\n",
       "   1268,\n",
       "   1252,\n",
       "   1278,\n",
       "   1354,\n",
       "   1210,\n",
       "   1404,\n",
       "   1362,\n",
       "   1333,\n",
       "   1440,\n",
       "   1271,\n",
       "   1419,\n",
       "   1460,\n",
       "   1453,\n",
       "   1178,\n",
       "   1400,\n",
       "   1374,\n",
       "   1288,\n",
       "   1335,\n",
       "   1364,\n",
       "   1204,\n",
       "   1376,\n",
       "   1267,\n",
       "   1212,\n",
       "   1348,\n",
       "   1285,\n",
       "   1156,\n",
       "   1375,\n",
       "   1215,\n",
       "   1372,\n",
       "   1381,\n",
       "   1332,\n",
       "   1256,\n",
       "   1229,\n",
       "   1430,\n",
       "   1191,\n",
       "   1251,\n",
       "   1222,\n",
       "   1351,\n",
       "   1429,\n",
       "   1216,\n",
       "   1192,\n",
       "   1182,\n",
       "   1244,\n",
       "   1320,\n",
       "   1265,\n",
       "   1293,\n",
       "   1176,\n",
       "   1402,\n",
       "   1276,\n",
       "   1447,\n",
       "   1243,\n",
       "   1195,\n",
       "   1159,\n",
       "   1367,\n",
       "   1249,\n",
       "   1408,\n",
       "   1301,\n",
       "   1236,\n",
       "   1270,\n",
       "   1391,\n",
       "   1395,\n",
       "   1449,\n",
       "   1220,\n",
       "   1377,\n",
       "   1266,\n",
       "   1304,\n",
       "   1231,\n",
       "   1448,\n",
       "   1257,\n",
       "   1254,\n",
       "   1241,\n",
       "   1359,\n",
       "   1407,\n",
       "   1196,\n",
       "   1461,\n",
       "   1302,\n",
       "   1443,\n",
       "   1318,\n",
       "   1326,\n",
       "   1329,\n",
       "   1451,\n",
       "   1411,\n",
       "   1225,\n",
       "   1459,\n",
       "   1308,\n",
       "   1158,\n",
       "   1328,\n",
       "   1378,\n",
       "   1152,\n",
       "   1221,\n",
       "   1280,\n",
       "   1213,\n",
       "   1262,\n",
       "   1343,\n",
       "   1207,\n",
       "   1274,\n",
       "   1240,\n",
       "   1282,\n",
       "   1356,\n",
       "   1200,\n",
       "   1286,\n",
       "   1283,\n",
       "   1425,\n",
       "   1317,\n",
       "   1423,\n",
       "   1235,\n",
       "   1330,\n",
       "   1363,\n",
       "   1410,\n",
       "   1432,\n",
       "   1162,\n",
       "   1233,\n",
       "   1392,\n",
       "   1313,\n",
       "   1379,\n",
       "   1299,\n",
       "   1452,\n",
       "   1250,\n",
       "   1253,\n",
       "   1439,\n",
       "   1455,\n",
       "   1341],\n",
       "  'talk.religion.misc': [4822,\n",
       "   4725,\n",
       "   4768,\n",
       "   4668,\n",
       "   4777,\n",
       "   4718,\n",
       "   4659,\n",
       "   4763,\n",
       "   4772,\n",
       "   4793,\n",
       "   4648,\n",
       "   4642,\n",
       "   4681,\n",
       "   4833,\n",
       "   4752,\n",
       "   4717,\n",
       "   4611,\n",
       "   4805,\n",
       "   4709,\n",
       "   4719,\n",
       "   4628,\n",
       "   4658,\n",
       "   4730,\n",
       "   4598,\n",
       "   4692,\n",
       "   4736,\n",
       "   4607,\n",
       "   4782,\n",
       "   4612,\n",
       "   4687,\n",
       "   4643,\n",
       "   4600,\n",
       "   4844,\n",
       "   4606,\n",
       "   4769,\n",
       "   4712,\n",
       "   4627,\n",
       "   4762,\n",
       "   4722,\n",
       "   4651,\n",
       "   4750,\n",
       "   4688,\n",
       "   4693,\n",
       "   4746,\n",
       "   4699,\n",
       "   4645,\n",
       "   4792,\n",
       "   4737,\n",
       "   4619,\n",
       "   4616,\n",
       "   4633,\n",
       "   4794,\n",
       "   4791,\n",
       "   4649,\n",
       "   4708,\n",
       "   4847,\n",
       "   4778,\n",
       "   4690,\n",
       "   4669,\n",
       "   4667,\n",
       "   4753,\n",
       "   4740,\n",
       "   4795,\n",
       "   4783,\n",
       "   4726,\n",
       "   4812,\n",
       "   4654,\n",
       "   4761,\n",
       "   4694,\n",
       "   4756,\n",
       "   4803,\n",
       "   4601,\n",
       "   4815,\n",
       "   4775,\n",
       "   4720,\n",
       "   4767,\n",
       "   4656,\n",
       "   4675,\n",
       "   4820,\n",
       "   4638,\n",
       "   4673,\n",
       "   4735,\n",
       "   4666,\n",
       "   4636,\n",
       "   4787,\n",
       "   4684,\n",
       "   4757,\n",
       "   4682,\n",
       "   4618,\n",
       "   4676,\n",
       "   4683,\n",
       "   4751,\n",
       "   4789,\n",
       "   4745,\n",
       "   4826,\n",
       "   4663,\n",
       "   4702,\n",
       "   4662,\n",
       "   4685,\n",
       "   4788,\n",
       "   4835,\n",
       "   4680,\n",
       "   4622,\n",
       "   4807,\n",
       "   4743,\n",
       "   4827,\n",
       "   4664,\n",
       "   4710,\n",
       "   4840,\n",
       "   4624,\n",
       "   4599,\n",
       "   4813]})"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_dict = {}\n",
    "test_dict = {}\n",
    "cat_ids = dev_df_select_.groupby('category').groups\n",
    "for cat,ids in cat_ids.items():\n",
    "    ids_li = ids.tolist()\n",
    "    test_ids_li = list(np.random.choice(ids_li, size=int(len(ids_li)/2), replace=False))\n",
    "    valid_ids_li = list(set(ids_li)-set(test_ids_li))\n",
    "    test_dict[cat] = test_ids_li\n",
    "    valid_dict[cat] = valid_ids_li\n",
    "#     print(cat)\n",
    "# cat_ids['alt.atheism'].tolist()\n",
    "valid_dict, test_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>category</th>\n",
       "      <th>preprocess_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>7215</td>\n",
       "      <td>12257</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>Bill ever heard of secular humanism Please che...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7218</td>\n",
       "      <td>12010</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>No wonder that we do n't see any detail for th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7219</td>\n",
       "      <td>12068</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>They must be theists in disguise In any event ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7220</td>\n",
       "      <td>12178</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>But it 's STILL HAPPENING That 's the entire p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7222</td>\n",
       "      <td>12276</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>reply to Keith M Ryan Jeez ca n't he get anyth...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4837</td>\n",
       "      <td>15465</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "      <td>Um I think you and the Bible are the ones insi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4841</td>\n",
       "      <td>15441</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "      <td>The killing of the ATF agents is a separate is...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4843</td>\n",
       "      <td>15365</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "      <td>Recently I 've asked myself a rather interesti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4845</td>\n",
       "      <td>15456</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "      <td>But neither of them claimed to have experiment...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4846</td>\n",
       "      <td>15477</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "      <td>Would someone be kind enought to document the ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3524 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         id            category  \\\n",
       "7215  12257         alt.atheism   \n",
       "7218  12010         alt.atheism   \n",
       "7219  12068         alt.atheism   \n",
       "7220  12178         alt.atheism   \n",
       "7222  12276         alt.atheism   \n",
       "...     ...                 ...   \n",
       "4837  15465  talk.religion.misc   \n",
       "4841  15441  talk.religion.misc   \n",
       "4843  15365  talk.religion.misc   \n",
       "4845  15456  talk.religion.misc   \n",
       "4846  15477  talk.religion.misc   \n",
       "\n",
       "                                        preprocess_text  \n",
       "7215  Bill ever heard of secular humanism Please che...  \n",
       "7218  No wonder that we do n't see any detail for th...  \n",
       "7219  They must be theists in disguise In any event ...  \n",
       "7220  But it 's STILL HAPPENING That 's the entire p...  \n",
       "7222  reply to Keith M Ryan Jeez ca n't he get anyth...  \n",
       "...                                                 ...  \n",
       "4837  Um I think you and the Bible are the ones insi...  \n",
       "4841  The killing of the ATF agents is a separate is...  \n",
       "4843  Recently I 've asked myself a rather interesti...  \n",
       "4845  But neither of them claimed to have experiment...  \n",
       "4846  Would someone be kind enought to document the ...  \n",
       "\n",
       "[3524 rows x 3 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_df = pd.DataFrame(columns=['id','category','preprocess_text'])\n",
    "for cat,id_li in valid_dict.items():\n",
    "    temp_df = dev_df_select_[dev_df_select_.index.isin(id_li)]\n",
    "    valid_df = pd.concat([valid_df,temp_df])\n",
    "valid_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>category</th>\n",
       "      <th>preprocess_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>7213</td>\n",
       "      <td>12298</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>That 's fascinating I take it that you 're exp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7214</td>\n",
       "      <td>12071</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>As per various threads on science and creation...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7216</td>\n",
       "      <td>12205</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>I suppose for the same reason that you do not ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7217</td>\n",
       "      <td>12070</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>What A Cook Off</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7221</td>\n",
       "      <td>12263</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>Sorry Fred but for the purposes under discussi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4833</td>\n",
       "      <td>15358</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "      <td>Baloney Either the programmer or the people wh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4835</td>\n",
       "      <td>15510</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "      <td>I 'm curious to know if Christians ever read b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4840</td>\n",
       "      <td>15482</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "      <td>If a Christian means someone who believes in t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4844</td>\n",
       "      <td>15537</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "      <td>Convenient It seems very appropriate that this...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4847</td>\n",
       "      <td>15403</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "      <td>Personally I think it was Mrs O'Leary 's cow t...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3515 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         id            category  \\\n",
       "7213  12298         alt.atheism   \n",
       "7214  12071         alt.atheism   \n",
       "7216  12205         alt.atheism   \n",
       "7217  12070         alt.atheism   \n",
       "7221  12263         alt.atheism   \n",
       "...     ...                 ...   \n",
       "4833  15358  talk.religion.misc   \n",
       "4835  15510  talk.religion.misc   \n",
       "4840  15482  talk.religion.misc   \n",
       "4844  15537  talk.religion.misc   \n",
       "4847  15403  talk.religion.misc   \n",
       "\n",
       "                                        preprocess_text  \n",
       "7213  That 's fascinating I take it that you 're exp...  \n",
       "7214  As per various threads on science and creation...  \n",
       "7216  I suppose for the same reason that you do not ...  \n",
       "7217                                    What A Cook Off  \n",
       "7221  Sorry Fred but for the purposes under discussi...  \n",
       "...                                                 ...  \n",
       "4833  Baloney Either the programmer or the people wh...  \n",
       "4835  I 'm curious to know if Christians ever read b...  \n",
       "4840  If a Christian means someone who believes in t...  \n",
       "4844  Convenient It seems very appropriate that this...  \n",
       "4847  Personally I think it was Mrs O'Leary 's cow t...  \n",
       "\n",
       "[3515 rows x 3 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df = pd.DataFrame(columns=['id','category','preprocess_text'])\n",
    "for cat,id_li in test_dict.items():\n",
    "    temp_df = dev_df_select_[dev_df_select_.index.isin(id_li)]\n",
    "    test_df = pd.concat([test_df,temp_df])\n",
    "test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>category</th>\n",
       "      <th>preprocess_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>7213</td>\n",
       "      <td>12298</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>That 's fascinating I take it that you 're exp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7214</td>\n",
       "      <td>12071</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>As per various threads on science and creation...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7216</td>\n",
       "      <td>12205</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>I suppose for the same reason that you do not ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7217</td>\n",
       "      <td>12070</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>What A Cook Off</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7221</td>\n",
       "      <td>12263</td>\n",
       "      <td>alt.atheism</td>\n",
       "      <td>Sorry Fred but for the purposes under discussi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4833</td>\n",
       "      <td>15358</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "      <td>Baloney Either the programmer or the people wh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4835</td>\n",
       "      <td>15510</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "      <td>I 'm curious to know if Christians ever read b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4840</td>\n",
       "      <td>15482</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "      <td>If a Christian means someone who believes in t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4844</td>\n",
       "      <td>15537</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "      <td>Convenient It seems very appropriate that this...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4847</td>\n",
       "      <td>15403</td>\n",
       "      <td>talk.religion.misc</td>\n",
       "      <td>Personally I think it was Mrs O'Leary 's cow t...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3515 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         id            category  \\\n",
       "7213  12298         alt.atheism   \n",
       "7214  12071         alt.atheism   \n",
       "7216  12205         alt.atheism   \n",
       "7217  12070         alt.atheism   \n",
       "7221  12263         alt.atheism   \n",
       "...     ...                 ...   \n",
       "4833  15358  talk.religion.misc   \n",
       "4835  15510  talk.religion.misc   \n",
       "4840  15482  talk.religion.misc   \n",
       "4844  15537  talk.religion.misc   \n",
       "4847  15403  talk.religion.misc   \n",
       "\n",
       "                                        preprocess_text  \n",
       "7213  That 's fascinating I take it that you 're exp...  \n",
       "7214  As per various threads on science and creation...  \n",
       "7216  I suppose for the same reason that you do not ...  \n",
       "7217                                    What A Cook Off  \n",
       "7221  Sorry Fred but for the purposes under discussi...  \n",
       "...                                                 ...  \n",
       "4833  Baloney Either the programmer or the people wh...  \n",
       "4835  I 'm curious to know if Christians ever read b...  \n",
       "4840  If a Christian means someone who believes in t...  \n",
       "4844  Convenient It seems very appropriate that this...  \n",
       "4847  Personally I think it was Mrs O'Leary 's cow t...  \n",
       "\n",
       "[3515 rows x 3 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.to_csv('data/20news-bydate-v3/valid_df_1.csv')\n",
    "valid_df.to_csv('data/20news-bydate-v3/test_df_1.csv')\n",
    "test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def convert(series):\n",
    "#     if type(series) is not int:\n",
    "#         try:\n",
    "#             return int(series.split(':')[-1].split()[0])\n",
    "#         except:\n",
    "#             print(series,type(series))\n",
    "#             return series\n",
    "#     return series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TEST & Deprecated\n",
    "* Playground"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/leoqaz12/.cache/torch/hub/pytorch_fairseq_master\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading archive file http://dl.fbaipublicfiles.com/fairseq/models/roberta.large.tar.gz from cache at /home/leoqaz12/.cache/torch/pytorch_fairseq/83e3a689e28e5e4696ecb0bbb05a77355444a5c8a3437e0f736d8a564e80035e.c687083d14776c1979f3f71654febb42f2bb3d9a94ff7ebdfe1ac6748dba89d2\n",
      "| dictionary: 50264 types\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RobertaHubInterface(\n",
       "  (model): RobertaModel(\n",
       "    (decoder): RobertaEncoder(\n",
       "      (sentence_encoder): TransformerSentenceEncoder(\n",
       "        (embed_tokens): Embedding(50265, 1024, padding_idx=1)\n",
       "        (embed_positions): LearnedPositionalEmbedding(514, 1024, padding_idx=1)\n",
       "        (layers): ModuleList(\n",
       "          (0): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (1): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (2): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (3): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (4): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (5): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (6): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (7): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (8): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (9): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (10): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (11): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (12): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (13): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (14): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (15): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (16): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (17): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (18): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (19): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (20): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (21): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (22): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "          (23): TransformerSentenceEncoderLayer(\n",
       "            (self_attn): MultiheadAttention(\n",
       "              (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "            )\n",
       "            (self_attn_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "            (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "            (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "            (final_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "          )\n",
       "        )\n",
       "        (emb_layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (lm_head): RobertaLMHead(\n",
       "        (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "        (layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "    )\n",
       "    (classification_heads): ModuleDict()\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roberta = torch.hub.load('pytorch/fairseq', 'roberta.large')\n",
    "roberta.eval() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens_ori = 'I have posted disp135.zip to alt.binaries.pictures.utilities ****** You may distribute this program freely for non-commercial use if no fee is gained. ****** There is no warranty. The author is not responsible for any damage caused by this program. Important changes since version 1,30: Fix bugs in file management system (file displaying). Improve file management system (more user-friendly). Fix bug in XPM version 3 reading. Fix bugs in TARGA reading/writng. Fix bug in GEM/IMG reading. Add support for PCX and GEM/IMG writing. Auto-skip macbinary header. (1) Introduction: This program can let you READ, WRITE and DISPLAY images with different formats. It also let you do some special effects(ROTATION, DITHERING ....) on image. Its main purpose is to let you convert image among different formts. Include simple file management system. Support \\'slide show\\'. There is NO LIMIT on image size. Currently this program supports 8, 15, 16, 24 bits display. If you want to use HiColor or TrueColor, you must have VESA driver. If you want to modify video driver, please read section (8). (2) Hardware Requirement: PC 386 or better. MSDOS 3,3 or higher. min amount of ram is 4M bytes(Maybe less memory will also work). (I recommend min 8M bytes for better performance). Hard disk for swapping(virtual memory). The following description is borrowed from DJGPP. Supported Wares: * Up to 128M of extended memory (expanded under VCPI) * Up to 128M of disk space used for swapping * SuperVGA 256-color mode up to 1024x768 * 80387 * XMS & VDISK memory allocation strategies * VCPI programs, such as QEMM, DESQview, and 386MAX Unsupported: * DPMI * Microsoft Windows Features: 80387 emulator, 32-bit unix-ish environment, flat memory model, SVGA graphics. (3) Installation: Video drivers, emu387 and go32.exe are borrowed from DJGPP. (If you use Western Digital VGA chips, read readme.wd) (This GO32.EXE is a modified version for vesa and is COMPLETELY compatible with original version) + *** But some people report that this go32.exe is not compatible with + other DJGPP programs in their system. If you encounter this problem, + DON\\'T put go32.exe within search path. *** Please read runme.bat for how to run this program. If you choose xxxxx.grn as video driver, add \\'nc 256\\' to environment GO32. For example, go32=driver x:/xxxxx/xxxxx.grn nc 256 If you don\\'t have 80x87, add \\'emu x:/xxxxx/emu387\\' to environment GO32. For example, go32=driver x:/xxxxx/xxxxx.grd emu x:/xxxxx/emu387 **** Notes: 1. I only test tr8900.grn, et4000.grn and vesa.grn. Other drivers are not tested. 2. I have modified et4000.grn to support 8, 15, 16, 24 bits display. You don\\'t need to use vesa driver. If et4000.grn doesn\\'t work, please try vesa.grn. 3. For those who want to use HiColor or TrueColor display, please use vesa.grn(except et4000 users). You can find vesa BIOS driver from : wuarchive.wustl.edu: /mirrors/msdos/graphics godzilla.cgl.rmit.oz.au: /kjb/MGL (4) Command Line Switch: + Usage : display [-d|--display initial_display_type] + [-s|--sort sort_method] + [-h|-?] Display type: 8(SVGA,default), 15, 16(HiColor), 24(TrueColor) + Sort method: \\'name\\', \\'ext\\' (5) Function Key: F2 : Change disk drive + CTRL-A -- CTRL-Z : change disk drive. F3 : Change filename mask (See match.doc) F4 : Change parameters F5 : Some effects on picture, eg. flip, rotate .... F7 : Make Directory t : Tag file + : Tag group files (See match.doc) T : Tag all files u : Untag file - : Untag group files (See match.doc) U : Untag all files Ins : Change display type (8,15,16,24) in \\'read\\' & \\'screen\\' menu. F6,m,M : Move file(s) F8,d,D : Delete file(s) r,R : Rename file c,C : Copy File(s) z,Z : Display first 10 bytes in Ascii, Hex and Dec modes. + f,F : Display disk free space. Page Up/Down : Move one page TAB : Change processing target. Arrow keys, Home, End, Page Up, Page Down: Scroll image. Home: Left Most. End: Right Most. Page Up: Top Most. Page Down: Bottom Most. in \\'screen\\' & \\'effect\\' menu : Left,Right arrow: Change display type(8, 15, 16, 24 bits) s,S : Slide Show. ESCAPE to terminate. ALT-X : Quit program without prompting. + ALT-A : Reread directory. Escape : Abort function and return. (6) Support Format: Read: GIF(.gif), Japan MAG(.mag), Japan PIC(.pic), Sun Raster(.ras), Jpeg(.jpg), XBM(.xbm), Utah RLE(.rle), PBM(.pbm), PGM(.pgm), PPM(.ppm), PM(.pm), PCX(.pcx), Japan MKI(.mki), Tiff(.tif), Targa(.tga), XPM(.xpm), Mac Paint(.mac), GEM/IMG(.img), IFF/ILBM(.lbm), Window BMP(.bmp), QRT ray tracing(.qrt), Mac PICT(.pct), VIS(.vis), PDS(.pds), VIKING(.vik), VICAR(.vic), FITS(.fit), Usenix FACE(.fac). the extensions in () are standard extensions. Write: GIF, Sun Raster, Jpeg, XBM, PBM, PGM, PPM, PM, Tiff, Targa, XPM, Mac Paint, Ascii, Laser Jet, IFF/ILBM, Window BMP, + Mac PICT, VIS, FITS, FACE, PCX, GEM/IMG. All Read/Write support full color(8 bits), grey scale, b/w dither, and 24 bits image, if allowed for that format. (7) Detail: Initialization: Set default display type to highest display type. Find allowable screen resolution(for .grn video driver only). 1. When you run this program, you will enter \\'read\\' menu. Whthin this menu you can press any function key except F5. If you move or copy files, you will enter \\'write\\' menu. the \\'write\\' menu is much like \\'read\\' menu, but only allow you to change directory. + The header line in \\'read\\' menu includes \"(d:xx,f:xx,t:xx)\". + d : display type. f: number of files. t: number of tagged files. pressing SPACE in \\'read\\' menu will let you select which format to use for reading current file. pressing RETURN in \\'read\\' menu will let you reading current file. This program will automatically determine which format this file is. The procedure is: First, check magic number. If fail, check standard extension. Still fail, report error. pressing s or S in \\'read\\' menu will do \\'Slide Show\\'. If delay time is 0, program will wait until you hit a key (except ESCAPE). If any error occurs, program will make a beep. ESCAPE to terminate. pressing Ins in \\'read\\' menu will change display type. pressing ALT-X in \\'read\\' menu will quit program without prompting. 2. Once image file is successfully read, you will enter \\'screen\\' menu. Within this menu F5 is turn on. You can do special effect on image. pressing RETURN: show image. in graphic mode, press RETURN, SPACE or ESCAPE to return to text mode. pressing TAB: change processing target. This program allows you to do special effects on 8-bit or 24-bit image. pressing Left,Right arrow: change display type. 8, 15, 16, 24 bits. pressing SPACE: save current image to file. B/W Dither: save as black/white image(1 bit). Grey Scale: save as grey image(8 bits). Full Color: save as color image(8 bits). True Color: save as 24-bit image. This program will ask you some questions if you want to write image to file. Some questions are format-dependent. Finally This program will prompt you a filename. If you want to save file under another directory other than current directory, please press SPACE. after pressing SPACE, you will enter \\'write2\\' menu. You can change directory to what you want. Then, pressing SPACE: this program will prompt you \\'original\\' filename. pressing RETURN: this program will prompt you \\'selected\\' filename (filename under bar). 3. This program supports 8, 15, 16, 24 bits display. 4. This Program is MEMORY GREEDY. If you don\\'t have enough memory, the performance is poor. 5. If you want to save 8 bits image : try GIF then TIFF(LZW) then TARGA then Sun Raster then BMP then ... If you want to save 24 bits image (lossless): try TIFF(LZW) or TARGA or ILBM or Sun Raster (No one is better for true 24bits image) 6. I recommend Jpeg for storing 24 bits images, even 8 bits images. 7. Not all subroutines are fully tested 8. This document is not well written. If you have any PROBLEM, SUGGESTION, COMMENT about this program, Please send to u7711501@bicmos.ee.nctu.edu.tw (140,113,11,13). I need your suggestion to improve this program. (There is NO anonymous ftp on this site) (8) Tech. information: Program (user interface and some subroutines) written by Jih-Shin Ho. Some subroutines are borrowed from XV(2,21) and PBMPLUS(dec 91). Tiff(V3,2) and Jpeg(V4) reading/writing are through public domain libraries. Compiled with DJGPP. You can get whole DJGPP package from SIMTEL20 or mirror sites. For example, wuarchive.wustl.edu: /mirrors/msdos/djgpp (9) For Thoese who want to modify video driver: 1. get GRX source code from SIMTEL20 or mirror sites. 2. For HiColor and TrueColor: 15 bits : # of colors is set to 32768. 16 bits : # of colors is set to 0xc010. 24 bits : # of colors is set to 0xc018. Acknowledgment: I would like to thank the authors of XV and PBMPLUS for their permission to let me use their subroutines. Also I will thank the authors who write Tiff and Jpeg libraries. Thank DJ. Without DJGPP I can\\'t do any thing on PC.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "tokens exceeds maximum length: 2550 > 512",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-d50bf0dfd79e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;31m#print(len(tokens) , tokens)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;31m#roberta.decode(tokens)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mdoc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mroberta\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextract_features_aligned_to_words\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtokens_ori\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdoc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtok\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdoc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.cache/torch/hub/pytorch_fairseq_master/fairseq/models/roberta/hub_interface.py\u001b[0m in \u001b[0;36mextract_features_aligned_to_words\u001b[0;34m(self, sentence, return_all_hiddens)\u001b[0m\n\u001b[1;32m    123\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    124\u001b[0m         \u001b[0;31m# extract features and align them\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 125\u001b[0;31m         \u001b[0mfeatures\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextract_features\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbpe_toks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_all_hiddens\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreturn_all_hiddens\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    126\u001b[0m         \u001b[0mfeatures\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfeatures\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    127\u001b[0m         \u001b[0maligned_feats\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0malignment_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0malign_features_to_words\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeatures\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malignment\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.cache/torch/hub/pytorch_fairseq_master/fairseq/models/roberta/hub_interface.py\u001b[0m in \u001b[0;36mextract_features\u001b[0;34m(self, tokens, return_all_hiddens)\u001b[0m\n\u001b[1;32m     80\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtokens\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_positions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     81\u001b[0m             raise ValueError('tokens exceeds maximum length: {} > {}'.format(\n\u001b[0;32m---> 82\u001b[0;31m                 \u001b[0mtokens\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_positions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     83\u001b[0m             ))\n\u001b[1;32m     84\u001b[0m         features, extra = self.model(\n",
      "\u001b[0;31mValueError\u001b[0m: tokens exceeds maximum length: 2550 > 512"
     ]
    }
   ],
   "source": [
    "# tokens_ori = train_df.loc[3,'preprocess_text']#'how\\'s are you'#\n",
    "#tokens = roberta.encode(tokens_ori)\n",
    "# assert tokens.tolist() == [0, 31414, 232, 328, 2]\n",
    "# assert roberta.decode(tokens) == 'Hello world!'\n",
    "#print(len(tokens) , tokens)\n",
    "#roberta.decode(tokens)\n",
    "doc = roberta.extract_features_aligned_to_words(tokens_ori)\n",
    "print(len(doc))\n",
    "for tok in doc:\n",
    "    print('{:10}{} (...)'.format(str(tok), tok.vector[:5]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8, ['I', 'said', ',', '\"', 'hello', 'RoBERTa', '.', '\"'])"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens = 'I said, \"hello RoBERTa.\"'\n",
    "tokens = sp_tokenizer(tokens)\n",
    "token_li = []\n",
    "for token in tokens:\n",
    "    token_li.append(token.text)\n",
    "len(token_li) , token_li"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "tokens exceeds maximum length: 592 > 512",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-167-7a2d631178ae>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlast_layer_features\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mroberta\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextract_features\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtokens\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlast_layer_features\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m# == torch.Size([1, 5, 1024])\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mall_layers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mroberta\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextract_features\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtokens\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_all_hiddens\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mall_layers\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m25\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.cache/torch/hub/pytorch_fairseq_master/fairseq/models/roberta/hub_interface.py\u001b[0m in \u001b[0;36mextract_features\u001b[0;34m(self, tokens, return_all_hiddens)\u001b[0m\n\u001b[1;32m     80\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtokens\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_positions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     81\u001b[0m             raise ValueError('tokens exceeds maximum length: {} > {}'.format(\n\u001b[0;32m---> 82\u001b[0;31m                 \u001b[0mtokens\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_positions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     83\u001b[0m             ))\n\u001b[1;32m     84\u001b[0m         features, extra = self.model(\n",
      "\u001b[0;31mValueError\u001b[0m: tokens exceeds maximum length: 592 > 512"
     ]
    }
   ],
   "source": [
    "last_layer_features = roberta.extract_features(tokens)\n",
    "print(last_layer_features.size())# == torch.Size([1, 5, 1024])\n",
    "\n",
    "all_layers = roberta.extract_features(tokens, return_all_hiddens=True)\n",
    "assert len(all_layers) == 25\n",
    "assert torch.all(all_layers[-1] == last_layer_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 19, 1024)"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_layer_features.detach().numpy().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pytorch_transformers import RobertaTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 898823/898823 [00:06<00:00, 134563.64B/s]\n",
      "100%|██████████| 456318/456318 [00:00<00:00, 545019.70B/s]\n"
     ]
    }
   ],
   "source": [
    "tokenizer = RobertaTokenizer.from_pretrained('roberta-large-mnli')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0925 15:40:29.729804 139833327294272 tokenization_utils.py:665] Token indices sequence length is longer than the specified maximum sequence length for this model (590 > 512). Running this sequence through the model will result in indexing errors\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "590"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tokenizer.encode(train_df.loc[0,'text']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "stop words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "594"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stop_words_1 = (stopwords.words('english')) #Stopword\n",
    "with open('data/preprocess/stop_words.txt') as f:\n",
    "    stop_words_2 = f.read().splitlines() #stop_list1\n",
    "stop_words_3 = pickle.load(open('data/preprocess/stop_list2.pkl','rb'))\n",
    "stop_words_all = set(stop_words_1 + stop_words_2 + stop_words_3)\n",
    "len(stop_words_all) # stopwords#"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "以下deprecated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def text_preprcess(text,tokenizers='nltk'):\n",
    "    '''\n",
    "    Input: string\n",
    "    Return: preprocessed list\n",
    "    '''\n",
    "    tokens = [i for i in text.split() if i not in stop_words_all]\n",
    "    tokens_str = ' '.join(tokens)\n",
    "    if tokenizers == 'keras':\n",
    "        tokens = T.text_to_word_sequence(tokens_str,filters=dil_filter) # lower casse & tokenize\n",
    "        tokens = list(filter(None, tokens))\n",
    "    elif tokenizers == 'nltk':\n",
    "#         tokens = re.sub(dil,\" \",tokens_str.lower())\n",
    "        tokens = [i for i in tokens_str.lower() if i not in dil_filter]\n",
    "        tokens = ''.join(tokens)\n",
    "        tokens = word_tokenize(tokens)\n",
    "        tokens = list(filter(None, tokens))\n",
    "#         tokens = [x.lower() for x in tokens]\n",
    "    elif tokenizers == 'spacy':\n",
    "#         tokens = re.sub(dil,\" \",tokens_str.lower())\n",
    "        tokens = [i for i in tokens_str.lower() if i not in dil_filter]\n",
    "        tokens = ''.join(tokens)\n",
    "        tokens = sp_tokenizer(tokens)\n",
    "        token_li = []\n",
    "        for token in tokens:\n",
    "            token_li.append(token.text)\n",
    "        tokens = list(filter(None, token_li))\n",
    "#         tokens = [x.lower() for x in token_li]\n",
    "    tokens_li = []\n",
    "    for token in tokens:\n",
    "        tokens_li.append(''.join([i for i in token if not i.isdigit()]))\n",
    "    tokens_str = ' '.join(tokens_li)\n",
    "    tokens_str = re.sub(dil,\" \",tokens_str.lower())\n",
    "    tokens_li = tokens_str.split()\n",
    "    tokens_li = list(filter(None, tokens_li))\n",
    "    ori_tokens_num = len(tokens_li)\n",
    "    tokens_li = [i for i in tokens_li if i in glove_words]\n",
    "    try:\n",
    "        rate = len(tokens_li)/ori_tokens_num\n",
    "    except ZeroDivisionError:\n",
    "        rate = np.nan\n",
    "    return tokens_li , rate#ori_tokens_num/len(tokens_li)\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▌       | 5/20 [01:02<03:16, 13.08s/it]"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-e45cfcfee62c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     21\u001b[0m             \u001b[0mnews_text_tok_keras\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrate_keras\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtext_preprcess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnews_text\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtokenizers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'keras'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m             \u001b[0mnews_text_tok_nltk\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrate_nltk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtext_preprcess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnews_text\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtokenizers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'nltk'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m             \u001b[0mnews_text_tok_spacy\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrate_spacy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtext_preprcess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnews_text\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtokenizers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'spacy'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m         train_df.loc[count] = [id_num,cat,news_text_ori,news_text_tok_num_ori,\n\u001b[1;32m     25\u001b[0m                                news_text_tok_keras,rate_keras,news_text_tok_nltk,rate_nltk,news_text_tok_spacy,rate_spacy]\n",
      "\u001b[0;32m<ipython-input-4-2e203f354942>\u001b[0m in \u001b[0;36mtext_preprcess\u001b[0;34m(text, tokenizers)\u001b[0m\n\u001b[1;32m     23\u001b[0m         \u001b[0mtoken_li\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mtoken\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtokens\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m             \u001b[0mtoken_li\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtoken\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m         \u001b[0mtokens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtoken_li\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[0;31m#         tokens = [x.lower() for x in token_li]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# train_df = pd.DataFrame(columns=['id','category','text','original_tokenize_num','preprocess_text','preprocss_tokenize_num'])\n",
    "train_df = pd.DataFrame(columns=['id','category','text','original_tokenize_num',\n",
    "                                 'text_keras','rate_keras','text_nltk','rate_nltk','text_apacy','rate_spacy'])\n",
    "train_dir = './data/20news-bydate-v3/20news-bydate-train/'\n",
    "cat_dir = next(os.walk(train_dir))[1]\n",
    "count = 0\n",
    "for cat in tqdm(cat_dir):\n",
    "    in_dir = train_dir + cat + '/'\n",
    "    news_list = next(os.walk(in_dir))[2]\n",
    "    news_list = [os.path.join(in_dir, f) for f in news_list]\n",
    "    news_list = list(filter(lambda f: f.endswith(\".txt\"), news_list))\n",
    "    for news_path in news_list:\n",
    "        id_num = news_path.split('/')[-1].split('.')[0]\n",
    "        with open(news_path,'r',encoding='latin1') as f:\n",
    "            news_text = f.read()\n",
    "            news_text = re.sub('\\n',' ',news_text)\n",
    "            news_text = re.sub('\\t',' ',news_text)\n",
    "            news_text_tok_ori = T.text_to_word_sequence(news_text,filters='',lower=False)\n",
    "            news_text_ori = \" \".join(news_text_tok_ori)\n",
    "            news_text_tok_num_ori = len(news_text_tok_ori)\n",
    "            news_text_tok_keras,rate_keras = text_preprcess(news_text,tokenizers='keras')\n",
    "            news_text_tok_nltk,rate_nltk = text_preprcess(news_text,tokenizers='nltk')\n",
    "            news_text_tok_spacy,rate_spacy = text_preprcess(news_text,tokenizers='spacy')\n",
    "        train_df.loc[count] = [id_num,cat,news_text_ori,news_text_tok_num_ori,\n",
    "                               news_text_tok_keras,rate_keras,news_text_tok_nltk,rate_nltk,news_text_tok_spacy,rate_spacy]\n",
    "        count+=1\n",
    "#         break\n",
    "    \n",
    "# train_df.loc[0] = [0,0,0]\n",
    "print(train_df.rate_keras.mean() , train_df.rate_nltk.mean(),train_df.rate_spacy.mean()  )\n",
    "train_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* keras tokenizer: 95.95% tokens in glove\n",
    "* nltk tokenizer: 95.96% tokens in glove\n",
    "* spacy tokenizer: 96.05% tokens in glove"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_df = pd.DataFrame(columns=['id','category','text','original_tokenize_num','preprocess_text','preprocss_tokenize_num'])\n",
    "train_dir = './data/20news-bydate-v3/20news-bydate-train/'\n",
    "cat_dir = next(os.walk(train_dir))[1]\n",
    "count = 0\n",
    "for cat in tqdm(cat_dir):\n",
    "    in_dir = train_dir + cat + '/'\n",
    "    news_list = next(os.walk(in_dir))[2]\n",
    "    news_list = [os.path.join(in_dir, f) for f in news_list]\n",
    "    news_list = list(filter(lambda f: f.endswith(\".txt\"), news_list))\n",
    "    for news_path in news_list:\n",
    "        id_num = news_path.split('/')[-1].split('.')[0]\n",
    "        with open(news_path,'r',encoding='latin1') as f:\n",
    "            news_text = f.read()\n",
    "            news_text = re.sub('\\n',' ',news_text)\n",
    "            news_text_tok_ori = T.text_to_word_sequence(news_text,filters='',lower=False)\n",
    "            news_text_ori = \" \".join(news_text_tok_ori)\n",
    "            news_text_tok_num_ori = len(news_text_tok_ori)\n",
    "#             news_text_tok_pre,rate_keras = text_preprcess(news_text,tokenizers='keras')\n",
    "#             news_text_tok_pre,rate_nltk = text_preprcess(news_text,tokenizers='nltk')\n",
    "            news_text_tok_pre,rate_spacy = text_preprcess(news_text,tokenizers='spacy')\n",
    "            news_text_pre = \" \".join(news_text_tok_pre)\n",
    "        train_df.loc[count] = [id_num,cat,news_text_ori,news_text_tok_num_ori,news_text_pre,len(news_text_tok_pre)]\n",
    "        count+=1\n",
    "        break\n",
    "    \n",
    "train_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "以上deprecated"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:root] *",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}