try-full-text.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "import pandas as pd\n",
    "import regex as re\n",
    "import numpy as np\n",
    "from itertools import chain\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.linear_model import SGDClassifier, LogisticRegression\n",
    "from sklearn.model_selection import cross_val_score\n",
    "import pickle\n",
    "from sklearn.metrics import fbeta_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_df size: (6027, 7), test_df size: (3826, 6)\n"
     ]
    }
   ],
   "source": [
    "train_df = pd.read_csv('data/train_v2.csv')\n",
    "test_df = pd.read_csv('data/test_v2.csv')\n",
    "\n",
    "print('train_df size: {}, test_df size: {}'.format(train_df.shape, test_df.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n",
      "done\n",
      "done\n",
      "done\n",
      "done\n"
     ]
    }
   ],
   "source": [
    "# In this part, we find most frequent unigram\n",
    "data = train_df[(train_df.category == 2)]['title']\n",
    "#df = train_df[(train_df.category > 1) & (train_df.category != 3)]\n",
    "\n",
    "cv = CountVectorizer()\n",
    "data = cv.fit_transform(data)\n",
    "\n",
    "tfidf_transformer = TfidfTransformer()\n",
    "tfidf_matrix = tfidf_transformer.fit_transform(data)\n",
    "word2tfidf = dict(zip(cv.get_feature_names(), tfidf_transformer.idf_))\n",
    "sorted_word2 = sorted(word2tfidf.items(), key=lambda kv: kv[1], reverse=False)\n",
    "\n",
    "def find_frequent_unigram(cat):\n",
    "    data = train_df[(train_df.category == cat)]['title']\n",
    "    cv = CountVectorizer()\n",
    "    data = cv.fit_transform(data)\n",
    "    \n",
    "    tfidf_transformer = TfidfTransformer()\n",
    "    tfidf_matrix = tfidf_transformer.fit_transform(data)\n",
    "    word2tfidf = dict(zip(cv.get_feature_names(), tfidf_transformer.idf_))\n",
    "    sorted_word2 = sorted(word2tfidf.items(), key=lambda kv: kv[1], reverse=False)\n",
    "    fp = 'data/cat-{}-frequent.txt'.format(cat)\n",
    "    with open(fp, 'w+') as wf:\n",
    "        wf.write('word, idf\\n')\n",
    "        for item in sorted_word2:\n",
    "            wf.write('{}, {}\\n'.format(item[0], item[1]))\n",
    "    print('done')\n",
    "            \n",
    "\n",
    "find_frequent_unigram(0)\n",
    "find_frequent_unigram(1)\n",
    "find_frequent_unigram(2)\n",
    "find_frequent_unigram(3)\n",
    "find_frequent_unigram(4)\n",
    "\n",
    "\n",
    "# 有一些stopwords 不能去掉，例如 down"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['to' 'in' 'for' 'on' 'us' 'of' 'video' 'china' 'the' 'billion'] ['to' 'in' 'of' 'for' 'on' 'the' 'us' 'china' 'missing' 'plane']\n",
      "cat-2: 2605, cat_4: 3828, intersection: 1731\n"
     ]
    }
   ],
   "source": [
    "cat_2_word = pd.read_csv('data/cat-2-frequent.txt')['word'].values\n",
    "cat_4_word = pd.read_csv('data/cat-4-frequent.txt')['word'].values\n",
    "print(cat_2_word[: 10], cat_4_word[: 10])\n",
    "intersection_2_4 = [item for item in cat_2_word if item in cat_4_word]\n",
    "residual_2_4 = [item for item in cat_2_word if item not in cat_4_word]\n",
    "print('cat-2: {}, cat_4: {}, intersection: {}'.format(len(cat_2_word), len(cat_4_word), len(intersection_2_4)))\n",
    "\n",
    "def calculate_intersection():\n",
    "    for i in range(4):\n",
    "        cat = pd.read_csv('data/cat-{}-frequent.txt'.format(i))['word'].values\n",
    "        cat_4 = pd.read_csv('data/cat-{}-frequent.txt'.format(4))['word'].values\n",
    "        intersection = [item for item in cat if item in cat_4]\n",
    "        print('cat{} unique word: {}, cat_4 unique word: {}, intersection word: {}, intersection percentage: {}'.format(i, len(cat), len(cat_4), len(intersection), len(intersection) / float(len(cat))))\n",
    "        \n",
    "#calculate_intersection()\n",
    "#print(intersection_2_4[: 200])\n",
    "#print('---')\n",
    "#print(residual_2_4[: 200])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "318\n",
      "315\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction import stop_words\n",
    "print(len(stop_words.ENGLISH_STOP_WORDS))\n",
    "my_stop_word = ['forty', 'much', 'whereafter', 'towards', 'our', 'during', 'done', 'one', 'please', 'whom', 'whereas', 'hers', 'thick', 'him', 'never', 'upon', 'are', 'whose', 'ten', 'except', 'than', 'those', 'whereupon', 'already', 'across', 're', 'somewhere', 'became', 'fire', 'six', 'without', 'although', 'third', 'eleven', 'amoungst', 'becomes', 'below', 'all', 'any', 'nothing', 'almost', 'how', 'in', 'he', 'over', 'seem', 'about', 'see', 'around', 'often', 'few', 'therefore', 'what', 'yourselves', 'both', 'keep', 'ours', 'always', 'once', 'others', 'seeming', 'none', 'perhaps', 'its', 'whether', 'whoever', 'be', 'etc', 'call', 'indeed', 'latter', 'further', 'last', 'three', 'eg', 'interest', 'other', 'but', 'your', 'here', 'per', 'wherein', 'against', 'sometime', 'within', 'name', 'former', 'may', 'when', 'will', 'whither', 'cry', 'beforehand', 'themselves', 'to', 'whereby', 'from', 'can', 'put', 'ie', 'bottom', 'which', 'have', 'not', 'seemed', 'until', 'somehow', 'was', 'cant', 'nobody', 'several', 'side', 'thru', 'now', 'four', 'there', 'neither', 'some', 'enough', 'otherwise', 'yet', 'into', 'still', 'anyone', 'made', 'via', 'everyone', 'give', 'take', 'thereby', 'amongst', 'thereafter', 'becoming', 'elsewhere', 'noone', 'no', 'onto', 'am', 'because', 'find', 'meanwhile', 'my', 'it', 'since', 'this', 'toward', 'anywhere', 'very', 'ever', 'another', 'formerly', 'her', 'hereby', 'me', 'an', 'himself', 'such', 'hereafter', 'might', 'afterwards', 'mine', 'nine', 'the', 'five', 'therein', 'where', 'why', 'anything', 'someone', 'that', 'after', 'fifteen', 'everywhere', 'herself', 'sometimes', 'nevertheless', 'mostly', 'show', 'a', 'part', 'before', 'together', 'either', 'of', 'whatever', 'de', 'eight', 'own', 'beyond', 'detail', 'even', 'were', 'cannot', 'fill', 'moreover', 'under', 'if', 'con', 'them', 'empty', 'get', 'system', 'nor', 'his', 'front', 'describe', 'must', 'then', 'between', 'been', 'bill', 'you', 'had', 'well', 'ourselves', 'alone', 'twelve', 'too', 'each', 'else', 'hundred', 'among', 'also', 'only', 'whole', 'less', 'hasnt', 'being', 'off', 'first', 'these', 'could', 'ltd', 'back', 'nowhere', 'with', 'everything', 'co', 'seems', 'sincere', 'myself', 'beside', 'go', 'every', 'rather', 'two', 'on', 'move', 'i', 'hence', 'who', 'due', 'amount', 'full', 'again', 'hereupon', 'fifty', 'whence', 'un', 'besides', 'latterly', 'mill', 'same', 'through', 'would', 'serious', 'sixty', 'top', 'most', 'we', 'anyway', 'couldnt', 'while', 'yourself', 'thus', 'become', 'their', 'thereupon', 'more', 'namely', 'above', 'wherever', 'found', 'itself', 'thin', 'thence', 'has', 'is', 'yours', 'by', 'for', 'though', 'along', 'out', 'she', 'next', 'twenty', 'however', 'something', 'herein', 'inc', 'at', 'do', 'throughout', 'as', 'anyhow', 'and', 'they', 'whenever', 'least', 'behind', 'or', 'should', 'so', 'many']\n",
    "print(len(my_stop_word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/diskA/yisong/anaconda3/lib/python3.6/site-packages/matplotlib/axes/_axes.py:6462: UserWarning: The 'normed' kwarg is deprecated, and has been replaced by the 'density' kwarg.\n",
      "  warnings.warn(\"The 'normed' kwarg is deprecated, and has been \"\n",
      "/diskA/yisong/anaconda3/lib/python3.6/site-packages/numpy/lib/histograms.py:823: RuntimeWarning: invalid value encountered in true_divide\n",
      "  return n/db/n.sum(), bin_edges\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAD8CAYAAABQFVIjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAEh5JREFUeJzt3XvsZGV9x/H3R7aClxZ2EQRZt4uBxCy9aJ1Cm17EC7caXCK0Qdu4tZptU01biYlYahHEFKyKtVrbDZhsaFqw2su2asiKYnrxwm/FRrct3XXVsAUFs9RKjVDqt3/M2Tr8nN/O7M4zMzvs+5VMzjnPeWbO9+GX8NlznjNnUlVIkjSpx827AEnSY4OBIklqwkCRJDVhoEiSmjBQJElNGCiSpCYMFElSEwaKJKkJA0WS1MSqeRcwS095ylNq/fr18y5DkhbKjh07vl5VJ4zqd0QFyvr161laWpp3GZK0UJJ8ZZx+XvKSJDVhoEiSmjBQJElNGCiSpCYMFElSEwaKJKkJA0WS1ISBIklqwkCRJDVhoEiSmjBQJElNGCiSpCYMFElSEwaKJKkJA0WS1ISBIklqwkCRJDVhoEiSmjBQJElNGCiSpCYMFElSEwaKJKkJA0WS1ISBIklqYq6BkuT8JHcl2Z3k8iH7j05yS7f/00nWL9u/LsmDSV43q5olScPNLVCSHAW8B7gA2AC8NMmGZd1eCTxQVacB1wPXLdt/PfCRadcqSRptnmcoZwK7q2pPVT0M3AxsXNZnI7C1W/8A8IIkAUhyEbAH2DmjeiVJBzDPQDkFuHtge2/XNrRPVT0CfAM4PsmTgNcDV82gTknSGOYZKBnSVmP2uQq4vqoeHHmQZHOSpSRL999//yGUKUkax6o5Hnsv8PSB7bXAPSv02ZtkFXAssA84C7gkyVuB44DvJPl2Vb17+UGqaguwBaDX6y0PLElSI/MMlDuA05OcCvwHcCnwsmV9tgGbgE8ClwAfq6oCfmZ/hyRvAh4cFiaSpNmZW6BU1SNJXgPcChwFvK+qdia5Gliqqm3AjcBNSXbTPzO5dF71SpIOLP1/8B8Zer1eLS0tzbsMSVooSXZUVW9UP78pL0lqwkCRJDVhoEiSmjBQJElNGCiSpCYMFElSEwaKJKkJA0WS1ISBIklqwkCRJDVhoEiSmjBQJElNGCiSpCYMFElSEwaKJKkJA0WS1ISBIklqwkCRJDVhoEiSmjBQJElNGCiSpCYMFElSEwaKJKkJA0WS1ISBIklqwkCRJDVhoEiSmjBQJElNGCiSpCYMFElSEwaKJKmJuQZKkvOT3JVkd5LLh+w/Oskt3f5PJ1nftZ+TZEeSz3fL58+6dknSo80tUJIcBbwHuADYALw0yYZl3V4JPFBVpwHXA9d17V8HLqyqHwY2ATfNpmpJ0krmeYZyJrC7qvZU1cPAzcDGZX02Alu79Q8AL0iSqrqzqu7p2ncCxyQ5eiZVS5KGmmegnALcPbC9t2sb2qeqHgG+ARy/rM/FwJ1V9dCU6pQkjWHVHI+dIW11MH2SnEH/Mti5Kx4k2QxsBli3bt3BVylJGss8z1D2Ak8f2F4L3LNSnySrgGOBfd32WuCvgJdX1RdXOkhVbamqXlX1TjjhhIblS5IGzTNQ7gBOT3JqkscDlwLblvXZRn/SHeAS4GNVVUmOAz4EvKGq/nFmFUuSVjS3QOnmRF4D3Ar8K/D+qtqZ5OokL+663Qgcn2Q3cBmw/9bi1wCnAW9M8rnudeKMhyBJGpCq5dMWj129Xq+WlpbmXYYkLZQkO6qqN6qf35SXJDVhoEiSmjBQJElNGCiSpCYMFElSEwaKJKkJA0WS1ISBIklqwkCRJDVhoEiSmhgrUJJ8MMmLkhhAkqShxg2I9wIvA3YluTbJM6dYkyRpAY0VKFX10ar6ReDHgC8D25P8U5JXJPm+aRYoSVoMY1/CSnI88MvAq4A7gT+gHzDbp1KZJGmhjPUTwEn+EngmcBNwYVXd2+26JYnPg5ckjf2b8jdU1YcHG5IcXVUPjfOMfEnSY9+4l7yuGdL2yZaFSJIW2wHPUJKcBJwCPCHJs4F0u34AeOKUa5MkLZBRl7zOoz8RvxZ4x0D7N4HfnlJNkqQFdMBAqaqtwNYkF1fVB2dUkyRpAY265PVLVfWnwPokly3fX1XvGPI2SdIRaNQlryd1yydPuxBJ0mIbdcnrT7rlVbMpR5K0qEZd8nrXgfZX1W+0LUeStKhGXfLaMZMqJEkLb5y7vCRJGmnUJa93VtVvJflboJbvr6oXT60ySdJCGXXJ66Zu+bZpFyJJWmyjLnnt6JafSPJ4+k8cLuCuqnp4BvVJkhbEuI+vfxHwx8AX6T/P69Qkv1pVH5lmcZKkxTHu04bfDjyvqs6uqucCzwOun/TgSc5PcleS3UkuH7L/6CS3dPs/nWT9wL43dO13JTlv0lokSZMZN1Duq6rdA9t7gPsmOXCSo4D3ABcAG4CXJtmwrNsrgQeq6jT6AXZd994NwKXAGcD5wB91nydJmpNRd3m9pFvdmeTDwPvpz6H8PHDHhMc+E9hdVXu6Y90MbAT+ZaDPRuBN3foHgHcnSdd+c1U9BHwpye7u8/yNFkmak1FzKBcOrH8NeG63fj+wesJjnwLcPbC9FzhrpT5V9UiSbwDHd+2fWvbeUyasR5I0gVF3eb1iisfOkLbl33VZqc847+1/QLIZ2Aywbt26g6lPknQQxr3L6xj68xlnAMfsb6+qX5ng2HuBpw9srwXuWaHP3iSrgGOBfWO+d3+NW4AtAL1eb2joSJImN+6k/E3ASfR/wfET9P8H/s0Jj30HcHqSU7vvuFwKbFvWZxuwqVu/BPhYVVXXfml3F9ipwOnAZyasR5I0gbHOUIDTqurnk2ysqq1J/gy4dZIDd3Mir+k+5yjgfVW1M8nVwFJVbQNuBG7qJt330Q8dun7vpz+B/wjw6qr630nqkSRNZtxA+Z9u+Z9Jfgj4KrB+0oNX1YeBDy9r+92B9W/Tv6Ns2HvfArxl0hokSW2MGyhbkqwG3kj/ctOTu3VJkoAxA6WqbuhWPwE8Y3rlSJIW1ViT8kmOT/KHST6bZEeSdyY5ftrFSZIWx7h3ed1M/1ErF9O/2+rrwC3TKkqStHjGnUNZU1VvHti+JslF0yhIkrSYxj1D+XiSS5M8rnv9AvChaRYmSVosox4O+U2++6iTy4A/7XY9DngQuHKq1UmSFsaoZ3l9/6wKkSQttnHnUEjyYuBnu83bq+rvplOSJGkRjXvb8LXAb9J/1Mm/AL/ZtUmSBIx/hvJzwLOq6jsASbYCdwLf87O9kqQj07h3eQEcN7B+bOtCJEmLbdwzlN8D7kzycfp3fP0s8IapVSVJWjgjA6X7Dfd/AH4C+HH6gfL6qvrqlGuTJC2QkYFSVZXkr6vqOXzvD2BJkgSMP4fyqSQ/PtVKJEkLbdw5lOcBv5bky8B/07/sVVX1I9MqTJK0WMYNlAumWoUkaeGNepbXMcCvAacBnwdurKpHZlGYJGmxjJpD2Qr06IfJBcDbp16RJGkhjbrktaGqfhggyY3AZ6ZfkiRpEY06Q/mf/Ste6pIkHcioM5QfTfJf3XqAJ3Tb++/y+oGpVidJWhijfg/lqFkVIklabAfzcEhJklZkoEiSmjBQJElNGCiSpCYMFElSEwaKJKkJA0WS1MRcAiXJmiTbk+zqlqtX6Lep67Mryaau7YlJPpTk35LsTHLtbKuXJA0zrzOUy4Hbqup04LZu+1GSrAGuBM4CzgSuHAiet1XVM4FnAz+VxMfrS9KczStQNtJ/kjHd8qIhfc4DtlfVvqp6ANgOnF9V36qqjwNU1cPAZ4G1M6hZknQA8wqUp1bVvQDd8sQhfU4B7h7Y3tu1/b8kxwEX0j/LkSTN0bi/2HjQknwUOGnIrivG/YghbTXw+auAPwfeVVV7DlDHZmAzwLp168Y8tCTpYE0tUKrqhSvtS/K1JCdX1b1JTgbuG9JtL3D2wPZa4PaB7S3Arqp654g6tnR96fV6daC+kqRDN69LXtuATd36JuBvhvS5FTg3yepuMv7cro0k1wDHAr81g1olSWOYV6BcC5yTZBdwTrdNkl6SGwCqah/wZuCO7nV1Ve1Lspb+ZbMNwGeTfC7Jq+YxCEnSd6XqyLkK1Ov1amlpad5lSNJCSbKjqnqj+vlNeUlSEwaKJKkJA0WS1ISBIklqwkCRJDVhoEiSmjBQJElNGCiSpCYMFElSEwaKJKkJA0WS1ISBIklqwkCRJDVhoEiSmjBQJElNGCiSpCYMFElSEwaKJKkJA0WS1ISBIklqwkCRJDVhoEiSmjBQJElNGCiSpCYMFElSEwaKJKkJA0WS1ISBIklqwkCRJDVhoEiSmphLoCRZk2R7kl3dcvUK/TZ1fXYl2TRk/7YkX5h+xZKkUeZ1hnI5cFtVnQ7c1m0/SpI1wJXAWcCZwJWDwZPkJcCDsylXkjTKvAJlI7C1W98KXDSkz3nA9qraV1UPANuB8wGSPBm4DLhmBrVKksYwr0B5alXdC9AtTxzS5xTg7oHtvV0bwJuBtwPfmmaRkqTxrZrWByf5KHDSkF1XjPsRQ9oqybOA06rqtUnWj1HHZmAzwLp168Y8tCTpYE0tUKrqhSvtS/K1JCdX1b1JTgbuG9JtL3D2wPZa4HbgJ4HnJPky/fpPTHJ7VZ3NEFW1BdgC0Ov16uBHIkkax7wueW0D9t+1tQn4myF9bgXOTbK6m4w/F7i1qt5bVU+rqvXATwP/vlKYSJJmZ16Bci1wTpJdwDndNkl6SW4AqKp99OdK7uheV3dtkqTDUKqOnKtAvV6vlpaW5l2GJC2UJDuqqjeqn9+UlyQ1YaBIkpowUCRJTRgokqQmDBRJUhMGiiSpCQNFktSEgSJJasJAkSQ1YaBIkpowUCRJTRgokqQmDBRJUhMGiiSpCQNFktSEgSJJasJAkSQ1YaBIkpowUCRJTRgokqQmDBRJUhMGiiSpCQNFktSEgSJJasJAkSQ1kaqadw0zk+R+4CvzruMgPQX4+ryLmDHHfGRwzIvjB6vqhFGdjqhAWURJlqqqN+86ZskxHxkc82OPl7wkSU0YKJKkJgyUw9+WeRcwB475yOCYH2OcQ5EkNeEZiiSpCQPlMJBkTZLtSXZ1y9Ur9NvU9dmVZNOQ/duSfGH6FU9ukjEneWKSDyX5tyQ7k1w72+oPTpLzk9yVZHeSy4fsPzrJLd3+TydZP7DvDV37XUnOm2XdkzjUMSc5J8mOJJ/vls+fde2HYpK/cbd/XZIHk7xuVjVPRVX5mvMLeCtwebd+OXDdkD5rgD3dcnW3vnpg/0uAPwO+MO/xTHvMwBOB53V9Hg/8PXDBvMe0wjiPAr4IPKOr9Z+BDcv6/Drwx936pcAt3fqGrv/RwKnd5xw17zFNeczPBp7Wrf8Q8B/zHs80xzuw/4PAXwCvm/d4Jnl5hnJ42Ahs7da3AhcN6XMesL2q9lXVA8B24HyAJE8GLgOumUGtrRzymKvqW1X1cYCqehj4LLB2BjUfijOB3VW1p6v1ZvpjHzT43+IDwAuSpGu/uaoeqqovAbu7zzvcHfKYq+rOqrqna98JHJPk6JlUfegm+RuT5CL6/1jaOaN6p8ZAOTw8taruBeiWJw7pcwpw98D23q4N4M3A24FvTbPIxiYdMwBJjgMuBG6bUp2TGjmGwT5V9QjwDeD4Md97OJpkzIMuBu6sqoemVGcrhzzeJE8CXg9cNYM6p27VvAs4UiT5KHDSkF1XjPsRQ9oqybOA06rqtcuvy87btMY88PmrgD8H3lVVew6+wpk44BhG9BnnvYejScbc35mcAVwHnNuwrmmZZLxXAddX1YPdCctCM1BmpKpeuNK+JF9LcnJV3ZvkZOC+Id32AmcPbK8Fbgd+EnhOki/T/3uemOT2qjqbOZvimPfbAuyqqnc2KHda9gJPH9heC9yzQp+9XUgeC+wb872Ho0nGTJK1wF8BL6+qL06/3IlNMt6zgEuSvBU4DvhOkm9X1bunX/YUzHsSx1cB/D6PnqB+65A+a4Av0Z+UXt2tr1nWZz2LMyk/0Zjpzxd9EHjcvMcyYpyr6F8fP5XvTtiesazPq3n0hO37u/UzePSk/B4WY1J+kjEf1/W/eN7jmMV4l/V5Ews+KT/3AnwV9K8d3wbs6pb7/6fZA24Y6Pcr9CdmdwOvGPI5ixQohzxm+v8CLOBfgc91r1fNe0wHGOvPAf9O/06gK7q2q4EXd+vH0L/DZzfwGeAZA++9onvfXRymd7K1HDPwO8B/D/xdPwecOO/xTPNvPPAZCx8oflNektSEd3lJkpowUCRJTRgokqQmDBRJUhMGiiSpCQNFktSEgSJJasJAkSQ18X+S44sl7g3AbgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "from nltk.stem.porter import *\n",
    "stemmer = PorterStemmer()\n",
    "\n",
    "#print(stemmer.stem('obamacare')) 得到 obamacar\n",
    "\n",
    "# 在这里，把 obamacar 当作 stopword!\n",
    "\n",
    "def get_new_stopwords():\n",
    "    manual_stopwords = ['obamacar', 'say', 'said', 'monday', 'tuesday',\n",
    "                       'wednesday', 'thursday', 'friday', 'saturday', 'sunday',\n",
    "                       'january', 'february', 'march', 'april', 'may', 'june',\n",
    "                       'july', 'april', 'may', 'june', 'july', 'august', 'september',\n",
    "                       'octerber', 'november', 'december']\n",
    "    words = [stemmer.stem(word) for word in manual_stopwords]\n",
    "    return words\n",
    "\n",
    "manual_stopwords = get_new_stopwords()\n",
    "\n",
    "def my_tokenizer(s):\n",
    "    words = re.findall(r'[A-Za-z]+', s)\n",
    "    words = [word.lower() for word in words]\n",
    "    words = [stemmer.stem(word) for word in words]\n",
    "    words = [word for word in words if (len(word) > 1 and word not in manual_stopwords)]\n",
    "    return words\n",
    "\n",
    "\n",
    "def show_tokenizer():\n",
    "    tag = []\n",
    "    for i in range(train_df.shape[0]):\n",
    "        #if i % 100 != 1:\n",
    "        #    continue\n",
    "        title = train_df.iloc[i]['title']\n",
    "        title_ = my_tokenizer(title)\n",
    "        #print('title: {}\\n{}'.format(train_df.iloc[i]['title'], title_))\n",
    "        if 'obamacar' in title_:\n",
    "            #print('tag: {}\\n{}'.format(train_df.iloc[i]['category'], title_))\n",
    "            tag.append(train_df.iloc[i]['category'])\n",
    "    print(len(tag))\n",
    "    plt.hist(tag, normed=True, bins=5)\n",
    "    plt.ylabel('Probability')\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show_tokenizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_obamacare():\n",
    "    index = []\n",
    "    for i in range(test_df.shape[0]):\n",
    "        title = train_df.iloc[i]['title']\n",
    "        title_ = my_tokenizer(title)\n",
    "        if 'obamacar' in title_:\n",
    "            index.append(i)\n",
    "    return index\n",
    "\n",
    "\n",
    "obama_index = find_obamacare()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6027,) (6027,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/diskA/yisong/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fit done\n",
      "f1 score is:0.681592039800995\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/diskA/yisong/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/diskA/yisong/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/diskA/yisong/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n",
      "/diskA/yisong/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.50413907 0.51409619 0.56716418 0.7358804  0.90024938] 0.6443058425839673\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/diskA/yisong/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:117: DeprecationWarning: n_iter parameter is deprecated in 0.19 and will be removed in 0.21. Use max_iter and tol instead.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "X = train_df['title']\n",
    "y = train_df['category']\n",
    "\n",
    "#X = train_df[((train_df.category == 2) | (train_df.category == 4))]['title']\n",
    "#y = train_df[((train_df.category == 2) | (train_df.category == 4))]['category']\n",
    "print(X.shape, y.shape)\n",
    "\n",
    "text_clf_svm = Pipeline([('vect', CountVectorizer(ngram_range=(1, 2), tokenizer=my_tokenizer, stop_words='english')),\n",
    "                      ('tfidf', TfidfTransformer(use_idf=True)),\n",
    "                      ('clf-svm', SGDClassifier(alpha=0.001, loss='hinge', penalty='l2', warm_start=True, n_iter=6, random_state=42)),])\n",
    "\n",
    "#('clf-lr', LogisticRegression(C=1, fit_intercept=True, penalty='l2', tol=0.01))\n",
    "#('clf-svm', SGDClassifier(alpha=0.001, loss='hinge', penalty='l2', warm_start=True, n_iter=6, random_state=42))\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20)\n",
    "\n",
    "text_clf_svm = text_clf_svm.fit(X_train.values.astype('U'), y_train.values)\n",
    "print('fit done')\n",
    "\n",
    "result = text_clf_svm.predict(X_test)\n",
    "print('f1 score is:{}'.format(fbeta_score(y_test, result, average='micro', beta=1)))\n",
    "\n",
    "scores = cross_val_score(text_clf_svm, X.values.astype('U'), y, cv=5, scoring='f1_micro')\n",
    "print(scores, np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16151\n",
      "finished\n"
     ]
    }
   ],
   "source": [
    "train_raw = pickle.load(open('data/train_v2.pk', 'rb'))\n",
    "test_raw = pickle.load(open('data/test_v2.pk', 'rb'))\n",
    "\n",
    "train_feature = text_clf_svm.steps[0][1].get_feature_names()\n",
    "print(len(train_feature))\n",
    "\n",
    "custom_tokens = dict()\n",
    "\n",
    "for i in range(train_df.shape[0]):\n",
    "    #if i % 50 != 0:\n",
    "    #    continue\n",
    "    title_ = my_tokenizer(train_df.iloc[i]['title'])\n",
    "    if train_raw[i] is None or len(train_raw[i]) < 500:\n",
    "        custom_tokens[''.join(('train', str(i)))] = title_\n",
    "        continue\n",
    "    extend = my_tokenizer(train_raw[i])[: 100000]\n",
    "    custom_tokens[''.join(('train-', str(i)))] = title_ + extend\n",
    "\n",
    "for i in range(test_df.shape[0]):\n",
    "    #if i % 50 != 0:\n",
    "    #    continue\n",
    "    title_ = my_tokenizer(test_df.iloc[i]['title'])\n",
    "    if test_raw[i] is None or len(test_raw[i]) < 500:\n",
    "        custom_tokens[''.join(('test', str(i)))] = title_\n",
    "        continue\n",
    "    extend = my_tokenizer(test_raw[i])[: 100000]\n",
    "    #new_word = [word for word in extend if (word in train_feature and word not in title_)]\n",
    "    #new_word = list(set(new_word))\n",
    "    #print(i, len(new_word), new_word)\n",
    "    #print(title_)\n",
    "    custom_tokens[''.join(('test', str(i)))] = title_ + extend\n",
    "\n",
    "print('finished')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6027 3826\n"
     ]
    }
   ],
   "source": [
    "train_key = [k for k in custom_tokens.keys() if k[: 2] == 'tr']\n",
    "test_key = [k for k in custom_tokens.keys() if k[: 2] == 'te']\n",
    "print(len(train_key), len(test_key))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fit done\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/diskA/yisong/anaconda3/lib/python3.6/site-packages/matplotlib/axes/_axes.py:6462: UserWarning: The 'normed' kwarg is deprecated, and has been replaced by the 'density' kwarg.\n",
      "  warnings.warn(\"The 'normed' kwarg is deprecated, and has been \"\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD8CAYAAACYebj1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAEjlJREFUeJzt3X+Q3Hd93/HnC9kOJBCg+DowluCcoExG/Ag/hOoOMwRSMpFxkDrF6chTmphCVZpqIONOGzk/nMHJTIE2hJBqShTwxIFQ24GUHrYYDzSGTNqa6IwdiOyovShufWNaHw7hR35gFN75Y1efbNar2+/J+u6eL8/HzM19f3x298XXrF73/ezud1NVSJIE8IR5B5AkbR6WgiSpsRQkSY2lIElqLAVJUmMpSJIaS0GS1FgKkqTGUpAkNRfMO8BGXXzxxbW4uDjvGJL0uHLXXXd9saoWpo173JXC4uIiy8vL844hSY8rSf5Pl3FOH0mSGktBktRYCpKkxlKQJDWWgiSpsRQkSY2lIElqLAVJUtNrKSTZm+RkkpUkh88y5h8nuTfJiSQf6jOPJGl9vX2iOck24Ajw/cAqcDzJUlXdOzJmJ3At8PKq+lKSv9tXHklby+Lh2+YdYebuf/sVvT9Gn2cKe4CVqjpVVY8ANwH7x8b8c+BIVX0JoKoe6jGPJGmKPkvhEuCBkfXV4bZR3wV8V5L/nuTOJHsn3VGSg0mWkyyvra31FFeS1GcpZMK2Glu/ANgJvBK4Cnhfkqc96kZVR6tqd1XtXliYepE/SdI56rMUVoEdI+vbgQcnjPmvVfWNqvoj4CSDkpAkzUGfpXAc2Jnk0iQXAQeApbExHwVeBZDkYgbTSad6zCRJWkdvpVBVp4FDwO3AfcAtVXUiyfVJ9g2H3Q48nORe4A7g31TVw31lkiStr9cv2amqY8CxsW3XjSwXcM3wR5I0Z36iWZLUWAqSpMZSkCQ1loIkqbEUJEmNpSBJaiwFSVJjKUiSGktBktRYCpKkxlKQJDWWgiSpsRQkSY2lIElqLAVJUmMpSJIaS0GS1FgKkqTGUpAkNZaCJKmxFCRJjaUgSWosBUlSYylIkhpLQZLU9FoKSfYmOZlkJcnhCfuvTrKW5J7hz5v6zCNJWt8Ffd1xkm3AEeD7gVXgeJKlqrp3bOjNVXWorxySpO76PFPYA6xU1amqegS4Cdjf4+NJkh6jPkvhEuCBkfXV4bZxr0vyuSQfTrKjxzySpCn6LIVM2FZj6x8DFqvqhcAngRsn3lFyMMlykuW1tbXzHFOSdEafpbAKjP7lvx14cHRAVT1cVV8frv4K8NJJd1RVR6tqd1XtXlhY6CWsJKnfUjgO7ExyaZKLgAPA0uiAJM8aWd0H3NdjHknSFL29+6iqTic5BNwObANuqKoTSa4HlqtqCXhLkn3AaeCPgav7yiNJmq63UgCoqmPAsbFt140sXwtc22cGSVJ3fqJZktRYCpKkxlKQJDWWgiSpsRQkSY2lIElqLAVJUmMpSJIaS0GS1FgKkqTGUpAkNZaCJKmxFCRJjaUgSWosBUlSYylIkhpLQZLUWAqSpMZSkCQ1loIkqbEUJEmNpSBJaiwFSVJjKUiSGktBktT0WgpJ9iY5mWQlyeF1xl2ZpJLs7jOPJGl9vZVCkm3AEeByYBdwVZJdE8Y9BXgL8Jm+skiSuunzTGEPsFJVp6rqEeAmYP+EcT8LvBP4ix6zSJI66LMULgEeGFlfHW5rkrwY2FFVt/aYQ5LUUZ+lkAnbqu1MngD8AvCvp95RcjDJcpLltbW18xhRkjSqz1JYBXaMrG8HHhxZfwrwfOBTSe4HLgOWJr3YXFVHq2p3Ve1eWFjoMbIk/e3WZykcB3YmuTTJRcABYOnMzqr6clVdXFWLVbUI3Ansq6rlHjNJktbRqRSSfCTJFcMpn06q6jRwCLgduA+4papOJLk+yb5ziytJ6tMFHcf9J+ANwHuS/Abwq1X1B9NuVFXHgGNj2647y9hXdswiSepJp7/8q+qTVfVPgJcA9wOfSPI/krwhyYV9BpQkzU7n6aAkzwCuBt4E3A38IoOS+EQvySRJM9dp+ijJbwLfDXwAeG1VfWG46+YkvjAsSVtE19cU3jd8faBJ8i1V9fWq8npFkrRFdJ0++rkJ2/7n+QwiSZq/dc8UkjyTwaUpnjS8JMWZTyl/O/CtPWeTJM3YtOmjH2Dw4vJ24F0j278K/ERPmSRJc7JuKVTVjcCNSV5XVR+ZUSZJ0pxMmz56fVV9EFhMcs34/qp614SbSZIep6ZNH33b8PeT+w4iSZq/adNHvzz8/bbZxJEkzdO06aP3rLe/qt5yfuNIkuZp2vTRXTNJIUnaFLq8+0iS9LfEtOmjd1fVjyX5GCNfpXlGVfm9CJK0hUybPvrA8Pd/6DuIJGn+pk0f3TX8/enhV2p+N4MzhpNV9cgM8kmSZqjrpbOvAN4L/CGD6x9dmuRfVNXH+wwnSZqtrpfO/nngVVW1ApDkO4HbAEtBkraQrpfOfuhMIQydAh7qIY8kaY6mvfvoHw0XTyQ5BtzC4DWFHwKO95xNkjRj06aPXjuy/P+B7x0urwFP7yWRJGlupr376A2zCiJJmr+u7z56IvBG4HnAE89sr6p/1lMuSdIcdH2h+QPAMxl8E9unGXwT21f7CiVJmo+upfDcqvpp4E+H10O6AnjBtBsl2ZvkZJKVJIcn7H9zks8nuSfJ7yTZtbH4kqTzqWspfGP4+0+SPB94KrC43g2SbAOOAJcDu4CrJvyj/6GqekFVvQh4J3/ze6AlSTPWtRSOJnk68NPAEnAv8I4pt9kDrFTVqeElMW4C9o8OqKqvjKx+GxMuuidJmp1OLzRX1fuGi58GvqPjfV8CPDCyvgr8vfFBSf4VcA1wEfB9He9bktSDTmcKSZ6R5JeSfDbJXUneneQZ0242Yduky28fqarvBH4c+KmzPP7BJMtJltfW1rpEliSdg67TRzcxuKzF64ArgS8CN0+5zSqwY2R9O/DglMf4h5N2VNXRqtpdVbsXFhY6RpYkbVTXUvg7VfWzVfVHw5+fA5425TbHgZ1JLh1edvsAg9cjmiQ7R1avAP531+CSpPOv61VS70hygMG1j2BwtnDbejeoqtNJDgG3A9uAG6rqRJLrgeWqWgIOJXk1g3c3fQn4kXP5HyFJOj+mXRDvqwxeBwiDF4M/ONz1BOBrwM+sd/uqOgYcG9t23cjyWzceWZLUl2nXPnrKrIJIkuav6/QRSfYBrxiufqqqbu0nkvTYLB5ed2ZzS7r/7VfMO4K2iK5vSX078FYGH1q7F3jrcJskaQvpeqbwGuBFVfVNgCQ3AncDj7qekSTp8avrW1Lhb74F9annO4gkaf66nin8O+DuJHcweCfSK4Bre0slSZqLqaWQJMDvAJcBL2NQCj9eVf+v52ySpBmbWgpVVUk+WlUvZewTyZKkraXrawp3JnlZr0kkSXPX9TWFVwFvTnI/8KcMppCqql7YVzBJ0ux1LYXLe00hSdoUpl376InAm4HnAp8H3l9Vp2cRTJI0e9NeU7gR2M2gEC4Hfr73RJKkuZk2fbSrql4AkOT9wO/2H0mSNC/TzhS+cWbBaSNJ2vqmnSl8T5KvDJcDPGm4fubdR9/eazpJ0kxN+z6FbbMKIkmav41cEE+StMVZCpKkxlKQJDWWgiSpsRQkSY2lIElqLAVJUtP1KqlbwuLh2+YdYebuf/sV844g6XGk1zOFJHuTnEyykuTwhP3XJLk3yeeS/Lckz+kzjyRpfb2VQpJtwBEGV1fdBVyVZNfYsLuB3cMv6/kw8M6+8kiSpuvzTGEPsFJVp6rqEeAmYP/ogKq6o6r+bLh6J7C9xzySpCn6LIVLgAdG1leH287mjcDHe8wjSZqizxeaM2FbTRyYvJ7Bl/l871n2HwQOAjz72c8+X/kkSWP6PFNYBXaMrG8HHhwflOTVwE8C+6rq65PuqKqOVtXuqtq9sLDQS1hJUr+lcBzYmeTSJBcBB4Cl0QFJXgz8MoNCeKjHLJKkDnorheE3tR0CbgfuA26pqhNJrk+ybzjs3wNPBn4jyT1Jls5yd5KkGej1w2tVdQw4NrbtupHlV/f5+JKkjfEyF5KkxlKQJDWWgiSpsRQkSY2lIElqLAVJUmMpSJIaS0GS1FgKkqTGUpAkNZaCJKmxFCRJjaUgSWosBUlSYylIkhpLQZLUWAqSpMZSkCQ1loIkqbEUJEmNpSBJaiwFSVJjKUiSGktBktRYCpKkptdSSLI3yckkK0kOT9j/iiSfTXI6yZV9ZpEkTddbKSTZBhwBLgd2AVcl2TU27P8CVwMf6iuHJKm7C3q87z3ASlWdAkhyE7AfuPfMgKq6f7jvmz3mkCR11Of00SXAAyPrq8NtkqRNqs9SyIRtdU53lBxMspxkeW1t7THGkiSdTZ+lsArsGFnfDjx4LndUVUerandV7V5YWDgv4SRJj9ZnKRwHdia5NMlFwAFgqcfHkyQ9Rr290FxVp5McAm4HtgE3VNWJJNcDy1W1lORlwH8Bng68Nsnbqup5fWWStqrFw7fNO4K2iD7ffURVHQOOjW27bmT5OINpJUnSJuAnmiVJjaUgSWosBUlSYylIkhpLQZLUWAqSpMZSkCQ1loIkqbEUJElNr59o1vx5+QNJG+GZgiSpsRQkSY2lIElqLAVJUmMpSJIaS0GS1FgKkqTGUpAkNZaCJKmxFCRJjaUgSWosBUlSYylIkhpLQZLUWAqSpKbXUkiyN8nJJCtJDk/Y/y1Jbh7u/0ySxT7zSJLW11spJNkGHAEuB3YBVyXZNTbsjcCXquq5wC8A7+grjyRpuj7PFPYAK1V1qqoeAW4C9o+N2Q/cOFz+MPAPkqTHTJKkdfRZCpcAD4ysrw63TRxTVaeBLwPP6DGTJGkdfX5H86S/+OscxpDkIHBwuPq1JCfPMdPFwBfP8bZ9MtfGmGvjNms2c21A3vGYcj2ny6A+S2EV2DGyvh148CxjVpNcADwV+OPxO6qqo8DRxxooyXJV7X6s93O+mWtjzLVxmzWbuTZmFrn6nD46DuxMcmmSi4ADwNLYmCXgR4bLVwK/VVWPOlOQJM1Gb2cKVXU6ySHgdmAbcENVnUhyPbBcVUvA+4EPJFlhcIZwoK88kqTp+pw+oqqOAcfGtl03svwXwA/1mWHMY56C6om5NsZcG7dZs5lrY3rPFWdrJElneJkLSVKzJUths15eo0Ouq5OsJbln+POmGeW6IclDSX7/LPuT5D3D3J9L8pJNkuuVSb48cryumzTuPGfakeSOJPclOZHkrRPGzPx4dcw1j+P1xCS/m+T3hrneNmHMzJ+PHXPN5fk4fOxtSe5OcuuEff0er6raUj8MXtT+Q+A7gIuA3wN2jY35UeC9w+UDwM2bJNfVwH+cwzF7BfAS4PfPsv81wMcZfK7kMuAzmyTXK4FbZ3ysngW8ZLj8FOB/TfjvOPPj1THXPI5XgCcPly8EPgNcNjZmHs/HLrnm8nwcPvY1wIcm/ffq+3htxTOFzXp5jS655qKqfpsJnw8ZsR/4tRq4E3hakmdtglwzV1VfqKrPDpe/CtzHoz+pP/Pj1THXzA2PwdeGqxcOf8ZfyJz587FjrrlIsh24AnjfWYb0ery2Yils1strdMkF8LrhlMOHk+yYsH8eumafh78/nAL4eJLnzfKBh6ftL2bwV+aouR6vdXLBHI7XcCrkHuAh4BNVddbjNcPnY5dcMJ/n47uBfwt88yz7ez1eW7EUztvlNc6zLo/5MWCxql4IfJK//mtg3uZxvLr4LPCcqvoe4JeAj87qgZM8GfgI8GNV9ZXx3RNuMpPjNSXXXI5XVf1lVb2IwVUN9iR5/tiQuRyvDrlm/nxM8oPAQ1V113rDJmw7b8drK5bCRi6vQda5vMasc1XVw1X19eHqrwAv7TlTV12O6cxV1VfOTAHU4DMxFya5uO/HTXIhg394f72qfnPCkLkcr2m55nW8Rh7/T4BPAXvHds3j+Tg115yejy8H9iW5n8EU8/cl+eDYmF6P11Yshc16eY2pucbmnfcxmBfeDJaAHx6+q+Yy4MtV9YV5h0ryzDNzqUn2MPj/88M9P2YYfBL/vqp611mGzfx4dck1p+O1kORpw+UnAa8G/mBs2Myfj11yzeP5WFXXVtX2qlpk8G/Eb1XV68eG9Xq8ev1E8zzUJr28Rsdcb0myDzg9zHV137kAkvxnBu9MuTjJKvAzDF54o6rey+BT6a8BVoA/A96wSXJdCfzLJKeBPwcOzKDcXw78U+Dzw/logJ8Anj2Sax7Hq0uueRyvZwE3ZvClW08AbqmqW+f9fOyYay7Px0lmebz8RLMkqdmK00eSpHNkKUiSGktBktRYCpKkxlKQJDWWgiSpsRQkSY2lIElq/go2spyt4iMbnQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "# for submit\n",
    "X_test = test_df['title']\n",
    "\n",
    "text_clf_svm = Pipeline([('vect', CountVectorizer(ngram_range=(1, 2), tokenizer=lambda key: custom_tokens[key], stop_words=my_stop_word)),\n",
    "                      ('tfidf', TfidfTransformer(use_idf=True)),\n",
    "                      ('NB', MultinomialNB(alpha=0.1, fit_prior=True))])\n",
    "\n",
    "#('NB', MultinomialNB(alpha=0.1, fit_prior=True))\n",
    "#('clf-lr', LogisticRegression(C=1, fit_intercept=True, penalty='l2', tol=0.01))\n",
    "#('clf-svm', SGDClassifier(alpha=0.001, loss='hinge', penalty='l2', warm_start=True, n_iter=6, random_state=42))\n",
    "text_clf_svm = text_clf_svm.fit(train_key, y.values)\n",
    "print('fit done')\n",
    "\n",
    "result = text_clf_svm.predict(test_key)\n",
    "\n",
    "write_file = open('improvement-add-all-nb.txt', 'w+')\n",
    "write_file.write('article_id,category\\n')\n",
    "i = 1\n",
    "for item in result:\n",
    "    write_file.write('{},{}\\n'.format(i, item))\n",
    "    i += 1\n",
    "write_file.close()\n",
    "\n",
    "plt.hist(result, normed=True, bins=5)\n",
    "plt.ylabel('Probability')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n"
     ]
    }
   ],
   "source": [
    "re0 = pd.read_csv('improvement-add-all.txt')['category'].values\n",
    "re1 = pd.read_csv('improvement-add-1000.txt')['category'].values\n",
    "\n",
    "difference = 0\n",
    "for i in range(0, len(re0)):\n",
    "    if re0[i] != re1[i]:\n",
    "        difference += 1\n",
    "print(difference)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total disagree: 400\n",
      "svm ≠ lr: 148, svm ≠ nb: 366, lr ≠ nb: 293\n",
      "nb and lr is same, overthrow smv: 107\n"
     ]
    }
   ],
   "source": [
    "re_all_nb = pd.read_csv('improvement-add-all-nb.txt')['category'].values\n",
    "re_all_lr = pd.read_csv('improvement-add-all-lr.txt')['category'].values\n",
    "re_all_svm = pd.read_csv('improvement-add-all-svm.txt')['category'].values\n",
    "\n",
    "#improvement-add-all-svm.txt 有0.87140\n",
    "\n",
    "c = 0\n",
    "for i in range(len(re_svm)):\n",
    "    if (re_all_lr[i] == re_all_svm[i] and re_all_nb[i] == re_all_svm[i]):\n",
    "        c += 1\n",
    "print('total disagree: {}'.format(len(re_all_nb) - c))\n",
    "\n",
    "svm_lr_dis = len([item for i, item in enumerate(re_all_svm) if item != re_all_lr[i]])\n",
    "svm_nb_dis = len([item for i, item in enumerate(re_all_svm) if item != re_all_nb[i]])\n",
    "lr_nb_dis = len([item for i, item in enumerate(re_all_lr) if item != re_all_nb[i]])\n",
    "print('svm ≠ lr: {}, svm ≠ nb: {}, lr ≠ nb: {}'.format(svm_lr_dis, svm_nb_dis, lr_nb_dis))\n",
    "\n",
    "nblr_vs_svm = len([item for i, item in enumerate(re_all_svm) if (item != re_all_lr[i] and item != re_all_nb[i])\n",
    "                  and (re_all_lr[i] == re_all_nb[i])])\n",
    "print('nb and lr is same, overthrow smv: {}'.format(nblr_vs_svm))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}