web scraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Want PPO:- Y/N \n",
      "y\n",
      "Enter Least Salary : 15000\n",
      "https://internshala.com/internships/backend%20development%2Cfront%20end%20development%2Cfull%20stack%20development%2Cweb%20development-internship/ppo-true\n",
      "https://internshala.com/internships/backend%20development%2Cfront%20end%20development%2Cfull%20stack%20development%2Cweb%20development-internship/ppo-true/page-2\n",
      "https://internshala.com/internships/backend%20development%2Cfront%20end%20development%2Cfull%20stack%20development%2Cweb%20development-internship/ppo-true/page-3\n",
      "https://internshala.com/internships/backend%20development%2Cfront%20end%20development%2Cfull%20stack%20development%2Cweb%20development-internship/ppo-true/page-4\n",
      "https://internshala.com/internships/backend%20development%2Cfront%20end%20development%2Cfull%20stack%20development%2Cweb%20development-internship/ppo-true/page-5\n",
      "https://internshala.com/internships/backend%20development%2Cfront%20end%20development%2Cfull%20stack%20development%2Cweb%20development-internship/ppo-true/page-6\n",
      "https://internshala.com/internships/backend%20development%2Cfront%20end%20development%2Cfull%20stack%20development%2Cweb%20development-internship/ppo-true/page-7\n",
      "https://internshala.com/internships/backend%20development%2Cfront%20end%20development%2Cfull%20stack%20development%2Cweb%20development-internship/ppo-true/page-8\n",
      "https://internshala.com/internships/backend%20development%2Cfront%20end%20development%2Cfull%20stack%20development%2Cweb%20development-internship/ppo-true/page-9\n",
      "https://internshala.com/internships/backend%20development%2Cfront%20end%20development%2Cfull%20stack%20development%2Cweb%20development-internship/ppo-true/page-10\n",
      "https://internshala.com/internships/backend%20development%2Cfront%20end%20development%2Cfull%20stack%20development%2Cweb%20development-internship/ppo-true/page-11\n",
      "https://internshala.com/internships/backend%20development%2Cfront%20end%20development%2Cfull%20stack%20development%2Cweb%20development-internship/ppo-true/page-12\n",
      "https://internshala.com/internships/backend%20development%2Cfront%20end%20development%2Cfull%20stack%20development%2Cweb%20development-internship/ppo-true/page-13\n",
      "77  Internship Found\n"
     ]
    }
   ],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import re\n",
    "import pandas as pd\n",
    "\n",
    " \n",
    "#filter url or normal url\n",
    "url = input('Enter the URL of Internshala \\n')\n",
    "#example URL\n",
    "#url=\"https://internshala.com/internships/backend%20development%2Cfront%20end%20development%2Cfull%20stack%20development%2Cweb%20development-internship/ppo-true\"\n",
    "\n",
    "\n",
    "next = True\n",
    "\n",
    "#salary for filter\n",
    "sal_must = int(input('Enter Least Salary : '))\n",
    "\n",
    "\n",
    "i=0\n",
    "internship_detail = {}\n",
    "while next:\n",
    "    print(url)\n",
    "    response = requests.get(url)\n",
    "    response\n",
    "    data = response.text\n",
    "    soup = BeautifulSoup(data,'html.parser')\n",
    "    intern_box = soup.find_all('div',{'class':'individual_internship'})\n",
    "\n",
    "    \n",
    "    \n",
    "    for intern in intern_box:\n",
    "        title = intern.find('a').text\n",
    "        link = \"https://internshala.com\" + intern.find('a').get('href')\n",
    "        company = intern.find('a',{\"class\":\"link_display_like_text\"}).text\n",
    "        location = intern.find('a',{\"class\":\"location_link\"}).text\n",
    "        duration = intern.find_all(\"td\")[1].text\n",
    "        salary = intern.find_all(\"td\")[2].text\n",
    "        \n",
    "        \n",
    "        \n",
    "        #salary find\n",
    "        matchob = re.match(r'(.*)-(.*) /Month (.*)',salary,re.M|re.I)\n",
    "        if matchob:\n",
    "            sal = int(matchob.group(2))\n",
    "        else:\n",
    "            matcho = re.match(r'(.*) /Month (.*)',salary,re.M|re.I)\n",
    "            if matcho:\n",
    "                sal = int(matcho.group(1))\n",
    "            else:\n",
    "                sal= 0\n",
    "                \n",
    "                \n",
    "        #find next page\n",
    "        next_page = soup.find('a',{\"id\":\"navigation-forward\"})\n",
    "        if next_page.get('href') =='#' or next_page.get('href') =='':\n",
    "            next = False\n",
    "        else:\n",
    "            url = \"https://internshala.com\" +  next_page.get('href')\n",
    "            \n",
    "\n",
    "        \n",
    "        if sal_must <= sal:\n",
    "            i+=1\n",
    "            #print(title +\" ----> \"+company +\"   -----> \"+location+\"  ----> \"+duration+\"  ----->  \"+salary+\"  ##### \"+str(sal))\n",
    "            internship_detail[i] = [title,company,location,duration,salary,link]\n",
    "\n",
    "            \n",
    "print(str(i)+\"  Internship Found\")\n",
    "\n",
    "\n",
    "internship = pd.DataFrame.from_dict(internship_detail, orient='index')\n",
    "internship.columns = ['Job Title','Comapny Name','Location','Duration','Salary','Link']\n",
    "\n",
    "#Export into CSV\n",
    "internship.to_csv('internship.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}