leg_extract.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = []\n",
    "\n",
    "# List of XML files for extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Extract_collapse(input_tag, target_tag, level_w, ignore_tag=False):\n",
    "    \n",
    "    for item in input_tag:\n",
    "        \n",
    "        if target_tag != \"\":\n",
    "            targets = item.find_all(target_tag)\n",
    "            for item in targets:\n",
    "                if ((item.find_all(ignore_tag) == []) or ignore_tag == False) and (item.find_all('BlockAmendment') == []) and (item.find_all('InlineAmendment') == []):\n",
    "                    txt_sec = item.find_all('Text')\n",
    "                    txt_mid = []\n",
    "                    try:\n",
    "                        section = item['id']\n",
    "                    except KeyError:\n",
    "                        section = \"Null\"\n",
    "                    for sub_item in txt_sec:\n",
    "                        txt = sub_item.text\n",
    "                        txt_con = \"\".join(txt)\n",
    "                        txt_mid.append(txt_con)\n",
    "                    txt_out = \" \".join(txt_mid)\n",
    "                    if (txt_out != \"\" and txt_out != \". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\"):\n",
    "                        legWriter.writerow([level_w, section, txt_out])\n",
    "\n",
    "        elif ((item.find_all(ignore_tag) == []) or ignore_tag == False) and (item.find_all('BlockAmendment') == []) and (item.find_all('InlineAmendment') == []):\n",
    "            txt_sec = item.find_all('Text')\n",
    "            txt_mid = []\n",
    "            try:\n",
    "                section = item['id']\n",
    "            except KeyError:\n",
    "                section = \"Null\"\n",
    "            for sub_item in txt_sec:\n",
    "                txt = sub_item.text\n",
    "                txt_con = \"\".join(txt)\n",
    "                txt_mid.append(txt_con)\n",
    "            txt_out = \" \".join(txt_mid)\n",
    "            if (txt_out != \"\" and txt_out != \". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\"):\n",
    "                legWriter.writerow([level_w, section, txt_out])\n",
    "                \n",
    "# Extract_collapse takes each tag in the input_tag set and extract any text within any Text tag in that tag. It also extracts\n",
    "# the section number and writes the output to a new line in the csv file. Target_tag can be set to sub-select a tag in the\n",
    "# input_tag set for processing. Ignore_tag can be set so that items in the input_tag set which contain the Ignore_tag are not\n",
    "# processed (useful for selecting sections with only 1 or 2 sub-sections levels etc.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Extract_fragment(input_tag, level_w):\n",
    "    for item in input_tag:\n",
    "        txt_sec = item.find('Text').text\n",
    "        txt_out = \"\".join(txt_sec)\n",
    "        try:\n",
    "            section = item['id']\n",
    "        except KeyError:\n",
    "            section = \"Null\"\n",
    "        if (txt_out != \"\" and txt_out != \". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .\"):\n",
    "            legWriter.writerow([level_w, section, txt_out])\n",
    "            \n",
    "# Extract_fragment takes each tag in the input_tag set and extracts any text within the first Text tag in that tag (whereas\n",
    "# Extract_collapse extract all text in all Text tags).It also extracts the section number and writes the output to a new line \n",
    "# in the csv file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Parsing...\")\n",
    "\n",
    "# Loop over documents. Open each document as a BeautifulSoup object.\n",
    "\n",
    "for doc in docs:\n",
    "    with open(doc, 'r', encoding = 'utf-8') as file:\n",
    "        xml = BeautifulSoup(file, \"xml\")\n",
    "    \n",
    "    # Open a csv and define the writer object. Writer encoding should match the open doc encoding.\n",
    "    \n",
    "    with open('leg_data.csv', 'a', newline='', encoding = 'utf-8') as outfile:\n",
    "        legWriter = csv.writer(outfile, delimiter='|', quotechar='\"',\n",
    "                               quoting=csv.QUOTE_MINIMAL)\n",
    "        title = xml.title.text\n",
    "        print(\"Parsed: \" + title)\n",
    "        \n",
    "        # Select all P1 tags.\n",
    "        \n",
    "        P1s = xml.find_all('P1')\n",
    "        \n",
    "        # Level 0 captures those provisions which are expected to exist as sentences at section level \n",
    "        # (i.e. either the section does not have sub-sections or sub-sections are within an over all sentence). \n",
    "        # Clauses which set out amendments are excluded (since structure is nested within these amendments).\n",
    "        \n",
    "        Extract_collapse(input_tag = P1s, target_tag = '', level_w = 'L0', ignore_tag = 'P2')\n",
    "                                \n",
    "        # Level 1 captures complete sub-sections (and condenses sub-sub-sections where necessary). Expect this\n",
    "        # text to be complete sentences.\n",
    "        # Clauses which set out amendments are excluded (since structure is nested within these amendments)\n",
    "        \n",
    "        Extract_collapse(input_tag = P1s, target_tag = 'P2', level_w = 'L1')\n",
    "        \n",
    "        # Level 2 captures sub-section level fragments (i.e. text at sub-section level which may or may not form a complete\n",
    "        # sentence.) E.g. the text following \"section 1(1)...\" until \"-:(a)\"\n",
    "        \n",
    "        for item in P1s:\n",
    "            P2s = item.find_all('P2')\n",
    "            Extract_fragment(input_tag = P2s, level_w = 'L2')\n",
    "            \n",
    "        # Level 3 captures sub-sub-section level fragments (i.e. text at sub-sub-section level which may or may not form a \n",
    "        # complete sentence.) E.g. the text following \"section 1(1)(a)...\" until \"-:(i)\"\n",
    "        \n",
    "            for item in P2s:\n",
    "                if item.find_all('P3') != []:\n",
    "                    P3s = item.find_all('P3')\n",
    "                    Extract_fragment(input_tag = P3s, level_w = 'L3')\n",
    "                    \n",
    "        # Level 4 captures sub-sub-sub-section level fragments, e.g. the text following \"section 1(1)(a)(i)\".\n",
    "                    \n",
    "                    for item in P3s:\n",
    "                        if item.find_all('P4') != []:\n",
    "                            P3s = item.find_all('P4')\n",
    "                            Extract_fragment(input_tag = P3s, level_w = 'L4')\n",
    "                            \n",
    "        # There is necessarily some duplication between layers. Every L1 should also be decomposed in L2, L3 and L4 as relevant.\n",
    "        # There should be no inherent duplication between L2, L3 and L4 text. There should be no duplication between L0 and L1.\n",
    "        # Note that legislation itself can also be repetitive (e.g. setting out identical provisions to apply to different \n",
    "        # areas of the UK.\n",
    "        \n",
    "file.close()\n",
    "outfile.close()\n",
    "        \n",
    "print(\"Done!\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}