2nd_classify_and_make_train_test.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","source":["# Environment"],"metadata":{"id":"sAhoqzSEVqoG"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"uW_62az96Tns"},"outputs":[],"source":["import csv\n","import psycopg2\n","from psycopg2 import Error\n","\n","import sys\n","import subprocess\n","\n","from sklearn.model_selection import train_test_split\n","import pandas as pd\n","import ast\n","import os"]},{"cell_type":"markdown","source":["# Helper Function"],"metadata":{"id":"PwoMfJNBVKZz"}},{"cell_type":"code","source":["def create_folders(folder_path='csv files', additional_folders=None):\n","    \"\"\"\n","    This function will check if a folder exists, if it does not it will create one with the path from the input, if a\n","    tuple of additional folders is defined, they are also created inside the main folder.\n","\n","    :param folder_path: Path to main folder to be created\n","    :param additional_folders: Tuple of strings with the name of the sub-folders to be created, if none are defined none\n","    are created\n","    \"\"\"\n","    if not os.path.exists(folder_path):\n","        os.mkdir(folder_path)\n","        if additional_folders is not None:\n","            for additional_folder in additional_folders:\n","                additional_folder_path = os.path.join(folder_path, additional_folder)\n","                os.mkdir(additional_folder_path)\n","        print(\"Folder %s created!\" % folder_path)\n","    else:\n","        print(\"Folder %s already exists\" % folder_path)\n","        if additional_folders is not None:\n","            for additional_folder in additional_folders:\n","                additional_folder_path = os.path.join(folder_path, additional_folder)\n","                os.mkdir(additional_folder_path)\n"],"metadata":{"id":"oXR1BffUVFS-"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Classify cases based on bug and smells flags"],"metadata":{"id":"vN4nOKCQV7av"}},{"cell_type":"code","source":["# check whether directory already exists and if it does not, create it\n","create_folders('csv files',('tokenizer data', 'tokenized'))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"XGn_l5pTW85b","executionInfo":{"status":"ok","timestamp":1696098670738,"user_tz":180,"elapsed":307,"user":{"displayName":"Orestes Bastos","userId":"10763828588366877923"}},"outputId":"1b62ff8a-d78a-41c6-df1a-683c42c24837"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Folder csv files created!\n"]}]},{"cell_type":"code","source":["# Connect to an existing database\n","connection = psycopg2.connect(user='postgres',\n","                              password='1234',\n","                              dbname='metrics')\n","# Create a cursor to perform database operations\n","cursor = connection.cursor()\n","\n","try:\n","\n","    # Fetch all cases that are a bug fix\n","    postgreSQL_select_Query = \"SELECT * FROM public.class WHERE bug_fix = %s\"\n","    cursor.execute(postgreSQL_select_Query, ('true',))\n","    cases = cursor.fetchall()\n","\n","    with open('csv files\\\\tokenizer data\\\\harmful_tokenizer_data.csv', 'w', encoding=\"utf-8\", newline='') as csvfile:\n","        fieldnames = ['id', 'language', 'text', 'smells', 'project']\n","        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\n","\n","        writer.writeheader()\n","        counter = 0\n","\n","        for case in cases:\n","            db_id = case[0]\n","            db_language = case[2]\n","            db_content = case[8]\n","            db_smells = case[13]\n","            db_project = case[3]\n","            # If there are any invalid values in that row skip it\n","            if db_id < 0 or db_language == '' or db_content == '' or db_smells == {}:\n","                continue\n","            # If there are no smells in row, skip it\n","            if not any(db_smells.values()):\n","                continue\n","            writer.writerow({'id': db_id, 'language': db_language, 'text': db_content, 'smells': db_smells, 'project': db_project})\n","            counter += 1\n","        else:\n","            print(' ')\n","            print('All harmful cases sorted, ' + str(counter) + ' total cases.')\n","\n","    # Fetch all cases that are not a bug fix for each language\n","    postgreSQL_select_Query = \"SELECT * FROM public.class WHERE bug_fix = %s\"\n","    cursor.execute(postgreSQL_select_Query, ('false',))\n","    cases = cursor.fetchall()\n","\n","    with open('csv files\\\\tokenizer data\\\\clean_tokenizer_data.csv', 'w', encoding=\"utf-8\", newline='') as csvfile:\n","        fieldnames = ['id', 'language', 'text', 'smells', 'project']\n","        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\n","\n","        writer.writeheader()\n","        counter = 0\n","\n","        for case in cases:\n","            db_id = case[0]\n","            db_language = case[2]\n","            db_content = case[8]\n","            db_smells = case[13]\n","            db_project = case[3]\n","            # If there are any invalid values in that row skip it\n","            if db_id < 0 or db_language == '' or db_content == '' or db_smells == {}:\n","                continue\n","            # If there are smells in row, skip it\n","            if any(db_smells.values()):\n","                continue\n","            writer.writerow({'id': db_id, 'language': db_language, 'text': db_content, 'smells': db_smells, 'project': db_project})\n","            counter += 1\n","        else:\n","            print(' ')\n","            print('All clean code cases sorted, ' + str(counter) + ' total cases.')\n","\n","except (Exception, Error) as error:\n","    print(' ')\n","    print('Error while connecting to PostgreSQL', error)\n","\n","finally:\n","    if connection:\n","        cursor.close()\n","        print(' ')\n","        print('PostgreSQL connection is closed')\n","        connection.close()\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":394},"id":"zYbC9xVVWwZL","executionInfo":{"status":"error","timestamp":1696098679950,"user_tz":180,"elapsed":372,"user":{"displayName":"Orestes Bastos","userId":"10763828588366877923"}},"outputId":"4742813e-fffd-4baa-f571-542df4f68f2c"},"execution_count":null,"outputs":[{"output_type":"error","ename":"OperationalError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mOperationalError\u001b[0m                          Traceback (most recent call last)","\u001b[0;32m<ipython-input-4-f7c62c9e6d45>\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Connect to an existing database\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m connection = psycopg2.connect(user='postgres',\n\u001b[0m\u001b[1;32m      3\u001b[0m                               \u001b[0mpassword\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'1234'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m                               dbname='metrics')\n\u001b[1;32m      5\u001b[0m \u001b[0;31m# Create a cursor to perform database operations\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/psycopg2/__init__.py\u001b[0m in \u001b[0;36mconnect\u001b[0;34m(dsn, connection_factory, cursor_factory, **kwargs)\u001b[0m\n\u001b[1;32m    120\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    121\u001b[0m     \u001b[0mdsn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_ext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake_dsn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdsn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 122\u001b[0;31m     \u001b[0mconn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_connect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdsn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconnection_factory\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconnection_factory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwasync\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    123\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcursor_factory\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    124\u001b[0m         \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcursor_factory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcursor_factory\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mOperationalError\u001b[0m: connection to server on socket \"/var/run/postgresql/.s.PGSQL.5432\" failed: No such file or directory\n\tIs the server running locally and accepting connections on that socket?\n"]}]},{"cell_type":"markdown","source":["# Tokenizer constants\n","### A tokenizer executable is available on the project repository"],"metadata":{"id":"kWjId1YSXczf"}},{"cell_type":"code","source":["# Set the string bellow to the path to a tokenizer executable file in Linux format\n","TOKENIZER_BIN = r\"./executable_folder_path\"\n","\n","# File names to run on tokenizer\n","FILE_NAMES = ['harmful', 'clean']"],"metadata":{"id":"voxVVrIgXnyu"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Tokenizer Helper Functions"],"metadata":{"id":"gzAjH2_tXyME"}},{"cell_type":"code","source":["def get_tokens(language, file):\n","    tokens = ''\n","    cmd = [TOKENIZER_BIN, '-l', language, file]\n","    process = subprocess.Popen(cmd, stdout=subprocess.PIPE)\n","\n","    try:\n","        output, error = process.communicate()\n","        tokens = output.decode('utf8')\n","        tokens = tokens.replace('\\t', ' ')\n","        tokens = tokens.replace('\\n', '')\n","    except Exception as e:\n","        print('Unexpected Error on Get Tokens', e)\n","\n","    return tokens\n","\n","\n","def create_tmp_file(code_text):\n","    try:\n","        with open('experiment.tmp', 'w+') as file:\n","            file.write(code_text)\n","            return 'experiment.tmp'\n","    except Exception as e:\n","        print('Unexpected Error on Create Tmp File', e)\n","        return None"],"metadata":{"id":"CbdhbpzOX5KM"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Run Tokenizer to get text tokens for each file"],"metadata":{"id":"_j61a_ujX-vs"}},{"cell_type":"code","source":["for file in FILE_NAMES:\n","    tokenizer_file_path = 'csv files\\\\tokenizer data\\\\' + file + '_tokenizer_data.csv'\n","    # Open csv with data to create tokens and save them all in a new file\n","    with open(tokenizer_file_path, encoding=\"utf-8\", newline='') as csvfile1:\n","        reader = csv.DictReader(csvfile1)\n","\n","        tokenized_file_path = 'csv files\\\\tokenized\\\\' + file + '_tokenized.csv'\n","        with open(tokenized_file_path, 'w', encoding=\"utf-8\", newline='') as csvfile2:\n","            fieldnames = ['id', 'language', 'text', 'smells', 'tokens']\n","            writer = csv.DictWriter(csvfile2, fieldnames=fieldnames)\n","            writer.writeheader()\n","\n","            for row in reader:\n","                csv_id = row['id']\n","                csv_language = row['language']\n","                csv_text = row['text']\n","                csv_smells = row['smells']\n","\n","                temp_file = create_tmp_file(csv_text)\n","                result_tokens = get_tokens(csv_language, temp_file)\n","                writer.writerow({'id': csv_id, 'language': csv_language, 'text': csv_text, 'smells': csv_smells,\n","                                 'tokens': result_tokens})"],"metadata":{"id":"_0ett2dOYe_k"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Create Train and Test files for each case"],"metadata":{"id":"9aBCN30_c0a1"}},{"cell_type":"code","source":["# check whether directory already exists and if it does not, create it\n","harmful_clean_path = os.path.join('csv files', 'harmful-clean')\n","create_folders(harmful_clean_path, ('all', 'train', 'test'))\n","\n","# TODO: Get max value from each case to set padding_tokens value in perceptron\n","# Get data from csv file\n","print(' ')\n","print('Largest token size:')\n","print(' ')\n","\n","print('harmful code:')\n","df1 = pd.read_csv(r'csv files\\tokenized\\harmful_tokenized.csv')\n","split_tokens = df1.tokens.str.split(' ')\n","print(split_tokens.str.len().max())\n","\n","print('clean code:')\n","df2 = pd.read_csv(r'csv files\\tokenized\\clean_tokenized.csv')\n","split_tokens = df2.tokens.str.split(' ')\n","print(split_tokens.str.len().max())\n","\n","# Check how many languages there are\n","languages = df1['language'].unique()\n","\n","# Separate in individual Data Frames for each language\n","for language in languages:\n","    lang_df1 = df1.loc[df1['language'] == language]\n","    lang_df2 = df2.loc[df2['language'] == language]\n","\n","    print(' ')\n","    print('[' + language + ']')\n","    print('harmful code cases:')\n","    print(len(lang_df1))\n","    print('clean code cases:')\n","    print(len(lang_df2))\n","\n","    file_name1 = language + '_' + 'Harmful' + '.csv'\n","\n","    # Open file inside new directory\n","    with open((harmful_clean_path + '\\\\all\\\\' + file_name1), 'w', encoding=\"utf-8\", newline='') as csvfile:\n","        fieldnames = ['id', 'language', 'text', 'smell', 'tokens']\n","        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\n","        writer.writeheader()\n","\n","        for index, row in lang_df1.iterrows():\n","            csv_id1 = row['id']\n","            text1 = row['text']\n","            tokens1 = row['tokens']\n","            smells1 = ast.literal_eval(row['smells'])\n","            if not any(smells1.values()):\n","                print('Error: row in data for harmful code without at least one smell')\n","                break\n","            smell_val1 = 1\n","\n","            writer.writerow(\n","                {'id': csv_id1, 'language': language, 'text': text1, 'smell': smell_val1, 'tokens': tokens1})\n","\n","    file_name2 = language + '_' + 'Clean' + '.csv'\n","\n","    # Open file inside new directory\n","    with open((harmful_clean_path + '\\\\all\\\\' + file_name2), 'w', encoding=\"utf-8\", newline='') as csvfile:\n","        fieldnames = ['id', 'language', 'text', 'smell', 'tokens']\n","        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\n","        writer.writeheader()\n","\n","        for index, row in lang_df2.iterrows():\n","            csv_id2 = row['id']\n","            text2 = row['text']\n","            tokens2 = row['tokens']\n","            smells2 = ast.literal_eval(row['smells'])\n","            if any(smells2.values()):\n","                print('Error: row in data for clean code with at least one smell')\n","                break\n","            smell_val2 = 0\n","\n","            writer.writerow(\n","                {'id': csv_id2, 'language': language, 'text': text2, 'smell': smell_val2, 'tokens': tokens2})\n","\n","for language in languages:\n","    harmful_name = language + '_' + 'Harmful'\n","    open_path = harmful_clean_path + '\\\\all\\\\' + harmful_name + '.csv'\n","    harmful_df = pd.read_csv(open_path)\n","\n","    clean_name = language + '_' + 'Clean'\n","    open_path = harmful_clean_path + '\\\\all\\\\' + clean_name + '.csv'\n","    clean_df = pd.read_csv(open_path)\n","\n","    # Check witch case is smaller and use its length\n","    if len(harmful_df) < len(clean_df):\n","        clean_df_harmful_vs_clean = clean_df[clean_df.index < len(harmful_df)]\n","        harmful_df_harmful_vs_clean = harmful_df\n","    else:\n","        harmful_df_harmful_vs_clean = harmful_df[harmful_df.index < len(clean_df)]\n","        clean_df_harmful_vs_clean = clean_df\n","\n","    train1_harmful_clean, test1_harmful_clean = train_test_split(clean_df_harmful_vs_clean, test_size=0.2)\n","    train2_harmful_clean, test2_harmful_clean = train_test_split(harmful_df_harmful_vs_clean, test_size=0.2)\n","\n","    concat_train_harmful_clean = pd.concat([train1_harmful_clean, train2_harmful_clean])\n","    concat_test_harmful_clean = pd.concat([test1_harmful_clean, test2_harmful_clean])\n","\n","    file_name = language + '_' + 'HarmfulVsClean'\n","    header = ['id', 'language', 'text', 'smell', 'tokens']\n","    train_path = harmful_clean_path + '\\\\train\\\\' + file_name + '_Train_1.csv'\n","    concat_train_harmful_clean.to_csv(train_path, header=header, encoding='utf-8', index=False)\n","    test_path = harmful_clean_path + '\\\\test\\\\' + file_name + '_Test_1.csv'\n","    concat_test_harmful_clean.to_csv(test_path, header=header, encoding='utf-8', index=False)\n"],"metadata":{"id":"WKRwR7nDc4Ps"},"execution_count":null,"outputs":[]}]}