prepareData.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare data for dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "from math import floor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define paths directly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = Path(\"data\", \"all_data\")\n",
    "vocab_output_path = data_path.parent / \"vocab.txt\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set the dataset splits percentages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_PER = 0.8\n",
    "TEST_PER = 0.1\n",
    "VAL_PER = 0.1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Iterate through files in the data_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "occurrences_count = {}\n",
    "\n",
    "for file in data_path.iterdir():\n",
    "    stem = file.stem\n",
    "    suffix = file.suffix\n",
    "    # Initialize the count dictionary as needed\n",
    "    occurrences_count.setdefault(stem, {}).setdefault(suffix, 0)\n",
    "    occurrences_count[stem][suffix] += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filter for valid pairs of .gui and .png files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_pairs = []\n",
    "for key, counts in occurrences_count.items():\n",
    "    # Check if exactly one .gui and one .png file exist for the key\n",
    "    has_one_gui = counts.get(\".gui\", 0) == 1\n",
    "    has_one_png = counts.get(\".png\", 0) == 1\n",
    "    if has_one_gui and has_one_png:\n",
    "        valid_pairs.append(key)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculate the number of valid examples and compute split indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "number_of_examples = len(valid_pairs)\n",
    "train_split = floor(number_of_examples * TRAIN_PER)\n",
    "validation_split = floor(number_of_examples * VAL_PER)\n",
    "test_split = floor(number_of_examples * TEST_PER)\n",
    "\n",
    "# Create datasets\n",
    "train_set = valid_pairs[:train_split]\n",
    "validation_set = valid_pairs[train_split:train_split + validation_split]\n",
    "test_set = valid_pairs[train_split + validation_split:]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save dataset splits to files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_splits = {\"train\": train_set, \"validation\": validation_set, \"test\": test_set}\n",
    "for key, value in dataset_splits.items():\n",
    "    filepath = data_path.parent / f'{key}_dataset.txt'\n",
    "    with open(filepath, \"w\") as writer:\n",
    "        for example in value:\n",
    "            writer.write(example + \"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write the set of all tokens to a vocab file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found a total of 1699 valid examples\n",
      "Writing vocab with 13 tokens to data\\vocab.txt\n"
     ]
    }
   ],
   "source": [
    "# Initialize a dictionary to keep unique tokens\n",
    "all_tokens = set()  \n",
    "\n",
    "# Iterate through .gui files to extract tokens\n",
    "for file in data_path.glob(\"*.gui\"):\n",
    "    with open(file, \"r\") as reader:\n",
    "        # Normalize space around commas and split by spaces\n",
    "        data = reader.read().replace('\\n', ' ').replace(', ', ' , ').split()\n",
    "        all_tokens.update(data)  # Update set with new tokens\n",
    "\n",
    "with open(vocab_output_path, \"w\") as writer:\n",
    "    writer.write(\" \".join(sorted(all_tokens)))  # Sort tokens for consistent ordering\n",
    "\n",
    "print(f'Found a total of {number_of_examples} valid examples')\n",
    "print(f'Writing vocab with {len(all_tokens)} tokens to {vocab_output_path}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}