diff --git a/choice_learn/datasets/data/ta_feng.csv.zip b/choice_learn/datasets/data/ta_feng.csv.zip new file mode 100644 index 00000000..7a255053 Binary files /dev/null and b/choice_learn/datasets/data/ta_feng.csv.zip differ diff --git a/choice_learn/datasets/examples.py b/choice_learn/datasets/examples.py new file mode 100644 index 00000000..aba64a97 --- /dev/null +++ b/choice_learn/datasets/examples.py @@ -0,0 +1,117 @@ +"""Some datasets used for personal examples.""" +import os + +import numpy as np +import pandas as pd + +from choice_learn.data.choice_dataset import ChoiceDataset + +DATA_MODULE = os.path.join(os.path.abspath(".."), "choice_learn", "datasets", "data") + + +def load_tafeng(as_frame=False, preprocessing=None): + """Function to load the TaFeng dataset. + + Orginal file and informations can be found here: + https://www.kaggle.com/datasets/chiranjivdas09/ta-feng-grocery-dataset/ + + Parameters + ---------- + as_frame : bool, optional + Whether to return the original file as pd.DF, by default False + preprocessing : str, optional + predefined pre-processing to apply, by default None + + Returns: + -------- + pd.DF or ChoiceDataset + TaFeng Grocery Dataset. + """ + filepath = os.path.join(DATA_MODULE, "ta_feng.csv.zip") + # url = "https://www.kaggle.com/datasets/chiranjivdas09/ta-feng-grocery-dataset/download?datasetVersionNumber=1" + # if not os.path.exists(filepath): + # with urllib.request.urlopen(url) as f: + # file = f.read().decode("utf-8") + + tafeng_df = pd.read_csv(filepath) + if as_frame: + return tafeng_df + + if preprocessing == "assort_example": + subdf = tafeng_df.loc[tafeng_df.PRODUCT_SUBCLASS == 100505] + prods = subdf.PRODUCT_ID.value_counts().index[ + (subdf.PRODUCT_ID.value_counts() > 20).to_numpy() + ] + subdf = tafeng_df.loc[tafeng_df.PRODUCT_ID.isin(prods)] + subdf = subdf.dropna() + subdf = subdf.reset_index(drop=True) + + # Create Prices + items = list(subdf.PRODUCT_ID.unique()) + init_prices = [] + for item in items: + first_price = subdf.loc[subdf.PRODUCT_ID == item].SALES_PRICE.to_numpy()[0] + init_prices.append(first_price) + + # Encode Age Groups + age_groups = {} + for i, j in enumerate(subdf.AGE_GROUP.unique()): + age_groups[j] = i + age_groups = { + "<25": 0, + "25-29": 0, + "30-34": 0, + "35-39": 1, + "40-44": 1, + "45-49": 1, + "50-54": 2, + "55-59": 2, + "60-64": 2, + ">65": 2, + } + age_groups = { + "<25": [1, 0, 0], + "25-29": [0, 1, 0], + "30-34": [0, 1, 0], + "35-39": [0, 1, 0], + "40-44": [0, 1, 0], + "45-49": [0, 1, 0], + "50-54": [0, 0, 1], + "55-59": [0, 0, 1], + "60-64": [0, 0, 1], + ">65": [0, 0, 1], + } + + all_prices = [] + customer_features = [] + choices = [] + + curr_prices = [i for i in init_prices] + + for n_row, row in subdf.iterrows(): + for _ in range(int(row.AMOUNT)): + item = row.PRODUCT_ID + price = row.SALES_PRICE / row.AMOUNT + age = row.AGE_GROUP + + item_index = items.index(item) + + # customer_features.append([age_groups[age]]) + customer_features.append(age_groups[age]) + choices.append(item_index) + curr_prices[item_index] = price + all_prices.append([i for i in curr_prices]) + + all_prices = np.expand_dims(np.array(all_prices), axis=-1) + customer_features = np.array(customer_features).astype("float32") + choices = np.array(choices) + + # Create Dataset + return ChoiceDataset( + contexts_features=customer_features, + choices=choices, + contexts_items_features=all_prices, + contexts_items_availabilities=np.ones((len(choices), 25)).astype("float32"), + ) + + return tafeng_df diff --git a/choice_learn/models/base_model.py b/choice_learn/models/base_model.py index c25162cc..cb87d805 100644 --- a/choice_learn/models/base_model.py +++ b/choice_learn/models/base_model.py @@ -1299,6 +1299,7 @@ def _em_fit(self, dataset, verbose=0): """ hist_logits = [] hist_loss = [] + # Initialization for model in self.models: # model.instantiate() diff --git a/choice_learn/toolbox/assortment_optimizer.py b/choice_learn/toolbox/assortment_optimizer.py new file mode 100644 index 00000000..c1dde0d5 --- /dev/null +++ b/choice_learn/toolbox/assortment_optimizer.py @@ -0,0 +1,135 @@ +"""Tool function for assortment optimization.""" +import gurobipy as gp +import numpy as np + +"""TODO: clarify outside good integration +TODO 2: ADD easy integration of additionnal constraints +""" + + +class AssortmentOptimizer(object): + """Base class for assortment optimization.""" + + def __init__(self, utilities, itemwise_values, assortment_size, outside_option_given=False): + """Initializes the AssortmentOptimizer object. + + Parameters + ---------- + utilities : Iterable + List of utilities for each item. + itemwise_values: Iterable + List of to-be-optimized values for each item, e.g. prices. + assortment_size : int + maximum size of the requested assortment. + outside_option_given : bool + Whether the outside option is given or not (and thus is automatically added). + """ + if len(utilities) != len(itemwise_values): + raise ValueError( + f"You should provide as many utilities as itemwise values.\ + Found {len(utilities)} and {len(itemwise_values)} instead." + ) + self.outside_option_given = outside_option_given + if not self.outside_option_given: + self.utilities = np.concatenate([[np.exp(0.0)], utilities], axis=0) + self.itemwise_values = np.concatenate([[0.0], itemwise_values], axis=0) + self.n_items = len(self.utilities) - 1 + self.assortment_size = assortment_size + + self.solver = self.base_instantiate() + self.set_base_constraints() + + def base_instantiate(self): + """Base instantiation of the solver. + + Returns: + -------- + gurobipy.Model + solver with basic variables and constraints. + """ + # Create a new model + solver = gp.Model("Assortment_IP") + solver.ModelSense = -1 + solver.setParam("OutputFlag", False) + + # Create variables + y = {} + + for j in range(self.n_items + 1): + y[j] = solver.addVar( + vtype=gp.GRB.CONTINUOUS, obj=self.itemwise_values[j], name="y_%s" % j + ) + self.y = y + # Integrate new variables + solver.update() + + return solver + + def set_base_constraints(self): + """Functions to set LP base constraints. + + In particular, ensures Charnes-Cooper transformation constraints + and assortment size constraint. + """ + # Base Charnes-Cooper Constraints for Integers + for j in range(1, self.n_items + 1): + self.solver.addConstr(self.y[j] <= self.y[0]) + + # Base Charnes-Cooper Constraint for Normalization + charnes_cooper = gp.quicksum(self.y[j] for j in range(self.n_items + 1)) + self.solver.addConstr(charnes_cooper == 1) + + # Assortment size constraint + if self.assortment_size is not None: + self.solver.addConstr( + gp.quicksum([self.y[j] for j in range(1, self.n_items)]) + <= self.assortment_size * self.y[0] + ) + self.solver.addConstr( + gp.quicksum([-self.y[j] for j in range(1, self.n_items)]) + <= -self.assortment_size * self.y[0] + ) + + # Integrate constraints + self.solver.update() + + def set_objective_function(self, itemwise_values): + """Function to define the objective function to maximize with the assortment. + + Parameters: + ----------- + itemwise_values : list-like + List of values for each item - total value to be optimized. + """ + raise NotImplementedError + + def add_constraint(self): + """Function to add constraints.""" + raise NotImplementedError + + def solve(self): + """Function to solve the optimization problem. + + Returns: + -------- + np.ndarray: + Array of 0s and 1s, indicating the presence of each item in the optimal assortment. + """ + self.solver.update() + + # -- Optimize -- + self.solver.optimize() + self.status = self.solver.Status + + if self.outside_option_given: + assortment = np.zeros(self.n_items + 1) + for i in range(0, self.n_items + 1): + if self.y[i].x > 0: + assortment[i - 1] = 1 + else: + assortment = np.zeros(self.n_items) + for i in range(1, self.n_items + 1): + if self.y[i].x > 0: + assortment[i] = 1 + + return assortment, self.solver.objVal diff --git a/notebooks/assortment_example.ipynb b/notebooks/assortment_example.ipynb new file mode 100644 index 00000000..f8b67fec --- /dev/null +++ b/notebooks/assortment_example.ipynb @@ -0,0 +1,331 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Assortment Example\n", + "A short example for assortment optimization under the conditional MNL." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Importing the right base libraries\n", + "import os\n", + "# Remove GPU use\n", + "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"\n", + "\n", + "import sys\n", + "sys.path.append(\"../\")\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will use the TaFeng Dataset that is available on [Kaggle](https://www.kaggle.com/datasets/chiranjivdas09/ta-feng-grocery-dataset). You can load it automatically with Choice-Learn !" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from choice_learn.datasets.examples import load_tafeng" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Short illustration of the dataset\n", + "tafeng_df = load_tafeng(as_frame=True)\n", + "tafeng_df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this example we will use the sales_price and age_group features to estimate a discrete choice model in the form of a conditional MNL:\n", + "\n", + "for a customer $z$ and a product $i$, we define the utility function:\n", + "\n", + "$$U(i, z) = u_i + e_{dem(z)} \\cdot p_i$$\n", + "\n", + "with:\n", + "- $u_i$ the base utility of product $i$\n", + "- $p_i$ the price of product $i$\n", + "- $e_{dem(z)}$ the price elasticity of customer $z$ depending of its age\n", + "\n", + "We decide to estimate three coefficients of price elasticity for customers <=25 y.o, 26<=.<=55 y.o. and =>56 y.o." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Let's reload the TaFeng dataset as a Choice Dataset\n", + "dataset = load_tafeng(as_frame=False, preprocessing=\"assort_example\")\n", + "\n", + "# The age categories are encoded as OneHot features:\n", + "print(\"Age Categories Encoding for choices 0, 4 and 16:\")\n", + "print(dataset.contexts_features[0][[0, 4, 16]])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's define a custom model that would fit our formulation using Choice-Learn's ChoiceModel inheritance:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "\n", + "from choice_learn.models.base_model import ChoiceModel\n", + "\n", + "\n", + "class TaFengMNL(ChoiceModel):\n", + " \"\"\"Custom model for the TaFeng dataset.\"\"\"\n", + "\n", + " def __init__(self, **kwargs):\n", + " \"\"\"Instantiation of our custom model.\"\"\"\n", + " # Standard inheritance stuff\n", + " super().__init__(**kwargs)\n", + "\n", + " # Instantiation of base utilties weights\n", + " # We have 25 items in the dataset making 25 weights\n", + " self.base_utilities = tf.Variable(\n", + " tf.random_normal_initializer(0.0, 0.02, seed=42)(shape=(1, 25))\n", + " )\n", + " # Instantiation of price elasticities weights\n", + " # We have 3 age categories making 3 weights\n", + " self.price_elasticities = tf.Variable(\n", + " tf.random_normal_initializer(0.0, 0.02, seed=42)(shape=(1, 3))\n", + " )\n", + " # Don't forget to add the weights to be optimized in self.weights !\n", + " self.weights = [self.base_utilities, self.price_elasticities]\n", + "\n", + " def compute_batch_utility(self,\n", + " fixed_items_features,\n", + " contexts_features,\n", + " contexts_items_features,\n", + " contexts_items_availabilities,\n", + " choices):\n", + " \"\"\"Function where to define the utility function for the model.\n", + "\n", + " It uses a standard ChoiceModel signature that needs to be adapted to our usecase.\n", + "\n", + " Parameters:\n", + " -----------\n", + " fixed_items_features : tf.Tensor\n", + " Fixed features of the items in the choice set. We do not have any here.\n", + " contexts_features : tf.Tensor\n", + " Features of the contexts. Here we have the customer age categories.\n", + " context_items_features : tf.Tensor\n", + " Features of the items in the choice set. Items Prices in our case.\n", + " contexts_items_availabilities : tf.Tensor\n", + " Availabilities of the items in the choice set. All items are always available in the dataset making it irrelevant.\n", + " choices : tf.Tensor\n", + " Choices made by the customers. Not relevant in utility computation here.\n", + "\n", + " Returns:\n", + " --------\n", + " tf.Tensor\n", + " Utilities for each item in each choice set.\n", + " \"\"\"\n", + " # Unused arguments\n", + " _ = (fixed_items_features, contexts_items_availabilities, choices)\n", + "\n", + " # Get the right price elasticity coefficient according to the age cateogry\n", + " price_coeffs = tf.tensordot(contexts_features,\n", + " tf.transpose(self.price_elasticities),\n", + " axes=1)\n", + " # Compute the utility: u_i + p_i * c\n", + " return tf.multiply(contexts_items_features[:, :, 0], price_coeffs) + self.base_utilities\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's estimate the model coefficients using the dataset !" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model = TaFengMNL(optimizer=\"lbfgs\", epochs=1000)\n", + "history = model.fit(dataset)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can observe each weight estimation with the .weights argument:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Model Negative Log-Likelihood: \", model.evaluate(dataset))\n", + "print(\"Model Weights:\")\n", + "print(\"Base Utilities u_i:\", model.weights[0].numpy())\n", + "print(\"Price Elasticities:\", model.weights[1].numpy())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As a short analysis we can observe that the price elasticiy in negative as expected and the younger the population the more impacted by the price.\\\n", + "Our models looks good good enough for a first and fast modelization.\n", + "Now let's see how to compute an optimal assortment using our model." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The first step is to compute the utility of each product. Here, let's consider that the last prices will also be the future prices of our products in our future assortment.\\\n", + "It can be easily change if theses prices were to be changed.\\\n", + "We can compute each age category utility using the *compute_batch_utility* method of our ChoiceModel:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "future_prices = np.stack([dataset.contexts_items_features[0][-1]]*3, axis=0)\n", + "age_category = np.array([[1., 0., 0.], [0., 1., 0.], [0., 0., 1.]]).astype(\"float32\")\n", + "predicted_utilities = model.compute_batch_utility(fixed_items_features=None,\n", + " contexts_features=age_category,\n", + " contexts_items_features=future_prices,\n", + " contexts_items_availabilities=None,\n", + " choices=None\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We compute the ratio of each age category appearance in our dataset to obtain an average utility for each product." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "age_frequencies = np.mean(dataset.contexts_features[0], axis=0)\n", + "\n", + "final_utilities = []\n", + "for freq, ut in zip(age_frequencies, predicted_utilities):\n", + " final_utilities.append(freq*ut)\n", + "final_utilities = np.mean(final_utilities, axis=0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We need to define what quantity needs to be optimized by our assortment. A usual answer is to optimize the revenue or margin. In our case we do not have these values, so let's say that we want to obtain the assortment with 12 products that will generate the highest turnover. # right word ?\\\n", + "We have everything we need to use Choice-Learn's AssortmentOptimizer !" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from choice_learn.toolbox.assortment_optimizer import AssortmentOptimizer\n", + "\n", + "opt = AssortmentOptimizer(\n", + " utilities=np.exp(final_utilities), # Utilities need to be transformed with exponential function\n", + " itemwise_values=future_prices[0][:, 0], # Values to optimize for each item, here price that is used to compute turnover\n", + " assortment_size=12) # Size of the assortment we want" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assortment, average_estimated_revenue = opt.solve()\n", + "print(\"Our Optimal Assortment is:\")\n", + "print(assortment)\n", + "print(\"With an estimated average reveue of:\", average_estimated_revenue)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Ending Notes\n", + "- In this example, the outside option is automatically integrated in the AssortmentOptimizer and not computed through the model. If you compute the outside option utility and give it to AssortmentOptimizer you can set its attribute *outside_option_given* to True.\n", + "- The current AssortmentOptimzer uses [Gurobi](https://www.gurobi.com/) for which you need a license. Future developments will integrate OR-Tools that is OpenSource.\n", + "- If you want to add custom constraints you can use the base code of the AssortmentOptimizer and manually add your constraints. Future developments will add an easy interface to integrate such needs.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "tf_env", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/pyproject.toml b/pyproject.toml index f4693f3b..861f14ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ select = [ "PTH", "PD", ] # See: https://beta.ruff.rs/docs/rules/ -ignore = ["D203", "D213", "ANN101", "ANN102", "ANN204", "ANN001", "ANN202", "ANN201", "ANN206", "ANN003"] +ignore = ["D203", "D213", "ANN101", "ANN102", "ANN204", "ANN001", "ANN202", "ANN201", "ANN206", "ANN003", "PTH100", "PTH118"] line-length = 100 target-version = "py310" exclude = [