notebooks/4_ip_computing_item_embeddings.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "notebook_start_time = time.time()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set up environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "⛳️ Local environment\n",
      "Adding the following directory to the PYTHONPATH: /Users/pauliusztin/Documents/01_projects/hopsworks_recsys/hands-on-recommender-system\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "from pathlib import Path\n",
    "\n",
    "\n",
    "def is_google_colab() -> bool:\n",
    "    if \"google.colab\" in str(get_ipython()):\n",
    "        return True\n",
    "    return False\n",
    "\n",
    "\n",
    "def clone_repository() -> None:\n",
    "    !git clone https://github.com/decodingml/hands-on-recommender-system.git\n",
    "    %cd hands-on-recommender-system/\n",
    "\n",
    "\n",
    "def install_dependencies() -> None:\n",
    "    !pip install --upgrade uv\n",
    "    !uv pip install --all-extras --system --requirement pyproject.toml\n",
    "\n",
    "\n",
    "if is_google_colab():\n",
    "    clone_repository()\n",
    "    install_dependencies()\n",
    "\n",
    "    root_dir = str(Path().absolute())\n",
    "    print(\"⛳️ Google Colab environment\")\n",
    "else:\n",
    "    root_dir = str(Path().absolute().parent)\n",
    "    print(\"⛳️ Local environment\")\n",
    "\n",
    "# Add the root directory to the `PYTHONPATH` to use the `recsys` Python module from the notebook.\n",
    "if root_dir not in sys.path:\n",
    "    print(f\"Adding the following directory to the PYTHONPATH: {root_dir}\")\n",
    "    sys.path.append(root_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 👩🏻‍🔬 Offline inference pipeline: Computing item embeddings\n",
    "\n",
    "In this notebook you will compute the candidate embeddings and populate a Hopsworks feature group with a vector index."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 📝 Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "from loguru import logger\n",
    "\n",
    "from recsys import features, hopsworks_integration\n",
    "from recsys.config import settings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'CUSTOMER_DATA_SIZE': <CustomerDatasetSize.SMALL: 'SMALL'>,\n",
      " 'CUSTOM_HOPSWORKS_INFERENCE_ENV': 'custom_env_name',\n",
      " 'FEATURES_EMBEDDING_MODEL_ID': 'all-MiniLM-L6-v2',\n",
      " 'HOPSWORKS_API_KEY': SecretStr('**********'),\n",
      " 'OPENAI_API_KEY': SecretStr('**********'),\n",
      " 'OPENAI_MODEL_ID': 'gpt-4o-mini',\n",
      " 'RANKING_DATASET_VALIDATON_SPLIT_SIZE': 0.1,\n",
      " 'RANKING_EARLY_STOPPING_ROUNDS': 5,\n",
      " 'RANKING_ITERATIONS': 100,\n",
      " 'RANKING_LEARNING_RATE': 0.2,\n",
      " 'RANKING_MODEL_TYPE': 'ranking',\n",
      " 'RANKING_SCALE_POS_WEIGHT': 10,\n",
      " 'RECSYS_DIR': PosixPath('/Users/pauliusztin/Documents/01_projects/hopsworks_recsys/hands-on-recommender-system/recsys'),\n",
      " 'TWO_TOWER_DATASET_TEST_SPLIT_SIZE': 0.1,\n",
      " 'TWO_TOWER_DATASET_VALIDATON_SPLIT_SIZE': 0.1,\n",
      " 'TWO_TOWER_LEARNING_RATE': 0.01,\n",
      " 'TWO_TOWER_MODEL_BATCH_SIZE': 2048,\n",
      " 'TWO_TOWER_MODEL_EMBEDDING_SIZE': 16,\n",
      " 'TWO_TOWER_NUM_EPOCHS': 10,\n",
      " 'TWO_TOWER_WEIGHT_DECAY': 0.001}\n"
     ]
    }
   ],
   "source": [
    "from pprint import pprint\n",
    "\n",
    "pprint(dict(settings))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <span style=\"color:#ff5f27\">🔮 Connect to Hopsworks Feature Store </span>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2024-12-24 13:04:50.353\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mrecsys.hopsworks_integration.feature_store\u001b[0m:\u001b[36mget_feature_store\u001b[0m:\u001b[36m13\u001b[0m - \u001b[1mLoging to Hopsworks using HOPSWORKS_API_KEY env var.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2024-12-24 13:04:50,354 INFO: Initializing external client\n",
      "2024-12-24 13:04:50,355 INFO: Base URL: https://c.app.hopsworks.ai:443\n",
      "2024-12-24 13:04:51,764 INFO: Python Engine initialized.\n",
      "\n",
      "Logged in to project, explore it here https://c.app.hopsworks.ai:443/p/1192098\n"
     ]
    }
   ],
   "source": [
    "project, fs = hopsworks_integration.get_feature_store()\n",
    "\n",
    "mr = project.get_model_registry()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Computing candidate embeddings\n",
    "\n",
    "You start by computing candidate embeddings for all items in the training data.\n",
    "\n",
    "First, you load your candidate model. Recall that you uploaded it to the Hopsworks Model Registry in previous steps:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2024-12-24 13:04:53.888\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mrecsys.hopsworks_integration.two_tower_serving\u001b[0m:\u001b[36mdownload\u001b[0m:\u001b[36m189\u001b[0m - \u001b[1mDownloading 'candidate_model' version 2\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading model artifact (2 dirs, 4 files)... DONE\r"
     ]
    }
   ],
   "source": [
    "candidate_model, candidate_features = (\n",
    "    hopsworks_integration.two_tower_serving.HopsworksCandidateModel.download(mr=mr)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get candidates data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we get the training retrieval data containing all the features required for the candidate embedding model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_view = fs.get_feature_view(\n",
    "    name=\"retrieval\",\n",
    "    version=1,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished: Reading data from Hopsworks, using Hopsworks Feature Query Service (5.28s) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "VersionWarning: Incremented version to `4`.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customer_id</th>\n",
       "      <th>article_id</th>\n",
       "      <th>t_dat</th>\n",
       "      <th>price</th>\n",
       "      <th>month_sin</th>\n",
       "      <th>month_cos</th>\n",
       "      <th>age</th>\n",
       "      <th>club_member_status</th>\n",
       "      <th>age_group</th>\n",
       "      <th>garment_group_name</th>\n",
       "      <th>index_group_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>f7048acb8188d98bde3a5c495475a3c86faafe0eede1f2...</td>\n",
       "      <td>670265002</td>\n",
       "      <td>0</td>\n",
       "      <td>0.013542</td>\n",
       "      <td>-8.660254e-01</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>48.0</td>\n",
       "      <td>ACTIVE</td>\n",
       "      <td>46-55</td>\n",
       "      <td>Under-, Nightwear</td>\n",
       "      <td>Ladieswear</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5d34f84e6cbe9ec4706872bb65376097af1e53f0c7dac5...</td>\n",
       "      <td>751471035</td>\n",
       "      <td>0</td>\n",
       "      <td>0.033881</td>\n",
       "      <td>1.224647e-16</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>30.0</td>\n",
       "      <td>ACTIVE</td>\n",
       "      <td>26-35</td>\n",
       "      <td>Trousers</td>\n",
       "      <td>Ladieswear</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>baf6dc7ea8575732794751bb80824fe84fd40e6af86193...</td>\n",
       "      <td>719308002</td>\n",
       "      <td>0</td>\n",
       "      <td>0.059305</td>\n",
       "      <td>5.000000e-01</td>\n",
       "      <td>-0.866025</td>\n",
       "      <td>48.0</td>\n",
       "      <td>ACTIVE</td>\n",
       "      <td>46-55</td>\n",
       "      <td>Dresses Ladies</td>\n",
       "      <td>Divided</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         customer_id article_id  t_dat  \\\n",
       "0  f7048acb8188d98bde3a5c495475a3c86faafe0eede1f2...  670265002      0   \n",
       "1  5d34f84e6cbe9ec4706872bb65376097af1e53f0c7dac5...  751471035      0   \n",
       "2  baf6dc7ea8575732794751bb80824fe84fd40e6af86193...  719308002      0   \n",
       "\n",
       "      price     month_sin  month_cos   age club_member_status age_group  \\\n",
       "0  0.013542 -8.660254e-01   0.500000  48.0             ACTIVE     46-55   \n",
       "1  0.033881  1.224647e-16  -1.000000  30.0             ACTIVE     26-35   \n",
       "2  0.059305  5.000000e-01  -0.866025  48.0             ACTIVE     46-55   \n",
       "\n",
       "  garment_group_name index_group_name  \n",
       "0  Under-, Nightwear       Ladieswear  \n",
       "1           Trousers       Ladieswear  \n",
       "2     Dresses Ladies          Divided  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df, val_df, test_df, _, _, _ = feature_view.train_validation_test_split(\n",
    "    validation_size=settings.TWO_TOWER_DATASET_VALIDATON_SPLIT_SIZE,\n",
    "    test_size=settings.TWO_TOWER_DATASET_TEST_SPLIT_SIZE,\n",
    "    description=\"Retrieval dataset splits\",\n",
    ")\n",
    "train_df.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute embeddings\n",
    "\n",
    "Next you compute the embeddings of all candidate items that were used to train the retrieval model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>garment_group_name</th>\n",
       "      <th>article_id</th>\n",
       "      <th>index_group_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Under-, Nightwear</td>\n",
       "      <td>670265002</td>\n",
       "      <td>Ladieswear</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Trousers</td>\n",
       "      <td>751471035</td>\n",
       "      <td>Ladieswear</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Dresses Ladies</td>\n",
       "      <td>719308002</td>\n",
       "      <td>Divided</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  garment_group_name article_id index_group_name\n",
       "0  Under-, Nightwear  670265002       Ladieswear\n",
       "1           Trousers  751471035       Ladieswear\n",
       "2     Dresses Ladies  719308002          Divided"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "item_df = features.embeddings.preprocess(train_df, candidate_features)\n",
    "item_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article_id</th>\n",
       "      <th>embeddings</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>670265002</td>\n",
       "      <td>[-0.2746514081954956, 0.8198990225791931, -1.0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>751471035</td>\n",
       "      <td>[2.1680235862731934, 1.021416187286377, -0.622...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>719308002</td>\n",
       "      <td>[0.9961531162261963, 0.14149034023284912, -0.6...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>759231002</td>\n",
       "      <td>[1.2411562204360962, -0.1422426700592041, -0.7...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>793911001</td>\n",
       "      <td>[2.1397600173950195, 0.920747697353363, 0.2480...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   article_id                                         embeddings\n",
       "0   670265002  [-0.2746514081954956, 0.8198990225791931, -1.0...\n",
       "1   751471035  [2.1680235862731934, 1.021416187286377, -0.622...\n",
       "2   719308002  [0.9961531162261963, 0.14149034023284912, -0.6...\n",
       "3   759231002  [1.2411562204360962, -0.1422426700592041, -0.7...\n",
       "4   793911001  [2.1397600173950195, 0.920747697353363, 0.2480..."
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeddings_df = features.embeddings.embed(df=item_df, candidate_model=candidate_model)\n",
    "embeddings_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <span style=\"color:#ff5f27\">Create Hopsworks Embedding Index </span>\n",
    "\n",
    "Now you are ready to create a feature group for your candidate embeddings.\n",
    "\n",
    "To begin with, you need to create your Embedding Index where you will specify the name of the embeddings feature and the embeddings length.\n",
    "Then you attach this index to the FG."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Uploading Dataframe: 100.00% |██████████| Rows 11820/11820 | Elapsed Time: 00:02 | Remaining Time: 00:00\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Launching job: candidate_embeddings_1_offline_fg_materialization\n",
      "Job started successfully, you can follow the progress at \n",
      "https://c.app.hopsworks.ai:443/p/1192098/jobs/named/candidate_embeddings_1_offline_fg_materialization/executions\n",
      "2024-12-24 13:05:27,637 INFO: Waiting for execution to finish. Current state: SUBMITTED. Final status: UNDEFINED\n",
      "2024-12-24 13:05:30,810 INFO: Waiting for execution to finish. Current state: RUNNING. Final status: UNDEFINED\n",
      "2024-12-24 13:07:02,804 INFO: Waiting for execution to finish. Current state: AGGREGATING_LOGS. Final status: SUCCEEDED\n",
      "2024-12-24 13:07:02,964 INFO: Waiting for log aggregation to finish.\n",
      "2024-12-24 13:07:14,855 INFO: Execution finished successfully.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2024-12-24 13:07:14.856\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m6\u001b[0m - \u001b[1m✅ Uploaded 'candidate_embeddings' Feature Group to Hopsworks!!\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "candidate_embeddings_fg = (\n",
    "    hopsworks_integration.feature_store.create_candidate_embeddings_feature_group(\n",
    "        fs=fs, df=embeddings_df, online_enabled=True\n",
    "    )\n",
    ")\n",
    "logger.info(\"✅ Uploaded 'candidate_embeddings' Feature Group to Hopsworks!!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Expose it to the online inference pipeline as a Feature View\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_view = (\n",
    "    hopsworks_integration.feature_store.create_candidate_embeddings_feature_view(\n",
    "        fs=fs, fg=candidate_embeddings_fg\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <span style=\"color:#ff5f27\"> Inspecting the embeddings in Hopsworks UI </span>\n",
    "\n",
    "View results in [Hopsworks Serverless](https://rebrand.ly/serverless-github): **Feature Store → Feature Groups**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2024-12-24 13:07:15.619\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m4\u001b[0m - \u001b[1m⌛️ Notebook Execution time: 149.17 seconds ~ 2.49 minutes\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "notebook_end_time = time.time()\n",
    "notebook_execution_time = notebook_end_time - notebook_start_time\n",
    "\n",
    "logger.info(\n",
    "    f\"⌛️ Notebook Execution time: {notebook_execution_time:.2f} seconds ~ {notebook_execution_time / 60:.2f} minutes\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <span style=\"color:#ff5f27\">→ Next Steps </span>\n",
    "\n",
    "Now that we have our vector index populated with item embeddings, everything is ready for production. In the next notebook, we will zoom in into the inference pipeline and how we can deploy it to Hopsworks as a real-time deployment."
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}