{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Bayesian hyperparameter optimization\n",
    "\n",
    "Hyperparameters are crucial for machine learning models. The optimal hyperparameters usually differ among different tasks. To find a better set of hyperparameters than the initial values, we provide an entry in the configuration to activate Bayesian hyperparameter optimization.\n",
    "\n",
    "The functionality is implemented based on `scikit-optimize` ([link](https://scikit-optimize.github.io/))."
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cuda device\n",
      "Downloading https://archive.ics.uci.edu/static/public/9/auto+mpg.zip to /tmp/tmphxmeot_i/data/Auto MPG.zip\n",
      "cylinders is Integer and will be treated as a continuous feature.\n",
      "model_year is Integer and will be treated as a continuous feature.\n",
      "origin is Integer and will be treated as a continuous feature.\n",
      "Unknown values are detected in ['horsepower']. They will be treated as np.nan.\n",
      "The project will be saved to /tmp/tmphxmeot_i/output/auto-mpg/2023-09-23-20-37-29-0_UserInputConfig\n",
      "Dataset size: 238 80 80\n",
      "Data saved to /tmp/tmphxmeot_i/output/auto-mpg/2023-09-23-20-37-29-0_UserInputConfig (data.csv and tabular_data.csv).\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from tabensemb.trainer import Trainer\n",
    "from tabensemb.model import *\n",
    "from tabensemb.config import UserConfig\n",
    "import tabensemb\n",
    "import os\n",
    "\n",
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "print(\"Using {} device\".format(device))\n",
    "\n",
    "from tempfile import TemporaryDirectory\n",
    "\n",
    "temp_path = TemporaryDirectory()\n",
    "tabensemb.setting[\"default_output_path\"] = os.path.join(temp_path.name, \"output\")\n",
    "tabensemb.setting[\"default_config_path\"] = os.path.join(temp_path.name, \"configs\")\n",
    "tabensemb.setting[\"default_data_path\"] = os.path.join(temp_path.name, \"data\")\n",
    "\n",
    "trainer = Trainer(device=device)\n",
    "mpg_columns = [\n",
    "    \"mpg\",\n",
    "    \"cylinders\",\n",
    "    \"displacement\",\n",
    "    \"horsepower\",\n",
    "    \"weight\",\n",
    "    \"acceleration\",\n",
    "    \"model_year\",\n",
    "    \"origin\",\n",
    "    \"car_name\",\n",
    "]\n",
    "cfg = UserConfig.from_uci(\"Auto MPG\", column_names=mpg_columns, sep=r\"\\s+\")\n",
    "trainer.load_config(cfg)\n",
    "trainer.load_data()\n",
    "models = [\n",
    "    PytorchTabular(trainer, model_subset=[\"Category Embedding\"]),\n",
    "]\n",
    "trainer.add_modelbases(models)"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "The initial hyperparameters can be seen using the following line."
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "data": {
      "text/plain": "{'dropout': 0.0,\n 'embedding_dropout': 0.1,\n 'lr': 0.001,\n 'weight_decay': 1e-09,\n 'batch_size': 1024}"
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "models[0]._get_params(\"Category Embedding\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Let us see the performance of the model using initial hyperparameters."
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "-------------Run PytorchTabular-------------\n",
      "\n",
      "Training Category Embedding\n",
      "Global seed set to 42\n",
      "2023-09-23 20:37:30,020 - {pytorch_tabular.tabular_model:473} - INFO - Preparing the DataLoaders\n",
      "2023-09-23 20:37:30,021 - {pytorch_tabular.tabular_datamodule:290} - INFO - Setting up the datamodule for regression task\n",
      "2023-09-23 20:37:30,029 - {pytorch_tabular.tabular_model:521} - INFO - Preparing the Model: CategoryEmbeddingModel\n",
      "2023-09-23 20:37:30,041 - {pytorch_tabular.tabular_model:268} - INFO - Preparing the Trainer\n",
      "/home/xlluo/anaconda3/envs/tabular_ensemble/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/accelerator_connector.py:589: LightningDeprecationWarning: The Trainer argument `auto_select_gpus` has been deprecated in v1.9.0 and will be removed in v2.0.0. Please use the function `pytorch_lightning.accelerators.find_usable_cuda_devices` instead.\n",
      "  rank_zero_deprecation(\n",
      "Auto select gpus: [0]\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "2023-09-23 20:37:31,045 - {pytorch_tabular.tabular_model:582} - INFO - Training Started\n",
      "You are using a CUDA device ('NVIDIA GeForce RTX 3090') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "\n",
      "  | Name             | Type                      | Params\n",
      "---------------------------------------------------------------\n",
      "0 | _backbone        | CategoryEmbeddingBackbone | 11.4 K\n",
      "1 | _embedding_layer | Embedding1dLayer          | 14    \n",
      "2 | head             | LinearHead                | 33    \n",
      "3 | loss             | MSELoss                   | 0     \n",
      "---------------------------------------------------------------\n",
      "11.4 K    Trainable params\n",
      "0         Non-trainable params\n",
      "11.4 K    Total params\n",
      "0.046     Total estimated model params size (MB)\n",
      "Epoch: 1/300, Train loss: 677.8015, Val loss: 582.9557, Min val loss: 582.9557, Epoch time: 0.016s.\n",
      "Epoch: 20/300, Train loss: 353.7851, Val loss: 302.0203, Min val loss: 302.0203, Epoch time: 0.010s.\n",
      "Epoch: 40/300, Train loss: 85.0776, Val loss: 62.1153, Min val loss: 62.1153, Epoch time: 0.010s.\n",
      "Epoch: 60/300, Train loss: 45.2654, Val loss: 34.2778, Min val loss: 34.2691, Epoch time: 0.010s.\n",
      "Epoch: 80/300, Train loss: 33.9537, Val loss: 26.8622, Min val loss: 26.8622, Epoch time: 0.010s.\n",
      "Epoch: 100/300, Train loss: 26.9038, Val loss: 23.2417, Min val loss: 23.2372, Epoch time: 0.009s.\n",
      "Epoch: 120/300, Train loss: 24.9622, Val loss: 20.4360, Min val loss: 20.4360, Epoch time: 0.011s.\n",
      "Epoch: 140/300, Train loss: 24.1636, Val loss: 19.4010, Min val loss: 19.4010, Epoch time: 0.009s.\n",
      "Epoch: 160/300, Train loss: 22.9200, Val loss: 18.0232, Min val loss: 17.9749, Epoch time: 0.008s.\n",
      "Epoch: 180/300, Train loss: 19.7677, Val loss: 16.9469, Min val loss: 16.9469, Epoch time: 0.009s.\n",
      "Epoch: 200/300, Train loss: 17.9390, Val loss: 16.6545, Min val loss: 16.4093, Epoch time: 0.012s.\n",
      "Epoch: 220/300, Train loss: 19.4496, Val loss: 15.4451, Min val loss: 15.1788, Epoch time: 0.011s.\n",
      "Epoch: 240/300, Train loss: 16.0483, Val loss: 14.5508, Min val loss: 14.5508, Epoch time: 0.010s.\n",
      "Epoch: 260/300, Train loss: 16.4672, Val loss: 13.8354, Min val loss: 13.8354, Epoch time: 0.019s.\n",
      "Epoch: 280/300, Train loss: 13.6031, Val loss: 12.9315, Min val loss: 12.9315, Epoch time: 0.017s.\n",
      "Epoch: 300/300, Train loss: 16.5369, Val loss: 12.3673, Min val loss: 12.3673, Epoch time: 0.010s.\n",
      "`Trainer.fit` stopped: `max_epochs=300` reached.\n",
      "2023-09-23 20:37:37,327 - {pytorch_tabular.tabular_model:584} - INFO - Training the model completed\n",
      "2023-09-23 20:37:37,327 - {pytorch_tabular.tabular_model:1258} - INFO - Loading the best model\n",
      "/home/xlluo/anaconda3/envs/tabular_ensemble/lib/python3.10/site-packages/pytorch_lightning/utilities/cloud_io.py:33: LightningDeprecationWarning: `pytorch_lightning.utilities.cloud_io.get_filesystem` has been deprecated in v1.8.0 and will be removed in v2.0.0. Please use `lightning_fabric.utilities.cloud_io.get_filesystem` instead.\n",
      "  rank_zero_deprecation(\n",
      "Training mse loss: 11.25175\n",
      "Validation mse loss: 12.36725\n",
      "Testing mse loss: 7.83801\n",
      "Trainer saved. To load the trainer, run trainer = load_trainer(path='/tmp/tmphxmeot_i/output/auto-mpg/2023-09-23-20-37-29-0_UserInputConfig/trainer.pkl')\n",
      "\n",
      "-------------PytorchTabular End-------------\n",
      "\n",
      "PytorchTabular metrics\n",
      "Category Embedding 1/1\n",
      "Trainer saved. To load the trainer, run trainer = load_trainer(path='/tmp/tmphxmeot_i/output/auto-mpg/2023-09-23-20-37-29-0_UserInputConfig/trainer.pkl')\n"
     ]
    },
    {
     "data": {
      "text/plain": "          Program               Model  Training RMSE  Training MSE  \\\n0  PytorchTabular  Category Embedding       3.354362     11.251746   \n\n   Training MAE  Training MAPE  Training R2  Training MEDIAN_ABSOLUTE_ERROR  \\\n0      2.445915       0.101659     0.825442                        1.775388   \n\n   Training EXPLAINED_VARIANCE_SCORE  Testing RMSE  ...  Testing R2  \\\n0                           0.854523      2.799644  ...    0.854221   \n\n   Testing MEDIAN_ABSOLUTE_ERROR  Testing EXPLAINED_VARIANCE_SCORE  \\\n0                       1.963455                          0.888258   \n\n   Validation RMSE  Validation MSE  Validation MAE  Validation MAPE  \\\n0          3.51671        12.36725        2.731159         0.125136   \n\n   Validation R2  Validation MEDIAN_ABSOLUTE_ERROR  \\\n0       0.779071                          2.375105   \n\n   Validation EXPLAINED_VARIANCE_SCORE  \n0                             0.808039  \n\n[1 rows x 23 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Program</th>\n      <th>Model</th>\n      <th>Training RMSE</th>\n      <th>Training MSE</th>\n      <th>Training MAE</th>\n      <th>Training MAPE</th>\n      <th>Training R2</th>\n      <th>Training MEDIAN_ABSOLUTE_ERROR</th>\n      <th>Training EXPLAINED_VARIANCE_SCORE</th>\n      <th>Testing RMSE</th>\n      <th>...</th>\n      <th>Testing R2</th>\n      <th>Testing MEDIAN_ABSOLUTE_ERROR</th>\n      <th>Testing EXPLAINED_VARIANCE_SCORE</th>\n      <th>Validation RMSE</th>\n      <th>Validation MSE</th>\n      <th>Validation MAE</th>\n      <th>Validation MAPE</th>\n      <th>Validation R2</th>\n      <th>Validation MEDIAN_ABSOLUTE_ERROR</th>\n      <th>Validation EXPLAINED_VARIANCE_SCORE</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>PytorchTabular</td>\n      <td>Category Embedding</td>\n      <td>3.354362</td>\n      <td>11.251746</td>\n      <td>2.445915</td>\n      <td>0.101659</td>\n      <td>0.825442</td>\n      <td>1.775388</td>\n      <td>0.854523</td>\n      <td>2.799644</td>\n      <td>...</td>\n      <td>0.854221</td>\n      <td>1.963455</td>\n      <td>0.888258</td>\n      <td>3.51671</td>\n      <td>12.36725</td>\n      <td>2.731159</td>\n      <td>0.125136</td>\n      <td>0.779071</td>\n      <td>2.375105</td>\n      <td>0.808039</td>\n    </tr>\n  </tbody>\n</table>\n<p>1 rows × 23 columns</p>\n</div>"
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train(stderr_to_stdout=True)\n",
    "trainer.get_leaderboard()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "To activate Bayesian hyperparameter optimization, change the value of `bayes_opt` to `True` in the configuration file or later in `trainer.args`. By improving the MSE loss (default for regression tasks) or log loss (default for classification tasks) on the validation set, the performance on the testing set might improve.\n",
    "\n",
    "**Remark**: But the improvement is not always guaranteed (neither on the validation set nor on the testing set):\n",
    "\n",
    "1. The number of epochs (the configuration `bayes_epoch`) used in an optimization iteration is less than that used in the final formal training.\n",
    "2. The validation set and the testing set might come from different distributions, which forms the so-called observational bias.\n",
    "\n",
    "The hyperparameters obtained by Bayesian hyperparameter optimization are saved and will be loaded and used automatically in the future."
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "-------------Run PytorchTabular-------------\n",
      "\n",
      "Training Category Embedding\n",
      "Bayes-opt 1/30, tot 0.62s, avg 0.62s/it: {'ls': 349.8274, 'param': [0.0, 0.1, 0.001, 0.0, 1024], 'min ls': 349.8274, 'min param': [0.0, 0.1, 0.001, 0.0, 1024], 'min at': 1}\n",
      "Bayes-opt 2/30, tot 1.16s, avg 0.58s/it: {'ls': 38.2165, 'param': [0.29642, 0.42213, 0.02068, 0.00333, 512], 'min ls': 38.2165, 'min param': [0.29642, 0.42213, 0.02068, 0.00333, 512], 'min at': 2}\n",
      "Bayes-opt 3/30, tot 1.81s, avg 0.60s/it: {'ls': 629.1395, 'param': [0.19219, 0.14877, 0.00014, 0.0, 256], 'min ls': 38.2165, 'min param': [0.29642, 0.42213, 0.02068, 0.00333, 512], 'min at': 2}\n",
      "Bayes-opt 4/30, tot 2.26s, avg 0.56s/it: {'ls': 324.7059, 'param': [0.40608, 0.23999, 0.00115, 0.00273, 256], 'min ls': 38.2165, 'min param': [0.29642, 0.42213, 0.02068, 0.00333, 512], 'min at': 2}\n",
      "Bayes-opt 5/30, tot 2.60s, avg 0.52s/it: {'ls': 30.744, 'param': [0.32409, 0.18412, 0.03831, 0.0, 1024], 'min ls': 30.744, 'min param': [0.32409, 0.18412, 0.03831, 0.0, 1024], 'min at': 5}\n",
      "Bayes-opt 6/30, tot 3.05s, avg 0.51s/it: {'ls': 60.8855, 'param': [0.2368, 0.40046, 0.00254, 0.00017, 1024], 'min ls': 30.744, 'min param': [0.32409, 0.18412, 0.03831, 0.0, 1024], 'min at': 5}\n",
      "Bayes-opt 7/30, tot 3.48s, avg 0.50s/it: {'ls': 48.1221, 'param': [0.29101, 0.26869, 0.01116, 0.0, 256], 'min ls': 30.744, 'min param': [0.32409, 0.18412, 0.03831, 0.0, 1024], 'min at': 5}\n",
      "Bayes-opt 8/30, tot 3.97s, avg 0.50s/it: {'ls': 544.2315, 'param': [0.09317, 0.36846, 0.00038, 0.0, 256], 'min ls': 30.744, 'min param': [0.32409, 0.18412, 0.03831, 0.0, 1024], 'min at': 5}\n",
      "Bayes-opt 9/30, tot 4.50s, avg 0.50s/it: {'ls': 315.017, 'param': [0.07484, 0.11116, 0.0011, 0.00889, 256], 'min ls': 30.744, 'min param': [0.32409, 0.18412, 0.03831, 0.0, 1024], 'min at': 5}\n",
      "Bayes-opt 10/30, tot 5.02s, avg 0.50s/it: {'ls': 616.4534, 'param': [0.30653, 0.45117, 0.00019, 0.02928, 512], 'min ls': 30.744, 'min param': [0.32409, 0.18412, 0.03831, 0.0, 1024], 'min at': 5}\n",
      "Bayes-opt 11/30, tot 6.17s, avg 0.56s/it: {'ls': 26.3264, 'param': [0.08545, 0.17908, 0.01062, 5e-05, 256], 'min ls': 26.3264, 'min param': [0.08545, 0.17908, 0.01062, 5e-05, 256], 'min at': 11}\n",
      "Bayes-opt 12/30, tot 7.01s, avg 0.58s/it: {'ls': 27.8408, 'param': [0.01454, 0.19714, 0.01039, 0.00065, 512], 'min ls': 26.3264, 'min param': [0.08545, 0.17908, 0.01062, 5e-05, 256], 'min at': 11}\n",
      "Bayes-opt 13/30, tot 7.89s, avg 0.61s/it: {'ls': 14.8487, 'param': [0.0, 0.0, 0.00464, 0.05, 64], 'min ls': 14.8487, 'min param': [0.0, 0.0, 0.00464, 0.05, 64], 'min at': 13}\n",
      "Bayes-opt 14/30, tot 8.85s, avg 0.63s/it: {'ls': 15.9753, 'param': [0.0, 0.5, 0.05, 0.05, 64], 'min ls': 14.8487, 'min param': [0.0, 0.0, 0.00464, 0.05, 64], 'min at': 13}\n",
      "Bayes-opt 15/30, tot 10.18s, avg 0.68s/it: {'ls': 9.0207, 'param': [0.0, 0.0, 0.03287, 0.05, 64], 'min ls': 9.0207, 'min param': [0.0, 0.0, 0.03287, 0.05, 64], 'min at': 15}\n",
      "Bayes-opt 16/30, tot 10.91s, avg 0.68s/it: {'ls': 42.5235, 'param': [0.0, 0.5, 0.03274, 0.0, 2048], 'min ls': 9.0207, 'min param': [0.0, 0.0, 0.03287, 0.05, 64], 'min at': 15}\n",
      "Bayes-opt 17/30, tot 11.98s, avg 0.70s/it: {'ls': 8.8255, 'param': [0.0, 0.0, 0.05, 0.05, 64], 'min ls': 8.8255, 'min param': [0.0, 0.0, 0.05, 0.05, 64], 'min at': 17}\n",
      "Bayes-opt 18/30, tot 12.99s, avg 0.72s/it: {'ls': 15.7142, 'param': [0.5, 0.0, 0.05, 0.05, 64], 'min ls': 8.8255, 'min param': [0.0, 0.0, 0.05, 0.05, 64], 'min at': 17}\n",
      "Bayes-opt 19/30, tot 13.98s, avg 0.74s/it: {'ls': 9.2985, 'param': [0.0, 0.0, 0.04368, 0.05, 64], 'min ls': 8.8255, 'min param': [0.0, 0.0, 0.05, 0.05, 64], 'min at': 17}\n",
      "Bayes-opt 20/30, tot 14.99s, avg 0.75s/it: {'ls': 9.3999, 'param': [0.0, 0.0, 0.04339, 0.05, 64], 'min ls': 8.8255, 'min param': [0.0, 0.0, 0.05, 0.05, 64], 'min at': 17}\n",
      "Bayes-opt 21/30, tot 15.95s, avg 0.76s/it: {'ls': 11.0562, 'param': [0.0, 0.0, 0.01061, 0.05, 64], 'min ls': 8.8255, 'min param': [0.0, 0.0, 0.05, 0.05, 64], 'min at': 17}\n",
      "Bayes-opt 22/30, tot 16.91s, avg 0.77s/it: {'ls': 8.7002, 'param': [0.0, 0.0, 0.04072, 0.05, 64], 'min ls': 8.7002, 'min param': [0.0, 0.0, 0.04072, 0.05, 64], 'min at': 22}\n",
      "/home/xlluo/anaconda3/envs/tabular_ensemble/lib/python3.10/site-packages/skopt/optimizer/optimizer.py:449: UserWarning: The objective has been evaluated at this point before.\n",
      "  warnings.warn(\"The objective has been evaluated \"\n",
      "Bayes-opt 23/30, tot 17.95s, avg 0.78s/it: {'ls': 8.8255, 'param': [0.0, 0.0, 0.05, 0.05, 64], 'min ls': 8.7002, 'min param': [0.0, 0.0, 0.04072, 0.05, 64], 'min at': 22}\n",
      "Bayes-opt 24/30, tot 18.89s, avg 0.79s/it: {'ls': 40.0748, 'param': [0.0, 0.0, 0.00441, 0.05, 512], 'min ls': 8.7002, 'min param': [0.0, 0.0, 0.04072, 0.05, 64], 'min at': 22}\n",
      "Bayes-opt 25/30, tot 20.11s, avg 0.80s/it: {'ls': 8.7661, 'param': [0.0855, 0.0, 0.03797, 0.05, 64], 'min ls': 8.7002, 'min param': [0.0, 0.0, 0.04072, 0.05, 64], 'min at': 22}\n",
      "Bayes-opt 26/30, tot 21.30s, avg 0.82s/it: {'ls': 76.1117, 'param': [0.12949, 0.5, 0.00367, 0.05, 64], 'min ls': 8.7002, 'min param': [0.0, 0.0, 0.04072, 0.05, 64], 'min at': 22}\n",
      "Bayes-opt 27/30, tot 22.52s, avg 0.83s/it: {'ls': 9.3727, 'param': [0.0, 0.0, 0.03705, 0.05, 64], 'min ls': 8.7002, 'min param': [0.0, 0.0, 0.04072, 0.05, 64], 'min at': 22}\n",
      "Bayes-opt 28/30, tot 23.60s, avg 0.84s/it: {'ls': 13.069, 'param': [0.0, 0.0, 0.00735, 0.05, 64], 'min ls': 8.7002, 'min param': [0.0, 0.0, 0.04072, 0.05, 64], 'min at': 22}\n",
      "Bayes-opt 29/30, tot 24.81s, avg 0.86s/it: {'ls': 9.1393, 'param': [0.0, 0.06589, 0.05, 0.05, 64], 'min ls': 8.7002, 'min param': [0.0, 0.0, 0.04072, 0.05, 64], 'min at': 22}\n",
      "Bayes-opt 30/30, tot 25.73s, avg 0.86s/it: {'ls': 16.2183, 'param': [0.0, 0.0, 0.0118, 0.05, 128], 'min ls': 8.7002, 'min param': [0.0, 0.0, 0.04072, 0.05, 64], 'min at': 22}\n",
      "Previous params loaded: {'dropout': 0.0, 'embedding_dropout': 0.0, 'lr': 0.040720275962475405, 'weight_decay': 0.04999999999999999, 'batch_size': 64, 'original_batch_size': 64}\n",
      "Global seed set to 42\n",
      "2023-09-23 20:38:03,278 - {pytorch_tabular.tabular_model:473} - INFO - Preparing the DataLoaders\n",
      "2023-09-23 20:38:03,279 - {pytorch_tabular.tabular_datamodule:290} - INFO - Setting up the datamodule for regression task\n",
      "2023-09-23 20:38:03,290 - {pytorch_tabular.tabular_model:521} - INFO - Preparing the Model: CategoryEmbeddingModel\n",
      "2023-09-23 20:38:03,303 - {pytorch_tabular.tabular_model:268} - INFO - Preparing the Trainer\n",
      "Auto select gpus: [0]\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "2023-09-23 20:38:03,319 - {pytorch_tabular.tabular_model:582} - INFO - Training Started\n",
      "You are using a CUDA device ('NVIDIA GeForce RTX 3090') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "\n",
      "  | Name             | Type                      | Params\n",
      "---------------------------------------------------------------\n",
      "0 | _backbone        | CategoryEmbeddingBackbone | 11.4 K\n",
      "1 | _embedding_layer | Embedding1dLayer          | 14    \n",
      "2 | head             | LinearHead                | 33    \n",
      "3 | loss             | MSELoss                   | 0     \n",
      "---------------------------------------------------------------\n",
      "11.4 K    Trainable params\n",
      "0         Non-trainable params\n",
      "11.4 K    Total params\n",
      "0.046     Total estimated model params size (MB)\n",
      "Epoch: 1/300, Train loss: 354.9424, Val loss: 140.1831, Min val loss: 140.1831, Epoch time: 0.031s.\n",
      "Epoch: 20/300, Train loss: 9.6374, Val loss: 8.8711, Min val loss: 8.7002, Epoch time: 0.020s.\n",
      "Epoch: 40/300, Train loss: 7.3040, Val loss: 9.2379, Min val loss: 8.6125, Epoch time: 0.024s.\n",
      "Epoch: 60/300, Train loss: 7.6705, Val loss: 12.4185, Min val loss: 8.3944, Epoch time: 0.021s.\n",
      "Epoch: 80/300, Train loss: 9.2612, Val loss: 9.6877, Min val loss: 8.1813, Epoch time: 0.022s.\n",
      "Epoch: 100/300, Train loss: 11.8986, Val loss: 13.2131, Min val loss: 8.1813, Epoch time: 0.020s.\n",
      "Epoch: 120/300, Train loss: 14.2237, Val loss: 9.1487, Min val loss: 8.1813, Epoch time: 0.021s.\n",
      "Epoch: 140/300, Train loss: 11.3343, Val loss: 9.3349, Min val loss: 8.1813, Epoch time: 0.024s.\n",
      "Epoch: 160/300, Train loss: 6.1817, Val loss: 8.9987, Min val loss: 8.1813, Epoch time: 0.021s.\n",
      "2023-09-23 20:38:07,718 - {pytorch_tabular.tabular_model:584} - INFO - Training the model completed\n",
      "2023-09-23 20:38:07,719 - {pytorch_tabular.tabular_model:1258} - INFO - Loading the best model\n",
      "Training mse loss: 5.30917\n",
      "Validation mse loss: 8.18127\n",
      "Testing mse loss: 4.27680\n",
      "Trainer saved. To load the trainer, run trainer = load_trainer(path='/tmp/tmphxmeot_i/output/auto-mpg/2023-09-23-20-37-29-0_UserInputConfig/trainer.pkl')\n",
      "\n",
      "-------------PytorchTabular End-------------\n",
      "\n",
      "PytorchTabular metrics\n",
      "Category Embedding 1/1\n",
      "Trainer saved. To load the trainer, run trainer = load_trainer(path='/tmp/tmphxmeot_i/output/auto-mpg/2023-09-23-20-37-29-0_UserInputConfig/trainer.pkl')\n"
     ]
    },
    {
     "data": {
      "text/plain": "          Program               Model  Training RMSE  Training MSE  \\\n0  PytorchTabular  Category Embedding       2.304163      5.309167   \n\n   Training MAE  Training MAPE  Training R2  Training MEDIAN_ABSOLUTE_ERROR  \\\n0      1.631622       0.065262     0.917634                         1.16303   \n\n   Training EXPLAINED_VARIANCE_SCORE  Testing RMSE  ...  Testing R2  \\\n0                           0.919889      2.068042  ...    0.920456   \n\n   Testing MEDIAN_ABSOLUTE_ERROR  Testing EXPLAINED_VARIANCE_SCORE  \\\n0                       1.294664                          0.923491   \n\n   Validation RMSE  Validation MSE  Validation MAE  Validation MAPE  \\\n0         2.860293        8.181274        2.009064         0.087603   \n\n   Validation R2  Validation MEDIAN_ABSOLUTE_ERROR  \\\n0        0.85385                           1.49169   \n\n   Validation EXPLAINED_VARIANCE_SCORE  \n0                             0.854234  \n\n[1 rows x 23 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Program</th>\n      <th>Model</th>\n      <th>Training RMSE</th>\n      <th>Training MSE</th>\n      <th>Training MAE</th>\n      <th>Training MAPE</th>\n      <th>Training R2</th>\n      <th>Training MEDIAN_ABSOLUTE_ERROR</th>\n      <th>Training EXPLAINED_VARIANCE_SCORE</th>\n      <th>Testing RMSE</th>\n      <th>...</th>\n      <th>Testing R2</th>\n      <th>Testing MEDIAN_ABSOLUTE_ERROR</th>\n      <th>Testing EXPLAINED_VARIANCE_SCORE</th>\n      <th>Validation RMSE</th>\n      <th>Validation MSE</th>\n      <th>Validation MAE</th>\n      <th>Validation MAPE</th>\n      <th>Validation R2</th>\n      <th>Validation MEDIAN_ABSOLUTE_ERROR</th>\n      <th>Validation EXPLAINED_VARIANCE_SCORE</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>PytorchTabular</td>\n      <td>Category Embedding</td>\n      <td>2.304163</td>\n      <td>5.309167</td>\n      <td>1.631622</td>\n      <td>0.065262</td>\n      <td>0.917634</td>\n      <td>1.16303</td>\n      <td>0.919889</td>\n      <td>2.068042</td>\n      <td>...</td>\n      <td>0.920456</td>\n      <td>1.294664</td>\n      <td>0.923491</td>\n      <td>2.860293</td>\n      <td>8.181274</td>\n      <td>2.009064</td>\n      <td>0.087603</td>\n      <td>0.85385</td>\n      <td>1.49169</td>\n      <td>0.854234</td>\n    </tr>\n  </tbody>\n</table>\n<p>1 rows × 23 columns</p>\n</div>"
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import warnings\n",
    "trainer.args[\"bayes_opt\"] = True\n",
    "trainer.args[\"bayes_epoch\"] = 20\n",
    "trainer.args[\"bayes_calls\"] = 30\n",
    "with warnings.catch_warnings():\n",
    "    warnings.filterwarnings(\"ignore\", module=\"pytorch_lightning\")\n",
    "    trainer.train(stderr_to_stdout=True)\n",
    "trainer.get_leaderboard()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}